diff --git a/.gitignore b/.gitignore index 9675516..8123d35 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ dist/ # Logs and runtime data *.log +*.pid /runtime/ /diagnostics/ /domain-api/runtime/ @@ -51,8 +52,27 @@ domain-api/deploy/multi-region/*.conf domainCheck/tools/node-v20.19.4-win-x64/ domainCheck/app/credentials.json domainCheck/credentials.json +domainCheck/runtime/ domainCheck/**/*.pkl domainCheck/domains.txt +thread_count.json +node_thread_counts.json +runtime_settings.json +runtime/runtime_settings.json +runtime/sensitive_words.json + +# Ops center generated night-run artifacts +docs/ops_center_runtime/night_runs/night_run_*/ +docs/ops_center_runtime/night_runs/night_run_*_summary.json +docs/ops_center_runtime/night_runs/*.log +docs/ops_center_runtime/night_runs/*.pid +docs/ops_center_runtime/night_runs/step_mix_*/ +docs/ops_center_runtime/chinaz_gray_runs/ +docs/_tmp_regression/ + +# Local probe / backup artifacts +.codex-release-probe.txt +*.bak_* # OS / editor .DS_Store diff --git a/docs/shenhe.md b/docs/shenhe.md new file mode 100644 index 0000000..29f6584 --- /dev/null +++ b/docs/shenhe.md @@ -0,0 +1,84 @@ +可以，按“尽量省额度但不丢高风险”的思路，我建议你把全项目审核拆成 3 轮，目标控制在 `800万 ~ 1200万 tokens`。 + +**先排除** +第一轮先不要审这些，不然额度会被白白吃掉： + +- `release/` +- `.venv/` +- `node_modules/` +- `domain-web/package-lock.json` +- `domainCheck/app/sdk_leg.js` +- `domainCheck/detect/sdk_leg.js` +- 运行时产物、日志、快照、历史 night runs +- 大 JSON 词库这类静态数据，除非代码直接依赖逻辑可疑 + +**三轮顺序** +1. 控制面与数据正确性 +预算：`300万 ~ 450万` +- `domain-api/app/services/detect_job_service.py` +- `domain-api/app/services/detect_service.py` +- `domain-api/app/services/runtime_status_service.py` +- `domain-api/app/services/dashboard.py` +- `domain-api/app/services/sync_record_service.py` +- `domain-api/app/services/sync_push_service.py` +- `domain-api/app/services/worker_control_service.py` +- `domain-api/app/services/settings_service.py` +- `domain-api/app/services/cluster_runtime_service.py` +- 对应 `routes/` 和关键测试 + +这一轮最值钱，因为它直接查： +- 页面显示为什么和现场不一致 +- sync 为什么会把旧状态盖新状态 +- runtime projection / queue health / active job 是否互相打架 +- 配置下发和节点实际执行是否一致 + +2. Worker 并发与执行链 +预算：`300万 ~ 400万` +- `domainCheck/detect_worker.py` +- `domainCheck/app/utils/database.py` +- `domainCheck/app/detectors/` +- `domainCheck/detect/` +- `domainCheck/tests/` 里和并发、连接池、超时、代理有关的测试 + +这一轮重点查： +- 多进程 / 多线程是否真能提升吞吐 +- DB 连接池、代理池、任务领取链有没有硬瓶颈 +- 内存为什么高、进程为什么空转 +- 超时、重试、降级逻辑是否会拖垮吞吐 + +3. 发布、运维、前端展示 +预算：`200万 ~ 300万` +- `domain-api/deploy/` +- `domain-api/app/node_agent.py` +- `domain-api/app/services/ops_*` +- `domain-web/src/views/detect/` +- `domain-web/src/views/runtime/` +- `domain-web/src/views/settings/` +- systemd 模板、迁移/发布脚本 + +这一轮重点查： +- 发布链和实际运行是否一致 +- 多实例 worker 的部署是否完整 +- 前端是否误导运维判断 +- 迁移、接管、rollout 有没有高风险坑 + +**模型建议** +- 第 1 轮：`gpt-5.4 + xhigh` +- 第 2 轮：`gpt-5.4 + xhigh` +- 第 3 轮：`gpt-5.4 + high` + +这样通常能把额度压在你要的区间里。 + +**输出方式** +每轮都只要这 3 类结果，最省额度： + +- `P0/P1` 真实问题 +- 影响面 +- 修复建议 + +不要第一轮就让模型写大篇架构说明，不然额度会烧很快。 + +一句话版： +先审 `domain-api` 的状态/同步链，再审 `domainCheck` 的并发执行链，最后审 `deploy + ops + 前端展示`；按这个顺序，`800万~1200万 tokens` 是有机会压住的。 + +如果你要，我下一步可以直接给你生成一份“第一轮审核清单”，精确到文件名单。 \ No newline at end of file diff --git a/docs/test.md b/docs/test.md index f99224f..6b22982 100644 --- a/docs/test.md +++ b/docs/test.md @@ -62,7 +62,65 @@ controller syncer 批量同步海外主库 ↓ controller finalizer 标记本次流程完成 +发现问题 +主体功能 +1.去后台点击 聚名获取删除域名入库 +2.按后台勾选 的检测选项，按顺序逐个去跑 +3.正常逻辑是按顺序 一步一步前面的处理完了，再往下一个处理，最大进程跑起来，直到所有任务跑完 +现在遇到的问题是： +1.后台展示文案的歧义非常大；运营很难理解 + 比喻托管节点：这个应该就是 服务器管理，现在你把进程也算是一个节点，全部放到这个列表，我看起来都蒙，如果需要你可以加多一个进程管理不就好了，不要混一起 + 现在后台应该有 机器/进程/线程，节点到底是啥？现在已经乱了， + + 运行配置这块也是 节点独立线程覆盖 你把所有进程都列出来配置 线程数量，这个不需要的，所有进程的线程数量全部走默认的，进程那么大不会人工管理的，设计很不合理， + + 检测管理页面：日志输出窗口 这块也是 + 参与节点：3 这个应该改成 参与 服务器 +参与进程：62 参与 进程 + 参与 线程 +运行中：127 +线程：127 / 74000（参与节点汇总） +要让人一眼看明白 + +2.最重要一点，后台页面的数据展示很多都是不符合实际的，很多一点，为啥黑名单一直是0，这块不肯定的，这个正常清空最少命中80%； +进程和线程一直跑不起来，这个最致命，优化了好几日了，知道目前还是没跑通完整流程 + + +3.概览 +步骤队列 +看每一步堆积、吞吐和失败，快速判断到底卡在注册、百度、360、爱站还是站长之家。 这块应该按后台勾选设置的顺序拍下来，正常任务完成也是，一个跑完才会往下推，才会往下一个跑 + + +4.检测的进程和线程 机器 不稳定，不会自动检测 自动跑起来，就是最大性能没有跑起来 + +5.如果大陆controller 的DB链接数量是瓶颈，那可以每个大陆 机器都开启db+redis 反正每个机器的配置都很高的，只要有效率能提速 + +6.还有一个但海外机器绝对不参与worker 检测 + + + + + + +一句话结论： + +线上 hotfix 收口基本完成 +下一步最该继续的是代理供给优化，不是回到代码审核 +我下一步建议就直接转到代理链路，继续收： + +为什么多实例同时刷新时会被代理源限流 +是否要继续压低单实例补货批量/频率 +是否要做更强的跨进程代理补货协调 + + + + + + + + +大陆处理好的跑完流程域名是否返回海外机器勾选状态 域名检测流程设计 diff --git a/docs/后台运行观察口径.md b/docs/后台运行观察口径.md new file mode 100644 index 0000000..d43812a --- /dev/null +++ b/docs/后台运行观察口径.md @@ -0,0 +1,229 @@ +# 后台运行观察口径 + +> 说明：这份文档主要解释页面怎么看。当前线上执行口径与判断顺序，统一以 [当前线上最终Runbook.md](/www/wwwroot/getDomain/docs/当前线上最终Runbook.md) 为准。 + +更新时间：`2026-04-24 14:24` + +## 先说结论 + +以后不要先盯散乱日志。 + +日志只是辅助定位。 + +现在优先看后台 `运行中心` 第一屏。 + +那里已经会直接告诉你： + +- 全库总盘子有多少 +- 当前活跃批次到底有多少 +- 近 15 分钟真实推进了多少 +- 当前到底算不算真跑起来了 +- 还有多少待处理 +- 真正有多少进程和线程在干活 +- 最近 15 分钟处理了多少 +- 失败多不多 +- 黑名单有没有推进 +- 当前最忙的是哪些进程 +- 主要积压卡在哪几个步骤 + +只有在后台页面临时不可用，或者你想在终端里持续盯时，才再用下面这条命令： + +```bash +cd /www/wwwroot/getDomain +./.venv/bin/python tools/runtime_observer.py +``` + +如果想持续盯着看，就用： + +```bash +cd /www/wwwroot/getDomain +./.venv/bin/python tools/runtime_observer.py --watch 5 +``` + +它会把和后台首屏同一套核心数字收成一屏，不用再自己去拼： + +- 当前任务是谁 +- 还有多少待处理 +- 真正有多少进程和线程在干活 +- 最近 15 分钟到底处理了多少 +- 失败多不多 +- 黑名单有没有推进 +- 哪几个节点真的在跑 +- 每一步卡在哪 + +## 先看哪 4 组数 + +脚本里最重要的是这 4 行： + +### 1. 任务积压 + +看： + +- `pending` +- `claimed` +- `running` + +这组数回答的是： + +- 还有多少活没做 +- 有没有任务已经被领走 +- 当前有多少任务处于运行中 + +### 2. 结果产出 + +看： + +- `completed` +- `failed` +- `blacklisted` + +这组数回答的是： + +- 成功推进了多少 +- 失败了多少 +- 黑名单命中了多少 + +### 3. 执行面 + +看： + +- `active_processes` +- `active_threads` +- `max_threads` + +这组数回答的是： + +- 现在到底有多少执行实例真的在动 +- 当前用了多少线程 +- 当前理论上限是多少 + +### 4. 最近吞吐 + +看： + +- `processed_recent` +- `per_minute` +- `failed_recent` +- `blacklisted_recent` + +这组数回答的是： + +- 最近 15 分钟有没有真实推进 +- 大概每分钟能跑多少 +- 最近失败是不是很多 +- 最近有没有新的黑名单命中 + +## 怎么判断“真跑起来了” + +满足下面这 3 条，才算真跑： + +1. `pending` 还有积压 +2. `active_threads` 不是 0 +3. `processed_recent` 持续增长 + +如果只是页面上显示 `running`，但： + +- `active_threads` 接近 0 +- `processed_recent` 也接近 0 + +那更像是： + +- 旧状态残影 +- 口径没收准 +- 或者任务挂着但没真正消化 + +## 怎么快速判断卡在哪 + +### 情况 1：`pending` 很高，`active_threads` 很低 + +说明更像： + +- 没真正拉起执行面 +- worker 没开始干活 +- 或者任务没正确分发进去 + +### 情况 2：`active_threads` 不低，但 `failed_recent` 很高 + +说明更像： + +- 不是没跑 +- 而是外部步骤在大量失败 +- 常见就是代理、RDAP、爱站、时光机这类链路超时 + +### 情况 3：`completed_recent` 很低，`blacklisted_recent` 也很低 + +说明更像： + +- 线程虽然在跑 +- 但结果大多没形成有效推进 +- 更像在外部失败里空转 + +### 情况 4：节点列表里只有一台在有动作 + +说明更像： + +- 当前真正承担执行的就一台 +- 其他节点可能只是在线 +- 或者只是历史残影，不是真正参与执行 + +## 为什么不要先盯日志 + +因为日志只能回答： + +- 某一步报了什么错 +- 某个实例刚才做了什么 + +但它回答不了： + +- 现在到底有多少进程真在跑 +- 一共跑成功了多少 +- 黑名单推进了多少 +- 当前整体吞吐是上升还是下降 + +所以顺序应该固定成： + +1. 先看后台 `运行中心` 第一屏 +2. 再看节点列表和步骤分布 +3. 只有需要深挖时，再看 `runtime_observer.py` 或具体日志 + +## 一句话记法 + +先看： + +- 有没有积压 +- 有没有执行面 +- 最近有没有吞吐 + +这 3 个一起动，才算真的在跑。 + +## 现在先看哪 3 块 + +后台第一屏已经拆成这 3 个核心口径： + +### 1. 总盘子（全库） + +回答的是： + +- 海外主库现在一共有多少域名 +- 全库还有多少待检测 +- 全库已经通过、失败、黑名单各多少 + +### 2. 当前活跃批次 + +回答的是： + +- 当前正在跑的是哪一批 +- 这一批的展示口径有多少 +- 这一批的原始口径有多少 + +这一块非常重要。 + +它不是全库总量。 + +### 3. 近窗吞吐 + +回答的是： + +- 最近 15 分钟到底推进了多少 +- 每分钟大概多少 +- 最近完成、失败、黑名单各多少 diff --git a/docs/审核顺序.md b/docs/审核顺序.md new file mode 100644 index 0000000..75f8c38 --- /dev/null +++ b/docs/审核顺序.md @@ -0,0 +1,103 @@ +# 第一轮审核清单 + +第一轮目标：先查控制面、运行态聚合、同步链、节点控制和配置分发。 + +建议模型与强度： +- `gpt-5.4` +- `xhigh` + +建议额度： +- `300万 ~ 450万 tokens` + +本轮不看： +- `release/` +- `.venv/` +- `node_modules/` +- 运行时日志、快照、night runs +- 大型静态资源和锁文件 + +## 审核顺序 + +### P0：运行态与页面口径 +1. [domain-api/app/services/detect_job_service.py](/www/wwwroot/getDomain/domain-api/app/services/detect_job_service.py) + 重点看：`active_job`、`runtime_snapshot`、`runtime_ingest`、`distributed_node_stats`、`display_*` 字段是否会互相覆盖。 +2. [domain-api/app/services/detect_service.py](/www/wwwroot/getDomain/domain-api/app/services/detect_service.py) + 重点看：检测控制页口径、聚合线程数、参与节点/参与进程统计、零值回退逻辑。 +3. [domain-api/app/services/runtime_status_service.py](/www/wwwroot/getDomain/domain-api/app/services/runtime_status_service.py) + 重点看：运行中心聚合、`queue_health` 与 backlog 对齐、活跃任务判定。 +4. [domain-api/app/services/dashboard.py](/www/wwwroot/getDomain/domain-api/app/services/dashboard.py) + 重点看：首页总览是否复用旧快照、`queue_display_*` 和 `active_job` 是否一致。 +5. [domain-api/app/api/routes/detect.py](/www/wwwroot/getDomain/domain-api/app/api/routes/detect.py) + 重点看：检测控制接口有没有直接透传脏口径。 +6. [domain-api/app/api/routes/runtime.py](/www/wwwroot/getDomain/domain-api/app/api/routes/runtime.py) + 重点看：运行中心接口是否二次加工错误。 +7. [domain-api/app/api/routes/dashboard.py](/www/wwwroot/getDomain/domain-api/app/api/routes/dashboard.py) + 重点看：首页口径是否和服务层一致。 + +### P0：同步链与投影链 +8. [domain-api/app/services/sync_record_service.py](/www/wwwroot/getDomain/domain-api/app/services/sync_record_service.py) + 重点看：`runtime_projection`、`runtime_ingest`、未来时间记录、重复投影、自愈逻辑。 +9. [domain-api/app/services/sync_push_service.py](/www/wwwroot/getDomain/domain-api/app/services/sync_push_service.py) + 重点看：同步触发、推送条件、失败重试、是否会把旧状态推成新状态。 +10. [domain-api/app/services/cluster_runtime_service.py](/www/wwwroot/getDomain/domain-api/app/services/cluster_runtime_service.py) + 重点看：集群节点汇总、心跳、stale/offline 判定。 +11. [domain-api/app/services/debug_event_service.py](/www/wwwroot/getDomain/domain-api/app/services/debug_event_service.py) + 重点看：调试事件是否参与运行态推断，是否会误导当前任务。 + +### P0：节点控制与配置分发 +12. [domain-api/app/services/worker_control_service.py](/www/wwwroot/getDomain/domain-api/app/services/worker_control_service.py) + 重点看：worker 进程探测、当前状态读取、systemd 口径和实际进程口径是否一致。 +13. [domain-api/app/services/settings_service.py](/www/wwwroot/getDomain/domain-api/app/services/settings_service.py) + 重点看：`process_count`、`thread_count`、节点级覆盖、多实例父子节点映射。 +14. [domain-api/app/services/runtime_settings_service.py](/www/wwwroot/getDomain/domain-api/app/services/runtime_settings_service.py) + 重点看：运行态配置来源、热配置优先级、页面修改后是否真生效。 +15. [domain-api/app/services/runtime_control_service.py](/www/wwwroot/getDomain/domain-api/app/services/runtime_control_service.py) + 重点看：启动/停止/恢复对多实例 worker 是否安全。 + +### P1：检测任务主链 +16. [domain-api/app/services/detect_run_service.py](/www/wwwroot/getDomain/domain-api/app/services/detect_run_service.py) + 重点看：运行记录、cycle 事件、页面日志来源。 +17. [domain-api/app/services/domains_service.py](/www/wwwroot/getDomain/domain-api/app/services/domains_service.py) + 重点看：域名主数据是否和检测任务状态有交叉写入风险。 +18. [domain-api/app/services/import_task_service.py](/www/wwwroot/getDomain/domain-api/app/services/import_task_service.py) + 重点看：导入任务是否影响 backlog、是否能造成页面统计失真。 + +## 必看测试 + +### 直接对应运行态/聚合 +1. [domain-api/tests/test_detect_job_service.py](/www/wwwroot/getDomain/domain-api/tests/test_detect_job_service.py) +2. [domain-api/tests/test_detect_service_status_fallback.py](/www/wwwroot/getDomain/domain-api/tests/test_detect_service_status_fallback.py) +3. [domain-api/tests/test_runtime_status_service.py](/www/wwwroot/getDomain/domain-api/tests/test_runtime_status_service.py) +4. [domain-api/tests/test_dashboard_service.py](/www/wwwroot/getDomain/domain-api/tests/test_dashboard_service.py) + +### 直接对应同步链 +5. [domain-api/tests/test_sync_record_service.py](/www/wwwroot/getDomain/domain-api/tests/test_sync_record_service.py) +6. [domain-api/tests/test_sync_push_service.py](/www/wwwroot/getDomain/domain-api/tests/test_sync_push_service.py) +7. [domain-api/tests/test_cluster_runtime_service.py](/www/wwwroot/getDomain/domain-api/tests/test_cluster_runtime_service.py) + +### 直接对应节点控制与配置 +8. [domain-api/tests/test_worker_control_service.py](/www/wwwroot/getDomain/domain-api/tests/test_worker_control_service.py) +9. [domain-api/tests/test_settings_service.py](/www/wwwroot/getDomain/domain-api/tests/test_settings_service.py) +10. [domain-api/tests/test_detect_api_routes.py](/www/wwwroot/getDomain/domain-api/tests/test_detect_api_routes.py) +11. [domain-api/tests/test_ops_api_routes.py](/www/wwwroot/getDomain/domain-api/tests/test_ops_api_routes.py) + +## 本轮输出要求 + +只输出这 3 类内容，避免烧额度： +- `P0 / P1` 真实问题，带文件和行号 +- 影响范围 +- 修复建议 + +不要在第一轮做这些： +- 大篇架构说明 +- 文件逐段复述 +- UI 细节优化建议 +- 发布脚本和迁移功能深挖 + +## 第一轮完成标准 + +满足以下条件就可以结束第一轮，转第二轮： +- 能回答“为什么页面显示会和现场不一致” +- 能回答“为什么 sync 会把旧运行态覆盖成新页面口径” +- 能回答“进程/线程配置、页面显示、节点实际执行三者有没有断层” +- 能列出前 `10` 个最值得先修的 `P0/P1` 问题 diff --git a/docs/当前已验证有效的线上参数与改动清单.md b/docs/当前已验证有效的线上参数与改动清单.md new file mode 100644 index 0000000..6db506a --- /dev/null +++ b/docs/当前已验证有效的线上参数与改动清单.md @@ -0,0 +1,180 @@ +# 当前已验证有效的线上参数与改动清单 + +> 说明：这份文档保留作阶段留档。后续执行和观察，统一以 [当前线上最终Runbook.md](/www/wwwroot/getDomain/docs/当前线上最终Runbook.md) 为准。 + +最后更新：2026-04-25 17:30 左右 +适用节点：`mainland-controller-01` +目标：给后续观察和继续优化留一个“当前已经验证有效”的固定基线，不再靠聊天记录回忆。 + +## 1. 当前线上基底 + +- 当前 `current` 指向： + `/opt/domaincheck/releases/domaincheck_release_20260424_220158` +- 当前核心服务状态： + - `domaincheck-worker.service = active` + - `domaincheck-sync-agent.service = active` + - `domaincheck-api.service = active` + +说明： +- 这台机器现在不是只跑旧 release 原样代码，而是“`220158` 基底 + 多轮热补”。 +- 后续如果要正式固化，应该把当前热补内容重新打一版正式 release。 + +## 2. 当前已确认生效的远端环境参数 + +以下参数已经在运行中的 `detect_worker.py` 进程环境里确认过，不是只改了文件没重启： + +### 单机性能/爱站相关 + +- `DOMAINCHECK_REGISTER_SINGLE_MACHINE_MODE=1` +- `DOMAINCHECK_SINGLE_MACHINE_AIZHAN_DIRECT_FIRST=1` +- `DOMAINCHECK_AIZHAN_REMOTE_DISCONNECT_DEGRADE=1` +- `DOMAINCHECK_AIZHAN_EXTERNAL_FAST_DEGRADE=1` +- `DOMAINCHECK_PROXY_STEP_MAX_ATTEMPTS_AIZHAN=2` +- `DOMAINCHECK_PROXY_STEP_MAX_SECONDS_AIZHAN=10` +- `DOMAINCHECK_AIZHAN_TIMEOUT_PROXY=1.8` +- `DOMAINCHECK_AIZHAN_TIMEOUT_DIRECT=2.4` + +### Wayback 相关 + +- `DOMAINCHECK_PROXY_STEP_MAX_ATTEMPTS_WAYBACK=2` +- `DOMAINCHECK_PROXY_STEP_MAX_SECONDS_WAYBACK=8` +- `WAYBACK_CDX_TIMEOUT=3` +- `WAYBACK_SNAPSHOT_TIMEOUT=2` +- `WAYBACK_RETRY_COUNT=1` +- `WAYBACK_DOMAIN_CONCURRENCY=1` +- `WAYBACK_MAX_RECORDS=2` +- `WAYBACK_TRANSIENT_BACKOFF_SECONDS=0.5` + +## 3. 当前已确认上线的代码改动方向 + +这些不是“计划中”，而是已经热补到 `mainland-controller-01` 并跑起来的： + +### 调度/分发侧 + +- 预补货提前触发，不再等批次快空了才补。 +- 预补货时不让拿到锁的那个 worker 先独吞下一批任务。 +- `pipeline` 推进和主动 `pull_tasks` 做了共享锁，减少 60 个 worker 同时拆批次。 +- `session_replaced -> queued_before_start` 这条重复唤起链已经加护栏压住。 +- `claim / finalize / release` 这些本地 DB 重链做过多轮减重，分钟级长尾已经明显掉下来。 + +### 注册/站点检测侧 + +- 注册检测保留单机性能模式。 +- 百度、360、爱站都已经不再停留在过于激进的超时口径。 +- `爱站` 已经补到： + - 单机模式下首轮可直连优先 + - `RemoteDisconnected` 可直接降级 + - 现在又进一步支持“外部依赖异常快速降级”，不再把整轮预算白白耗完 + +### Wayback 侧 + +- `wayback` 已经从“自己 backoff 把自己打死”收住。 +- 当前远端新版逻辑里： + - `latest_cdx` transient 时，不再硬打额外的 `records_cdx` + - backoff 已收短到 `0.5s` + - 当前剩下主要是 `web.archive.org` 本身慢/拒绝连接 + +## 4. 这轮真正验证出来的效果 + +下面这些是已经实测到的，不是预估： + +### 4.1 机器确实吃起来了 + +较早一轮稳定快照里，`mainland-controller-01` 已经出现： + +- `60` 个节点全在线 +- `53` 个子 worker 的 `active_threads > 0` +- `active_thread_count = 13531` + +后续观察里，最近 2 分钟也长期能看到： + +- `59` 个 fresh worker +- 大约 `51 ~ 56` 个 worker 有真实线程数大于 `0` +- 真实线程总和大约在 `3.8 万 ~ 4.2 万` + +这说明当前不是“进程活着但没吃活”，而是确实在跑。 + +### 4.2 内存高位问题先被压下来了 + +滚动重启并替换旧 RSS 后，现场有过一轮明显回落： + +- `Mem used ≈ 31Gi` +- `available ≈ 93Gi` +- `60` 个 worker 总 RSS 约 `23.9Gi` + +说明“旧 RSS 不退”这层已经不是最早那种危险状态。 + +### 4.3 爱站与 Wayback 的尾部外部链，已经从“卡住不产出”变成“持续产出” + +观察窗：`2026-04-25 17:17:18` 到 `17:29:23` + +- `completed_total: 50786 -> 51298`，净增 `512` +- `aizhan_completed: 7718 -> 7936`，净增 `218` +- `wayback_completed: 3068 -> 3362`，净增 `294` +- 这段窗口里： + - `aizhan_failed = 0` + - `wayback_failed = 0` + +这个结论很重要： + +- 现在已经不是“压住 failed，但 completed 不动” +- 而是“failed 压住了，同时 completed 也在持续增长” + +### 4.4 爱站快速降级这刀确实有作用 + +更早一段观察里，`爱站 failed` 还是会堆。 +在把 `DOMAINCHECK_AIZHAN_EXTERNAL_FAST_DEGRADE=1` 灰到远端后，后面的观察窗里： + +- `aizhan_failed` 被压到 `0` +- `aizhan_completed` 持续增长 + +这说明当前主链里，对爱站这类外部依赖异常，“快速降级继续”是有效的。 + +## 5. 当前怎么理解这些数字 + +当前最应该盯的不是： + +- 单次 `claimed` +- 单次 `pending` +- 某一个瞬间的 `running` + +因为这是 `domain_pipeline`，前一步跑完会继续生成下一步，数字天然会波动。 + +当前更应该盯的是： + +1. `completed_total` 是否持续增长 +2. `aizhan_completed / wayback_completed` 是否持续增长 +3. `aizhan_failed / wayback_failed` 是否继续维持低位 + +如果这三组数继续保持当前趋势，就说明这套配置是对的。 + +## 6. 当前不要再乱动的东西 + +在下一轮更长观察结束前，建议先不要再继续频繁改： + +- worker 进程数 +- 每进程线程上限 +- claim / backlog 批量参数 +- wayback / 爱站超时参数 + +原因很简单： +现在已经进入“参数开始生效、completed 在涨”的阶段，再频繁动，会把已验证有效的窗口打碎。 + +## 7. 下一步建议 + +当前建议先停手观察，不再继续大改参数。 + +优先做两件事： + +1. 继续观察更长窗口 + 建议至少再看 `30 ~ 60` 分钟，确认： + - `completed_total` 继续增长 + - `aizhan_completed / wayback_completed` 继续增长 + - `aizhan_failed / wayback_failed` 不重新抬头 + +2. 之后再做正式固化 + 把当前这些热补过的代码和远端有效参数，整理进正式 release，避免后续机器重启或重新发版时丢失。 + +## 8. 当前一句话结论 + +**这轮已经验证到：当前 `mainland-controller-01` 上这套“单机性能模式 + 外部依赖快速降级继续”的组合是有效的，主链已经从“线程忙但不出结果”切到“线程忙，同时持续产出 completed”。** diff --git a/docs/当前线上最终Runbook.md b/docs/当前线上最终Runbook.md new file mode 100644 index 0000000..9000ed6 --- /dev/null +++ b/docs/当前线上最终Runbook.md @@ -0,0 +1,273 @@ +# 当前线上最终 Runbook + +最后更新：2026-04-26 18:12 左右 +适用节点：`mainland-controller-01` + +这份文档的目标很简单： +后面不再靠聊天记录回忆，也不在多份文档里来回翻。 +只看这一份，就知道： + +1. 当前线上到底跑在什么基线上 +2. 现在先看什么数 +3. 什么情况继续观察 +4. 什么情况才值得继续动刀 + +## 1. 当前结论 + +当前这套线上方案，已经从： + +- 线程忙但不出结果 + +推进到了： + +- 线程忙，同时持续产出 `completed` + +所以现在的正确策略不是继续乱改参数，而是： + +**先稳住，先观察，确认这套已经验证有效的配置能不能持续出结果。** + +## 2. 当前线上基线 + +### 当前基底版本 + +- `current` 指向： + `/opt/domaincheck/releases/domaincheck_release_20260424_220158` + +说明： + +- 当前不是纯旧 release 原样运行 +- 真实线上状态是： + `220158 基底 + 多轮已热补代码 + 已生效 env` + +### 当前服务状态 + +- `domaincheck-worker.service = active` +- `domaincheck-sync-agent.service = active` +- `domaincheck-api.service = active` + +## 3. 当前已确认生效的关键参数 + +这些参数是**已经在运行进程环境里确认过**的，不是只改了文件。 + +### 单机性能与爱站相关 + +- `DOMAINCHECK_REGISTER_SINGLE_MACHINE_MODE=1` +- `DOMAINCHECK_SINGLE_MACHINE_AIZHAN_DIRECT_FIRST=1` +- `DOMAINCHECK_AIZHAN_REMOTE_DISCONNECT_DEGRADE=1` +- `DOMAINCHECK_AIZHAN_EXTERNAL_FAST_DEGRADE=1` +- `DOMAINCHECK_PROXY_STEP_MAX_ATTEMPTS_AIZHAN=2` +- `DOMAINCHECK_PROXY_STEP_MAX_SECONDS_AIZHAN=10` +- `DOMAINCHECK_AIZHAN_TIMEOUT_PROXY=1.8` +- `DOMAINCHECK_AIZHAN_TIMEOUT_DIRECT=2.4` + +### Wayback 相关 + +- `DOMAINCHECK_PROXY_STEP_MAX_ATTEMPTS_WAYBACK=2` +- `DOMAINCHECK_PROXY_STEP_MAX_SECONDS_WAYBACK=8` +- `WAYBACK_CDX_TIMEOUT=3` +- `WAYBACK_SNAPSHOT_TIMEOUT=2` +- `WAYBACK_RETRY_COUNT=1` +- `WAYBACK_DOMAIN_CONCURRENCY=1` +- `WAYBACK_MAX_RECORDS=2` +- `WAYBACK_TRANSIENT_BACKOFF_SECONDS=0.5` + +## 4. 当前已确认上线的代码方向 + +### 调度 / 分发侧 + +- 预补货提前触发 +- 预补货时不再让补货 worker 先独吞下一批 +- `pipeline` 推进和主动 `pull_tasks` 做了共享锁 +- `session_replaced -> queued_before_start` 这条重复唤起链已压住 +- `claim / finalize / release` 这些本地 DB 重链已经做过减重 + +### 外部步骤侧 + +- 注册检测保留单机性能模式 +- 百度、360、爱站都不再是早期那套过于激进的超时口径 +- `爱站` 当前已经具备： + - 首轮可直连优先 + - `RemoteDisconnected` 可直接降级 + - 外部依赖异常可快速降级继续 +- `wayback` 当前已经具备： + - `latest_cdx` transient 时不再硬打额外 `records_cdx` + - backoff 已收短 + - 主要剩下 `web.archive.org` 本身慢/拒绝连接 + +## 5. 当前已经验证出来的效果 + +### 5.1 机器已经真正吃起来 + +已观察到： + +- `60` 个节点全在线 +- 多次快照里 `51 ~ 56` 个 worker 有真实线程 +- 真实线程和长期在 `3.8 万 ~ 4.2 万` + +所以当前不是“进程活着但没干活”，而是确实在跑。 + +### 5.2 内存高位问题先被压下来 + +滚动重启释放旧 RSS 后，现场出现过一轮明显回落： + +- `Mem used ≈ 31Gi` +- `available ≈ 93Gi` +- `60` 个 worker 总 RSS ≈ `23.9Gi` + +说明“旧 RSS 不退”这层已经先被压住。 + +### 5.3 当前真正最重要的验证结果 + +观察窗：`2026-04-25 17:17:18` 到 `17:29:23` + +- `completed_total: 50786 -> 51298`，净增 `512` +- `aizhan_completed: 7718 -> 7936`，净增 `218` +- `wayback_completed: 3068 -> 3362`，净增 `294` +- 这段窗口里： + - `aizhan_failed = 0` + - `wayback_failed = 0` + +这说明： + +- 现在已经不是“压住 failed，但 completed 不动” +- 而是“failed 被压住，同时 completed 持续增长” + +## 6. 当前最该看的数 + +后面观察时，优先级固定按这个顺序： + +1. `completed_total` +2. `aizhan_completed` +3. `wayback_completed` +4. `aizhan_failed` +5. `wayback_failed` +6. `running / claimed / pending` + +顺序不要反过来。 + +原因： + +- `claimed / pending` 在 `domain_pipeline` 下天然会波动 +- `completed` 才是最真实的产出 + +## 7. 当前正确的判断方法 + +### 7.1 什么叫“继续观察就行” + +只要看到下面这种趋势，就先不要再改： + +- `completed_total` 持续增长 +- `aizhan_completed` 持续增长 +- `wayback_completed` 持续增长 +- `aizhan_failed / wayback_failed` 维持低位或归零 + +这说明： + +- 当前主链是通的 +- 当前配置方向是对的 +- 现在最应该做的是稳住，不是继续乱调 + +### 7.2 什么叫“又卡住了” + +只有出现下面这些情况，才值得继续下刀： + +#### 情况一 + +`completed_total` 在 `30 ~ 60` 分钟窗口里基本不动 + +说明： + +- 虽然线程在跑 +- 但结果没有真正落地 + +#### 情况二 + +`aizhan_failed` 或 `wayback_failed` 又明显抬头 + +说明： + +- 当前“快速降级继续”的策略还不够稳 + +#### 情况三 + +`running` 长时间很高，但 `aizhan_completed / wayback_completed` 不再增长 + +说明： + +- 新的主瓶颈又出现了 + +## 8. 当前建议观察窗口 + +建议至少继续看 `30 ~ 60` 分钟。 + +正确做法： + +1. 记录一组起始值 +2. 每隔 `10 ~ 15` 分钟看一次 +3. 至少看 `3 ~ 4` 个点 + +不要只看一两个瞬间。 + +## 9. 当前先不要再碰的东西 + +在这轮观察结束前，先不要继续改： + +- worker 进程数 +- 每进程线程数 +- claim 批量 +- backlog 批量 +- 爱站超时 +- wayback 超时 +- 其他步骤的重试窗口 + +原因很简单： + +- 现在已经进入“开始稳定产出”的阶段 +- 再频繁改参数，只会把已经验证有效的窗口打碎 + +## 10. 当前一句话执行原则 + +**先稳住，先看 `completed` 能不能持续增长；只要结果还在持续出来，就先不要再乱改。** + +## 11. `job 876` 尾项释放的当前判断 + +观察时间：`2026-04-26 18:12` 左右 + +当前不是“又卡死了”，而是： + +- `idx_detect_job_items_release_node_job` 还没转成 `valid=1` +- 但它已经稳定进入 `index validation: scanning table` +- 最新验证进度已经到 `229991 / 6197384` + +`codex-job876-tail-watch.service` 这条后台链当前是活的，而且在连续记进度： + +- `18:09:51 -> 126725` +- `18:10:52 -> 169564` +- `18:11:52 -> 210067` +- `18:12:22 -> 227856` + +同时现场另外两组状态也正常： + +- `job 876 = running` +- worker 已恢复到： + - `base=active` + - `templated=199` + - `node_agent=active` + +这次现场更准确的判断应该是： + +- 已经从“卡住”切成“后台验证扫描中” +- 当前最合理的动作不是再人工乱动 +- 应该继续让索引自己扫完 + +一旦 `idx_detect_job_items_release_node_job` 转成 `valid=1`，`codex-job876-tail-watch.service` 就会自动继续释放 `876` 的尾项。 + +## 12. 如果后面还要继续优化，下一刀顺序 + +只有在更长观察窗口里确认又卡住时，才按这个顺序继续： + +1. 先看 `completed_total` 为什么不涨 +2. 再看 `aizhan_completed / wayback_completed` 哪个先停 +3. 再决定是继续打外部链，还是回头查回写 / 聚合语义 + +不要一上来就回去改线程、claim、并发。 diff --git a/docs/当前线上运维执行清单.md b/docs/当前线上运维执行清单.md new file mode 100644 index 0000000..168307f --- /dev/null +++ b/docs/当前线上运维执行清单.md @@ -0,0 +1,149 @@ +# 当前线上运维执行清单 + +> 说明：这份文档保留作阶段执行稿。后续执行和观察，统一以 [当前线上最终Runbook.md](/www/wwwroot/getDomain/docs/当前线上最终Runbook.md) 为准。 + +最后更新：2026-04-25 17:30 左右 +适用节点：`mainland-controller-01` + +## 1. 当前目标 + +当前先不要继续乱调参数。 +先确认这一套已经生效的配置，能不能稳定把结果持续跑出来。 + +现在最重要的不是： + +- 再加线程 +- 再改 claim +- 再改批量 +- 再改超时 + +现在最重要的是： + +1. `completed_total` 能不能持续增长 +2. `aizhan_completed / wayback_completed` 能不能继续增长 +3. `aizhan_failed / wayback_failed` 会不会重新抬头 + +## 2. 当前已经验证有效的方向 + +这一轮已经确认有效的不是单点，而是整套思路： + +- 单机性能模式保留 +- 任务已经能铺到更多 worker 上 +- `爱站` 支持更快降级继续 +- `wayback` 已经减少了额外的 `records_cdx` 压力 +- 当前主链已经从“线程忙但不出结果”变成“线程忙，同时持续产出 completed” + +## 3. 当前观察时先看什么 + +优先看这 6 组数： + +1. `completed_total` +2. `aizhan_completed` +3. `wayback_completed` +4. `aizhan_failed` +5. `wayback_failed` +6. `running / claimed / pending` + +判断顺序不要乱： + +1. 先看 `completed_total` 有没有涨 +2. 再看 `aizhan_completed / wayback_completed` 有没有涨 +3. 最后才看 `failed` 和 `pending` + +原因： + +- `claimed / pending` 在 `domain_pipeline` 下会波动，不能单独看 +- `completed` 才是最真实的产出 + +## 4. 当前正确的判断口径 + +### 说明一 + +如果看到： + +- `running` 很高 +- `claimed / pending` 在波动 +- 但 `completed_total` 也在持续涨 + +这不是问题。 +这说明当前机器正在持续消化任务。 + +### 说明二 + +如果看到： + +- `aizhan_failed = 0` +- `wayback_failed = 0` +- `aizhan_completed / wayback_completed` 继续涨 + +这说明现在这套“外部依赖快速降级继续”是对的。 + +### 说明三 + +如果看到： + +- `running` 很高 +- 但 `completed_total` 长时间完全不涨 + +这才说明主链又卡住了，需要继续排。 + +## 5. 当前建议观察窗口 + +建议至少看 `30 ~ 60` 分钟，不要只看一两个瞬间。 + +推荐做法： + +1. 先记一组起始值 +2. 每隔 `10 ~ 15` 分钟看一次 +3. 连看至少 `3 ~ 4` 个点 + +只看单点，很容易误判。 + +## 6. 什么情况说明继续观察就行 + +只要出现下面这种趋势，就先不要动参数： + +- `completed_total` 持续增长 +- `aizhan_completed` 持续增长 +- `wayback_completed` 持续增长 +- `aizhan_failed / wayback_failed` 维持低位或归零 + +这说明当前主链是通的，应该先稳住。 + +## 7. 什么情况才需要继续下刀 + +只有出现下面这些情况，才值得继续改： + +### 情况一 + +`completed_total` 在 `30 ~ 60` 分钟窗口里基本不动 +说明虽然线程在跑，但结果没有真正落地。 + +### 情况二 + +`aizhan_failed` 或 `wayback_failed` 又明显抬头 +说明当前“快速降级继续”还不够稳。 + +### 情况三 + +`running` 长时间很高，但 `aizhan_completed / wayback_completed` 不再增长 +说明新的主瓶颈又出现了。 + +## 8. 当前不要再碰的东西 + +在这轮观察结束前，先不要继续改： + +- worker 进程数 +- 每进程线程数 +- claim 批量 +- backlog 批量 +- wayback 参数 +- 爱站参数 + +原因很简单： + +现在已经进到“开始稳定产出”的阶段，频繁动参数，只会把已验证有效的窗口打碎。 + +## 9. 当前一句话执行原则 + +**先稳住，先观察 completed 是否持续增长；只要结果还在持续出来，就先不要再乱改。** diff --git a/docs/新机器接手交接_20260427.md b/docs/新机器接手交接_20260427.md new file mode 100644 index 0000000..b319d3e --- /dev/null +++ b/docs/新机器接手交接_20260427.md @@ -0,0 +1,162 @@ +# 新机器接手交接 + +## 1. 目标 + +这次不是继续在当前机器上排障，而是： + +- 把当前代码整理后提交到 Git +- 在新服务器上重新部署验证 +- 让新的 Codex 接手继续推进 + +当前最重要的未完成主线只有一条： + +- `domain-api/app/services/sync_push_service.py` 中 `_load_pushable_projections()` 的修复，需要在远端稳定部署后验证： + - `detect_result_projection` push 是否恢复 + - `detect_result_ingest` 是否出现 + +## 2. 当前代码结论 + +已经确认并完成的代码侧修复： + +- `domain-api/app/services/sync_push_service.py` + - 修复 `_load_pushable_projections()` 选择逻辑 + - 旧问题：最新 `detect_result_projection` 明明存在且没有 push attempt，但候选集为空 + - 现修复：候选改为优先按最新 projection 选择 +- `domain-api/tests/test_sync_push_service.py` + - 已补回归测试 + +本地验证已完成： + +- `py_compile` 通过 +- `python -m unittest tests.test_sync_push_service` 通过 + +## 3. 当前卡点 + +不是代码不清楚，而是当前远端 SSH 不稳定： + +- `121.204.244.188:22` TCP 可连接 +- 但 SSH banner 经常超时 +- Paramiko 常见报错： + - `SSHException('No existing session')` + - `Error reading SSH protocol banner` + +因此当前没有拿到“远端已发布并生效”的硬结果。 + +## 4. 新机器接手后的首要动作 + +新机器接手后，先不要扩散排查面，只做这一个最小动作： + +1. 发布 `domain-api/app/services/sync_push_service.py` +2. 重启 `domaincheck-sync-agent.service` +3. 只验证两项： + - `detect_result_projection` push 是否恢复 + - `detect_result_ingest` 是否出现 + +## 5. 关键背景 + +项目根目录： + +- `/www/wwwroot/getDomain` + +主要子目录： + +- `domain-api/` +- `domain-web/` +- `domainCheck/` +- `docs/` + +当前远端主机： + +- `mainland-controller-01` +- `121.204.244.188` + +已确认的远端运行时 region 配置： + +- `NODE_REGION=mainland` +- `SYNC_SOURCE_REGION=mainland` +- `SYNC_TARGET_REGION=overseas` + +已确认的数据事实： + +- mainland 本地已经生成了 `detect_result_projection` +- surrogate job 例如： + - `job_id=909` + - `job_code=sync-overseas-28618` +- worker 事件已写入 mainland 本地库 +- 真正未打通的是 `detect_result_projection` 的 push 选择阶段 + +## 6. 建议提交范围 + +建议优先提交真正的代码改动，不要把运行产物一起带上。 + +建议提交： + +- `domain-api/` +- `domain-web/` +- `domainCheck/` +- `tools/` +- 需要保留的文档 `.md` + +建议重点确认本次必须包含： + +- `domain-api/app/services/sync_push_service.py` +- `domain-api/tests/test_sync_push_service.py` + +## 7. 明确不要提交的内容 + +以下属于运行产物、临时文件或本地探针，不建议提交： + +- `docs/ops_center_runtime/night_runs/step_mix_*/` +- `docs/ops_center_runtime/chinaz_gray_runs/` +- `docs/_tmp_regression/` +- `.codex-release-probe.txt` +- `*.bak_*` +- `*.log` +- `*.pid` + +当前工作区里特别要排除的项目： + +- `domainCheck/app/utils/database.py.bak_20260426_220818_active_job_cache_tune` +- `domainCheck/detect_worker.py.bak_20260426_220818_active_job_cache_tune` +- `docs/ops_center_runtime/night_runs/step_mix_20260426_051718_smoke_p200/` +- `docs/ops_center_runtime/night_runs/step_mix_20260426_051754_smoke_p200/` +- `docs/ops_center_runtime/night_runs/step_mix_20260426_052641_debug_stepmix/` +- `docs/ops_center_runtime/night_runs/step_mix_20260426_053252_mainland_step_mix_p200_t800/` +- `docs/ops_center_runtime/chinaz_gray_runs/` +- `.codex-release-probe.txt` + +## 8. 已跟踪但不建议把这次删除提交进去的文件 + +这些文件目前显示为已删除，但更像本地运行态/配置文件，不建议在这次“迁移到新服务器”的提交里带上删除： + +- `domainCheck/app/thread_count.json` +- `domainCheck/node_thread_counts.json` +- `domainCheck/runtime/runtime_settings.json` +- `domainCheck/runtime/sensitive_words.json` +- `domainCheck/runtime_settings.json` +- `domainCheck/thread_count.json` + +处理建议： + +- 如果这些删除不是你明确想做的配置收口，请在提交前恢复 +- 不要把“本地运行时删掉了某个 JSON”误当作产品代码改动提交 + +## 9. 提交前建议 + +提交前建议至少做这几步： + +1. `git status --short` +2. 把运行产物和备份文件排除掉 +3. 只保留代码、测试、必要文档 +4. 单独复查： + - `sync_push_service.py` + - `test_sync_push_service.py` + - `.gitignore` + - 本交接文档 + +## 10. 给新机器上的 Codex 的一句话 + +接手后不要重新发散排查 worker、projection 生成、事件落库链；这些已经基本收敛。优先完成 `sync_push_service.py` 的远端生效验证，只盯： + +- `detect_result_projection` push +- `detect_result_ingest` diff --git a/docs/测试优化分析.md b/docs/测试优化分析.md new file mode 100644 index 0000000..029dc95 --- /dev/null +++ b/docs/测试优化分析.md @@ -0,0 +1,417 @@ +# 测试优化分析 + +> 说明：这份文档保留作历史分析记录。当前线上执行基线与观察方法，统一以 [当前线上最终Runbook.md](/www/wwwroot/getDomain/docs/当前线上最终Runbook.md) 为准。 + +更新时间：`2026-04-24 14:00` + +## 这份文档是干什么的 + +这份文档不是讲理想情况，而是把这次已经做过的单机压测结果，和当前现场“根本没真正跑起来”的现状，放在一起说清楚。 + +目的很直接： + +- 先把已经验证过的参数结论留档 +- 再把现在为什么看起来线程上限很高、实际只跑了很少线程说清楚 +- 给后面继续优化的人一个明确顺序，别再一边主链没跑通，一边继续盲目加并发 + +## 一句话结论 + +早上那轮“只测一台大陆机器”的压测，已经得出过一个单机最优值： + +- `mainland-controller-01` 单机跑时，当前最优参数是 `35 进程 / 900 线程` + +但这不等于项目现在已经能按这个参数稳定跑。 + +按 `2026-04-24 13:22` 左右控制面看到的实时状态，这套链路目前更像是： + +- 任务还挂着 +- 页面也还能显示“运行中” +- 但真正参与执行的只有很少几个桶 +- 总体上属于“没有真正跑起来” + +所以现在的主矛盾不是“继续把线程调大”，而是“先让主流程真的跑起来”。 + +## 这次单机压测是怎么测的 + +这轮压测不是全链路压测，而是一个故意收缩后的测试环境。 + +当时做了这几件事： + +- 先停掉 `121.204.244.248`，也就是 `mainland-worker-01` +- 只保留 `mainland-controller-01` 单机承担检测 +- 停掉 controller 上的 `domaincheck-sync-agent.service` +- 不再继续额外增加新的 Redis 限制动作 +- 直接在 controller 本地数据库里看 60 秒窗口真实完成量和更新量 + +这么做的目的，不是模拟线上真实规模，而是先排除多机互相干扰，看单机能跑到什么水平。 + +## 单机压测结果 + +压测时间段大致是 `2026-04-24 05:14 - 05:28`。 + +当时 controller 本地 60 秒窗口结果如下： + +| 参数 | 60 秒完成量 | 60 秒更新量 | Redis 连接数 | 内存已用 | 可用内存 | 实际活跃 worker | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| `35/900` | `4092` | `22267` | `751` | `73.6G` | `54.3G` | `35` | +| `40/1000` | `1885` | `1544` | `793` | `75.1G` | `52.9G` | `40` | +| `45/1100` | `388` | `11347` | `818` | `75.7G` | `52.3G` | `45` | +| `50/1200` | `0` | `12118` | `823` | `75.9G` | `52.1G` | `45` | + +## 压测结论 + +从这轮结果里，可以直接得出 4 个结论： + +### 1. 单机最优值不是越大越好 + +当前这套链路下，`35/900` 明显优于后面的更大参数。 + +也就是说，继续加进程、加线程，并没有带来更高完成量，反而更差。 + +### 2. 当前单机稳定上限大约在 `45/1100` + +`45/1100` 还能勉强活住。 + +但它已经不是“跑得更快”，只是“还能挂着不死”。 + +### 3. `50/1200` 已经没有意义 + +虽然当时把参数抬到了 `50/1200`，但最终实际只活成了 `45` 个 worker，而且完成量已经掉到 `0`。 + +这说明不是配置没写进去，而是链路已经先撞到别的瓶颈了。 + +### 4. 当时真正的瓶颈已经不是 Redis + +从这轮压测看，Redis 连接数有上升，但没有先爆。 + +更像是外部检测链路本身先撑不住了，尤其是： + +- 注册检测 +- 代理池质量 +- 会话切换抖动 + +## 为什么当时测出来 `35/900`，现在页面却只剩 `52` 线程 + +这就是最容易误判的地方。 + +早上压测结论成立的前提是： + +- 只测 controller 单机 +- `mainland-worker-01` 被停掉 +- `sync-agent` 被停掉 +- 现场是专门为压测收缩过的 + +而现在你看到的实时状态，已经不是那个压测场景了。 + +按 `2026-04-24 13:22` 左右接口返回的状态，现在的现场是： + +- `detect/job/active` 里的活动任务还是 `job_id=1` +- 这条任务从 `2026-04-21 23:30:42` 就已经是 `running` +- `raw_items_pending=9478` +- `raw_items_running=1` +- `raw_items_completed=4996` +- `raw_items_failed=486` + +同时，`runtime/status` 给出的聚合状态是： + +- `参与服务器：4` +- `参与进程：4` +- `参与线程：52` +- `运行中：52` +- `线程上限：52 / 2701` + +这几组数字合起来说明的不是“系统已经高效跑起来”，而是： + +- 页面还能看到一些参与桶 +- 但真正持续工作的执行面很薄 +- 大量任务还躺在队列里，没有被健康地持续消化 + +## 当前为什么可以直接判断“根本跑不起来” + +不是因为页面数字小，而是因为几条关键事实同时出现了： + +### 1. 活动任务太老 + +当前活动任务还是 `2026-04-21` 开出来的老任务。 + +如果主链真的健康，它不应该拖这么久还挂在 `running`。 + +### 2. 待处理量还很大 + +当前还剩： + +- `9478` 条原始 pending +- `3779` 条 claimed +- `1028` 条 running backlog + +但真正展示出来的活跃线程只有 `52`。 + +这个比例说明队列不是空了，而是消化不动。 + +### 3. 当前节点自己根本不跑本机 worker + +当前 `overseas-control-01` 的 `runtime/status` 里明确写着： + +- `worker_online=false` +- `worker_process_count=0` +- `worker_runtime_message=inactive/dead` + +也就是说，现在这个页面看到的“运行中”，本来就不是本机在跑，而是靠远端汇总。 + +### 4. 远端参与桶很少，而且口径不稳 + +现在控制面聚合出的参与节点只有： + +- `mainland-worker-01` +- `mainland-controller-01-k` +- `mainland-controller-01-a` +- `mainland-controller-01-s` + +这和早上单机压测时的 `35` 个 worker，根本不是一个级别。 + +所以你现在看到“线程上限 2701”，不要被这个数字迷惑。 + +它只是汇总上限，不代表这些线程真的都在跑。 + +## 现在最像什么问题 + +如果用人话说，现在最像下面这种情况： + +- 任务队列里还有很多活 +- 但是执行面只剩很薄一层还在动 +- 页面还能看到“运行中”，所以看起来像没完全死 +- 可真正吞吐已经低到接近“没跑起来” + +所以这不是单纯的参数问题，而是主流程状态已经歪了。 + +## 当前最该先做的事 + +下一步优化顺序建议固定成这样： + +### 第 1 步：先恢复“能持续跑” + +先不要继续加进程、加线程。 + +先确认 4 件事： + +- 当前到底是哪几个节点真的在跑 +- `mainland-worker-01` 是不是还应该继续停着 +- controller 上当前实际活着多少 worker +- 老任务里这些 `claimed/running` 有没有大量卡死项 + +### 第 2 步：把老任务和会话问题理顺 + +现在这条 `2026-04-21` 的老任务已经拖太久了。 + +要优先确认： + +- 是不是有很多旧 `session_replaced` +- 是不是有大量任务项一直卡在 `claimed/running` +- 是不是页面显示还在跑，但实际 worker 已经不工作了 + +### 第 3 步：确认代理链是不是主瓶颈 + +早上单机压测已经说明，Redis 不是先撞墙的那个点。 + +下一步更值得盯的是： + +- `detect_register` 的真实完成率 +- 代理池是否经常空 +- RDAP 访问是否大量超时 + +### 第 4 步：只有主链恢复后，才重新做性能优化 + +## 13:48 补充结论 + +这轮又确认了一件非常关键的事： + +- `mainland-worker-01` 不是“已经彻底消失”，而是“被标记为停用，但之前还在继续上报运行态” + +后来实际做了两步处理： + +- 先通过远端 `ops job` 让 `mainland-worker-01` 本机执行 `systemctl stop domaincheck-worker` +- 再把控制面的状态汇总改成：只要节点在 `ops_managed_nodes` 里是 `is_enabled=false`，就不再算进在线 Worker、参与节点和活跃线程 + +处理完成后，控制面实时口径已经从“2 台参与节点 / 2 个检测进程 / 2 线程”收到了： + +- `参与节点：1` +- `参与进程：1` +- `运行中：1` +- 只剩 `mainland-controller-01` + +这说明前面那种“明明已经准备单机测了，页面还一直把停用节点算进去”的问题，确实是状态口径 bug，不是现场真的还有两台健康执行机一起在跑。 + +所以从现在开始再看单机测试结果时，要以这条新口径为准： + +- `mainland-worker-01` 已停用，不再参与单机压测统计 +- 当前有效执行面只剩 `mainland-controller-01` + +等主流程恢复成“真的能持续吃队列”之后，再拿 `35/900` 作为第一版基线。 + +到那时可以按下面顺序再调： + +1. 先验证 `35/900` 在恢复后的现场还能不能稳定 +2. 如果稳定，再测 `40/1000` +3. 只有吞吐确实提高了，才继续往上加 + +## 这份文档的最终结论 + +可以把这次结果浓缩成两句话： + +第一句： + +- 单机压测已经证明，当前代码和链路下，`mainland-controller-01` 的最佳参数是 `35 进程 / 900 线程` + +第二句： + +- 但当前线上真正的问题不是“参数不够大”，而是“主流程根本没有持续跑起来”，所以现在继续加线程没有意义 + +后面再优化时，应该先解决“为什么只剩 52 线程在动”，再谈如何把吞吐重新抬高。 + +## 2026-04-24 13:35 补充结论 + +这份文档写完以后，又继续往下排了一轮，现场有两个非常关键的新结论。 + +### 1. `52` 线程里有明显口径错配 + +后来继续核对后发现，页面里那组： + +- `参与服务器：4` +- `参与进程：4` +- `参与线程：52` + +并不是来自 `detect_job_items` 真表本身。 + +真实数据库里，`job_id=1` 其实只剩： + +- `1` 条 `running` + +而且这条记录最后更新时间还停在 `2026-04-22 15:00:56`，本质上已经是老僵死项。 + +真正把页面抬到 `52` 的，是一条口径 bug： + +- 当前 active job 还是老任务 `detect-20260421232649-acfa3d` +- 但控制面又拿到了别的 sync job 的 runtime overlay +- 两套数据被硬合到了一起 + +简单说就是： + +- 任务表是老 job +- 线程数却借用了别的 job 的运行态 + +### 2. 这条口径 bug 已经修掉了 + +现在代码已经补成： + +- 只有 runtime overlay 的 `job_code / job_id` 和当前 active job 对得上时，才允许覆盖 `queue_health` +- 对不上时，宁可退回真实任务表，也不再把别的 job 的运行态硬套到当前页面上 + +修完并重启海外控制面 API 之后，实时口径已经从之前的 `52` 收回到了： + +- `参与节点：2` +- `参与进程：2` +- `参与线程：2` + +当前这 `2` 个真实信号分别来自： + +- `mainland-controller-01` 那条老 `running` +- `mainland-worker-01` 当前 heartbeat 上报的 `1` 条活跃线程 + +这说明一件事： + +- 之前那组 `52` 的确主要是错口径，不是实际吞吐 + +## 目前还剩的真实问题 + +虽然 `52 -> 2` 这一步已经收准了，但项目还是没有真正恢复健康。 + +当前还剩 2 个真实问题： + +### 1. `mainland-worker-01` 明明已停用，却还在持续推运行态 + +当前控制面日志里还能持续看到： + +- `121.204.244.248` 往海外控制面打 `runtime/debug-ingest` +- 同时也还在打 `ops/agent/pull` + +这说明这台机器不是“旧残影”，而是还真的在线、还真的在报活。 + +所以“已停用”这件事，目前只是在托管配置层停了，但没有真正把它从 Agent / runtime 上报链里摘干净。 + +### 2. `job_id=1` 这条老任务本身也还没收口 + +当前老任务依然挂着： + +- `job_id=1` +- `job_code=detect-20260421232649-acfa3d` +- `started_at=2026-04-21 23:30:42` + +而真实任务表里，它现在已经不是“很多线程在跑”，而是： + +- 大部分还在 `pending` +- 只剩 1 条老 `running` + +这说明这条老任务本身也需要后续专门处理，不能继续长期挂在 `running`。 + +## 现在更准确的下一步 + +到这里为止，下一步就更明确了： + +1. 先把 `mainland-worker-01` 真的摘掉。 +2. 再处理 `job_id=1` 这条老任务的遗留 `running` 项。 +3. 等这两件事收完，再重新看当前真实执行面到底还有没有持续吞吐。 + +也就是说，现在已经不是“继续猜线程参数”的阶段，而是“先把错误执行面和老任务残留清掉”的阶段。 + +## 新增一版“单机性能模式”给注册检测 + +这次为了后面继续压单机吞吐，我已经在代码里补了一版只影响“注册状态检测”的性能模式。 + +它的目标不是改全局架构，而是先把当前最像瓶颈的那一段链缩短： + +- 只改 `detect_register` +- 不碰百度、360、站长、爱站、时光机这些步骤 +- 保留当前版的连接复用和失败闭环 +- 只把注册检测改得更像老版本那种“先尽快打出去，再说” + +### 这版模式做了什么 + +打开后，注册检测会变成下面这个思路： + +- 即使全局 `allow_direct=false`，注册检测这一步也允许直连 +- 默认先连续直连 `2` 次 +- 这两次之间不再等代理补货 +- 只有前面的直连没打通，后面才继续走代理兜底 + +简单说就是： + +- 当前默认模式：代理优先，直连兜底 +- 单机性能模式：注册检测直连优先，代理兜底 + +### 怎么开 + +这版先做成环境变量开关： + +- `DOMAINCHECK_REGISTER_SINGLE_MACHINE_MODE=1` +- `DOMAINCHECK_REGISTER_DIRECT_STREAK_ATTEMPTS=2` + +当前默认建议先用这组： + +- `进程：35` +- `线程：900` +- `DOMAINCHECK_REGISTER_SINGLE_MACHINE_MODE=1` +- `DOMAINCHECK_REGISTER_DIRECT_STREAK_ATTEMPTS=2` + +如果后面继续试更激进一点，再考虑把直连连打次数从 `2` 提到 `3`，但不建议一上来就抬。 + +### 为什么不是直接回退老版本 + +老版本真正有参考价值的是“链短、切得快”，不是它的实现本身更先进。 + +所以这次没有回退这几样东西： + +- 没回退到 `20s * 3` 的长超时 +- 没回退到“注册失败也继续往后跑” +- 没丢掉当前的 `Session` 复用 + +换句话说，这次是“借老策略”，不是“退老实现”。 diff --git a/docs/项目当前运行流程说明.md b/docs/项目当前运行流程说明.md new file mode 100644 index 0000000..6f50ae4 --- /dev/null +++ b/docs/项目当前运行流程说明.md @@ -0,0 +1,405 @@ +# 项目当前运行流程说明 + +> 说明：这份文档主要回答“项目现在怎么跑”。当前线上执行基线、观察顺序和是否继续调参，统一以 [当前线上最终Runbook.md](/www/wwwroot/getDomain/docs/当前线上最终Runbook.md) 为准。 + +更新时间：`2026-04-24` + +## 这份文档是干什么的 + +这不是一份“理想设计稿”，而是按项目现在的真实跑法整理出来的说明。 + +目标只有一个：让人用人话看明白，这个项目现在到底怎么跑，任务从哪里来，谁在执行，结果又是怎么回来的。 + +## 先用一句话讲明白 + +这套系统现在不是“点一下开始，然后一台机器自己跑完”。 + +它现在的实际跑法是： + +海外控制面先挑出要检测的域名，打成一批任务，然后把这批任务交给大陆节点；大陆节点把任务拉下来后，由 worker 一步一步去跑检测；跑出来的结果，再同步回海外控制面，最后由页面统一展示。 + +## 先认识 5 个角色 + +### 1. 海外控制面 + +可以把它理解成“总调度台”。 + +它主要负责： + +- 创建检测任务 +- 给大陆节点派动作 +- 接收大陆回传的运行状态和检测结果 +- 在页面上展示当前进度 + +当前本机 `overseas-control-01` 就是这个角色。 + +它本机一般不直接跑检测。 + +### 2. 大陆控制节点 + +可以把它理解成“大陆现场调度员”。 + +它主要负责： + +- 去海外把待检测批次拉下来 +- 启动大陆这边的 sync-agent、worker +- 把大陆现场的运行情况回传出去 + +### 3. 大陆 worker 节点 + +这才是真正“干活”的机器。 + +它负责： + +- 领取待执行任务 +- 开线程跑检测 +- 把每一步的结果写回本地 + +### 4. sync-agent + +它就是“搬运工”。 + +负责在海外和大陆之间搬 3 类东西： + +- 待检测任务批次 +- 当前运行状态 +- 检测结果 + +### 5. node-agent + +它就是“远程执行员”。 + +它会在节点上定时来领控制命令，然后执行，比如： + +- 启动 worker +- 开始检测 +- 拉取任务 +- 重启服务 + +## 当前现场快照 + +下面这段，是按 `2026-04-24 01:44 - 01:45` 左右控制面看到的实时数据整理的。 + +- 当前控制面节点是 `overseas-control-01` +- 角色是 `overseas / control` +- 本机 API 在线 +- 本机 worker 不承担实际检测 + +当时控制面看到的活动任务是： + +- `job_id = 208` +- `job_code = sync-overseas-1430` +- 状态是 `running` +- 这一批一共 `5000` 个任务项 +- 其中 `4435` 个已经被领取 +- `575` 个正在执行 + +当时控制面汇总看到的执行规模大致是： + +- 约 `28` 个参与节点 +- 约 `28` 个检测进程 +- 约 `575` 个活跃线程 + +当时积压里最大头还是“注册状态检测”： + +- `detect_register` 待处理约 `8235` +- 后续步骤待处理约 `1243` + +同一时间段里，系统的 readiness 仍然提示： + +- 大陆节点心跳并不稳定 +- 还有结果批次待推送 + +这说明了一件很关键的事： + +不是完全没跑，而是“链路在跑，但跑得不稳”。所以现在的主矛盾，确实还是稳定性，不是单纯把线程数继续往上调。 + +## 整个项目现在是怎么跑的 + +下面按真实流程，一步一步讲。 + +### 第 1 步：海外控制面先挑出“需要再检测”的域名 + +简单理解就是： + +- 还没检测过的 +- 之前检测失败过的 +- 或者状态需要重新确认的 + +这些域名会先被挑出来，形成一份“待处理名单”。 + +这时候只是“选名单”，还没有真正开始跑检测。 + +### 第 2 步：海外控制面创建一轮检测任务 + +系统会先创建一条“本轮任务”记录。 + +然后把这批域名拆成很多个“任务项”。 + +这里要特别注意： + +它不是一次就给某个域名下发“整套检测”。 + +它做的是： + +- 先判断这个域名下一步该做什么 +- 只给它安排“下一步” + +所以这个系统现在跑的是“分步骤推进”，不是“单个域名一次性从头跑到尾”。 + +### 第 3 步：海外控制面自己不跑，而是把动作派到大陆 + +当你在页面上点“开始检测”后，海外控制面会去排队发送几类动作给大陆节点： + +- 让大陆启动 sync-agent +- 让大陆去拉取待检测批次 +- 让大陆启动 worker +- 让大陆开始执行检测 + +这些动作不是直接 ssh 过去硬敲命令。 + +而是先进入远程动作队列，再由大陆节点上的 node-agent 定时来领。 + +所以你可以把它理解成： + +海外控制面负责“发指令”，大陆节点负责“取指令并执行”。 + +### 第 4 步：大陆控制节点先把任务批次拉下来 + +大陆控制节点会去海外控制面拿一批待检测域名。 + +拿到以后，会做几件事： + +- 把这批域名落到大陆本地库里 +- 在大陆本地创建一条对应的 job +- 给每个域名生成“下一步要做什么”的任务项 +- 再回头告诉海外：这批任务我已经收到了 + +这个“确认收到”很重要。 + +因为如果不确认，海外会以为这批任务还没被接走，后面就可能重复下发。 + +另外，大陆控制节点不会无脑一直拉新任务。 + +如果它发现本地已经堆了很多待处理任务，它会先停一下，不再继续拉新批次，避免越堆越多。 + +### 第 5 步：worker 启动后，先做准备，不会立刻开跑 + +worker 真正开始检测前，会先做一轮准备动作。 + +大致包括： + +- 重新读取最新配置 +- 重新读取线程数和进程数 +- 重新加载 cookies +- 刷新代理池 +- 回收上次异常退出留下来的遗留任务 + +所以你看到“worker 已经启动”，并不等于“已经开始稳定出结果”。 + +它中间还有一个准备阶段。 + +### 第 6 步：worker 真正领的，是任务队列里的“下一步” + +worker 现在不是直接扫整张域名表。 + +它主要是从任务队列里领取待执行任务。 + +领取后的状态大致可以这样理解： + +- `pending`：还没被谁接手 +- `claimed`：已经被某个节点领走了 +- `running`：已经开始跑了 +- `completed / failed / blacklisted`：这一小步跑完了 + +也就是说，系统现在盯的不是“这个域名整体做完没”，而是“这个域名现在跑到哪一步了”。 + +### 第 7 步：一个域名不是一次跑完，而是一小步一小步推进 + +当前默认的主流程顺序，大致是： + +1. 注册状态检测 +2. 百度 site 检测 +3. 360 site 检测 +4. 站长之家检测 +5. 爱站检测 +6. 时光机检测 + +有些来源的域名，会跳过注册状态这一步，直接从后面的步骤开始。 + +所以你不能把它理解成“所有域名一定都从第一步开始”。 + +更准确地说，是系统会先判断这个域名“现在最该补哪一步”。 + +### 第 8 步：流程顺序，和 worker 实际优先领什么，不完全一样 + +这个地方很容易误会，所以单独说一下。 + +流程顺序上，域名一般是先过前面的关，再去后面的关。 + +但 worker 实际领任务时，会优先照顾已经走到后面的域名。 + +为什么要这样做？ + +因为如果完全按最前面的步骤一直领，后面的域名会永远被堵住，怎么都跑不到尾部。 + +所以现在的真实策略更像是： + +- 流程上按顺序推进 +- 调度上优先让已经走到后面的域名尽快跑完 + +这也是为什么你现在会看到“注册状态检测积压很大”，但后面的步骤也还在继续跑。 + +### 第 9 步：每一步跑完后，系统会决定下一步怎么走 + +某一步做完后，系统不会简单地只记一个“成功 / 失败”。 + +它还会决定后面怎么走。 + +大致有几种情况： + +- 这一步通过了：给这个域名创建“下一步”的任务项 +- 这一步命中黑名单：后面的步骤就不再继续 +- 这一步属于外部站点异常、超时、代理问题：可能会重试，也可能降级，也可能转成人工复核 +- 这一步明确不通过：流程在这里终止 + +所以这个项目现在不是“跑完就完”，而是“每走完一步，再决定下一步”。 + +### 第 10 步：结果先写回大陆本地 + +worker 跑完某一步后，会先把结果写回大陆本地。 + +写回去的内容包括： + +- 这一个任务项是什么结果 +- 这个域名当前是什么状态 +- 这一步的详细信息 +- 是否需要人工复核 + +所以大陆本地库，先是“第一落点”。 + +### 第 11 步：大陆再把结果和运行状态同步回海外 + +大陆这边不是只回传“最终结果”。 + +它还会把两类信息往海外送： + +- 当前运行状态 +- 最近有哪些域名开始了、完成了、失败了、进黑名单了 + +然后海外控制面收到后，再去更新自己的展示和汇总。 + +这也是为什么页面上看到的数字，不是某一台机器的原始数字。 + +它其实是“控制面汇总后的结果”。 + +### 第 12 步：海外页面最后展示出来的，是一份汇总快照 + +所以现在页面上的数据，本质上是几部分拼起来的： + +- 控制面自己知道的任务信息 +- 大陆回传的运行状态 +- 大陆回传的结果事件 +- 当前 job 的统计汇总 + +这就会带来一个现实情况： + +如果大陆心跳断一下，或者结果同步晚一点，页面看起来就会突然不稳定，甚至和现场有一点时间差。 + +这不一定代表 worker 完全没跑。 + +很多时候，只是“页面依赖的回传链路断了一下”。 + +## 你可以把现在的项目理解成两条并行链 + +### 第一条：任务执行链 + +海外选域名 -> 建任务 -> 大陆拉批次 -> worker 领任务 -> 按步骤推进 -> 写结果 + +### 第二条：运行状态和结果同步链 + +大陆上报运行状态 -> 大陆回推结果事件 -> 海外接收并汇总 -> 页面刷新展示 + +这两条链只要有一条不稳，使用感受就会变差。 + +如果任务执行链慢，你会感觉“没速度”。 + +如果同步链不稳，你会感觉“页面不准、看不清到底跑没跑”。 + +## 为什么你会一直觉得“没有速度、没有效率” + +因为现在影响体验的，不只是“worker 够不够快”。 + +这套系统至少要同时经过下面这些环节： + +- 海外选任务 +- 海外派动作 +- 大陆领动作 +- 大陆拉批次 +- worker 领任务 +- 外部站点检测 +- 结果回传 +- 页面汇总展示 + +任何一段不稳，最后给人的感觉都会像是“整套系统没跑起来”。 + +所以从当前现场来看，先把主链路稳定跑顺，仍然比继续往上冲性能更重要。 + +## 如果你只想用最简单的方法判断现在卡在哪 + +可以按下面这个顺序看： + +1. 先看有没有新的活动 job + - 没有的话，说明卡在“任务还没真正建起来” + +2. 再看大陆有没有把任务批次收下 + - 没收下的话，说明卡在“海外发过去了，但大陆没接住” + +3. 再看 `claimed` 和 `running` 有没有开始增长 + - 不增长的话，说明 worker 没真正开始领任务 + +4. 再看 `completed / failed / blacklisted` 有没有开始变化 + - 长时间不变的话，说明任务虽然在跑，但结果没持续产出 + +5. 最后看海外页面有没有跟着更新 + - 大陆明明在跑，海外不更新，通常就是“结果同步链”有问题 + +## 最后只记住 4 句话就够了 + +1. 现在这套项目是“海外控制，大陆执行”，不是单机直跑。 +2. 一个域名不是一次跑完，而是按步骤一小步一小步推进。 +3. 页面看到的是汇总快照，不是某台机器的原始现场数字。 +4. 当前最大的矛盾仍然是链路稳定，不是单纯把性能参数继续调大。 + +继续看完了，当前是稳推进，不是又卡死。 + +现在现场是： +- `idx_detect_job_items_release_node_job` 还没转 `valid=1` +- 但已经稳定在 `index validation: scanning table` +- 最新进度到： + - `229991 / 6197384` +- `watch` 这条后台链是活的，而且一直在连续记进度： + - `18:09:51 -> 126725` + - `18:10:52 -> 169564` + - `18:11:52 -> 210067` + - `18:12:22 -> 227856` + +另外两点也正常： +- `876` 现在还显示 `running` +- worker 已经全恢复： + - `base=active` + - `templated=199` + - `node_agent=active` + +一句话说： +**现在已经从“卡住”切成“后台验证扫描中”，而且进度在稳定往前走。** + +下一步最合理的动作不是再人工乱动，而是继续让它扫；一旦索引转成 `valid=1`，`codex-job876-tail-watch.service` 就会自动接着去释放 `876` 的尾项。 + + + +发布 sync_push_service.py +重启 domaincheck-sync-agent.service +验证： +detect_result_projection push 是否恢复 +detect_result_ingest 是否出现 \ No newline at end of file diff --git a/domain-api/app/api/routes/detect.py b/domain-api/app/api/routes/detect.py index 31d2ad8..6cafb3b 100644 --- a/domain-api/app/api/routes/detect.py +++ b/domain-api/app/api/routes/detect.py @@ -2,7 +2,7 @@ from __future__ import annotations from uuid import uuid4 -from fastapi import APIRouter +from fastapi import APIRouter, Body from app.core.config import settings from app.schemas.common import ApiResponse @@ -18,7 +18,7 @@ from app.services.detect_job_service import ( from app.services.detect_service import get_detect_status from app.services.detect_run_service import create_detect_run_snapshot, finalize_detect_run, mark_detect_run_stopping from app.services.ops_job_service import create_ops_job, list_managed_nodes -from app.services.settings_service import get_settings_payload, resolve_thread_count +from app.services.settings_service import get_settings_payload, resolve_process_count, resolve_thread_count from app.services.worker_control_service import send_worker_command, start_worker router = APIRouter(tags=["detect"]) @@ -66,7 +66,13 @@ def _build_detect_action_result( def _build_settings_summary(settings_payload: dict) -> dict: thread_count_resolution = resolve_thread_count(settings_payload=settings_payload) + process_count_resolution = resolve_process_count(settings_payload=settings_payload) return { + "process_count": int(process_count_resolution["effective_process_count"]), + "process_count_default": int(process_count_resolution["default_process_count"]), + "process_count_source": str(process_count_resolution["source"]), + "process_count_override": process_count_resolution["override_process_count"], + "process_count_node_code": str(process_count_resolution["node_code"]), "thread_count": int(thread_count_resolution["effective_thread_count"]), "thread_count_default": int(thread_count_resolution["default_thread_count"]), "thread_count_source": str(thread_count_resolution["source"]), @@ -78,6 +84,38 @@ def _build_settings_summary(settings_payload: dict) -> dict: } +def _normalize_target_node_codes(payload: dict | None) -> list[str]: + if not isinstance(payload, dict): + return [] + + normalized_targets: list[str] = [] + + def append_target(raw_value: object) -> None: + normalized_value = str(raw_value or "").strip() + if normalized_value and normalized_value not in normalized_targets: + normalized_targets.append(normalized_value) + + for key in ("target_node_codes", "node_codes"): + raw_value = payload.get(key) + if isinstance(raw_value, (list, tuple, set)): + for item in raw_value: + append_target(item) + elif isinstance(raw_value, str) and raw_value.strip(): + for item in raw_value.split(","): + append_target(item) + if normalized_targets: + return normalized_targets + + for key in ("target_node_code", "node_code"): + raw_value = payload.get(key) + if raw_value not in (None, ""): + append_target(raw_value) + if normalized_targets: + return normalized_targets + + return normalized_targets + + def _mainland_detect_targets() -> dict[str, list[dict]]: controllers: list[dict] = [] workers: list[dict] = [] @@ -212,15 +250,18 @@ def _dispatch_remote_detect_start(*, job_summary: dict, cycle_token: str) -> dic } -def _dispatch_remote_detect_stop(*, active_job: dict | None, cycle_token: str = "") -> dict: +def _dispatch_remote_detect_stop(*, active_job: dict | None, cycle_token: str = "", payload: dict | None = None) -> dict: targets = _mainland_detect_targets() job_summary = active_job or {} queued: list[dict] = [] + target_node_codes = _normalize_target_node_codes(payload) for node in [*targets["controllers"], *targets["workers"]]: node_code = str(node.get("node_code") or "").strip() if not node_code: continue + if target_node_codes and node_code not in target_node_codes: + continue queued.append( _queue_remote_detect_job( node_code=node_code, @@ -317,26 +358,63 @@ def start_detect(step_code: str | None = None) -> ApiResponse: local_worker_expected = _local_worker_expected_on_this_node() if local_worker_expected: - ok, message = start_worker() - if not ok: - result = _build_detect_action_result( - action="start", - ok=False, - message=message, - data={"job": job_summary}, - ) + prestart_snapshot = get_detect_status() + worker_already_running = bool(prestart_snapshot.get("worker_online", False)) or int( + prestart_snapshot.get("worker_process_count", 0) or 0 + ) > 0 + if worker_already_running: + ok = True + message = "检测端已在运行，跳过重复启动，直接发送控制指令" append_detect_job_event( job_summary["job_id"], - event_type="job_dispatch_failed", - level="error", - message=f"启动 Worker 失败: {message}", - payload={"cycle_token": cycle_token}, - ) - return ApiResponse( - code=1, + event_type="job_dispatch_start_skipped", + level="info", message=message, - data=result, + payload={ + "cycle_token": cycle_token, + "worker_process_count": int(prestart_snapshot.get("worker_process_count", 0) or 0), + }, ) + else: + ok, message = start_worker() + if not ok: + degraded_snapshot = get_detect_status() + worker_already_running = bool(degraded_snapshot.get("worker_online", False)) or int( + degraded_snapshot.get("worker_process_count", 0) or 0 + ) > 0 + if not worker_already_running: + result = _build_detect_action_result( + action="start", + ok=False, + message=message, + data={"job": job_summary}, + ) + append_detect_job_event( + job_summary["job_id"], + event_type="job_dispatch_failed", + level="error", + message=f"启动 Worker 失败: {message}", + payload={"cycle_token": cycle_token}, + ) + return ApiResponse( + code=1, + message=message, + data=result, + ) + + degraded_message = f"{message}；检测端已在运行，改为直接发送控制指令" + append_detect_job_event( + job_summary["job_id"], + event_type="job_dispatch_start_degraded", + level="warning", + message=degraded_message, + payload={ + "cycle_token": cycle_token, + "worker_process_count": int(degraded_snapshot.get("worker_process_count", 0) or 0), + }, + ) + ok = True + message = degraded_message command_ok, command_message = send_worker_command( "start_detection", @@ -372,18 +450,19 @@ def start_detect(step_code: str | None = None) -> ApiResponse: settings_summary = _build_settings_summary(settings_payload) if command_ok: remote_dispatch = _dispatch_remote_detect_start(job_summary=job_summary, cycle_token=cycle_token) - create_detect_run_snapshot( - message=f"{message}；{command_message}", - runtime={ - "mode": snapshot.get("worker_mode", ""), - "running": snapshot.get("worker_online", False), - "process_count": snapshot.get("worker_process_count", 0), - "latest_start_time": snapshot.get("worker_latest_start_time", ""), - "message": snapshot.get("worker_runtime_message", ""), - }, - progress=snapshot.get("progress", {}), - settings_summary=settings_summary, - ) + if local_worker_expected: + create_detect_run_snapshot( + message=f"{message}；{command_message}", + runtime={ + "mode": snapshot.get("worker_mode", ""), + "running": snapshot.get("worker_online", False), + "process_count": snapshot.get("worker_process_count", 0), + "latest_start_time": snapshot.get("worker_latest_start_time", ""), + "message": snapshot.get("worker_runtime_message", ""), + }, + progress=snapshot.get("progress", {}), + settings_summary=settings_summary, + ) append_detect_job_event( job_summary["job_id"], event_type="job_dispatch_remote_queued", @@ -411,11 +490,16 @@ def start_detect(step_code: str | None = None) -> ApiResponse: @router.post("/detect/stop", response_model=ApiResponse) -def stop_detect() -> ApiResponse: +def stop_detect(payload: dict | None = Body(default=None)) -> ApiResponse: active_job = get_active_detect_job_summary(event_limit=10) - ok, message = send_worker_command("stop_detection") + normalized_payload = { + key: value + for key, value in dict(payload or {}).items() + if value not in (None, "") + } + ok, message = send_worker_command("stop_detection", payload=normalized_payload) cycle_token = str((active_job or {}).get("current_cycle_token") or "").strip() - remote_dispatch = _dispatch_remote_detect_stop(active_job=active_job, cycle_token=cycle_token) + remote_dispatch = _dispatch_remote_detect_stop(active_job=active_job, cycle_token=cycle_token, payload=normalized_payload) if active_job: append_detect_job_event( active_job["job_id"], diff --git a/domain-api/app/api/routes/ops.py b/domain-api/app/api/routes/ops.py index fe9bfc3..2d9da55 100644 --- a/domain-api/app/api/routes/ops.py +++ b/domain-api/app/api/routes/ops.py @@ -31,6 +31,11 @@ from app.services.ops_job_service import ( sync_managed_nodes_from_cluster, upsert_managed_node, ) +from app.services.ops_migration_service import ( + execute_ops_migration, + get_ops_migration_source_profile, + preview_ops_migration, +) from app.services.ops_playbook_service import ( cancel_ops_playbook_run, execute_ops_playbook, @@ -80,6 +85,23 @@ def ops_overview() -> ApiResponse: return ApiResponse(data=get_ops_overview()) +@router.get("/ops/migration/source-profile", response_model=ApiResponse) +def ops_migration_source_profile() -> ApiResponse: + return ApiResponse(data=get_ops_migration_source_profile()) + + +@router.post("/ops/migration/preview", response_model=ApiResponse) +def ops_migration_preview(payload: dict | None = None) -> ApiResponse: + ok, message, data = preview_ops_migration(payload or {}) + return ApiResponse(code=0 if ok else 1, message=message, data=data) + + +@router.post("/ops/migration/execute", response_model=ApiResponse) +def ops_migration_execute(payload: dict | None = None) -> ApiResponse: + ok, message, data = execute_ops_migration(payload or {}) + return ApiResponse(code=0 if ok else 1, message=message, data=data) + + @router.get("/ops/link-snapshot", response_model=ApiResponse) def ops_link_snapshot() -> ApiResponse: return ApiResponse(data=get_ops_link_snapshot()) diff --git a/domain-api/app/api/routes/runtime.py b/domain-api/app/api/routes/runtime.py index ac400a5..86edffc 100644 --- a/domain-api/app/api/routes/runtime.py +++ b/domain-api/app/api/routes/runtime.py @@ -19,6 +19,55 @@ from app.services.sync_record_service import get_sync_summary, list_sync_records router = APIRouter(tags=["runtime"]) +def _filter_debug_handover_by_node_code(payload: dict, node_code: str) -> dict: + normalized_node_code = str(node_code or "").strip() + if not normalized_node_code: + return dict(payload or {}) + + normalized_payload = dict(payload or {}) + + def _matches_node(item: object) -> bool: + if not isinstance(item, dict): + return False + return str(item.get("node_code") or "").strip() == normalized_node_code + + overview = dict(normalized_payload.get("overview") or {}) + if overview: + overview["recent_issues"] = [ + item + for item in list(overview.get("recent_issues") or []) + if _matches_node(item) + ] + normalized_payload["overview"] = overview + + normalized_payload["recent_issues"] = [ + item + for item in list(normalized_payload.get("recent_issues") or []) + if _matches_node(item) + ] + normalized_payload["issue_groups"] = [ + item + for item in list(normalized_payload.get("issue_groups") or []) + if _matches_node(item) + ] + + failure_handoff = dict(normalized_payload.get("failure_handoff") or {}) + if failure_handoff: + failure_handoff["recent_issues"] = [ + item + for item in list(failure_handoff.get("recent_issues") or []) + if _matches_node(item) + ] + failure_handoff["issue_groups"] = [ + item + for item in list(failure_handoff.get("issue_groups") or []) + if _matches_node(item) + ] + normalized_payload["failure_handoff"] = failure_handoff + + return normalized_payload + + @router.get("/runtime/status", response_model=ApiResponse) def runtime_status() -> ApiResponse: return ApiResponse(data=get_runtime_status()) @@ -60,6 +109,7 @@ def runtime_debug_events( service: Optional[str] = None, event_type: Optional[str] = None, source_region: Optional[str] = None, + node_code: Optional[str] = None, level: Optional[str] = None, before_id: Optional[int] = None, after_id: Optional[int] = None, @@ -71,6 +121,7 @@ def runtime_debug_events( service=service, event_type=event_type, source_region=source_region, + node_code=node_code, level=level, before_id=before_id, after_id=after_id, @@ -125,12 +176,7 @@ def runtime_health_handover( ) normalized_node_code = str(node_code or "").strip() if normalized_node_code: - data = dict(data) - data["nodes"] = [ - item - for item in list(data.get("nodes") or []) - if str(item.get("node_code") or "").strip() == normalized_node_code - ] + data = _filter_debug_handover_by_node_code(data, normalized_node_code) return ApiResponse(data=data) diff --git a/domain-api/app/core/redis_client.py b/domain-api/app/core/redis_client.py index e0ebc08..72a09b3 100644 --- a/domain-api/app/core/redis_client.py +++ b/domain-api/app/core/redis_client.py @@ -1,17 +1,64 @@ from __future__ import annotations +import os +import threading + import redis from app.core.config import settings +_REDIS_CLIENT: redis.Redis | None = None +_LOCK = threading.Lock() + + +def _safe_int(raw_value: object, default: int, minimum: int) -> int: + try: + parsed = int(raw_value) + except Exception: + parsed = default + return max(minimum, parsed) + + +def _safe_float(raw_value: object, default: float, minimum: float) -> float: + try: + parsed = float(raw_value) + except Exception: + parsed = default + return max(minimum, parsed) + + def get_redis() -> redis.Redis: - return redis.Redis( - host=settings.redis_host, - port=settings.redis_port, - password=settings.redis_password or None, - db=settings.redis_db, - decode_responses=True, - socket_connect_timeout=5, - socket_timeout=5, - ) + global _REDIS_CLIENT + with _LOCK: + if _REDIS_CLIENT is not None: + return _REDIS_CLIENT + + pool = redis.BlockingConnectionPool( + host=settings.redis_host, + port=settings.redis_port, + password=settings.redis_password or None, + db=settings.redis_db, + decode_responses=True, + socket_connect_timeout=5, + socket_timeout=5, + health_check_interval=_safe_int(os.getenv("DOMAIN_API_REDIS_HEALTH_CHECK_INTERVAL", "30"), 30, 0), + retry_on_timeout=True, + max_connections=_safe_int(os.getenv("DOMAIN_API_REDIS_MAX_CONNECTIONS", "32"), 32, 1), + timeout=_safe_float(os.getenv("DOMAIN_API_REDIS_POOL_TIMEOUT", "1.5"), 1.5, 0.1), + client_name=f"domain-api:{settings.node_code}:{os.getpid()}", + ) + _REDIS_CLIENT = redis.Redis(connection_pool=pool) + return _REDIS_CLIENT + + +def reset_redis_client_for_tests() -> None: + global _REDIS_CLIENT + with _LOCK: + client = _REDIS_CLIENT + _REDIS_CLIENT = None + if client is not None: + try: + client.close() + except Exception: + pass diff --git a/domain-api/app/main.py b/domain-api/app/main.py index e8728ec..4ca95d4 100644 --- a/domain-api/app/main.py +++ b/domain-api/app/main.py @@ -11,6 +11,23 @@ from app.services.ops_agent_service import ensure_ops_agent_schema from app.services.ops_job_service import ensure_ops_schema from app.services.ops_release_service import ensure_ops_release_schema _heartbeat_stop_event = threading.Event() +_bootstrap_started = False +_bootstrap_lock = threading.Lock() + + +def _bootstrap_runtime_prerequisites() -> None: + for task in ( + ensure_runtime_schema, + ensure_ops_schema, + ensure_ops_agent_schema, + ensure_ops_release_schema, + register_local_control_heartbeat, + ): + try: + task() + except Exception: + # Startup must not block API listening on slow schema checks or DB stalls. + pass def _control_heartbeat_loop() -> None: @@ -39,14 +56,18 @@ app.add_middleware( @app.on_event("startup") def on_startup() -> None: - ensure_runtime_schema() - ensure_ops_schema() - ensure_ops_agent_schema() - ensure_ops_release_schema() + global _bootstrap_started remember_registered_route_paths(route.path for route in app.routes) - register_local_control_heartbeat() _heartbeat_stop_event.clear() - threading.Thread(target=_control_heartbeat_loop, name="control-heartbeat", daemon=True).start() + with _bootstrap_lock: + if not _bootstrap_started: + threading.Thread( + target=_bootstrap_runtime_prerequisites, + name="api-bootstrap", + daemon=True, + ).start() + threading.Thread(target=_control_heartbeat_loop, name="control-heartbeat", daemon=True).start() + _bootstrap_started = True @app.on_event("shutdown") diff --git a/domain-api/app/node_agent.py b/domain-api/app/node_agent.py index c975344..6d262db 100644 --- a/domain-api/app/node_agent.py +++ b/domain-api/app/node_agent.py @@ -10,6 +10,7 @@ import time import urllib.error import urllib.request from datetime import datetime +from pathlib import Path from urllib.parse import urlparse from uuid import uuid4 @@ -678,6 +679,265 @@ def _publish_local_config_update(config_type: str) -> None: return +def _normalize_process_count(value: object, default: int = 1) -> int: + try: + normalized = int(value) + except Exception: + return max(1, int(default or 1)) + return max(1, normalized) + + +def _resolve_desired_process_count(bundle: dict) -> int: + normalized_bundle = dict(bundle or {}) + node_process_counts = normalized_bundle.get("node_process_counts") + if isinstance(node_process_counts, dict): + override_value = node_process_counts.get(NODE_CODE) + if override_value not in (None, ""): + return _normalize_process_count(override_value, default=80) + return _normalize_process_count(normalized_bundle.get("process_count", 80), default=80) + + +def _worker_instance_suffixes(extra_count: int) -> list[str]: + alphabet = "abcdefghijklmnopqrstuvwxyz" + suffixes: list[str] = [] + normalized_extra_count = max(0, int(extra_count or 0)) + if normalized_extra_count <= 0: + return suffixes + + for char in alphabet: + suffixes.append(char) + if len(suffixes) >= normalized_extra_count: + return suffixes + for first in alphabet: + for second in alphabet: + suffixes.append(f"{first}{second}") + if len(suffixes) >= normalized_extra_count: + return suffixes + return suffixes[:normalized_extra_count] + + +def _worker_env_path(service_name: str, suffix: str = "") -> Path: + normalized_service_name = str(service_name or "").strip() or WORKER_SERVICE_NAME + suffix_text = f"-{suffix}" if str(suffix or "").strip() else "" + return Path("/etc/default") / f"{normalized_service_name}{suffix_text}" + + +def _worker_instance_unit_path(service_name: str) -> Path: + normalized_service_name = str(service_name or "").strip() or WORKER_SERVICE_NAME + return Path("/etc/systemd/system") / f"{normalized_service_name}@.service" + + +def _worker_instance_unit_name(service_name: str, suffix: str) -> str: + normalized_service_name = str(service_name or "").strip() or WORKER_SERVICE_NAME + return f"{normalized_service_name}@{suffix}" + + +def _template_worker_unit_source_path() -> Path: + return Path(_PROJECT_DIR) / "deploy" / "systemd" / "domain-worker@.service" + + +def _render_worker_instance_env(base_env_text: str, *, instance_node_code: str, parent_node_code: str) -> str: + lines: list[str] = [] + saw_node_code = False + saw_parent = False + for raw_line in str(base_env_text or "").splitlines(): + if raw_line.startswith("NODE_CODE="): + lines.append(f"NODE_CODE={instance_node_code}") + saw_node_code = True + continue + if raw_line.startswith("WORKER_PARENT_NODE_CODE="): + lines.append(f"WORKER_PARENT_NODE_CODE={parent_node_code}") + saw_parent = True + continue + lines.append(raw_line) + if not saw_node_code: + lines.append(f"NODE_CODE={instance_node_code}") + if not saw_parent: + lines.append(f"WORKER_PARENT_NODE_CODE={parent_node_code}") + return "\n".join(lines).rstrip() + "\n" + + +def _ensure_worker_instance_unit_template(service_name: str) -> tuple[bool, str]: + unit_path = _worker_instance_unit_path(service_name) + source_path = _template_worker_unit_source_path() + if not source_path.exists(): + return False, f"worker instance template missing: {source_path}" + try: + template_text = source_path.read_text(encoding="utf-8") + normalized_service_name = str(service_name or "").strip() or WORKER_SERVICE_NAME + rendered = ( + template_text + .replace("domaincheck-worker-%i", f"{normalized_service_name}-%i") + .replace("domaincheck-worker@%i", f"{normalized_service_name}@%i") + ) + if unit_path.exists(): + existing = unit_path.read_text(encoding="utf-8") + if existing == rendered: + return True, str(unit_path) + unit_path.write_text(rendered, encoding="utf-8") + return True, str(unit_path) + except Exception as exc: + return False, f"write worker instance template failed: {exc}" + + +def _worker_instance_start_batch_size() -> int: + raw_value = str(os.getenv("NODE_AGENT_WORKER_RECONCILE_BATCH_SIZE", "") or "").strip() + try: + return max(1, min(32, int(raw_value or 6))) + except Exception: + return 6 + + +def _worker_instance_start_batch_delay_seconds() -> float: + raw_value = str(os.getenv("NODE_AGENT_WORKER_RECONCILE_BATCH_DELAY_SECONDS", "") or "").strip() + try: + return max(0.0, min(30.0, float(raw_value or 1.0))) + except Exception: + return 1.0 + + +def _chunked_units(units: list[str], size: int) -> list[list[str]]: + batch_size = max(1, int(size or 1)) + return [units[index:index + batch_size] for index in range(0, len(units), batch_size)] + + +def _reconcile_worker_instances(bundle: dict) -> dict: + runtime_settings = dict(bundle.get("runtime_settings") or {}) + worker_mode = str(runtime_settings.get("worker_mode") or "").strip() or "windows-local" + worker_service_name = str(runtime_settings.get("worker_service_name") or WORKER_SERVICE_NAME).strip() or WORKER_SERVICE_NAME + desired_process_count = _resolve_desired_process_count(bundle) + + if worker_mode != "linux-systemd": + return { + "applied": False, + "reason": f"worker_mode={worker_mode}", + "desired_process_count": desired_process_count, + } + if NODE_REGION != "mainland": + return { + "applied": False, + "reason": f"region={NODE_REGION}", + "desired_process_count": desired_process_count, + } + + base_env_path = _worker_env_path(worker_service_name) + if not base_env_path.exists(): + return { + "applied": False, + "reason": f"base env missing: {base_env_path}", + "desired_process_count": desired_process_count, + } + + ok, template_message = _ensure_worker_instance_unit_template(worker_service_name) + if not ok: + return { + "applied": False, + "reason": template_message, + "desired_process_count": desired_process_count, + } + + desired_suffixes = _worker_instance_suffixes(max(0, desired_process_count - 1)) + desired_units = [_worker_instance_unit_name(worker_service_name, suffix) for suffix in desired_suffixes] + desired_env_paths = {_worker_env_path(worker_service_name, suffix) for suffix in desired_suffixes} + managed_prefix = f"{worker_service_name}-" + + try: + base_env_text = base_env_path.read_text(encoding="utf-8") + for suffix in desired_suffixes: + env_path = _worker_env_path(worker_service_name, suffix) + env_path.write_text( + _render_worker_instance_env( + base_env_text, + instance_node_code=f"{NODE_CODE}-{suffix}", + parent_node_code=NODE_CODE, + ), + encoding="utf-8", + ) + except Exception as exc: + return { + "applied": False, + "reason": f"write worker env failed: {exc}", + "desired_process_count": desired_process_count, + } + + existing_env_paths: list[Path] = [] + try: + for candidate in Path("/etc/default").iterdir(): + if not candidate.is_file(): + continue + if not candidate.name.startswith(managed_prefix): + continue + existing_env_paths.append(candidate) + except Exception: + existing_env_paths = [] + + stale_env_paths = [ + candidate + for candidate in existing_env_paths + if candidate not in desired_env_paths + ] + + rc, stdout, stderr = _run(["systemctl", "daemon-reload"], timeout=90) + if rc != 0: + return { + "applied": False, + "reason": stderr or stdout or "systemctl daemon-reload failed", + "desired_process_count": desired_process_count, + } + + if desired_units: + rc, stdout, stderr = _run(["systemctl", "enable", *desired_units], timeout=180) + if rc != 0: + return { + "applied": False, + "reason": stderr or stdout or "systemctl enable worker instances failed", + "desired_process_count": desired_process_count, + } + batch_size = _worker_instance_start_batch_size() + batch_delay_seconds = _worker_instance_start_batch_delay_seconds() + for batch_index, unit_batch in enumerate(_chunked_units(desired_units, batch_size), start=1): + rc, stdout, stderr = _run( + ["systemctl", "start", *unit_batch], + timeout=max(120, 30 * len(unit_batch)), + ) + if rc != 0: + return { + "applied": False, + "reason": ( + stderr + or stdout + or f"systemctl start worker instances failed at batch {batch_index}" + ), + "desired_process_count": desired_process_count, + } + if batch_delay_seconds > 0 and batch_index * batch_size < len(desired_units): + time.sleep(batch_delay_seconds) + + stale_units = [ + f"{worker_service_name}@{candidate.name[len(managed_prefix):]}" + for candidate in stale_env_paths + if candidate.name[len(managed_prefix):] + ] + if stale_units: + _run(["systemctl", "stop", *stale_units], timeout=180) + _run(["systemctl", "disable", *stale_units], timeout=180) + for candidate in stale_env_paths: + try: + candidate.unlink() + except Exception: + continue + + return { + "applied": True, + "reason": "reconciled", + "desired_process_count": desired_process_count, + "instance_service_name": f"{worker_service_name}@.service", + "extra_instances": len(desired_suffixes), + "stale_instances_removed": len(stale_units), + "template_path": template_message, + } + + def _apply_runtime_config(bundle: dict) -> bool: global _LAST_RUNTIME_CONFIG_HASH @@ -688,6 +948,13 @@ def _apply_runtime_config(bundle: dict) -> bool: json.dumps(normalized_bundle, ensure_ascii=False, sort_keys=True).encode("utf-8") ).hexdigest() if bundle_hash and bundle_hash == _LAST_RUNTIME_CONFIG_HASH: + reconcile_summary = _reconcile_worker_instances(normalized_bundle) + if reconcile_summary.get("applied"): + _log( + "worker instance reconcile refreshed: " + f"desired={reconcile_summary.get('desired_process_count', 1)} " + f"extra={reconcile_summary.get('extra_instances', 0)}" + ) return False from app.core.files import write_json @@ -696,8 +963,10 @@ def _apply_runtime_config(bundle: dict) -> bool: detect_options = dict(normalized_bundle.get("detect_options") or {}) proxy_config = dict(normalized_bundle.get("proxy_config") or {}) - thread_count = int(normalized_bundle.get("thread_count", 2) or 2) + thread_count = int(normalized_bundle.get("thread_count", 1000) or 1000) node_thread_counts = dict(normalized_bundle.get("node_thread_counts") or {}) + process_count = int(normalized_bundle.get("process_count", 80) or 80) + node_process_counts = dict(normalized_bundle.get("node_process_counts") or {}) runtime_settings = dict(normalized_bundle.get("runtime_settings") or {}) sensitive_words = dict(normalized_bundle.get("sensitive_words") or {}) sensitive_words_text = str(sensitive_words.get("text") or "") @@ -707,6 +976,8 @@ def _apply_runtime_config(bundle: dict) -> bool: write_json("proxy_config.json", proxy_config) write_json("thread_count.json", {"thread_count": str(thread_count)}) write_json("node_thread_counts.json", node_thread_counts) + write_json("process_count.json", {"process_count": str(process_count)}) + write_json("node_process_counts.json", node_process_counts) write_json("runtime_settings.json", runtime_settings) write_json("runtime/runtime_settings.json", runtime_settings) write_json( @@ -736,12 +1007,16 @@ def _apply_runtime_config(bundle: dict) -> bool: redis_client.set("domain_tool:proxy_config", json.dumps(proxy_config, ensure_ascii=False)) redis_client.set("domain_tool:thread_count", thread_count) redis_client.set("domain_tool:node_thread_counts", json.dumps(node_thread_counts, ensure_ascii=False)) + redis_client.set("domain_tool:process_count", process_count) + redis_client.set("domain_tool:node_process_counts", json.dumps(node_process_counts, ensure_ascii=False)) redis_client.set("domain_tool:runtime_settings", json.dumps(runtime_settings, ensure_ascii=False)) redis_client.set("domain_tool:sensitive_words", json.dumps(sensitive_word_items, ensure_ascii=False)) redis_client.publish("domain_tool:config_update", "detect_options") redis_client.publish("domain_tool:config_update", "proxy_config") redis_client.publish("domain_tool:config_update", "thread_count") redis_client.publish("domain_tool:config_update", "node_thread_counts") + redis_client.publish("domain_tool:config_update", "process_count") + redis_client.publish("domain_tool:config_update", "node_process_counts") redis_client.publish("domain_tool:config_update", "runtime_settings") redis_client.publish("domain_tool:config_update", "sensitive_words") except Exception: @@ -749,16 +1024,30 @@ def _apply_runtime_config(bundle: dict) -> bool: _publish_local_config_update("proxy_config") _publish_local_config_update("thread_count") _publish_local_config_update("node_thread_counts") + _publish_local_config_update("process_count") + _publish_local_config_update("node_process_counts") _publish_local_config_update("runtime_settings") _publish_local_config_update("sensitive_words") + reconcile_summary = _reconcile_worker_instances(normalized_bundle) _LAST_RUNTIME_CONFIG_HASH = bundle_hash _log( "runtime config applied: " f"thread_count={thread_count} " + f"process_count={process_count} " f"node_override={node_thread_counts.get(NODE_CODE)} " + f"process_override={node_process_counts.get(NODE_CODE)} " f"sensitive_words={int(sensitive_words.get('total', 0) or 0)}" ) + if reconcile_summary.get("applied"): + _log( + "worker instance reconcile: " + f"desired={reconcile_summary.get('desired_process_count', 1)} " + f"extra={reconcile_summary.get('extra_instances', 0)} " + f"removed={reconcile_summary.get('stale_instances_removed', 0)}" + ) + elif reconcile_summary.get("reason"): + _log(f"worker instance reconcile skipped: {reconcile_summary.get('reason')}") return True @@ -914,6 +1203,15 @@ def _detect_runtime_snapshot() -> dict: "phase_detail": phase_detail, "recent_warning": str(detect_status.get("recent_warning") or "").strip(), "updated_at": str(runtime_state.get("updated_at") or "").strip(), + "available_proxy_count": int(detect_status.get("available_proxy_count", 0) or 0), + "proxy_runtime_label": str(detect_status.get("proxy_runtime_label") or "").strip(), + "proxy_runtime_reason": str(detect_status.get("proxy_runtime_reason") or "").strip(), + "proxy_last_refresh_status": str(detect_status.get("proxy_last_refresh_status") or "").strip(), + "proxy_last_refresh_time": str(detect_status.get("proxy_last_refresh_time") or "").strip(), + "proxy_last_refresh_source_count": int(detect_status.get("proxy_last_refresh_source_count", 0) or 0), + "proxy_last_refresh_total_items": int(detect_status.get("proxy_last_refresh_total_items", 0) or 0), + "proxy_last_validated_count": int(detect_status.get("proxy_last_validated_count", 0) or 0), + "proxy_last_available_count": int(detect_status.get("proxy_last_available_count", 0) or 0), "detect_participating": bool( detect_status.get("detect_participating", False) or current_load > 0 @@ -934,6 +1232,15 @@ def _detect_runtime_snapshot() -> dict: "phase_detail": worker_message, "recent_warning": "", "updated_at": str(worker_runtime.get("latest_start_time") or "").strip(), + "available_proxy_count": 0, + "proxy_runtime_label": "", + "proxy_runtime_reason": "", + "proxy_last_refresh_status": "", + "proxy_last_refresh_time": "", + "proxy_last_refresh_source_count": 0, + "proxy_last_refresh_total_items": 0, + "proxy_last_validated_count": 0, + "proxy_last_available_count": 0, "detect_participating": False, "error": str(exc), } diff --git a/domain-api/app/services/build_info_service.py b/domain-api/app/services/build_info_service.py index 09e0d74..bb13778 100644 --- a/domain-api/app/services/build_info_service.py +++ b/domain-api/app/services/build_info_service.py @@ -164,6 +164,9 @@ def _expected_route_paths() -> dict[str, str]: "ops_contracts": f"{prefix}/ops/contracts", "ops_contract_detail": f"{prefix}/ops/contracts/{{contract_key}}", "ops_stack_diagnosis": f"{prefix}/ops/stack-diagnosis", + "ops_migration_source_profile": f"{prefix}/ops/migration/source-profile", + "ops_migration_preview": f"{prefix}/ops/migration/preview", + "ops_migration_execute": f"{prefix}/ops/migration/execute", "ops_node_handover": f"{prefix}/ops/nodes/{{node_code}}/handover", "ops_node_onboarding": f"{prefix}/ops/nodes/{{node_code}}/onboarding", "ops_node_onboarding_bootstrap_preview": f"{prefix}/ops/nodes/{{node_code}}/onboarding/bootstrap/preview", diff --git a/domain-api/app/services/cluster_runtime_service.py b/domain-api/app/services/cluster_runtime_service.py index 72700d3..8bb0701 100644 --- a/domain-api/app/services/cluster_runtime_service.py +++ b/domain-api/app/services/cluster_runtime_service.py @@ -5,6 +5,8 @@ import socket import threading from datetime import datetime, timedelta +from psycopg2 import errors + from app.core.config import settings from app.core.db import db_read_retry, get_db @@ -63,6 +65,22 @@ CREATE TABLE IF NOT EXISTS detect_job_items ( CREATE INDEX IF NOT EXISTS idx_detect_job_items_status_lease ON detect_job_items(status, lease_expires_at); +CREATE INDEX IF NOT EXISTS idx_detect_job_items_claim_ready +ON detect_job_items(status, create_time, id) +WHERE step_code <> ''; + +CREATE INDEX IF NOT EXISTS idx_detect_job_items_claim_job_ready +ON detect_job_items(job_id, status, create_time, id) +WHERE step_code <> ''; + +CREATE INDEX IF NOT EXISTS idx_detect_job_items_claim_step_ready +ON detect_job_items(status, step_code, lease_expires_at, create_time, id) +WHERE step_code <> '' AND status IN ('pending', 'failed'); + +CREATE INDEX IF NOT EXISTS idx_detect_job_items_claim_job_step_ready +ON detect_job_items(job_id, status, step_code, lease_expires_at, create_time, id) +WHERE step_code <> '' AND status IN ('pending', 'failed'); + ALTER TABLE detect_jobs ADD COLUMN IF NOT EXISTS task_mode VARCHAR(32) NOT NULL DEFAULT 'domain_pipeline', ADD COLUMN IF NOT EXISTS step_code VARCHAR(64) NOT NULL DEFAULT ''; @@ -108,11 +126,100 @@ CREATE TABLE IF NOT EXISTS detect_sync_records ( _STALE_AFTER_SECONDS = 90 _OFFLINE_AFTER_MINUTES = 5 +_IMPORTED_RUNTIME_STALE_AFTER_MINUTES = 10 +_IMPORTED_RUNTIME_OFFLINE_AFTER_MINUTES = 30 _PRUNE_IMPORTED_AFTER_MINUTES = 30 _PRUNE_GENERAL_AFTER_HOURS = 6 _RUNTIME_SCHEMA_READY = False _RUNTIME_SCHEMA_LOCK = threading.Lock() _RUNTIME_SCHEMA_ADVISORY_LOCK_ID = 62021001 +_RUNTIME_SCHEMA_INDEX_ADVISORY_LOCK_ID = 62021002 +_DISABLED_MANAGED_NODE_CACHE: set[str] = set() +_DISABLED_MANAGED_NODE_CACHE_EXPIRES_AT = 0.0 +_RUNTIME_REQUIRED_TABLES = ( + "detect_worker_nodes", + "detect_jobs", + "detect_job_items", + "detect_run_events", + "detect_sync_records", +) +_RUNTIME_REQUIRED_COLUMNS = { + "detect_jobs": {"task_mode", "step_code"}, + "detect_job_items": {"step_code", "step_payload_json", "result_payload_json"}, +} +_RUNTIME_REQUIRED_INDEXES = ( + "idx_detect_job_items_job_domain_step", + "idx_detect_job_items_claim_step_ready", + "idx_detect_job_items_claim_job_step_ready", + "idx_detect_sync_records_scope_created", + "idx_detect_sync_records_source_record_created", + "idx_detect_sync_records_source_record_hash_created", + "idx_detect_sync_records_runtime_push_lookup", +) +_RUNTIME_REQUIRED_INDEX_TABLES = { + "idx_detect_job_items_job_domain_step": "detect_job_items", + "idx_detect_job_items_claim_step_ready": "detect_job_items", + "idx_detect_job_items_claim_job_step_ready": "detect_job_items", + "idx_detect_sync_records_scope_created": "detect_sync_records", + "idx_detect_sync_records_source_record_created": "detect_sync_records", + "idx_detect_sync_records_source_record_hash_created": "detect_sync_records", + "idx_detect_sync_records_runtime_push_lookup": "detect_sync_records", +} +_RUNTIME_REQUIRED_INDEX_DDL = { + "idx_detect_job_items_job_domain_step": """ + CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS idx_detect_job_items_job_domain_step + ON detect_job_items(job_id, domain_id, step_code) + """, + "idx_detect_job_items_claim_step_ready": """ + CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_detect_job_items_claim_step_ready + ON detect_job_items(status, step_code, lease_expires_at, create_time, id) + WHERE step_code <> '' AND status IN ('pending', 'failed') + """, + "idx_detect_job_items_claim_job_step_ready": """ + CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_detect_job_items_claim_job_step_ready + ON detect_job_items(job_id, status, step_code, lease_expires_at, create_time, id) + WHERE step_code <> '' AND status IN ('pending', 'failed') + """, + "idx_detect_sync_records_scope_created": """ + CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_detect_sync_records_scope_created + ON detect_sync_records(sync_type, source_region, target_region, created_at DESC, id DESC) + """, + "idx_detect_sync_records_source_record_created": """ + CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_detect_sync_records_source_record_created + ON detect_sync_records( + sync_type, + source_region, + target_region, + ((payload_json->>'source_record_id')), + created_at DESC, + id DESC + ) + """, + "idx_detect_sync_records_source_record_hash_created": """ + CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_detect_sync_records_source_record_hash_created + ON detect_sync_records( + sync_type, + source_region, + target_region, + ((payload_json->>'source_record_id')), + ((payload_json->>'projection_hash')), + created_at DESC, + id DESC + ) + """, + "idx_detect_sync_records_runtime_push_lookup": """ + CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_detect_sync_records_runtime_push_lookup + ON detect_sync_records( + source_region, + target_region, + ((payload_json->>'sync_type')), + ((payload_json->>'source_record_id')), + created_at DESC, + id DESC + ) + WHERE sync_type = 'runtime_push' + """, +} def _resolve_local_ip() -> str: @@ -133,6 +240,17 @@ def _decode_json(value: object) -> dict: return {} +def _parse_runtime_timestamp(value: object) -> datetime | None: + raw = str(value or "").strip() + if not raw: + return None + normalized = raw.replace("Z", "+00:00") + try: + return datetime.fromisoformat(normalized) + except Exception: + return None + + def _control_node_supports_worker(*, region: object, metadata: dict | None) -> bool: normalized_region = str(region or "").strip() runtime_metadata = dict(metadata or {}) @@ -198,6 +316,35 @@ def _load_managed_node_overlays() -> dict[str, dict]: return overlays +def _load_disabled_managed_node_codes() -> set[str]: + global _DISABLED_MANAGED_NODE_CACHE, _DISABLED_MANAGED_NODE_CACHE_EXPIRES_AT + now_ts = datetime.now().timestamp() + if now_ts < _DISABLED_MANAGED_NODE_CACHE_EXPIRES_AT: + return set(_DISABLED_MANAGED_NODE_CACHE) + try: + with get_db() as conn: + with conn.cursor() as cur: + cur.execute( + """ + SELECT node_code + FROM ops_managed_nodes + WHERE is_enabled = FALSE + """ + ) + rows = cur.fetchall() + except Exception: + return set(_DISABLED_MANAGED_NODE_CACHE) + + disabled_codes = { + str(row[0] or "").strip() + for row in list(rows or []) + if str(row[0] or "").strip() + } + _DISABLED_MANAGED_NODE_CACHE = disabled_codes + _DISABLED_MANAGED_NODE_CACHE_EXPIRES_AT = now_ts + 5.0 + return set(disabled_codes) + + def ensure_runtime_schema() -> None: global _RUNTIME_SCHEMA_READY if _RUNTIME_SCHEMA_READY: @@ -206,14 +353,131 @@ def ensure_runtime_schema() -> None: if _RUNTIME_SCHEMA_READY: return with get_db() as conn: - conn.autocommit = False with conn.cursor() as cur: - cur.execute("SELECT pg_advisory_xact_lock(%s)", (_RUNTIME_SCHEMA_ADVISORY_LOCK_ID,)) - cur.execute(_RUNTIME_SCHEMA_SQL) - conn.commit() + if _runtime_schema_basics_present(cur): + missing_indexes = list(_runtime_missing_indexes(cur)) + if not missing_indexes: + _RUNTIME_SCHEMA_READY = True + return + else: + missing_indexes = [] + if missing_indexes: + _ensure_runtime_schema_indexes(missing_indexes) + with conn.cursor() as cur: + if _runtime_schema_basics_present(cur) and not list(_runtime_missing_indexes(cur)): + _RUNTIME_SCHEMA_READY = True + return + else: + try: + with conn.cursor() as cur: + cur.execute("SELECT pg_advisory_xact_lock(%s)", (_RUNTIME_SCHEMA_ADVISORY_LOCK_ID,)) + cur.execute(_RUNTIME_SCHEMA_SQL) + conn.commit() + except Exception as exc: + recoverable = isinstance(exc, (errors.DeadlockDetected, errors.LockNotAvailable)) + try: + conn.rollback() + except Exception: + pass + if not recoverable: + raise + with conn.cursor() as cur: + if not _runtime_schema_basics_present(cur): + raise _RUNTIME_SCHEMA_READY = True +def _runtime_schema_basics_present(cur) -> bool: + for table_name in _RUNTIME_REQUIRED_TABLES: + cur.execute("SELECT to_regclass(%s)", (f"public.{table_name}",)) + row = cur.fetchone() + if not row or not row[0]: + return False + + for table_name, required_columns in _RUNTIME_REQUIRED_COLUMNS.items(): + cur.execute( + """ + SELECT column_name + FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = %s + """, + (table_name,), + ) + existing_columns = {str(row[0] or "").strip() for row in list(cur.fetchall() or [])} + if not set(required_columns).issubset(existing_columns): + return False + return True + +def _runtime_missing_indexes(cur): + states = _runtime_index_states(cur) + for index_name in _RUNTIME_REQUIRED_INDEXES: + if not bool((states.get(index_name) or {}).get("valid")): + yield index_name + + +def _runtime_index_states(cur) -> dict[str, dict[str, bool]]: + table_names = sorted(set(_RUNTIME_REQUIRED_INDEX_TABLES.values())) + cur.execute( + """ + SELECT + idx.relname AS index_name, + pg_index.indisvalid AS is_valid, + pg_index.indisready AS is_ready, + pg_index.indislive AS is_live + FROM pg_class AS idx + JOIN pg_index ON pg_index.indexrelid = idx.oid + JOIN pg_class AS tbl ON tbl.oid = pg_index.indrelid + JOIN pg_namespace AS ns ON ns.oid = tbl.relnamespace + WHERE ns.nspname = 'public' + AND tbl.relname = ANY(%s) + AND idx.relname = ANY(%s) + """, + (table_names, list(_RUNTIME_REQUIRED_INDEXES)), + ) + states = { + index_name: {"valid": False, "ready": False, "live": False} + for index_name in _RUNTIME_REQUIRED_INDEXES + } + for row in list(cur.fetchall() or []): + index_name = str(row[0] or "").strip() + if index_name not in states: + continue + states[index_name] = { + "valid": bool(row[1]), + "ready": bool(row[2]), + "live": bool(row[3]), + } + return states + + +def _ensure_runtime_schema_indexes(index_names: list[str] | tuple[str, ...]) -> None: + normalized_indexes = [ + index_name + for index_name in list(index_names or []) + if str(index_name or "").strip() in _RUNTIME_REQUIRED_INDEX_DDL + ] + if not normalized_indexes: + return + with get_db() as conn: + conn.autocommit = True + with conn.cursor() as cur: + cur.execute("SELECT pg_try_advisory_lock(%s)", (_RUNTIME_SCHEMA_INDEX_ADVISORY_LOCK_ID,)) + row = cur.fetchone() + if not bool((row or [False])[0]): + return + try: + current_states = _runtime_index_states(cur) + for index_name in normalized_indexes: + index_state = current_states.get(index_name) or {} + if bool(index_state.get("valid")): + continue + if bool(index_state.get("ready")) or bool(index_state.get("live")): + cur.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {index_name}") + cur.execute(_RUNTIME_REQUIRED_INDEX_DDL[index_name]) + finally: + cur.execute("SELECT pg_advisory_unlock(%s)", (_RUNTIME_SCHEMA_INDEX_ADVISORY_LOCK_ID,)) + + def register_node_heartbeat( *, node_code: str, @@ -351,14 +615,11 @@ def prune_expired_runtime_nodes() -> None: def register_local_control_heartbeat() -> None: - from app.services.detect_job_service import get_active_detect_job_summary from app.services.detect_service import get_detect_status - from app.services.worker_control_service import detect_worker_runtime - worker_runtime = detect_worker_runtime() - worker_online = bool(worker_runtime.get("running", False)) detect_status = get_detect_status() - active_job = get_active_detect_job_summary(event_limit=5) or {} + worker_online = bool(detect_status.get("worker_online", False)) + active_job = dict(detect_status.get("active_job") or {}) node_stats = list(active_job.get("node_stats") or []) local_bucket = next( (item for item in node_stats if str(item.get("node_code") or "").strip() == settings.node_code), @@ -385,7 +646,7 @@ def register_local_control_heartbeat() -> None: "api_port": settings.api_port, "worker_mode": settings.worker_mode, "worker_online": worker_online, - "worker_process_count": int(worker_runtime.get("process_count", 0) or 0), + "worker_process_count": int(detect_status.get("worker_process_count", 0) or 0), "detect_participating": detect_participating, "active_job_code": str(active_job.get("job_code") or ""), "active_job_status": str(active_job.get("status") or ""), @@ -395,6 +656,15 @@ def register_local_control_heartbeat() -> None: "job_items_completed": items_completed, "active_threads": active_threads, "max_threads": max_threads, + "available_proxy_count": int(detect_status.get("available_proxy_count", 0) or 0), + "proxy_runtime_label": str(detect_status.get("proxy_runtime_label") or "").strip(), + "proxy_runtime_reason": str(detect_status.get("proxy_runtime_reason") or "").strip(), + "proxy_last_refresh_status": str(detect_status.get("proxy_last_refresh_status") or "").strip(), + "proxy_last_refresh_time": str(detect_status.get("proxy_last_refresh_time") or "").strip(), + "proxy_last_refresh_source_count": int(detect_status.get("proxy_last_refresh_source_count", 0) or 0), + "proxy_last_refresh_total_items": int(detect_status.get("proxy_last_refresh_total_items", 0) or 0), + "proxy_last_validated_count": int(detect_status.get("proxy_last_validated_count", 0) or 0), + "proxy_last_available_count": int(detect_status.get("proxy_last_available_count", 0) or 0), "phase_label": str(detect_status.get("phase_label") or ""), "phase_detail": str(detect_status.get("phase_detail") or ""), "updated_at": datetime.now().isoformat(timespec="seconds"), @@ -402,15 +672,42 @@ def register_local_control_heartbeat() -> None: ) -def _normalize_node_status(raw_status: str, last_heartbeat_at: datetime | None) -> str: +def _resolve_effective_runtime_heartbeat( + *, + metadata: dict | None, + last_heartbeat_at: datetime | None, + update_time: datetime | None, +) -> datetime | None: + effective_last_heartbeat = last_heartbeat_at + runtime_metadata = dict(metadata or {}) + if str(runtime_metadata.get("service") or "").strip() != "runtime-ingest": + return effective_last_heartbeat + + metadata_updated_at = _parse_runtime_timestamp(runtime_metadata.get("updated_at")) + for candidate in (update_time, metadata_updated_at): + if not candidate: + continue + if effective_last_heartbeat is None or candidate > effective_last_heartbeat: + effective_last_heartbeat = candidate + return effective_last_heartbeat + + +def _normalize_node_status(raw_status: str, last_heartbeat_at: datetime | None, *, metadata: dict | None = None) -> str: status = str(raw_status or "").strip() or "unknown" if not last_heartbeat_at: return status now = datetime.now(last_heartbeat_at.tzinfo) if last_heartbeat_at.tzinfo else datetime.now() age = now - last_heartbeat_at - if age > timedelta(minutes=_OFFLINE_AFTER_MINUTES): + runtime_metadata = dict(metadata or {}) + service_name = str(runtime_metadata.get("service") or "").strip() + stale_after = timedelta(seconds=_STALE_AFTER_SECONDS) + offline_after = timedelta(minutes=_OFFLINE_AFTER_MINUTES) + if service_name == "runtime-ingest": + stale_after = timedelta(minutes=_IMPORTED_RUNTIME_STALE_AFTER_MINUTES) + offline_after = timedelta(minutes=_IMPORTED_RUNTIME_OFFLINE_AFTER_MINUTES) + if age > offline_after: return "offline" - if age > timedelta(seconds=_STALE_AFTER_SECONDS): + if age > stale_after: return "stale" return status @@ -420,11 +717,12 @@ def get_cluster_snapshot() -> dict: prune_expired_runtime_nodes() register_local_control_heartbeat() managed_overlays = _load_managed_node_overlays() + disabled_node_codes = _load_disabled_managed_node_codes() with get_db() as conn: with conn.cursor() as cur: cur.execute( """ - SELECT node_code, region, role, hostname, ip, status, worker_version, current_load, metadata_json, last_heartbeat_at + SELECT node_code, region, role, hostname, ip, status, worker_version, current_load, metadata_json, last_heartbeat_at, update_time FROM detect_worker_nodes ORDER BY CASE WHEN status = 'busy' THEN 0 WHEN status = 'online' THEN 1 ELSE 2 END, @@ -434,14 +732,33 @@ def get_cluster_snapshot() -> dict: LIMIT 100 """ ) - rows = cur.fetchall() + rows = [ + row + for row in list(cur.fetchall() or []) + if str((row or [""])[0] or "").strip() not in disabled_node_codes + ] + cur.execute( + """ + SELECT node_code, region, role, hostname, ip, status, worker_version, current_load, metadata_json, last_heartbeat_at, update_time + FROM detect_worker_nodes + ORDER BY + CASE WHEN status = 'busy' THEN 0 WHEN status = 'online' THEN 1 ELSE 2 END, + region ASC, + role ASC, + node_code ASC + """ + ) + summary_rows = [ + row + for row in list(cur.fetchall() or []) + if str((row or [""])[0] or "").strip() not in disabled_node_codes + ] cur.execute("SELECT count(*) FROM detect_jobs") jobs_total = cur.fetchone()[0] cur.execute("SELECT count(*) FROM detect_job_items WHERE status IN ('pending', 'claimed', 'running')") active_items = cur.fetchone()[0] - nodes = [] - for row in rows: + def _build_node_payload(row: tuple) -> dict: node_code = str(row[0] or "").strip() metadata = _decode_json(row[8]) current_load = int(row[7] or 0) @@ -452,14 +769,20 @@ def get_cluster_snapshot() -> dict: metadata["detect_participating"] = False metadata["sanitized_runtime_state"] = "idle_phase_zeroed" runtime_last_heartbeat = row[9] + runtime_update_time = row[10] managed_overlay = managed_overlays.get(node_code) or {} managed_last_seen = managed_overlay.get("last_seen_at") + effective_last_heartbeat = _resolve_effective_runtime_heartbeat( + metadata=metadata, + last_heartbeat_at=runtime_last_heartbeat, + update_time=runtime_update_time, + ) overlay_is_newer = bool( managed_last_seen - and (not runtime_last_heartbeat or managed_last_seen > runtime_last_heartbeat) + and (not effective_last_heartbeat or managed_last_seen > effective_last_heartbeat) ) - effective_last_heartbeat = managed_last_seen if overlay_is_newer else runtime_last_heartbeat - normalized_status = _normalize_node_status(row[5], effective_last_heartbeat) + effective_last_heartbeat = managed_last_seen if overlay_is_newer else effective_last_heartbeat + normalized_status = _normalize_node_status(row[5], effective_last_heartbeat, metadata=metadata) if sanitized_idle_runtime and normalized_status == "busy": normalized_status = "online" if overlay_is_newer and normalized_status in {"offline", "stale"}: @@ -468,22 +791,23 @@ def get_cluster_snapshot() -> dict: metadata["agent_last_seen_at"] = managed_last_seen.isoformat(sep=" ", timespec="seconds") if overlay_is_newer: metadata["cluster_status_source"] = "managed-agent-overlay" - nodes.append( - { - "node_code": node_code, - "region": row[1], - "role": row[2], - "hostname": row[3], - "ip": row[4], - "status": normalized_status, - "worker_version": row[6], - "current_load": current_load, - "metadata": metadata, - "last_heartbeat_at": effective_last_heartbeat.isoformat(sep=" ", timespec="seconds") - if effective_last_heartbeat - else "", - } - ) + return { + "node_code": node_code, + "region": row[1], + "role": row[2], + "hostname": row[3], + "ip": row[4], + "status": normalized_status, + "worker_version": row[6], + "current_load": current_load, + "metadata": metadata, + "last_heartbeat_at": effective_last_heartbeat.isoformat(sep=" ", timespec="seconds") + if effective_last_heartbeat + else "", + } + + nodes = [_build_node_payload(row) for row in rows] + summary_nodes = [_build_node_payload(row) for row in summary_rows] status_counts: dict[str, int] = {} role_counts: dict[str, int] = {} region_counts: dict[str, int] = {} @@ -494,7 +818,7 @@ def get_cluster_snapshot() -> dict: dedicated_online_worker_nodes = 0 online_control_nodes = 0 - for node in nodes: + for node in summary_nodes: node_status = str(node.get("status") or "unknown") node_role = str(node.get("role") or "unknown") node_region = str(node.get("region") or "unknown") @@ -534,9 +858,25 @@ def get_cluster_snapshot() -> dict: effective_worker and (metadata.get("detect_participating", False) or node_current_load > 0) ) + summary_node_map = { + str(item.get("node_code") or "").strip(): item + for item in summary_nodes + if str(item.get("node_code") or "").strip() + } + for node in nodes: + summary_node = summary_node_map.get(str(node.get("node_code") or "").strip()) + if not summary_node: + continue + node["current_load"] = summary_node.get("current_load", node.get("current_load", 0)) + node["status"] = summary_node.get("status", node.get("status", "unknown")) + node["metadata"] = summary_node.get("metadata", node.get("metadata") or {}) + node["last_heartbeat_at"] = summary_node.get("last_heartbeat_at", node.get("last_heartbeat_at", "")) + node["is_effective_worker"] = bool(summary_node.get("is_effective_worker", False)) + node["detect_participating"] = bool(summary_node.get("detect_participating", False)) + return { "nodes": nodes, - "nodes_total": len(nodes), + "nodes_total": len(summary_nodes), "jobs_total": jobs_total, "active_job_items": active_items, "summary": { diff --git a/domain-api/app/services/dashboard.py b/domain-api/app/services/dashboard.py index b3074ef..59c44f0 100644 --- a/domain-api/app/services/dashboard.py +++ b/domain-api/app/services/dashboard.py @@ -1,13 +1,18 @@ from __future__ import annotations +from app.core.config import settings from app.core.db import get_db +from app.services.cluster_runtime_service import get_cluster_snapshot +from app.services.detect_service import get_detect_status from app.services.detect_job_service import ( _build_step_bucket, + order_step_buckets, get_active_detect_job_summary, get_detect_capacity_plan, get_detect_queue_health, ) -from app.services.runtime_status_service import get_runtime_status +from app.services.runtime_settings_service import get_runtime_settings +from app.services.worker_control_service import detect_worker_runtime def _empty_active_jobs_aggregate(window_minutes: int) -> dict: @@ -93,16 +98,7 @@ def _merge_step_queues_with_runtime_activity( round(processed_recent / safe_window_minutes, 2), ) - return sorted( - step_map.values(), - key=lambda item: ( - -int(item.get("items_pending", 0) or 0), - -int(item.get("items_running", 0) or 0), - -int(item.get("started_recent", 0) or 0), - -int(item.get("processed_recent", 0) or 0), - str(item.get("step_code") or ""), - ), - )[:normalized_limit] + return order_step_buckets(list(step_map.values()), limit=normalized_limit) def _align_active_jobs_aggregate_with_runtime( @@ -169,6 +165,58 @@ def _align_active_jobs_aggregate_with_runtime( return normalized +def _build_dashboard_runtime_summary(*, queue_health: dict) -> dict: + runtime_settings = get_runtime_settings() + worker_runtime = detect_worker_runtime() + worker_expected_on_this_node = not ( + str(settings.node_region or "").strip() == "overseas" + and str(settings.node_role or "").strip() == "control" + ) + return { + "node": { + "region": settings.node_region, + "role": settings.node_role, + }, + "worker": { + "running": bool(worker_runtime.get("running", False)), + "mode": worker_runtime.get("mode", runtime_settings.get("worker_mode", "windows-local")), + "expected_on_this_node": worker_expected_on_this_node, + }, + "cluster": get_cluster_snapshot(), + "detect": { + "backlog": dict((queue_health or {}).get("runtime_snapshot_backlog") or {}), + }, + } + + +def _resolve_server_code(node_code: str | None) -> str: + normalized_node_code = str(node_code or "").strip() + if not normalized_node_code: + return "" + parent_node_code, separator, suffix = normalized_node_code.rpartition("-") + if separator and parent_node_code and suffix.isalpha() and len(suffix) <= 3: + if any(char.isdigit() for char in parent_node_code): + return parent_node_code + return normalized_node_code + + +def _count_active_execution_servers(rows: list[dict] | None) -> int: + active_servers: set[str] = set() + for item in list(rows or []): + if not isinstance(item, dict): + continue + server_code = _resolve_server_code(item.get("node_code")) + if not server_code: + continue + if ( + int(item.get("items_running", 0) or 0) > 0 + or int(item.get("items_claimed", 0) or 0) > 0 + or int(item.get("processed_recent", 0) or 0) > 0 + ): + active_servers.add(server_code) + return len(active_servers) + + def _fetch_active_jobs_aggregate(window_minutes: int = 15) -> dict: safe_window_minutes = max(5, min(int(window_minutes or 15), 120)) payload = _empty_active_jobs_aggregate(safe_window_minutes) @@ -303,7 +351,7 @@ def _fetch_active_jobs_aggregate(window_minutes: int = 15) -> dict: } ) steps.append(bucket) - payload["steps"] = steps + payload["steps"] = order_step_buckets(steps, limit=8) cur.execute( """ @@ -409,8 +457,28 @@ def fetch_overview() -> dict: active_jobs_aggregate = _fetch_active_jobs_aggregate(window_minutes=window_minutes) active_job = get_active_detect_job_summary(event_limit=20) or {} aggregate_queue = active_jobs_aggregate.get("queue") or {} + active_job_display_claimed = int(active_job.get("display_items_claimed", active_job.get("items_claimed", 0)) or 0) + active_job_display_running = int( + active_job.get("display_items_running", active_job.get("display_active_threads", active_job.get("items_running", 0))) + or 0 + ) + active_job_display_active_threads = int( + active_job.get("display_active_threads", active_job.get("display_items_running", active_job.get("items_running", 0))) + or 0 + ) + active_job_display_max_threads = int(active_job.get("display_max_threads", 0) or 0) + active_job_distributed_node_stats = [ + dict(item) + for item in list(active_job.get("distributed_node_stats") or []) + if isinstance(item, dict) + ] - runtime = get_runtime_status() + queue_health = get_detect_queue_health(window_minutes=window_minutes) + runtime = _build_dashboard_runtime_summary(queue_health=queue_health) + try: + detect_status = get_detect_status() + except Exception: + detect_status = {} cluster_summary = ((runtime.get("cluster") or {}).get("summary") or {}) online_worker_nodes = int(cluster_summary.get("online_worker_nodes", 0) or 0) dedicated_online_worker_nodes = int(cluster_summary.get("dedicated_online_worker_nodes", 0) or 0) @@ -425,7 +493,6 @@ def fetch_overview() -> dict: result["node_region"] = runtime["node"]["region"] result["node_role"] = runtime["node"]["role"] - queue_health = get_detect_queue_health(window_minutes=window_minutes) active_jobs_aggregate = _align_active_jobs_aggregate_with_runtime( active_jobs_aggregate, runtime=runtime, @@ -460,6 +527,25 @@ def fetch_overview() -> dict: throughput_payload = queue_health.get("throughput") or {} runtime_job_code = str(job_payload.get("runtime_job_code") or "").strip() display_job_code = runtime_job_code or str(job_payload.get("job_code") or "") + active_job_matches_display_job = display_job_code == str(active_job.get("job_code") or "").strip() + queue_nodes = [ + dict(item) + for item in list(queue_health.get("nodes") or []) + if isinstance(item, dict) + ] + summary_display_running = max( + int(queue_payload.get("display_running", queue_payload.get("running", 0)) or 0), + active_job_display_running if active_job_matches_display_job else 0, + ) + summary_display_active_threads = max( + int(queue_payload.get("display_running", queue_payload.get("running", 0)) or 0), + active_job_display_active_threads if active_job_matches_display_job else 0, + ) + summary_display_max_threads = max( + sum(int(item.get("max_threads", 0) or 0) for item in queue_nodes), + active_job_display_max_threads if active_job_matches_display_job else 0, + ) + summary_distributed_node_stats = active_job_distributed_node_stats if active_job_matches_display_job else queue_nodes active_job_summary = { "job_id": int(job_payload.get("job_id", 0) or 0), "job_code": display_job_code, @@ -469,9 +555,19 @@ def fetch_overview() -> dict: "progress_percent": float(job_payload.get("progress_percent", 0) or 0), "items_total": int(queue_payload.get("items_total", 0) or 0), "items_pending": int(queue_payload.get("pending", 0) or 0), - "items_claimed": int(queue_payload.get("display_claimed", queue_payload.get("claimed", 0)) or 0), - "items_running": int(queue_payload.get("running", 0) or 0), - "items_display_running": int(queue_payload.get("display_running", queue_payload.get("running", 0)) or 0), + "items_claimed": max( + int(queue_payload.get("display_claimed", queue_payload.get("claimed", 0)) or 0), + active_job_display_claimed if active_job_matches_display_job else 0, + ), + "items_running": max( + int(queue_payload.get("running", 0) or 0), + int(active_job.get("items_running", 0) or 0) if active_job_matches_display_job else 0, + ), + "items_display_running": summary_display_running, + "display_items_running": summary_display_running, + "display_active_threads": summary_display_active_threads, + "display_max_threads": summary_display_max_threads, + "distributed_node_stats": summary_distributed_node_stats, "items_completed": int(queue_payload.get("completed", 0) or 0), "items_blacklisted": int(queue_payload.get("blacklisted", 0) or 0), "items_failed": int(queue_payload.get("failed", 0) or 0), @@ -482,6 +578,33 @@ def fetch_overview() -> dict: "blacklisted_recent": int(throughput_payload.get("blacklisted_recent", 0) or 0), "active_jobs_total": int(active_jobs_aggregate.get("active_jobs_total", 0) or 0), } + elif active_job: + active_job_summary = { + "job_id": int(active_job.get("job_id", 0) or 0), + "job_code": str(active_job.get("job_code") or active_job.get("runtime_job_code") or ""), + "db_job_code": str(active_job.get("job_code") or ""), + "runtime_job_code": str(active_job.get("runtime_job_code") or ""), + "status": str(active_job.get("status") or ""), + "progress_percent": float(active_job.get("progress_percent", 0) or 0), + "items_total": int(active_job.get("items_total", 0) or 0), + "items_pending": int(active_job.get("items_pending", 0) or 0), + "items_claimed": active_job_display_claimed, + "items_running": int(active_job.get("items_running", 0) or 0), + "items_display_running": active_job_display_running, + "display_items_running": active_job_display_running, + "display_active_threads": active_job_display_active_threads, + "display_max_threads": active_job_display_max_threads, + "distributed_node_stats": active_job_distributed_node_stats, + "items_completed": int(active_job.get("items_completed", 0) or 0), + "items_blacklisted": int(active_job.get("items_blacklisted", 0) or 0), + "items_failed": int(active_job.get("items_failed", 0) or 0), + "processed_per_minute": float((active_jobs_aggregate.get("throughput") or {}).get("processed_per_minute", 0) or 0), + "processed_recent": int((active_jobs_aggregate.get("throughput") or {}).get("processed_recent", 0) or 0), + "completed_recent": int((active_jobs_aggregate.get("throughput") or {}).get("completed_recent", 0) or 0), + "failed_recent": int((active_jobs_aggregate.get("throughput") or {}).get("failed_recent", 0) or 0), + "blacklisted_recent": int((active_jobs_aggregate.get("throughput") or {}).get("blacklisted_recent", 0) or 0), + "active_jobs_total": int(active_jobs_aggregate.get("active_jobs_total", 0) or 0), + } queue_pending_total = 0 queue_claimed_total = 0 @@ -514,9 +637,15 @@ def fetch_overview() -> dict: if queue_health.get("has_active_job"): queue_payload = queue_health.get("queue") or {} queue_pending_total = int(queue_payload.get("pending", 0) or 0) - queue_claimed_total = int(queue_payload.get("display_claimed", queue_payload.get("claimed", 0)) or 0) - queue_running_total = int(queue_payload.get("running", 0) or 0) - queue_display_running_total = int(queue_payload.get("display_running", queue_payload.get("running", 0)) or 0) + queue_claimed_total = max( + int(queue_payload.get("display_claimed", queue_payload.get("claimed", 0)) or 0), + active_job_display_claimed, + ) + queue_running_total = max(int(queue_payload.get("running", 0) or 0), int(active_job.get("items_running", 0) or 0)) + queue_display_running_total = max( + int(queue_payload.get("display_running", queue_payload.get("running", 0)) or 0), + active_job_display_running, + ) queue_completed_total = int(queue_payload.get("completed", 0) or 0) queue_blacklist_total = int(queue_payload.get("blacklisted", 0) or 0) queue_failed_total = int(queue_payload.get("failed", 0) or 0) @@ -558,7 +687,7 @@ def fetch_overview() -> dict: } for item in list(active_jobs_aggregate.get("steps") or [])[:8] ] - aggregate_node_throughput = [ + aggregate_node_throughput_all = [ { "node_code": str(item.get("node_code") or ""), "items_pending": int(item.get("items_pending", 0) or 0), @@ -574,8 +703,9 @@ def fetch_overview() -> dict: "failed_recent": int(item.get("failed_recent", 0) or 0), "blacklisted_recent": int(item.get("blacklisted_recent", 0) or 0), } - for item in list(active_jobs_aggregate.get("nodes") or [])[:8] + for item in list(active_jobs_aggregate.get("nodes") or []) ] + aggregate_node_throughput = aggregate_node_throughput_all[:8] queue_step_queue = [ { "step_code": str(item.get("step_code") or ""), @@ -600,7 +730,7 @@ def fetch_overview() -> dict: limit=8, ) ] - queue_node_throughput = [ + queue_node_throughput_all = [ { "node_code": str(item.get("node_code") or ""), "items_pending": int(item.get("items_pending", 0) or 0), @@ -616,15 +746,18 @@ def fetch_overview() -> dict: "failed_recent": int(item.get("failed_recent", 0) or 0), "blacklisted_recent": int(item.get("blacklisted_recent", 0) or 0), } - for item in list(queue_health.get("nodes") or [])[:8] + for item in list(queue_health.get("nodes") or []) ] + queue_node_throughput = queue_node_throughput_all[:8] step_queue = aggregate_step_queue node_throughput = aggregate_node_throughput + active_execution_node_source = aggregate_node_throughput_all aggregate_ppm = float((active_jobs_aggregate.get("throughput") or {}).get("processed_per_minute", 0) or 0) queue_ppm = float((queue_health.get("throughput") or {}).get("processed_per_minute", 0) or 0) if queue_health.get("has_active_job") or queue_ppm > aggregate_ppm: step_queue = queue_step_queue node_throughput = queue_node_throughput + active_execution_node_source = queue_node_throughput_all if step_queue: bottleneck_step = max( step_queue, @@ -668,13 +801,7 @@ def fetch_overview() -> dict: "recommended_additional_workers": int(capacity_plan.get("recommended_additional_workers", 0) or 0), "online_worker_nodes": online_worker_nodes, "dedicated_online_worker_nodes": dedicated_online_worker_nodes, - "active_execution_nodes": sum( - 1 - for item in node_throughput - if int(item.get("items_running", 0) or 0) > 0 - or int(item.get("items_claimed", 0) or 0) > 0 - or int(item.get("processed_recent", 0) or 0) > 0 - ), + "active_execution_nodes": _count_active_execution_servers(active_execution_node_source), } result["processed_per_minute"] = ops_processed_per_minute result["processed_recent"] = ops_processed_recent @@ -686,12 +813,26 @@ def fetch_overview() -> dict: result["queue_claimed_total"] = queue_claimed_total result["queue_running_total"] = queue_running_total result["queue_display_running_total"] = max(queue_display_running_total, queue_running_total) + result["queue_display_max_threads"] = max( + int((active_job_summary or {}).get("display_max_threads", 0) or 0), + sum(int(item.get("max_threads", 0) or 0) for item in list(queue_health.get("nodes") or []) if isinstance(item, dict)), + ) result["queue_completed_total"] = queue_completed_total result["queue_blacklist_total"] = queue_blacklist_total result["queue_failed_total"] = queue_failed_total + result["current_job_blacklisted"] = queue_blacklist_total + result["recent_blacklisted_total"] = ops_blacklisted_recent + result["cumulative_blacklisted_total"] = int(result.get("blacklist_total", 0) or 0) result["backlog_pending_total"] = max(backlog_pending_total, queue_pending_total) result["backlog_claimed_total"] = max(backlog_claimed_total, queue_claimed_total) result["backlog_running_total"] = max(backlog_running_total, queue_running_total) result["backlog_register_pending_total"] = backlog_register_pending_total result["backlog_downstream_pending_total"] = backlog_downstream_pending_total + result["cluster_proxy_available_count"] = int(detect_status.get("available_proxy_count", 0) or 0) + result["cluster_proxy_runtime_label"] = str(detect_status.get("proxy_runtime_label") or "").strip() + result["cluster_proxy_runtime_detail"] = str(detect_status.get("proxy_runtime_detail") or "").strip() + result["cluster_proxy_last_refresh_status"] = str(detect_status.get("proxy_last_refresh_status") or "").strip() + result["aggregate_process_count"] = int(detect_status.get("aggregate_process_count", 0) or 0) + result["aggregate_participating_node_count"] = int(detect_status.get("aggregate_participating_node_count", 0) or 0) + result["aggregate_active_thread_count"] = int(detect_status.get("active_thread_count", 0) or 0) return result diff --git a/domain-api/app/services/debug_event_service.py b/domain-api/app/services/debug_event_service.py index 713cfc7..886a67a 100644 --- a/domain-api/app/services/debug_event_service.py +++ b/domain-api/app/services/debug_event_service.py @@ -242,6 +242,105 @@ def _normalize_worker_log_event(debug_event: dict) -> dict | None: } +def _normalize_debug_event_job_identity(payload: dict | None) -> dict: + normalized_payload = dict(payload or {}) if isinstance(payload, dict) else {} + nested_job = normalized_payload.get("job") if isinstance(normalized_payload.get("job"), dict) else {} + + raw_job_id = normalized_payload.get("job_id") + if raw_job_id in (None, "", 0, "0"): + raw_job_id = normalized_payload.get("target_job_id") + if raw_job_id in (None, "", 0, "0"): + raw_job_id = nested_job.get("job_id") + try: + job_id = int(raw_job_id or 0) + except Exception: + job_id = 0 + + job_code = str( + normalized_payload.get("job_code") + or normalized_payload.get("target_job_code") + or nested_job.get("job_code") + or "" + ).strip() + cycle_token = str(normalized_payload.get("cycle_token") or nested_job.get("cycle_token") or "").strip() + + return { + "job_id": job_id, + "job_code": job_code, + "cycle_token": cycle_token, + "has_identity": bool(job_id > 0 or job_code), + } + + +def _load_detect_job_summary_by_job_code(job_code: str, *, event_limit: int = 1) -> dict | None: + normalized_job_code = str(job_code or "").strip() + if not normalized_job_code: + return None + + from app.services.detect_job_service import get_detect_job_summary + + with get_db() as conn: + with conn.cursor() as cur: + cur.execute( + """ + SELECT id + FROM detect_jobs + WHERE job_code = %s + ORDER BY id DESC + LIMIT 1 + """, + (normalized_job_code,), + ) + row = cur.fetchone() + if not row: + return None + return get_detect_job_summary(int(row[0]), event_limit=event_limit) + + +def _resolve_target_job_for_debug_event(event_payload: dict | None) -> tuple[dict | None, str]: + from app.services.detect_job_service import get_active_detect_job_summary, get_detect_job_summary + + identity = _normalize_debug_event_job_identity(event_payload) + if not identity["has_identity"]: + return None, "missing_job_identity" + + payload_job_id = int(identity["job_id"] or 0) + payload_job_code = str(identity["job_code"] or "").strip() + payload_cycle_token = str(identity["cycle_token"] or "").strip() + + active_job = get_active_detect_job_summary(event_limit=1) or {} + active_job_id = int(active_job.get("job_id") or 0) + active_job_code = str(active_job.get("job_code") or active_job.get("runtime_job_code") or "").strip() + + target_job: dict | None = None + if payload_job_id > 0: + if active_job_id == payload_job_id: + target_job = active_job + else: + target_job = get_detect_job_summary(payload_job_id, event_limit=1) + elif payload_job_code: + if active_job_code and active_job_code == payload_job_code: + target_job = active_job + else: + target_job = _load_detect_job_summary_by_job_code(payload_job_code, event_limit=1) + + if not target_job: + return None, "job_not_found" + + target_job_id = int(target_job.get("job_id") or 0) + target_job_code = str(target_job.get("job_code") or target_job.get("runtime_job_code") or "").strip() + target_cycle_token = str(target_job.get("current_cycle_token") or "").strip() + + if payload_job_id > 0 and target_job_id > 0 and target_job_id != payload_job_id: + return None, "job_mismatch" + if payload_job_code and target_job_code and target_job_code != payload_job_code: + return None, "job_mismatch" + if payload_cycle_token and target_cycle_token and payload_cycle_token != target_cycle_token: + return None, "cycle_mismatch" + + return target_job, "matched" + + def _ingest_worker_log_into_active_job(debug_event: dict) -> dict: if str(debug_event.get("event_type") or "").strip() != "worker_log": return {"imported": False, "reason": "not_worker_log"} @@ -249,15 +348,16 @@ def _ingest_worker_log_into_active_job(debug_event: dict) -> dict: normalized_event = _normalize_worker_log_event(debug_event) if not normalized_event: return {"imported": False, "reason": "not_domain_progress_event"} - - from app.services.detect_job_service import get_active_detect_job_summary from app.services.sync_push_service import ( _apply_detect_result_event_to_domain, _apply_detect_result_event_to_job_item, ) - active_job = get_active_detect_job_summary(event_limit=1) or {} - target_job_id = int(active_job.get("job_id") or 0) + target_job, resolve_reason = _resolve_target_job_for_debug_event(normalized_event.get("payload")) + if not target_job: + return {"imported": False, "reason": resolve_reason} + + target_job_id = int(target_job.get("job_id") or 0) if target_job_id <= 0: return {"imported": False, "reason": "no_active_job"} @@ -334,6 +434,7 @@ def _ingest_worker_log_into_active_job(debug_event: dict) -> dict: "imported": True, "reason": "imported", "target_job_id": target_job_id, + "target_job_code": str(target_job.get("job_code") or ""), "detect_run_event_id": detect_run_event_id, "updated_job_items": updated_job_items, "event_type": normalized_event["event_type"], @@ -847,7 +948,9 @@ def get_debug_handoff_report( def ingest_debug_event(payload: dict, *, shared_token: str | None = None) -> tuple[bool, str, dict]: configured_token = str(settings.sync_shared_token or "").strip() incoming_token = str(shared_token or "").strip() - if configured_token and incoming_token != configured_token: + if not configured_token: + return False, "调试事件共享 token 未配置，拒绝远端写入", {"configuration_required": True} + if incoming_token != configured_token: return False, "调试事件 token 校验失败", {} record_id = append_debug_event( diff --git a/domain-api/app/services/detect_job_service.py b/domain-api/app/services/detect_job_service.py index 3e02113..042b0e0 100644 --- a/domain-api/app/services/detect_job_service.py +++ b/domain-api/app/services/detect_job_service.py @@ -1,11 +1,15 @@ from __future__ import annotations -import json -import time -from datetime import datetime +import hashlib import math +import json +import threading +import time +from datetime import datetime, timedelta from uuid import uuid4 +from psycopg2 import errors + from app.core.config import settings from app.core.db import db_read_retry, get_db, is_retryable_db_error from app.services.debug_event_service import push_debug_event @@ -14,7 +18,24 @@ from app.services.settings_service import get_settings_payload ACTIVE_JOB_STATUSES = ("pending", "running") _RUNTIME_NODE_STALE_MINUTES = 10 +_PREFERRED_RUNTIME_SNAPSHOT_GRACE = timedelta(minutes=10) +_LIVE_RUNTIME_NODE_STALE_AFTER = timedelta(seconds=90) _DEFAULT_TASK_MODE = "domain_pipeline" +_DETECT_JOBS_LEGACY_SELECT_MODE: bool | None = None +_DETECT_JOB_ITEM_RECYCLE_LOCK = threading.Lock() +_DETECT_JOB_ITEM_RECYCLE_LAST_RUN_TS = 0.0 +_DETECT_JOB_ITEM_RECYCLE_INTERVAL_SECONDS = 15.0 +_DETECT_JOB_ITEM_RECYCLE_ADVISORY_LOCK_KEY = 0 +_DETECT_JOB_ITEM_RECYCLE_BATCH_SIZE = 2000 +_DETECT_JOB_ITEM_RECYCLE_MAX_BATCHES = 6 +_ACTIVE_JOB_SUMMARY_CACHE_LOCK = threading.Lock() +_ACTIVE_JOB_SUMMARY_CACHE_TTL_SECONDS = 3.0 +_ACTIVE_JOB_SUMMARY_CACHE: dict[tuple[int], tuple[float, dict | None]] = {} +_DETECT_QUEUE_HEALTH_CACHE_LOCK = threading.Lock() +_DETECT_QUEUE_HEALTH_CACHE_TTL_SECONDS = 3.0 +_DETECT_QUEUE_HEALTH_CACHE: dict[tuple[int], tuple[float, dict]] = {} +_DISABLED_MANAGED_NODE_CACHE: set[str] = set() +_DISABLED_MANAGED_NODE_CACHE_EXPIRES_AT = 0.0 _PIPELINE_STEP_ORDER = [ "detect_register", "detect_baidu_site", @@ -64,6 +85,27 @@ _SINGLE_STEP_JOB_SPECS = { } +def _build_pg_advisory_lock_key(scope: str) -> int: + normalized_scope = str(scope or "").strip() or "domaincheck-default" + digest = hashlib.sha1(normalized_scope.encode("utf-8")).digest() + raw_value = int.from_bytes(digest[:8], "big", signed=False) + return raw_value - (1 << 64) if raw_value >= (1 << 63) else raw_value + + +_DETECT_JOB_ITEM_RECYCLE_ADVISORY_LOCK_KEY = _build_pg_advisory_lock_key("detect-job-items-recycle-expired") + + +def _clone_cacheable_payload(value): + try: + return json.loads(json.dumps(value, ensure_ascii=False)) + except Exception: + if isinstance(value, dict): + return dict(value) + if isinstance(value, list): + return list(value) + return value + + def _selection_sql() -> str: return """ SELECT id @@ -76,10 +118,322 @@ def _selection_sql() -> str: """ +def _refresh_detect_job_status_with_cursor(cur, job_id: int) -> None: + cur.execute( + """ + SELECT + job.task_mode, + count(*) FILTER (WHERE item.status = 'pending') AS pending_count, + count(*) FILTER (WHERE item.status IN ('claimed', 'running')) AS dispatch_active_count, + count(*) FILTER (WHERE item.status = 'failed') AS failed_count, + count(*) FILTER (WHERE item.status IN ('completed', 'blacklisted')) AS done_count, + count(*) FILTER ( + WHERE item.status IN ('completed', 'blacklisted', 'failed') + AND COALESCE(item.step_code, '') <> '' + AND COALESCE(item.result_payload_json->>'controller_processed', 'false') <> 'true' + ) AS unprocessed_terminal_count + FROM detect_job_items AS item + JOIN detect_jobs AS job ON job.id = item.job_id + WHERE item.job_id = %s + GROUP BY job.task_mode + """, + (int(job_id),), + ) + refresh_row = cur.fetchone() + if not refresh_row: + return + task_mode = str((refresh_row or [""])[0] or "").strip() + pending_count = int((refresh_row or ["", 0])[1] or 0) + dispatch_active_count = int((refresh_row or ["", 0, 0])[2] or 0) + failed_count = int((refresh_row or ["", 0, 0, 0])[3] or 0) + done_count = int((refresh_row or ["", 0, 0, 0, 0])[4] or 0) + unprocessed_terminal_count = int((refresh_row or ["", 0, 0, 0, 0, 0])[5] or 0) + if dispatch_active_count > 0 or (task_mode == "domain_pipeline" and unprocessed_terminal_count > 0): + cur.execute( + """ + UPDATE detect_jobs + SET status = 'running', + started_at = COALESCE(started_at, CURRENT_TIMESTAMP), + finished_at = NULL + WHERE id = %s + AND ( + status <> 'running' + OR started_at IS NULL + OR finished_at IS NOT NULL + ) + """, + (int(job_id),), + ) + return + if pending_count > 0: + cur.execute( + """ + UPDATE detect_jobs + SET status = 'pending', + finished_at = NULL + WHERE id = %s + """, + (int(job_id),), + ) + return + + final_status = "completed" + if failed_count > 0 and done_count > 0: + final_status = "partial_failed" + elif failed_count > 0: + final_status = "failed" + cur.execute( + """ + UPDATE detect_jobs + SET status = %s, + finished_at = CURRENT_TIMESTAMP, + started_at = COALESCE(started_at, CURRENT_TIMESTAMP) + WHERE id = %s + """, + (final_status, int(job_id)), + ) + + +def _recycle_expired_detect_job_items_once() -> int: + conn = None + advisory_locked = False + try: + with get_db() as conn: + conn.autocommit = False + with conn.cursor() as cur: + cur.execute( + "SELECT pg_try_advisory_lock(%s)", + (_DETECT_JOB_ITEM_RECYCLE_ADVISORY_LOCK_KEY,), + ) + lock_row = cur.fetchone() + advisory_locked = bool((lock_row or [False])[0]) + if not advisory_locked: + conn.rollback() + return 0 + recycled_count = 0 + touched_job_ids: set[int] = set() + for _ in range(_DETECT_JOB_ITEM_RECYCLE_MAX_BATCHES): + cur.execute( + """ + WITH expired_candidates AS ( + SELECT id, job_id, status + FROM detect_job_items + WHERE status IN ('claimed', 'running') + AND lease_expires_at IS NOT NULL + AND lease_expires_at < CURRENT_TIMESTAMP + ORDER BY lease_expires_at ASC, id ASC + FOR UPDATE SKIP LOCKED + LIMIT %s + ), + recycled AS ( + UPDATE detect_job_items AS item + SET status = 'pending', + claimed_by = '', + claim_token = '', + lease_expires_at = NULL, + updated_at = CURRENT_TIMESTAMP, + last_error = CASE + WHEN expired_candidates.status = 'running' THEN 'lease expired while running' + WHEN expired_candidates.status = 'claimed' THEN 'lease expired before running' + ELSE item.last_error + END + FROM expired_candidates + WHERE item.id = expired_candidates.id + RETURNING expired_candidates.job_id + ) + SELECT count(*), array_remove(array_agg(DISTINCT job_id), NULL) + FROM recycled + """, + (_DETECT_JOB_ITEM_RECYCLE_BATCH_SIZE,), + ) + row = cur.fetchone() + batch_count = int((row or [0])[0] or 0) + recycled_count += batch_count + touched_job_ids.update( + int(item) + for item in list((row or [0, []])[1] or []) + if item is not None + ) + if batch_count < _DETECT_JOB_ITEM_RECYCLE_BATCH_SIZE: + break + for job_id in sorted(touched_job_ids): + _refresh_detect_job_status_with_cursor(cur, job_id) + conn.commit() + + if recycled_count > 0: + try: + push_debug_event( + service="detect-job", + event_type="expired_item_recycled", + level="info", + message=f"控制面回收过期检测任务项 {recycled_count} 个", + payload={ + "recycled_count": recycled_count, + "job_ids": touched_job_ids, + }, + ) + except Exception: + pass + return recycled_count + except Exception as exc: + try: + if conn is not None: + conn.rollback() + except Exception: + pass + try: + push_debug_event( + service="detect-job", + event_type="expired_item_recycle_failed", + level="warning", + message="控制面回收过期检测任务项失败", + payload={"error": str(exc or "")[:500]}, + ) + except Exception: + pass + return 0 + finally: + if conn is not None and advisory_locked: + try: + with conn.cursor() as cur: + cur.execute( + "SELECT pg_advisory_unlock(%s)", + (_DETECT_JOB_ITEM_RECYCLE_ADVISORY_LOCK_KEY,), + ) + conn.commit() + except Exception: + try: + conn.rollback() + except Exception: + pass + + +def _maybe_recycle_expired_detect_job_items() -> int: + global _DETECT_JOB_ITEM_RECYCLE_LAST_RUN_TS + + now_ts = time.monotonic() + with _DETECT_JOB_ITEM_RECYCLE_LOCK: + if now_ts - _DETECT_JOB_ITEM_RECYCLE_LAST_RUN_TS < _DETECT_JOB_ITEM_RECYCLE_INTERVAL_SECONDS: + return 0 + _DETECT_JOB_ITEM_RECYCLE_LAST_RUN_TS = now_ts + return _recycle_expired_detect_job_items_once() + + def _format_time(value: datetime | None) -> str: return value.isoformat(sep=" ", timespec="seconds") if value else "" +def _load_disabled_managed_node_codes(node_codes: list[str] | None = None) -> set[str]: + global _DISABLED_MANAGED_NODE_CACHE, _DISABLED_MANAGED_NODE_CACHE_EXPIRES_AT + normalized_codes = [ + str(item or "").strip() + for item in list(node_codes or []) + if str(item or "").strip() + ] + now_ts = time.time() + if not normalized_codes and now_ts < _DISABLED_MANAGED_NODE_CACHE_EXPIRES_AT: + return set(_DISABLED_MANAGED_NODE_CACHE) + try: + with get_db() as conn: + with conn.cursor() as cur: + if normalized_codes: + cur.execute( + """ + SELECT node_code + FROM ops_managed_nodes + WHERE is_enabled = FALSE + AND node_code = ANY(%s) + """, + (normalized_codes,), + ) + else: + cur.execute( + """ + SELECT node_code + FROM ops_managed_nodes + WHERE is_enabled = FALSE + """ + ) + rows = list(cur.fetchall() or []) + except Exception: + if normalized_codes: + return {code for code in normalized_codes if code in _DISABLED_MANAGED_NODE_CACHE} + return set(_DISABLED_MANAGED_NODE_CACHE) + + disabled_codes = { + str(row[0] or "").strip() + for row in rows + if str(row[0] or "").strip() + } + if normalized_codes: + return disabled_codes + _DISABLED_MANAGED_NODE_CACHE = disabled_codes + _DISABLED_MANAGED_NODE_CACHE_EXPIRES_AT = now_ts + 5.0 + return set(disabled_codes) + + +def _detect_jobs_select_columns(*, alias: str = "", legacy_mode: bool = False) -> str: + prefix = f"{str(alias).strip()}." if str(alias or "").strip() else "" + task_mode_expr = f"'{_DEFAULT_TASK_MODE}' AS task_mode" if legacy_mode else f"{prefix}task_mode" + step_code_expr = "'' AS step_code" if legacy_mode else f"{prefix}step_code" + return ( + f"{prefix}id, {prefix}job_code, {prefix}source, " + f"{task_mode_expr}, {step_code_expr}, " + f"{prefix}status, {prefix}created_by, {prefix}created_at, {prefix}started_at, {prefix}finished_at" + ) + + +def _execute_detect_jobs_select( + cur, + *, + from_clause: str, + where_clause: str = "", + order_clause: str = "", + limit_clause: str = "", + params: tuple | list | None = None, + alias: str = "", +) -> None: + global _DETECT_JOBS_LEGACY_SELECT_MODE + + safe_params = tuple(params or ()) + candidate_modes = ( + [_DETECT_JOBS_LEGACY_SELECT_MODE] + if _DETECT_JOBS_LEGACY_SELECT_MODE is not None + else [False, True] + ) + last_error: Exception | None = None + + for legacy_mode in candidate_modes: + parts = [ + f"SELECT {_detect_jobs_select_columns(alias=alias, legacy_mode=bool(legacy_mode))}", + from_clause, + ] + if where_clause: + parts.append(f"WHERE {where_clause}") + if order_clause: + parts.append(f"ORDER BY {order_clause}") + if limit_clause: + normalized_limit = str(limit_clause or "").strip() + if normalized_limit: + parts.append( + normalized_limit + if normalized_limit.upper().startswith("LIMIT") + else f"LIMIT {normalized_limit}" + ) + sql = "\n".join(parts) + try: + cur.execute(sql, safe_params) + _DETECT_JOBS_LEGACY_SELECT_MODE = bool(legacy_mode) + return + except errors.UndefinedColumn as exc: + last_error = exc + _DETECT_JOBS_LEGACY_SELECT_MODE = True + if bool(legacy_mode): + raise + if last_error is not None: + raise last_error + + def _normalize_sync_region(value: str | None, fallback: str) -> str: text = str(value or "").strip() if not text or text == "unknown": @@ -189,6 +543,31 @@ def _load_pipeline_order(settings_payload: dict | None = None) -> list[str]: return [item for item in normalized_order if bool(detect_options.get(item, False))] +def order_step_buckets( + step_items: list[dict] | None, + *, + settings_payload: dict | None = None, + limit: int | None = None, +) -> list[dict]: + normalized_items = [dict(item) for item in list(step_items or []) if isinstance(item, dict)] + if not normalized_items: + return [] + + pipeline_order = _load_pipeline_order(settings_payload=settings_payload) + order_index = {step_code: index for index, step_code in enumerate(pipeline_order)} + + ordered = sorted( + normalized_items, + key=lambda item: ( + int(order_index.get(str(item.get("step_code") or "").strip(), 10_000)), + str(item.get("step_code") or ""), + ), + ) + if limit is not None and int(limit or 0) > 0: + return ordered[: int(limit)] + return ordered + + def _load_domain_pipeline_snapshot(cur, domain_id: int) -> dict | None: cur.execute( """ @@ -358,6 +737,10 @@ def _int_value(value: object) -> int: return 0 +def _max_runtime_metric(*values: object) -> int: + return max((_int_value(value) for value in values), default=0) + + def _effective_runtime_load(*, items_running: object, active_threads: object) -> int: return max(_int_value(items_running), _int_value(active_threads), 0) @@ -523,6 +906,7 @@ def _merge_display_node_stats(*, local_node_stats: list[dict], runtime_node_rows def _load_runtime_display_rows(cur) -> list[tuple]: + disabled_node_codes = _load_disabled_managed_node_codes() cur.execute( f""" SELECT node_code, region, role, status, current_load, metadata_json, last_heartbeat_at @@ -531,7 +915,11 @@ def _load_runtime_display_rows(cur) -> list[tuple]: ORDER BY last_heartbeat_at DESC, node_code ASC """ ) - return list(cur.fetchall()) + return [ + row + for row in list(cur.fetchall() or []) + if str((row or [""])[0] or "").strip() not in disabled_node_codes + ] def _build_display_summary(node_stats: list[dict]) -> dict: @@ -871,9 +1259,19 @@ def _load_runtime_activity_snapshot(window_minutes: int) -> dict: ) rows = cur.fetchall() + snapshot_job_codes = _load_recent_runtime_snapshot_job_codes(safe_window_minutes) job_activity: dict[str, dict] = {} - node_stats: dict[str, dict] = {} - step_activity: dict[str, dict] = {} + overall_totals = { + "processed_recent": 0, + "completed_recent": 0, + "failed_recent": 0, + "blacklisted_recent": 0, + } + overall_node_stats: dict[str, dict] = {} + overall_step_activity: dict[str, dict] = {} + per_job_totals: dict[str, dict] = {} + per_job_node_stats: dict[str, dict[str, dict]] = {} + per_job_step_activity: dict[str, dict[str, dict]] = {} for row in rows: node_code = str(row[0] or "unknown").strip() or "unknown" event_type = str(row[1] or "").strip() @@ -904,8 +1302,8 @@ def _load_runtime_activity_snapshot(window_minutes: int) -> dict: bucket["latest_at"] = created_at if classified.get("terminal"): - snapshot["processed_recent"] += 1 - step_bucket = step_activity.setdefault( + overall_totals["processed_recent"] += 1 + step_bucket = overall_step_activity.setdefault( step_code, { "step_code": step_code, @@ -917,7 +1315,7 @@ def _load_runtime_activity_snapshot(window_minutes: int) -> dict: }, ) step_bucket["processed_recent"] += 1 - node_bucket = node_stats.setdefault( + node_bucket = overall_node_stats.setdefault( node_code, { "node_code": node_code, @@ -928,22 +1326,68 @@ def _load_runtime_activity_snapshot(window_minutes: int) -> dict: }, ) node_bucket["processed_recent"] += 1 + if job_code: + job_totals = per_job_totals.setdefault( + job_code, + { + "processed_recent": 0, + "completed_recent": 0, + "failed_recent": 0, + "blacklisted_recent": 0, + }, + ) + job_totals["processed_recent"] += 1 + job_step_bucket = per_job_step_activity.setdefault(job_code, {}).setdefault( + step_code, + { + "step_code": step_code, + "started_recent": 0, + "processed_recent": 0, + "completed_recent": 0, + "failed_recent": 0, + "blacklisted_recent": 0, + }, + ) + job_step_bucket["processed_recent"] += 1 + job_node_bucket = per_job_node_stats.setdefault(job_code, {}).setdefault( + node_code, + { + "node_code": node_code, + "processed_recent": 0, + "completed_recent": 0, + "failed_recent": 0, + "blacklisted_recent": 0, + }, + ) + job_node_bucket["processed_recent"] += 1 terminal_status = str(classified.get("terminal_status") or "").strip() if terminal_status == "completed": - snapshot["completed_recent"] += 1 + overall_totals["completed_recent"] += 1 node_bucket["completed_recent"] += 1 step_bucket["completed_recent"] += 1 + if job_code: + job_totals["completed_recent"] += 1 + job_node_bucket["completed_recent"] += 1 + job_step_bucket["completed_recent"] += 1 elif terminal_status == "failed": - snapshot["failed_recent"] += 1 + overall_totals["failed_recent"] += 1 node_bucket["failed_recent"] += 1 step_bucket["failed_recent"] += 1 + if job_code: + job_totals["failed_recent"] += 1 + job_node_bucket["failed_recent"] += 1 + job_step_bucket["failed_recent"] += 1 elif terminal_status == "blacklisted": - snapshot["blacklisted_recent"] += 1 + overall_totals["blacklisted_recent"] += 1 node_bucket["blacklisted_recent"] += 1 step_bucket["blacklisted_recent"] += 1 + if job_code: + job_totals["blacklisted_recent"] += 1 + job_node_bucket["blacklisted_recent"] += 1 + job_step_bucket["blacklisted_recent"] += 1 if classified.get("started"): - step_bucket = step_activity.setdefault( + step_bucket = overall_step_activity.setdefault( step_code, { "step_code": step_code, @@ -955,10 +1399,71 @@ def _load_runtime_activity_snapshot(window_minutes: int) -> dict: }, ) step_bucket["started_recent"] += 1 + if job_code: + job_step_bucket = per_job_step_activity.setdefault(job_code, {}).setdefault( + step_code, + { + "step_code": step_code, + "started_recent": 0, + "processed_recent": 0, + "completed_recent": 0, + "failed_recent": 0, + "blacklisted_recent": 0, + }, + ) + job_step_bucket["started_recent"] += 1 - if step_activity: + ordered_jobs = [] + if job_activity: + ordered_jobs = sorted( + job_activity.values(), + key=lambda item: ( + -int(item.get("terminal_count", 0) or 0), + -int(item.get("activity_count", 0) or 0), + str(item.get("latest_at") or ""), + str(item.get("job_code") or ""), + ), + ) + ordered_activity_codes = [ + str(item.get("job_code") or "").strip() + for item in ordered_jobs + if str(item.get("job_code") or "").strip() + ] + merged_job_codes: list[str] = [] + seen_job_codes: set[str] = set() + for job_code in [*snapshot_job_codes, *ordered_activity_codes]: + normalized_job_code = str(job_code or "").strip() + if not normalized_job_code or normalized_job_code in seen_job_codes: + continue + merged_job_codes.append(normalized_job_code) + seen_job_codes.add(normalized_job_code) + focus_job_code = merged_job_codes[0] if merged_job_codes else "" + if focus_job_code: + snapshot["focus_job_code"] = focus_job_code + snapshot["job_codes"] = merged_job_codes[:5] + elif ordered_activity_codes: + snapshot["focus_job_code"] = ordered_activity_codes[0] + snapshot["job_codes"] = ordered_activity_codes[:5] + + selected_totals = overall_totals + selected_step_activity = overall_step_activity + selected_node_stats = overall_node_stats + if focus_job_code: + if focus_job_code in per_job_totals: + selected_totals = dict(per_job_totals.get(focus_job_code) or {}) + if focus_job_code in per_job_step_activity: + selected_step_activity = dict(per_job_step_activity.get(focus_job_code) or {}) + if focus_job_code in per_job_node_stats: + selected_node_stats = dict(per_job_node_stats.get(focus_job_code) or {}) + + snapshot["processed_recent"] = int(selected_totals.get("processed_recent", 0) or 0) + snapshot["completed_recent"] = int(selected_totals.get("completed_recent", 0) or 0) + snapshot["failed_recent"] = int(selected_totals.get("failed_recent", 0) or 0) + snapshot["blacklisted_recent"] = int(selected_totals.get("blacklisted_recent", 0) or 0) + + if selected_step_activity: ordered_steps = sorted( - step_activity.values(), + selected_step_activity.values(), key=lambda item: ( -int(item.get("processed_recent", 0) or 0), -int(item.get("started_recent", 0) or 0), @@ -978,28 +1483,84 @@ def _load_runtime_activity_snapshot(window_minutes: int) -> dict: for item in ordered_steps if str(item.get("step_code") or "").strip() } - if job_activity: - ordered_jobs = sorted( - job_activity.values(), - key=lambda item: ( - -int(item.get("terminal_count", 0) or 0), - -int(item.get("activity_count", 0) or 0), - str(item.get("latest_at") or ""), - str(item.get("job_code") or ""), - ), - ) - snapshot["focus_job_code"] = str(ordered_jobs[0].get("job_code") or "") - snapshot["job_codes"] = [str(item.get("job_code") or "") for item in ordered_jobs[:5] if str(item.get("job_code") or "")] - snapshot["nodes"] = node_stats + snapshot["nodes"] = selected_node_stats return snapshot -@db_read_retry() -def _load_latest_runtime_active_job_snapshot(window_minutes: int) -> dict: +def _load_recent_runtime_snapshot_job_codes(window_minutes: int, *, limit: int = 5) -> list[str]: + safe_window_minutes = max(5, min(int(window_minutes or 15), 12 * 60)) + safe_limit = max(1, min(int(limit or 5), 10)) + if not (settings.node_region == "overseas" and settings.node_role == "control"): + return [] + + entries: list[tuple[datetime, str]] = [] + with get_db() as conn: + with conn.cursor() as cur: + cur.execute( + """ + SELECT payload_json, created_at + FROM detect_debug_events + WHERE source_region = 'mainland' + AND service = 'detect-runtime' + AND event_type = 'active_job_snapshot' + AND created_at >= CURRENT_TIMESTAMP - (%s || ' minutes')::interval + ORDER BY created_at DESC, id DESC + LIMIT 12 + """, + (safe_window_minutes,), + ) + for payload_json, created_at in list(cur.fetchall() or []): + payload = _decode_payload(payload_json) + job_code = str((payload.get("job") or {}).get("job_code") or "").strip() if isinstance(payload, dict) else "" + if job_code and created_at: + entries.append((created_at, job_code)) + + cur.execute( + """ + SELECT payload_json, COALESCE(updated_at, created_at) + FROM detect_sync_records + WHERE sync_type = 'runtime_ingest' + AND source_region = 'mainland' + AND target_region = 'overseas' + AND created_at >= CURRENT_TIMESTAMP - (%s || ' minutes')::interval + ORDER BY COALESCE(updated_at, created_at) DESC, id DESC + LIMIT 12 + """, + (safe_window_minutes,), + ) + for payload_json, created_at in list(cur.fetchall() or []): + payload = _decode_payload(payload_json) + projection = payload.get("projection") if isinstance(payload, dict) else {} + job_code = str((projection.get("active_job") or {}).get("job_code") or "").strip() if isinstance(projection, dict) else "" + if job_code and created_at: + entries.append((created_at, job_code)) + + codes: list[str] = [] + seen_codes: set[str] = set() + for _created_at, job_code in sorted(entries, key=lambda item: item[0], reverse=True): + if job_code in seen_codes: + continue + codes.append(job_code) + seen_codes.add(job_code) + if len(codes) >= safe_limit: + break + return codes + + +def _load_latest_runtime_debug_active_job_snapshot( + window_minutes: int, + *, + preferred_job_codes: list[str] | None = None, +) -> dict: safe_window_minutes = max(5, min(int(window_minutes or 15), 120)) if not (settings.node_region == "overseas" and settings.node_role == "control"): return {} + preferred = { + str(item or "").strip() + for item in list(preferred_job_codes or []) + if str(item or "").strip() + } source_region = "mainland" with get_db() as conn: with conn.cursor() as cur: @@ -1012,20 +1573,704 @@ def _load_latest_runtime_active_job_snapshot(window_minutes: int) -> dict: AND event_type = 'active_job_snapshot' AND created_at >= CURRENT_TIMESTAMP - (%s || ' minutes')::interval ORDER BY created_at DESC, id DESC - LIMIT 1 + LIMIT 12 """, (source_region, safe_window_minutes), ) - row = cur.fetchone() + rows = list(cur.fetchall() or []) - if not row: + fallback_snapshot: dict = {} + for row in rows: + payload = _decode_payload(row[0]) + if not isinstance(payload, dict): + continue + snapshot = dict(payload) + snapshot["_created_at"] = _format_time(row[1]) + snapshot["_snapshot_source"] = "active_job_snapshot" + if not fallback_snapshot: + fallback_snapshot = snapshot + job_code = str((snapshot.get("job") or {}).get("job_code") or "").strip() + if preferred and job_code in preferred: + return snapshot + return fallback_snapshot + + +def _normalize_runtime_projection_nodes( + *, + active_job: dict, + progress: dict, + projection: dict, +) -> list[dict]: + node_info = dict(projection.get("node") or {}) + fallback_node_code = str(node_info.get("node_code") or "").strip() + fallback_region = str(node_info.get("region") or "").strip() + fallback_role = str(node_info.get("role") or "").strip() + active_thread_count = _int_value(projection.get("active_thread_count")) + max_thread_count = _int_value(projection.get("max_thread_count")) + + nodes: list[dict] = [] + for raw_item in list(active_job.get("distributed_node_stats") or active_job.get("node_stats") or []): + if not isinstance(raw_item, dict): + continue + node_code = str(raw_item.get("node_code") or "").strip() + if not node_code: + continue + nodes.append( + _normalize_node_bucket( + { + **raw_item, + "node_code": node_code, + "metrics_source": str(raw_item.get("metrics_source") or "runtime_ingest").strip() or "runtime_ingest", + "region": str(raw_item.get("region") or (fallback_region if node_code == fallback_node_code else "")).strip(), + "role": str(raw_item.get("role") or (fallback_role if node_code == fallback_node_code else "")).strip(), + "status": str(raw_item.get("status") or "").strip(), + } + ) + ) + + if fallback_node_code: + matched = False + for item in nodes: + if str(item.get("node_code") or "").strip() != fallback_node_code: + continue + matched = True + item["region"] = str(item.get("region") or fallback_region).strip() + item["role"] = str(item.get("role") or fallback_role).strip() + item["active_threads"] = max(_int_value(item.get("active_threads")), active_thread_count) + item["max_threads"] = max(_int_value(item.get("max_threads")), max_thread_count) + item["current_load"] = _effective_runtime_load( + items_running=item.get("items_running"), + active_threads=item.get("active_threads"), + ) + item["display_running"] = item["current_load"] + if not str(item.get("status") or "").strip(): + item["status"] = "busy" if item["current_load"] > 0 else "online" + break + has_child_instances = any( + str(item.get("node_code") or "").strip().startswith(f"{fallback_node_code}-") + for item in nodes + ) + if not matched and not has_child_instances and ( + int(active_job.get("items_total", 0) or 0) > 0 + or active_thread_count > 0 + or max_thread_count > 0 + ): + nodes.append( + _normalize_node_bucket( + { + "node_code": fallback_node_code, + "items_total": int(active_job.get("items_total", 0) or 0), + "items_pending": int(progress.get("pending", 0) or 0), + "items_claimed": max( + 0, + int(active_job.get("items_total", 0) or 0) + - int(progress.get("pending", 0) or 0) + - int(progress.get("running", 0) or 0) + - int(active_job.get("items_terminal", 0) or 0), + ), + "items_running": int(progress.get("running", 0) or 0), + "items_completed": int(progress.get("completed", 0) or 0), + "items_blacklisted": int(progress.get("blacklisted", 0) or 0), + "items_failed": max( + int(progress.get("failed", 0) or 0), + int(active_job.get("items_failed", 0) or 0), + ), + "active_threads": active_thread_count, + "max_threads": max_thread_count, + "metrics_source": "runtime_ingest", + "region": fallback_region, + "role": fallback_role, + "status": "busy" if max(active_thread_count, int(progress.get("running", 0) or 0)) > 0 else "online", + } + ) + ) + + items_total = max(0, int(active_job.get("items_total", 0) or 0)) + pending = max(0, int(progress.get("pending", 0) or 0)) + running = max(0, int(progress.get("running", 0) or 0)) + completed = max(0, int(progress.get("completed", 0) or 0)) + blacklisted = max(0, int(progress.get("blacklisted", 0) or 0)) + failed = max( + max(0, int(progress.get("failed", 0) or 0)), + max(0, int(active_job.get("items_failed", 0) or 0)), + ) + terminal = max( + max(0, int(active_job.get("items_terminal", 0) or 0)), + completed + blacklisted + failed, + ) + claimed = max(0, items_total - pending - running - terminal) + assigned_totals = { + "items_total": sum(_int_value(item.get("items_total")) for item in nodes), + "items_pending": sum(_int_value(item.get("items_pending")) for item in nodes), + "items_claimed": sum(_int_value(item.get("items_claimed")) for item in nodes), + "items_running": sum(_int_value(item.get("items_running")) for item in nodes), + "items_completed": sum(_int_value(item.get("items_completed")) for item in nodes), + "items_blacklisted": sum(_int_value(item.get("items_blacklisted")) for item in nodes), + "items_failed": sum(_int_value(item.get("items_failed")) for item in nodes), + } + unassigned_bucket = { + "node_code": "unassigned", + "items_total": max(0, items_total - assigned_totals["items_total"]), + "items_pending": max(0, pending - assigned_totals["items_pending"]), + "items_claimed": max(0, claimed - assigned_totals["items_claimed"]), + "items_running": max(0, running - assigned_totals["items_running"]), + "items_completed": max(0, completed - assigned_totals["items_completed"]), + "items_blacklisted": max(0, blacklisted - assigned_totals["items_blacklisted"]), + "items_failed": max(0, failed - assigned_totals["items_failed"]), + "metrics_source": "runtime_ingest", + "status": "online", + } + if any(int(unassigned_bucket.get(key, 0) or 0) > 0 for key in ( + "items_total", + "items_pending", + "items_claimed", + "items_running", + "items_completed", + "items_blacklisted", + "items_failed", + )): + nodes.append(_normalize_node_bucket(unassigned_bucket)) + return nodes + + +def _build_runtime_snapshot_from_projection( + projection: dict, + *, + created_at: datetime | None = None, + window_minutes: int = 15, +) -> dict: + normalized_projection = dict(projection or {}) + active_job = dict(normalized_projection.get("active_job") or {}) + progress = dict(normalized_projection.get("progress") or {}) + backlog = dict(normalized_projection.get("backlog") or {}) + nodes = _normalize_runtime_projection_nodes( + active_job=active_job, + progress=progress, + projection=normalized_projection, + ) + + items_total = max(0, int(active_job.get("items_total", 0) or 0)) + pending = max(0, int(progress.get("pending", active_job.get("items_pending", 0)) or 0)) + running = max(0, int(progress.get("running", active_job.get("items_running", 0)) or 0)) + completed = max(0, int(progress.get("completed", 0) or 0)) + blacklisted = max(0, int(progress.get("blacklisted", 0) or 0)) + failed = max( + max(0, int(progress.get("failed", 0) or 0)), + max(0, int(active_job.get("items_failed", 0) or 0)), + ) + terminal = max( + max(0, int(active_job.get("items_terminal", 0) or 0)), + completed + blacklisted + failed, + ) + if items_total <= 0: + items_total = pending + running + terminal + claimed = max(0, items_total - pending - running - terminal) + display_running = _max_runtime_metric( + running, + normalized_projection.get("active_thread_count"), + active_job.get("display_active_threads"), + active_job.get("display_items_running"), + ) + display_claimed = _max_runtime_metric(claimed, active_job.get("display_items_claimed")) + progress_percent = float(active_job.get("progress_percent", 0) or 0) + if progress_percent <= 0 and items_total > 0 and terminal > 0: + progress_percent = round((terminal / items_total) * 100, 2) + + queue_health = { + "window_minutes": max(5, min(int(window_minutes or 15), 120)), + "has_active_job": bool(str(active_job.get("job_code") or "").strip() or items_total > 0), + "job": { + "job_id": int(active_job.get("job_id", 0) or 0), + "job_code": str(active_job.get("job_code") or "").strip(), + "runtime_job_code": str(active_job.get("job_code") or "").strip(), + "status": str(active_job.get("status") or "").strip(), + "progress_percent": progress_percent, + }, + "queue": { + "items_total": items_total, + "pending": pending, + "claimed": claimed, + "running": running, + "display_claimed": display_claimed, + "display_running": display_running, + "completed": completed, + "blacklisted": blacklisted, + "failed": failed, + "terminal": terminal, + "terminal_percent": round((terminal / items_total) * 100, 2) if items_total else 0.0, + "oldest_pending_at": "", + "oldest_pending_age_minutes": 0, + "nearest_lease_expiry_at": "", + "overdue_leases": 0, + "expiring_soon_leases": 0, + }, + "throughput": { + "processed_recent": 0, + "processed_per_minute": 0.0, + "completed_recent": 0, + "blacklisted_recent": 0, + "failed_recent": 0, + }, + "nodes": nodes, + "steps": [], + "runtime_activity": {}, + } + snapshot = { + "job": { + "job_id": int(active_job.get("job_id", 0) or 0), + "job_code": str(active_job.get("job_code") or "").strip(), + "status": str(active_job.get("status") or "").strip(), + "items_total": items_total, + "items_pending": pending, + "items_claimed": claimed, + "items_running": running, + "items_completed": completed, + "items_blacklisted": blacklisted, + "items_failed": failed, + "items_terminal": terminal, + "progress_percent": progress_percent, + "node_stats": nodes, + }, + "queue_health": queue_health, + "backlog": backlog if isinstance(backlog, dict) else {}, + "recent_events": [], + "_snapshot_source": "runtime_ingest", + } + if created_at: + snapshot["_created_at"] = _format_time(created_at) + return snapshot + + +def _snapshot_created_at(snapshot: dict) -> datetime | None: + return _parse_runtime_timestamp(snapshot.get("_created_at")) + + +def _parse_runtime_timestamp(value: object) -> datetime | None: + raw_value = str(value or "").strip() + if not raw_value: + return None + normalized_value = raw_value.replace("T", " ") + if normalized_value.endswith("Z"): + normalized_value = f"{normalized_value[:-1]}+00:00" + try: + return datetime.fromisoformat(normalized_value) + except ValueError: + for fmt in ("%Y-%m-%d %H:%M:%S.%f", "%Y-%m-%d %H:%M:%S"): + try: + return datetime.strptime(normalized_value, fmt) + except ValueError: + continue + return None + + +def _runtime_timestamp_is_fresh(value: datetime | None, *, max_age: timedelta) -> bool: + if value is None: + return False + now = datetime.now(value.tzinfo) if value.tzinfo else datetime.now() + return now - value <= max_age + + +def _runtime_node_display_running(item: dict) -> int: + return _max_runtime_metric( + item.get("display_running"), + item.get("current_load"), + item.get("active_threads"), + item.get("items_running"), + ) + + +def _load_runtime_node_overlay_map(node_codes: list[str]) -> dict[str, dict]: + normalized_codes = [ + str(item or "").strip() + for item in list(node_codes or []) + if str(item or "").strip() and str(item or "").strip() != "unassigned" + ] + if not normalized_codes: + return {} + if not (settings.node_region == "overseas" and settings.node_role == "control"): + return {} + disabled_node_codes = _load_disabled_managed_node_codes(normalized_codes) + try: + with get_db() as conn: + with conn.cursor() as cur: + cur.execute( + """ + SELECT node_code, region, role, status, current_load, metadata_json, last_heartbeat_at + FROM detect_worker_nodes + WHERE node_code = ANY(%s) + """, + (normalized_codes,), + ) + rows = list(cur.fetchall() or []) + except Exception: return {} - payload = _decode_payload(row[0]) - if not isinstance(payload, dict): + overlays: dict[str, dict] = {} + for row in rows: + node_code = str((row or [""])[0] or "").strip() + if not node_code or node_code in disabled_node_codes: + continue + metadata = _decode_payload((row or ["", "", "", "", 0, {}])[5]) + overlays[node_code] = { + "region": str((row or ["", "", ""])[1] or "").strip(), + "role": str((row or ["", "", "", ""])[2] or "").strip(), + "status": str((row or ["", "", "", "", ""])[3] or "").strip(), + "current_load": _int_value((row or ["", "", "", "", 0])[4]), + "active_threads": _int_value(metadata.get("active_threads")), + "max_threads": _int_value(metadata.get("max_threads")), + "last_heartbeat_at": _format_time((row or ["", "", "", "", 0, {}, None])[6]) if len(row or []) > 6 and row[6] else "", + "metrics_source": str(metadata.get("service") or "runtime").strip() or "runtime", + } + return overlays + + +def _merge_runtime_node_overlay(item: dict, overlay: dict | None) -> dict: + merged = dict(item or {}) + normalized_overlay = dict(overlay or {}) + if not normalized_overlay: + return merged + for key in ("region", "role", "status", "last_heartbeat_at", "metrics_source"): + if not str(merged.get(key) or "").strip() and str(normalized_overlay.get(key) or "").strip(): + merged[key] = normalized_overlay.get(key) + for key in ("current_load", "active_threads", "max_threads"): + merged[key] = max(_int_value(merged.get(key)), _int_value(normalized_overlay.get(key))) + return merged + + +def _filter_live_runtime_nodes(nodes: list[dict], *, snapshot_created_at: datetime | None) -> list[dict]: + filtered: list[dict] = [] + for raw_item in list(nodes or []): + if not isinstance(raw_item, dict): + continue + item = dict(raw_item) + node_code = str(item.get("node_code") or "").strip() + if not node_code: + continue + if node_code == "unassigned": + filtered.append(item) + continue + status = str(item.get("status") or "").strip().lower() + if status in {"stale", "offline"}: + continue + reference_at = _parse_runtime_timestamp(item.get("last_heartbeat_at")) + if reference_at is None: + filtered.append(item) + continue + if not _runtime_timestamp_is_fresh(reference_at, max_age=_LIVE_RUNTIME_NODE_STALE_AFTER): + continue + filtered.append(item) + return filtered + + +def _build_live_runtime_snapshot(snapshot: dict | None) -> dict: + normalized_snapshot = dict(snapshot or {}) + if not normalized_snapshot: return {} - payload["_created_at"] = _format_time(row[1]) - return payload + + snapshot_created_at = _snapshot_created_at(normalized_snapshot) + job = dict(normalized_snapshot.get("job") or {}) + queue_health = dict(normalized_snapshot.get("queue_health") or {}) + queue = dict(queue_health.get("queue") or {}) + queue_nodes = [dict(item) for item in list(queue_health.get("nodes") or []) if isinstance(item, dict)] + job_nodes = [dict(item) for item in list(job.get("node_stats") or []) if isinstance(item, dict)] + overlay_map = {} + if str(normalized_snapshot.get("_snapshot_source") or "").strip() == "runtime_ingest": + overlay_map = _load_runtime_node_overlay_map( + [ + *(item.get("node_code") for item in queue_nodes), + *(item.get("node_code") for item in job_nodes), + ] + ) + queue_nodes = [ + _merge_runtime_node_overlay(item, overlay_map.get(str(item.get("node_code") or "").strip())) + for item in queue_nodes + ] + job_nodes = [ + _merge_runtime_node_overlay(item, overlay_map.get(str(item.get("node_code") or "").strip())) + for item in job_nodes + ] + live_queue_nodes = _filter_live_runtime_nodes(queue_nodes, snapshot_created_at=snapshot_created_at) + live_job_nodes = _filter_live_runtime_nodes(job_nodes, snapshot_created_at=snapshot_created_at) + has_runtime_node_rows = bool(queue_nodes or job_nodes) + raw_node_count = len([item for item in queue_nodes or job_nodes if str(item.get("node_code") or "").strip() and str(item.get("node_code") or "").strip() != "unassigned"]) + live_nodes_for_display = live_queue_nodes if queue_nodes else live_job_nodes + live_participant_nodes = [ + item + for item in live_nodes_for_display + if str(item.get("node_code") or "").strip() + and str(item.get("node_code") or "").strip() != "unassigned" + ] + dropped_node_count = max(0, raw_node_count - len(live_participant_nodes)) + + display_running = sum(_runtime_node_display_running(item) for item in live_participant_nodes) + display_claimed = sum( + _max_runtime_metric(item.get("display_claimed"), item.get("items_claimed")) + for item in live_participant_nodes + ) + display_max_threads = sum(max(0, _int_value(item.get("max_threads"))) for item in live_participant_nodes) + allow_raw_display_fallback = dropped_node_count <= 0 and ( + not has_runtime_node_rows or bool(live_participant_nodes) + ) + if allow_raw_display_fallback: + display_running = _max_runtime_metric( + display_running, + queue.get("display_running"), + job.get("display_items_running"), + job.get("display_active_threads"), + normalized_snapshot.get("active_thread_count"), + job.get("items_running"), + queue.get("running"), + ) + display_claimed = _max_runtime_metric( + display_claimed, + queue.get("display_claimed"), + job.get("display_items_claimed"), + queue.get("claimed"), + job.get("items_claimed"), + ) + display_max_threads = _max_runtime_metric( + display_max_threads, + job.get("display_max_threads"), + normalized_snapshot.get("max_thread_count"), + ) + + if queue_nodes or job_nodes: + queue_health["nodes"] = list(live_queue_nodes or live_job_nodes) + queue["display_running"] = display_running + queue["display_claimed"] = display_claimed + queue_health["queue"] = queue + job["node_stats"] = list(live_job_nodes or live_queue_nodes) + job["display_items_running"] = display_running + job["display_active_threads"] = display_running + job["display_max_threads"] = display_max_threads + + normalized_snapshot["job"] = job + normalized_snapshot["queue_health"] = queue_health + normalized_snapshot["_raw_runtime_node_count"] = raw_node_count + normalized_snapshot["_live_runtime_node_count"] = len(live_participant_nodes) + normalized_snapshot["_dropped_runtime_node_count"] = dropped_node_count + normalized_snapshot["_has_runtime_node_rows"] = has_runtime_node_rows + return normalized_snapshot + + +def _snapshot_display_running(snapshot: dict) -> int: + queue = dict((snapshot.get("queue_health") or {}).get("queue") or {}) + job = dict(snapshot.get("job") or {}) + return max( + _int_value(queue.get("display_running")), + _int_value(job.get("display_items_running")), + _int_value(job.get("display_active_threads")), + _int_value(snapshot.get("active_thread_count")), + ) + + +def _runtime_job_matches_active_job( + active_job: dict | None, + *, + runtime_job_code: object = "", + runtime_job_id: object = 0, +) -> bool: + normalized_active_job = dict(active_job or {}) + active_job_code = str(normalized_active_job.get("job_code") or "").strip() + active_job_id = _int_value(normalized_active_job.get("job_id")) + normalized_runtime_job_code = str(runtime_job_code or "").strip() + normalized_runtime_job_id = _int_value(runtime_job_id) + if active_job_id > 0 and normalized_runtime_job_id > 0 and active_job_id == normalized_runtime_job_id: + return True + if active_job_code and normalized_runtime_job_code and active_job_code == normalized_runtime_job_code: + return True + return False + + +def _snapshot_node_count(snapshot: dict) -> int: + queue_nodes = list((snapshot.get("queue_health") or {}).get("nodes") or []) + if queue_nodes: + return len( + [ + item + for item in queue_nodes + if isinstance(item, dict) and str(item.get("node_code") or "").strip() != "unassigned" + ] + ) + job_nodes = list((snapshot.get("job") or {}).get("node_stats") or []) + return len( + [ + item + for item in job_nodes + if isinstance(item, dict) and str(item.get("node_code") or "").strip() != "unassigned" + ] + ) + + +def _snapshot_is_stale_for_live_display(snapshot: dict) -> bool: + normalized_snapshot = dict(snapshot or {}) + if str(normalized_snapshot.get("_snapshot_source") or "").strip() != "runtime_ingest": + return False + created_at = _snapshot_created_at(normalized_snapshot) + if created_at is None: + return False + return not _runtime_timestamp_is_fresh(created_at, max_age=_PREFERRED_RUNTIME_SNAPSHOT_GRACE) + + +def _should_prefer_fallback_runtime_snapshot(*, fallback_snapshot: dict, matched_snapshot: dict) -> bool: + fallback_nodes = _snapshot_node_count(fallback_snapshot) + matched_nodes = _snapshot_node_count(matched_snapshot) + fallback_display_running = _snapshot_display_running(fallback_snapshot) + matched_display_running = _snapshot_display_running(matched_snapshot) + if fallback_nodes <= matched_nodes and fallback_display_running <= matched_display_running: + return False + + fallback_created_at = _snapshot_created_at(fallback_snapshot) + matched_created_at = _snapshot_created_at(matched_snapshot) + if fallback_created_at and matched_created_at: + return fallback_created_at - matched_created_at > _PREFERRED_RUNTIME_SNAPSHOT_GRACE + return matched_nodes <= 0 and matched_display_running <= 0 + + +def _load_latest_runtime_ingest_active_job_snapshot( + window_minutes: int, + *, + preferred_job_codes: list[str] | None = None, +) -> dict: + safe_window_minutes = max(5, min(int(window_minutes or 15), 120)) + if not (settings.node_region == "overseas" and settings.node_role == "control"): + return {} + + preferred = { + str(item or "").strip() + for item in list(preferred_job_codes or []) + if str(item or "").strip() + } + query_sql = """ + SELECT payload_json, COALESCE(updated_at, created_at) + FROM detect_sync_records + WHERE sync_type = 'runtime_ingest' + AND source_region = 'mainland' + AND target_region = 'overseas' + AND created_at >= CURRENT_TIMESTAMP - (%s || ' minutes')::interval + ORDER BY COALESCE(updated_at, created_at) DESC, id DESC + LIMIT 20 + """ + + def _select_rows(lookback_minutes: int) -> list[tuple]: + with get_db() as conn: + with conn.cursor() as cur: + cur.execute(query_sql, (int(lookback_minutes),)) + return list(cur.fetchall() or []) + + def _pick_snapshot(rows: list[tuple]) -> tuple[dict, dict]: + fallback_snapshot: dict = {} + matched_snapshot: dict = {} + for row in rows: + payload = _decode_payload(row[0]) + projection = payload.get("projection") if isinstance(payload, dict) else {} + if not isinstance(projection, dict): + continue + snapshot = _build_runtime_snapshot_from_projection( + projection, + created_at=row[1], + window_minutes=safe_window_minutes, + ) + if not fallback_snapshot: + fallback_snapshot = snapshot + job_code = str((snapshot.get("job") or {}).get("job_code") or "").strip() + if preferred and job_code in preferred: + matched_snapshot = snapshot + break + return fallback_snapshot, matched_snapshot + + recent_rows = _select_rows(safe_window_minutes) + fallback_snapshot, matched_snapshot = _pick_snapshot(recent_rows) + if matched_snapshot: + if ( + fallback_snapshot + and fallback_snapshot is not matched_snapshot + and _should_prefer_fallback_runtime_snapshot( + fallback_snapshot=fallback_snapshot, + matched_snapshot=matched_snapshot, + ) + ): + return fallback_snapshot + return matched_snapshot + + if preferred: + extended_rows = _select_rows(max(safe_window_minutes, 12 * 60)) + extended_fallback, matched_snapshot = _pick_snapshot(extended_rows) + if matched_snapshot: + if ( + extended_fallback + and extended_fallback is not matched_snapshot + and _should_prefer_fallback_runtime_snapshot( + fallback_snapshot=extended_fallback, + matched_snapshot=matched_snapshot, + ) + ): + return extended_fallback + return matched_snapshot + if not fallback_snapshot: + fallback_snapshot = extended_fallback + + return fallback_snapshot + + +@db_read_retry() +def _load_latest_runtime_active_job_snapshot(window_minutes: int) -> dict: + safe_window_minutes = max(5, min(int(window_minutes or 15), 120)) + if not (settings.node_region == "overseas" and settings.node_role == "control"): + return {} + + runtime_activity = _load_runtime_activity_snapshot(safe_window_minutes) + focus_job_code = str(runtime_activity.get("focus_job_code") or "").strip() + preferred_job_codes = [focus_job_code] + [ + str(item or "").strip() + for item in list(runtime_activity.get("job_codes") or []) + if str(item or "").strip() + ] + debug_snapshot = _load_latest_runtime_debug_active_job_snapshot( + safe_window_minutes, + preferred_job_codes=preferred_job_codes, + ) + ingest_snapshot = _load_latest_runtime_ingest_active_job_snapshot( + safe_window_minutes, + preferred_job_codes=preferred_job_codes, + ) + debug_has_payload = bool(debug_snapshot.get("job")) or bool((debug_snapshot.get("queue_health") or {}).get("queue")) + ingest_has_payload = bool(ingest_snapshot.get("job")) or bool((ingest_snapshot.get("queue_health") or {}).get("queue")) + + if ingest_has_payload and not debug_has_payload: + return _build_live_runtime_snapshot(ingest_snapshot) + if debug_has_payload and not ingest_has_payload: + return _build_live_runtime_snapshot(debug_snapshot) + if not debug_has_payload and not ingest_has_payload: + return {} + if focus_job_code: + debug_job_code = str((debug_snapshot.get("job") or {}).get("job_code") or "").strip() + ingest_job_code = str((ingest_snapshot.get("job") or {}).get("job_code") or "").strip() + debug_matches_focus = bool(debug_job_code and debug_job_code == focus_job_code) + ingest_matches_focus = bool(ingest_job_code and ingest_job_code == focus_job_code) + if ingest_matches_focus and not debug_matches_focus: + return _build_live_runtime_snapshot(ingest_snapshot) + if debug_matches_focus and not ingest_matches_focus: + return _build_live_runtime_snapshot(debug_snapshot) + debug_queue = dict((debug_snapshot.get("queue_health") or {}).get("queue") or {}) + ingest_queue = dict((ingest_snapshot.get("queue_health") or {}).get("queue") or {}) + debug_nodes = list((debug_snapshot.get("queue_health") or {}).get("nodes") or (debug_snapshot.get("job") or {}).get("node_stats") or []) + ingest_nodes = list((ingest_snapshot.get("queue_health") or {}).get("nodes") or (ingest_snapshot.get("job") or {}).get("node_stats") or []) + debug_display_running = max( + _int_value(debug_queue.get("display_running")), + _int_value(((debug_snapshot.get("job") or {}).get("display_items_running"))), + _int_value(((debug_snapshot.get("job") or {}).get("display_active_threads"))), + ) + ingest_display_running = max( + _int_value(ingest_queue.get("display_running")), + _int_value(((ingest_snapshot.get("job") or {}).get("display_items_running"))), + _int_value(((ingest_snapshot.get("job") or {}).get("display_active_threads"))), + ) + if len(ingest_nodes) > len(debug_nodes) or ingest_display_running > debug_display_running: + return _build_live_runtime_snapshot(ingest_snapshot) + if str(ingest_snapshot.get("_created_at") or "") > str(debug_snapshot.get("_created_at") or ""): + return _build_live_runtime_snapshot(ingest_snapshot) + return _build_live_runtime_snapshot(debug_snapshot) def _select_runtime_snapshot_events( @@ -1104,6 +2349,142 @@ def _filter_events_for_job(events: list[dict], *, job_code: str = "", job_id: in return filtered +def _build_active_job_summary_from_runtime_snapshot(runtime_snapshot: dict, *, event_limit: int = 20) -> dict | None: + normalized_snapshot = _build_live_runtime_snapshot(runtime_snapshot) + runtime_job = dict(normalized_snapshot.get("job") or {}) + runtime_queue = dict((normalized_snapshot.get("queue_health") or {}).get("queue") or {}) + if not runtime_job and not runtime_queue: + return None + + runtime_nodes = list((normalized_snapshot.get("queue_health") or {}).get("nodes") or runtime_job.get("node_stats") or []) + runtime_steps = list((normalized_snapshot.get("queue_health") or {}).get("steps") or []) + runtime_throughput = dict((normalized_snapshot.get("queue_health") or {}).get("throughput") or {}) + normalized_nodes = [ + _normalize_node_bucket(item) + for item in runtime_nodes + if isinstance(item, dict) and str(item.get("node_code") or "").strip() + ] + display_summary = _build_display_summary(normalized_nodes) + runtime_allow_raw_display_fallback = ( + int(normalized_snapshot.get("_dropped_runtime_node_count", 0) or 0) <= 0 + and ( + not bool(normalized_snapshot.get("_has_runtime_node_rows")) + or bool(display_summary.get("active_nodes") or []) + or any(str(item.get("node_code") or "").strip() != "unassigned" for item in normalized_nodes) + ) + ) + display_running_candidates = [ + runtime_queue.get("display_running"), + display_summary.get("display_running"), + runtime_job.get("display_items_running"), + runtime_job.get("display_active_threads"), + ] + if runtime_allow_raw_display_fallback: + display_running_candidates.extend((runtime_job.get("items_running"), runtime_queue.get("running"))) + display_items_running = _max_runtime_metric( + *display_running_candidates, + ) + display_active_thread_candidates = [ + display_summary.get("active_threads"), + runtime_job.get("display_active_threads"), + runtime_job.get("display_items_running"), + runtime_queue.get("display_running"), + ] + if runtime_allow_raw_display_fallback: + display_active_thread_candidates.extend((runtime_job.get("items_running"), runtime_queue.get("running"))) + display_active_threads = _max_runtime_metric( + *display_active_thread_candidates, + ) + display_max_threads = _max_runtime_metric( + display_summary.get("max_threads"), + runtime_job.get("display_max_threads"), + ) + + items_total = _int_value(runtime_queue.get("items_total", runtime_job.get("items_total", 0))) + items_pending = _int_value(runtime_queue.get("pending", runtime_job.get("items_pending", 0))) + items_claimed = _int_value(runtime_queue.get("claimed", runtime_job.get("items_claimed", 0))) + items_running = _int_value(runtime_queue.get("running", runtime_job.get("items_running", 0))) + items_completed = _int_value(runtime_queue.get("completed", runtime_job.get("items_completed", 0))) + items_blacklisted = _int_value(runtime_queue.get("blacklisted", runtime_job.get("items_blacklisted", 0))) + items_failed = _int_value(runtime_queue.get("failed", runtime_job.get("items_failed", 0))) + items_terminal = _int_value( + runtime_queue.get( + "terminal", + runtime_job.get("items_terminal", items_completed + items_blacklisted + items_failed), + ) + ) + display_current_load = _max_runtime_metric( + display_summary.get("current_load"), + display_items_running, + display_active_threads, + items_running, + ) + recent_events = _select_runtime_snapshot_events( + normalized_snapshot, + job_code=str(runtime_job.get("job_code") or "").strip(), + job_id=int(runtime_job.get("job_id", 0) or 0), + event_limit=event_limit, + ) + cycle_token, current_cycle_events = _extract_current_cycle_events(recent_events) + return { + "job_id": int(runtime_job.get("job_id", 0) or 0), + "job_code": str(runtime_job.get("job_code") or "").strip(), + "source": str(normalized_snapshot.get("_snapshot_source") or "runtime").strip() or "runtime", + "task_mode": _DEFAULT_TASK_MODE, + "step_code": "", + "status": str(runtime_job.get("status") or "").strip() or "running", + "created_by": str(normalized_snapshot.get("_snapshot_source") or "runtime").strip() or "runtime", + "created_at": str(normalized_snapshot.get("_created_at") or "").strip(), + "started_at": "", + "finished_at": "", + "items_total": items_total, + "items_pending": items_pending, + "items_claimed": items_claimed, + "items_running": items_running, + "items_completed": items_completed, + "items_blacklisted": items_blacklisted, + "items_failed": items_failed, + "items_terminal": items_terminal, + "progress_percent": float(runtime_job.get("progress_percent", 0) or 0), + "raw_items_total": items_total, + "raw_items_pending": items_pending, + "raw_items_claimed": items_claimed, + "raw_items_running": items_running, + "raw_items_completed": items_completed, + "raw_items_blacklisted": items_blacklisted, + "raw_items_failed": items_failed, + "raw_items_terminal": items_terminal, + "raw_node_stats": list(normalized_nodes), + "raw_step_stats": list(runtime_steps), + "node_stats": list(normalized_nodes), + "distributed_node_stats": list(normalized_nodes), + "step_stats": list(runtime_steps), + "display_items_claimed": _max_runtime_metric( + runtime_queue.get("display_claimed"), + display_summary.get("items_claimed"), + items_claimed, + ), + "display_items_running": display_items_running, + "display_current_load": display_current_load, + "display_active_threads": display_active_threads, + "display_max_threads": display_max_threads, + "display_items_completed": int(display_summary.get("items_completed", items_completed) or 0), + "display_items_failed": int(display_summary.get("items_failed", items_failed) or 0), + "display_active_node_codes": list(display_summary.get("active_nodes") or []), + "processed_recent": int(runtime_throughput.get("processed_recent", 0) or 0), + "processed_per_minute": float(runtime_throughput.get("processed_per_minute", 0) or 0), + "completed_recent": int(runtime_throughput.get("completed_recent", 0) or 0), + "blacklisted_recent": int(runtime_throughput.get("blacklisted_recent", 0) or 0), + "failed_recent": int(runtime_throughput.get("failed_recent", 0) or 0), + "recent_events": recent_events, + "latest_event": recent_events[0] if recent_events else None, + "current_cycle_token": cycle_token, + "current_cycle_events": current_cycle_events, + "latest_cycle_event": current_cycle_events[0] if current_cycle_events else (recent_events[0] if recent_events else None), + "recent_domain_events": [], + } + + def _fetch_job_summary(cur, job_row, event_limit: int = 20) -> dict: job_id = job_row[0] cur.execute( @@ -1287,7 +2668,7 @@ def _enrich_active_job_summary_with_runtime( return summary enriched = dict(summary) runtime_activity = dict(runtime_activity or _load_runtime_activity_snapshot(window_minutes)) - runtime_snapshot = dict(runtime_snapshot or _load_latest_runtime_active_job_snapshot(window_minutes)) + runtime_snapshot = _build_live_runtime_snapshot(runtime_snapshot or _load_latest_runtime_active_job_snapshot(window_minutes)) runtime_job_code = str(runtime_activity.get("focus_job_code") or "").strip() runtime_job_codes = [ str(item or "").strip() @@ -1319,6 +2700,8 @@ def _enrich_active_job_summary_with_runtime( int(enriched.get("blacklisted_recent", 0) or 0), int(runtime_activity.get("blacklisted_recent", 0) or 0), ) + runtime_allow_raw_display_fallback = True + runtime_missing_participant_attribution = False if recent_domain_events is not None: enriched["recent_domain_events"] = list(recent_domain_events) @@ -1338,7 +2721,27 @@ def _enrich_active_job_summary_with_runtime( runtime_nodes = list((runtime_snapshot.get("queue_health") or {}).get("nodes") or []) runtime_steps = list((runtime_snapshot.get("queue_health") or {}).get("steps") or []) runtime_job_nodes = list(runtime_job.get("node_stats") or []) - if runtime_job: + runtime_snapshot_source = str(runtime_snapshot.get("_snapshot_source") or "").strip() + runtime_snapshot_stale_for_live_display = _snapshot_is_stale_for_live_display(runtime_snapshot) + if runtime_snapshot_stale_for_live_display and str(enriched.get("created_by") or "").strip() == runtime_snapshot_source: + runtime_snapshot_stale_for_live_display = False + has_runtime_participant_nodes = any( + str(item.get("node_code") or "").strip() and str(item.get("node_code") or "").strip() != "unassigned" + for item in [*runtime_nodes, *runtime_job_nodes] + if isinstance(item, dict) + ) + runtime_missing_participant_attribution = bool(runtime_snapshot.get("_has_runtime_node_rows")) and not has_runtime_participant_nodes + runtime_allow_raw_display_fallback = ( + int(runtime_snapshot.get("_dropped_runtime_node_count", 0) or 0) <= 0 + and ( + not bool(runtime_snapshot.get("_has_runtime_node_rows")) + or has_runtime_participant_nodes + ) + ) + if runtime_snapshot_stale_for_live_display: + runtime_missing_participant_attribution = True + runtime_allow_raw_display_fallback = False + if runtime_job and not runtime_snapshot_stale_for_live_display: runtime_snapshot_job_id = int(runtime_job.get("job_id", 0) or 0) enriched["runtime_snapshot_job_id"] = runtime_snapshot_job_id runtime_snapshot_job_code = str(runtime_job.get("job_code") or "").strip() @@ -1357,7 +2760,7 @@ def _enrich_active_job_summary_with_runtime( enriched["status"] = runtime_status if float(runtime_job.get("progress_percent", 0) or 0) > float(enriched.get("progress_percent", 0) or 0): enriched["progress_percent"] = float(runtime_job.get("progress_percent", 0) or 0) - if runtime_queue: + if runtime_queue and not runtime_snapshot_stale_for_live_display: enriched["runtime_snapshot_queue"] = runtime_queue runtime_items_total = int(runtime_queue.get("items_total", 0) or 0) if runtime_items_total > 0: @@ -1369,38 +2772,71 @@ def _enrich_active_job_summary_with_runtime( enriched["items_blacklisted"] = int(runtime_queue.get("blacklisted", 0) or 0) enriched["items_failed"] = int(runtime_queue.get("failed", 0) or 0) enriched["items_terminal"] = int(runtime_queue.get("terminal", 0) or 0) - enriched["display_items_claimed"] = int(runtime_queue.get("display_claimed", enriched.get("items_claimed", 0)) or 0) - enriched["display_items_running"] = int(runtime_queue.get("display_running", enriched.get("items_running", 0)) or 0) - if runtime_job_nodes: + enriched["display_items_claimed"] = _max_runtime_metric( + enriched.get("display_items_claimed"), + runtime_queue.get("display_claimed"), + enriched.get("items_claimed"), + ) + display_running_candidates = [ + enriched.get("display_items_running"), + runtime_queue.get("display_running"), + enriched.get("display_active_threads"), + ] + if runtime_allow_raw_display_fallback: + display_running_candidates.append(enriched.get("items_running")) + enriched["display_items_running"] = _max_runtime_metric(*display_running_candidates) + if runtime_job_nodes and not runtime_snapshot_stale_for_live_display: enriched["node_stats"] = list(runtime_job_nodes) enriched["distributed_node_stats"] = list(runtime_job_nodes) runtime_display_summary = _build_display_summary(list(runtime_job_nodes)) - enriched["display_items_claimed"] = int(runtime_display_summary.get("items_claimed", enriched.get("display_items_claimed", 0)) or 0) - enriched["display_items_running"] = int(runtime_display_summary.get("display_running", enriched.get("display_items_running", 0)) or 0) - enriched["display_current_load"] = int(runtime_display_summary.get("current_load", enriched.get("display_current_load", 0)) or 0) - enriched["display_active_threads"] = int(runtime_display_summary.get("active_threads", enriched.get("display_active_threads", 0)) or 0) - enriched["display_max_threads"] = int(runtime_display_summary.get("max_threads", enriched.get("display_max_threads", 0)) or 0) + enriched["display_items_claimed"] = _max_runtime_metric( + enriched.get("display_items_claimed"), + runtime_display_summary.get("items_claimed"), + enriched.get("items_claimed"), + ) + enriched["display_items_running"] = _max_runtime_metric( + enriched.get("display_items_running"), + runtime_display_summary.get("display_running"), + enriched.get("display_active_threads"), + *([] if not runtime_allow_raw_display_fallback else [enriched.get("items_running")]), + ) + enriched["display_current_load"] = _max_runtime_metric( + enriched.get("display_current_load"), + runtime_display_summary.get("current_load"), + enriched.get("display_items_running"), + ) + enriched["display_active_threads"] = _max_runtime_metric( + enriched.get("display_active_threads"), + runtime_display_summary.get("active_threads"), + enriched.get("display_items_running"), + *([] if not runtime_allow_raw_display_fallback else [enriched.get("items_running")]), + ) + enriched["display_max_threads"] = _max_runtime_metric( + enriched.get("display_max_threads"), + runtime_display_summary.get("max_threads"), + ) enriched["display_items_completed"] = int(runtime_display_summary.get("items_completed", enriched.get("display_items_completed", 0)) or 0) enriched["display_items_failed"] = int(runtime_display_summary.get("items_failed", enriched.get("display_items_failed", 0)) or 0) enriched["display_active_node_codes"] = list(runtime_display_summary.get("active_nodes") or []) - elif runtime_nodes: + elif runtime_nodes and not runtime_snapshot_stale_for_live_display: enriched["node_stats"] = list(runtime_nodes) enriched["distributed_node_stats"] = list(runtime_nodes) - if runtime_steps: + if runtime_steps and not runtime_snapshot_stale_for_live_display: enriched["step_stats"] = list(runtime_steps) - runtime_recent_events = _select_runtime_snapshot_events( - runtime_snapshot, - job_code=str(runtime_job.get("job_code") or "").strip(), - job_id=int(runtime_job.get("job_id", 0) or 0), - event_limit=event_limit, - ) - if runtime_recent_events: - enriched["recent_events"] = runtime_recent_events - current_cycle_token, current_cycle_events = _extract_current_cycle_events(runtime_recent_events) - enriched["current_cycle_token"] = current_cycle_token - enriched["current_cycle_events"] = current_cycle_events - enriched["latest_event"] = runtime_recent_events[0] - enriched["latest_cycle_event"] = current_cycle_events[0] if current_cycle_events else runtime_recent_events[0] + if not runtime_snapshot_stale_for_live_display: + runtime_recent_events = _select_runtime_snapshot_events( + runtime_snapshot, + job_code=str(runtime_job.get("job_code") or "").strip(), + job_id=int(runtime_job.get("job_id", 0) or 0), + event_limit=event_limit, + ) + if runtime_recent_events: + enriched["recent_events"] = runtime_recent_events + current_cycle_token, current_cycle_events = _extract_current_cycle_events(runtime_recent_events) + enriched["current_cycle_token"] = current_cycle_token + enriched["current_cycle_events"] = current_cycle_events + enriched["latest_event"] = runtime_recent_events[0] + enriched["latest_cycle_event"] = current_cycle_events[0] if current_cycle_events else runtime_recent_events[0] aligned_job_code = str(enriched.get("job_code") or enriched.get("runtime_job_code") or "").strip() aligned_job_id = int(enriched.get("job_id", 0) or 0) aligned_recent_events = _filter_events_for_job( @@ -1422,6 +2858,36 @@ def _enrich_active_job_summary_with_runtime( enriched["current_cycle_events"] = current_cycle_events enriched["latest_event"] = aligned_recent_events[0] enriched["latest_cycle_event"] = current_cycle_events[0] if current_cycle_events else aligned_recent_events[0] + enriched["display_items_running"] = _max_runtime_metric( + enriched.get("display_items_running"), + enriched.get("display_active_threads"), + *([] if not runtime_allow_raw_display_fallback else [enriched.get("items_running")]), + ) + enriched["display_current_load"] = _max_runtime_metric( + enriched.get("display_current_load"), + enriched.get("display_items_running"), + enriched.get("display_active_threads"), + *([] if not runtime_allow_raw_display_fallback else [enriched.get("items_running")]), + ) + enriched["display_active_threads"] = _max_runtime_metric( + enriched.get("display_active_threads"), + enriched.get("display_items_running"), + *([] if not runtime_allow_raw_display_fallback else [enriched.get("items_running")]), + ) + if runtime_missing_participant_attribution: + enriched["display_items_running"] = 0 + enriched["display_current_load"] = 0 + enriched["display_active_threads"] = 0 + enriched["display_active_node_codes"] = [] + enriched["display_max_threads"] = 0 + enriched["display_max_threads"] = _max_runtime_metric( + enriched.get("display_max_threads"), + sum( + _int_value(item.get("max_threads")) + for item in list(enriched.get("distributed_node_stats") or []) + if isinstance(item, dict) + ), + ) return enriched @@ -1429,14 +2895,12 @@ def _enrich_active_job_summary_with_runtime( def get_detect_job_summary(job_id: int, event_limit: int = 20) -> dict | None: with get_db() as conn: with conn.cursor() as cur: - cur.execute( - """ - SELECT id, job_code, source, task_mode, step_code, status, created_by, created_at, started_at, finished_at - FROM detect_jobs - WHERE id = %s - LIMIT 1 - """, - (int(job_id),), + _execute_detect_jobs_select( + cur, + from_clause="FROM detect_jobs", + where_clause="id = %s", + limit_clause="1", + params=(int(job_id),), ) row = cur.fetchone() if not row: @@ -1444,26 +2908,102 @@ def get_detect_job_summary(job_id: int, event_limit: int = 20) -> dict | None: return _enrich_active_job_summary_with_runtime(_fetch_job_summary(cur, row, event_limit=event_limit), event_limit=event_limit) +def _select_runtime_aligned_active_job_row( + job_rows: list[tuple], + runtime_activity: dict | None = None, + runtime_snapshot: dict | None = None, +): + rows = list(job_rows or []) + if not rows: + return None + activity = dict(runtime_activity or {}) + snapshot = dict(runtime_snapshot or {}) + snapshot_job = dict(snapshot.get("job") or {}) + preferred_job_codes = [ + str(snapshot_job.get("job_code") or "").strip(), + str(activity.get("focus_job_code") or "").strip(), + *[ + str(item or "").strip() + for item in list(activity.get("job_codes") or []) + if str(item or "").strip() + ], + ] + normalized_codes: list[str] = [] + seen_codes: set[str] = set() + for item in preferred_job_codes: + if not item or item in seen_codes: + continue + normalized_codes.append(item) + seen_codes.add(item) + if not normalized_codes: + return rows[0] + for preferred_code in normalized_codes: + for row in rows: + if str(row[1] or "").strip() == preferred_code: + return row + return rows[0] + + @db_read_retry() def get_active_detect_job_summary(event_limit: int = 20) -> dict | None: + safe_event_limit = max(1, min(int(event_limit or 20), 100)) + cache_key = (safe_event_limit,) + now_ts = time.monotonic() + with _ACTIVE_JOB_SUMMARY_CACHE_LOCK: + cached = _ACTIVE_JOB_SUMMARY_CACHE.get(cache_key) + if cached and now_ts < float(cached[0] or 0.0): + return _clone_cacheable_payload(cached[1]) + + _maybe_recycle_expired_detect_job_items() + result: dict | None = None with get_db() as conn: with conn.cursor() as cur: - cur.execute( - """ - SELECT id, job_code, source, task_mode, step_code, status, created_by, created_at, started_at, finished_at - FROM detect_jobs - WHERE status IN ('pending', 'running') - ORDER BY + runtime_snapshot = _load_latest_runtime_active_job_snapshot(15) + runtime_activity = _load_runtime_activity_snapshot(15) + _execute_detect_jobs_select( + cur, + from_clause="FROM detect_jobs", + where_clause="status IN ('pending', 'running')", + order_clause=""" CASE WHEN status = 'running' THEN 0 ELSE 1 END ASC, COALESCE(started_at, created_at) DESC, id DESC - LIMIT 1 - """ + """, + limit_clause="20", ) - row = cur.fetchone() - if not row: - return None - return _enrich_active_job_summary_with_runtime(_fetch_job_summary(cur, row, event_limit=event_limit), event_limit=event_limit) + rows = list(cur.fetchall() or []) + if not rows: + runtime_summary = _build_active_job_summary_from_runtime_snapshot( + runtime_snapshot, + event_limit=safe_event_limit, + ) + if not runtime_summary: + result = None + else: + result = _enrich_active_job_summary_with_runtime( + runtime_summary, + event_limit=safe_event_limit, + runtime_activity=runtime_activity, + runtime_snapshot=runtime_snapshot, + ) + else: + row = _select_runtime_aligned_active_job_row( + rows, + runtime_activity=runtime_activity, + runtime_snapshot=runtime_snapshot, + ) + result = _enrich_active_job_summary_with_runtime( + _fetch_job_summary(cur, row, event_limit=safe_event_limit), + event_limit=safe_event_limit, + runtime_activity=runtime_activity, + runtime_snapshot=runtime_snapshot, + ) + with _ACTIVE_JOB_SUMMARY_CACHE_LOCK: + _ACTIVE_JOB_SUMMARY_CACHE[cache_key] = ( + time.monotonic() + _ACTIVE_JOB_SUMMARY_CACHE_TTL_SECONDS, + _clone_cacheable_payload(result), + ) + return _clone_cacheable_payload(result) @db_read_retry() @@ -1495,12 +3035,11 @@ def get_latest_detect_job_summary( with get_db() as conn: with conn.cursor() as cur: - cur.execute( - f""" - SELECT id, job_code, source, task_mode, step_code, status, created_by, created_at, started_at, finished_at - FROM detect_jobs - WHERE {' AND '.join(where_clauses)} - ORDER BY + _execute_detect_jobs_select( + cur, + from_clause="FROM detect_jobs", + where_clause=" AND ".join(where_clauses), + order_clause=""" CASE WHEN status = 'running' THEN 0 WHEN status = 'pending' THEN 1 @@ -1511,9 +3050,9 @@ def get_latest_detect_job_summary( END ASC, COALESCE(finished_at, started_at, created_at) DESC, id DESC - LIMIT 1 """, - tuple(params), + limit_clause="1", + params=tuple(params), ) row = cur.fetchone() if not row: @@ -1573,17 +3112,17 @@ def get_latest_unprojected_detect_job_summary( with get_db() as conn: with conn.cursor() as cur: - cur.execute( - f""" - SELECT job.id, job.job_code, job.source, job.task_mode, job.step_code, job.status, job.created_by, job.created_at, job.started_at, job.finished_at - FROM detect_jobs AS job - WHERE {' AND '.join(where_clauses)} - ORDER BY + _execute_detect_jobs_select( + cur, + from_clause="FROM detect_jobs AS job", + where_clause=" AND ".join(where_clauses), + order_clause=""" COALESCE(job.finished_at, job.started_at, job.created_at) DESC, job.id DESC - LIMIT 1 """, - tuple(params), + limit_clause="1", + params=tuple(params), + alias="job", ) row = cur.fetchone() if not row: @@ -1594,14 +3133,12 @@ def get_latest_unprojected_detect_job_summary( def list_detect_jobs(limit: int = 20) -> list[dict]: with get_db() as conn: with conn.cursor() as cur: - cur.execute( - """ - SELECT id, job_code, source, task_mode, step_code, status, created_by, created_at, started_at, finished_at - FROM detect_jobs - ORDER BY created_at DESC, id DESC - LIMIT %s - """, - (max(1, min(int(limit or 20), 100)),), + _execute_detect_jobs_select( + cur, + from_clause="FROM detect_jobs", + order_clause="created_at DESC, id DESC", + limit_clause="%s", + params=(max(1, min(int(limit or 20), 100)),), ) rows = cur.fetchall() return [_fetch_job_summary(cur, row, event_limit=10) for row in rows] @@ -1638,11 +3175,62 @@ def list_recent_detect_run_events(limit: int = 20) -> list[dict]: @db_read_retry() def get_detect_queue_health(window_minutes: int = 15) -> dict: window_minutes = max(5, min(int(window_minutes or 15), 120)) + cache_key = (window_minutes,) + now_ts = time.monotonic() + with _DETECT_QUEUE_HEALTH_CACHE_LOCK: + cached = _DETECT_QUEUE_HEALTH_CACHE.get(cache_key) + if cached and now_ts < float(cached[0] or 0.0): + return _clone_cacheable_payload(cached[1]) + active_job = get_active_detect_job_summary(event_limit=10) runtime_activity = _load_runtime_activity_snapshot(window_minutes) runtime_snapshot = _load_latest_runtime_active_job_snapshot(window_minutes) + active_job_runtime_code = str( + (active_job or {}).get("runtime_snapshot_job_code") + or (active_job or {}).get("runtime_job_code") + or (active_job or {}).get("job_code") + or "" + ).strip() + if active_job_runtime_code: + merged_runtime_codes = [active_job_runtime_code] + [ + str(item or "").strip() + for item in list(runtime_activity.get("job_codes") or []) + if str(item or "").strip() and str(item or "").strip() != active_job_runtime_code + ] + runtime_activity = { + **dict(runtime_activity or {}), + "focus_job_code": active_job_runtime_code, + "job_codes": merged_runtime_codes[:5], + } + + runtime_snapshot_job = dict((runtime_snapshot or {}).get("job") or {}) + runtime_snapshot_matches_active_job = _runtime_job_matches_active_job( + active_job, + runtime_job_code=runtime_snapshot_job.get("job_code"), + runtime_job_id=runtime_snapshot_job.get("job_id"), + ) + runtime_activity_matches_active_job = _runtime_job_matches_active_job( + active_job, + runtime_job_code=(runtime_activity or {}).get("focus_job_code"), + ) + if runtime_snapshot and not runtime_snapshot_matches_active_job: + runtime_snapshot = {} + if runtime_activity and not runtime_activity_matches_active_job and not runtime_snapshot_matches_active_job: + runtime_activity = { + **dict(runtime_activity or {}), + "focus_job_code": "", + "job_codes": [], + "processed_recent": 0, + "completed_recent": 0, + "failed_recent": 0, + "blacklisted_recent": 0, + "step_code": "", + "step_stats": {}, + "nodes": {}, + } + if not active_job: - return { + result = { "window_minutes": window_minutes, "has_active_job": False, "job": None, @@ -1673,6 +3261,12 @@ def get_detect_queue_health(window_minutes: int = 15) -> dict: "steps": [], "runtime_activity": runtime_activity, } + with _DETECT_QUEUE_HEALTH_CACHE_LOCK: + _DETECT_QUEUE_HEALTH_CACHE[cache_key] = ( + time.monotonic() + _DETECT_QUEUE_HEALTH_CACHE_TTL_SECONDS, + _clone_cacheable_payload(result), + ) + return _clone_cacheable_payload(result) job_id = int(active_job["job_id"]) with get_db() as conn: @@ -1766,6 +3360,14 @@ def get_detect_queue_health(window_minutes: int = 15) -> dict: "completed_recent": 0, "blacklisted_recent": 0, "failed_recent": 0, + "display_running": _int_value(item.get("display_running")), + "current_load": _int_value(item.get("current_load")), + "active_threads": _int_value(item.get("active_threads")), + "max_threads": _int_value(item.get("max_threads")), + "region": str(item.get("region") or "").strip(), + "role": str(item.get("role") or "").strip(), + "status": str(item.get("status") or "").strip(), + "last_heartbeat_at": str(item.get("last_heartbeat_at") or "").strip(), } for item in active_job.get("node_stats") or [] } @@ -1787,6 +3389,14 @@ def get_detect_queue_health(window_minutes: int = 15) -> dict: "blacklisted_recent": 0, "failed_recent": 0, "metrics_source": str(item.get("metrics_source") or "").strip(), + "display_running": _int_value(item.get("display_running")), + "current_load": _int_value(item.get("current_load")), + "active_threads": _int_value(item.get("active_threads")), + "max_threads": _int_value(item.get("max_threads")), + "region": str(item.get("region") or "").strip(), + "role": str(item.get("role") or "").strip(), + "status": str(item.get("status") or "").strip(), + "last_heartbeat_at": str(item.get("last_heartbeat_at") or "").strip(), } for item in distributed_nodes } @@ -2000,7 +3610,7 @@ def get_detect_queue_health(window_minutes: int = 15) -> dict: failed_items = int(active_job.get("items_failed", 0) or 0) display_claimed = int(active_job.get("display_items_claimed", active_job.get("items_claimed", 0)) or 0) display_running = int( - active_job.get("display_active_threads", active_job.get("display_items_running", active_job.get("items_running", 0))) + active_job.get("display_active_threads", active_job.get("display_items_running", 0)) or 0 ) progress_percent = float(active_job.get("progress_percent", 0) or 0) @@ -2016,7 +3626,7 @@ def get_detect_queue_health(window_minutes: int = 15) -> dict: blacklisted_items = int(runtime_snapshot_queue.get("blacklisted", 0) or 0) failed_items = int(runtime_snapshot_queue.get("failed", 0) or 0) display_claimed = int(runtime_snapshot_queue.get("display_claimed", claimed_items) or 0) - display_running = int(runtime_snapshot_queue.get("display_running", running_items) or 0) + display_running = int(runtime_snapshot_queue.get("display_running", display_running) or 0) progress_percent = float(runtime_snapshot_job.get("progress_percent", progress_percent) or 0) if runtime_snapshot_nodes: @@ -2036,6 +3646,14 @@ def get_detect_queue_health(window_minutes: int = 15) -> dict: "blacklisted_recent": _int_value(item.get("blacklisted_recent")), "failed_recent": _int_value(item.get("failed_recent")), "metrics_source": str(item.get("metrics_source") or "").strip(), + "display_running": _int_value(item.get("display_running")), + "current_load": _int_value(item.get("current_load")), + "active_threads": _int_value(item.get("active_threads")), + "max_threads": _int_value(item.get("max_threads")), + "region": str(item.get("region") or "").strip(), + "role": str(item.get("role") or "").strip(), + "status": str(item.get("status") or "").strip(), + "last_heartbeat_at": str(item.get("last_heartbeat_at") or "").strip(), } for item in runtime_snapshot_nodes } @@ -2180,6 +3798,28 @@ def get_detect_queue_health(window_minutes: int = 15) -> dict: bucket["blacklisted_recent"] = int(runtime_bucket.get("blacklisted_recent", 0) or 0) bucket["failed_recent"] = int(runtime_bucket.get("failed_recent", 0) or 0) + snapshot_created_at = _snapshot_created_at(runtime_snapshot) + raw_nodes = list(node_map.values()) + live_nodes = _filter_live_runtime_nodes(raw_nodes, snapshot_created_at=snapshot_created_at) + raw_participant_count = sum( + 1 + for item in raw_nodes + if str(item.get("node_code") or "").strip() + and str(item.get("node_code") or "").strip() != "unassigned" + ) + live_participant_count = sum( + 1 + for item in live_nodes + if str(item.get("node_code") or "").strip() + and str(item.get("node_code") or "").strip() != "unassigned" + ) + dropped_runtime_node_count = max(0, raw_participant_count - live_participant_count) + if raw_nodes: + node_map = { + str(item.get("node_code") or "unknown"): dict(item) + for item in live_nodes + } + normalized_running_items = sum( int(item.get("items_running", 0) or 0) for item in node_map.values() @@ -2193,9 +3833,9 @@ def get_detect_queue_health(window_minutes: int = 15) -> dict: # Prefer normalized per-node sums once runtime heartbeats are present. # Stale snapshot queue rows can temporarily over-report thread load from an # older cycle, which is exactly what misleads the ops dashboard. - if normalized_running_items > 0: + if normalized_running_items > 0 or dropped_runtime_node_count > 0: running_items = normalized_running_items - if normalized_display_running > 0: + if normalized_display_running > 0 or dropped_runtime_node_count > 0: display_running = normalized_display_running nodes = sorted( @@ -2208,16 +3848,8 @@ def get_detect_queue_health(window_minutes: int = 15) -> dict: ), ) terminal = completed_items + blacklisted_items + failed_items - steps = sorted( - step_map.values(), - key=lambda item: ( - -int(item.get("processed_recent", 0) or 0), - -int(item.get("items_running", 0) or 0), - -int(item.get("items_pending", 0) or 0), - str(item.get("step_code") or ""), - ), - ) - return { + steps = order_step_buckets(list(step_map.values())) + result = { "window_minutes": window_minutes, "has_active_job": True, "job": { @@ -2258,6 +3890,12 @@ def get_detect_queue_health(window_minutes: int = 15) -> dict: "runtime_activity": runtime_activity, "runtime_snapshot_backlog": dict(runtime_snapshot.get("backlog") or {}), } + with _DETECT_QUEUE_HEALTH_CACHE_LOCK: + _DETECT_QUEUE_HEALTH_CACHE[cache_key] = ( + time.monotonic() + _DETECT_QUEUE_HEALTH_CACHE_TTL_SECONDS, + _clone_cacheable_payload(result), + ) + return _clone_cacheable_payload(result) def get_detect_capacity_plan(*, queue_health: dict | None = None, online_worker_nodes: int = 0, target_finish_hours: int = 6) -> dict: @@ -2669,70 +4307,7 @@ def process_detect_pipeline(limit: int = 5000, job_id: int | None = None) -> dic summary["terminal_items"] += 1 for job_id in touched_job_ids: - cur.execute( - """ - SELECT - job.task_mode, - count(*) FILTER (WHERE item.status = 'pending') AS pending_count, - count(*) FILTER (WHERE item.status IN ('claimed', 'running')) AS dispatch_active_count, - count(*) FILTER (WHERE item.status = 'failed') AS failed_count, - count(*) FILTER (WHERE item.status IN ('completed', 'blacklisted')) AS done_count, - count(*) FILTER ( - WHERE item.status IN ('completed', 'blacklisted', 'failed') - AND COALESCE(item.step_code, '') <> '' - AND COALESCE(item.result_payload_json->>'controller_processed', 'false') <> 'true' - ) AS unprocessed_terminal_count - FROM detect_job_items AS item - JOIN detect_jobs AS job ON job.id = item.job_id - WHERE item.job_id = %s - GROUP BY job.task_mode - """, - (job_id,), - ) - refresh_row = cur.fetchone() - task_mode = str((refresh_row or [""])[0] or "").strip() - pending_count = int((refresh_row or ["", 0])[1] or 0) - dispatch_active_count = int((refresh_row or ["", 0, 0])[2] or 0) - failed_count = int((refresh_row or ["", 0, 0, 0])[3] or 0) - done_count = int((refresh_row or ["", 0, 0, 0, 0])[4] or 0) - unprocessed_terminal_count = int((refresh_row or ["", 0, 0, 0, 0, 0])[5] or 0) - if dispatch_active_count > 0 or (task_mode == "domain_pipeline" and unprocessed_terminal_count > 0): - cur.execute( - """ - UPDATE detect_jobs - SET status = 'running', - started_at = COALESCE(started_at, CURRENT_TIMESTAMP), - finished_at = NULL - WHERE id = %s - """, - (job_id,), - ) - elif pending_count > 0: - cur.execute( - """ - UPDATE detect_jobs - SET status = 'pending', - finished_at = NULL - WHERE id = %s - """, - (job_id,), - ) - else: - final_status = "completed" - if failed_count > 0 and done_count > 0: - final_status = "partial_failed" - elif failed_count > 0: - final_status = "failed" - cur.execute( - """ - UPDATE detect_jobs - SET status = %s, - finished_at = CURRENT_TIMESTAMP, - started_at = COALESCE(started_at, CURRENT_TIMESTAMP) - WHERE id = %s - """, - (final_status, job_id), - ) + _refresh_detect_job_status_with_cursor(cur, job_id) summary["job_ids"].append(job_id) conn.commit() diff --git a/domain-api/app/services/detect_run_service.py b/domain-api/app/services/detect_run_service.py index 3ad0ebd..ab931d3 100644 --- a/domain-api/app/services/detect_run_service.py +++ b/domain-api/app/services/detect_run_service.py @@ -1,5 +1,6 @@ from __future__ import annotations +import threading from datetime import datetime from uuid import uuid4 @@ -10,6 +11,7 @@ _MAX_LOG_LINES = 240 _LOG_TAIL_LINES = 1200 _ACTIVE_STATUSES = {"starting", "running", "stopping"} _TIMESTAMP_FORMATS = ("%Y-%m-%d %H:%M:%S.%f", "%Y-%m-%d %H:%M:%S") +_DETECT_RUNS_LOCK = threading.RLock() def _now() -> str: @@ -234,192 +236,196 @@ def _sync_record( def create_detect_run_snapshot(message: str, runtime: dict, progress: dict, settings_summary: dict) -> dict: - records = _load() - active = _find_active(records) - current_logs = _capture_worker_logs() - if active: - if active.get("status") == "stopping" and runtime.get("running"): - active["status"] = "running" + with _DETECT_RUNS_LOCK: + records = _load() + active = _find_active(records) + current_logs = _capture_worker_logs() + if active: + if active.get("status") == "stopping" and runtime.get("running"): + active["status"] = "running" + _sync_record( + active, + status=active.get("status", "starting"), + message=message, + runtime=runtime, + progress=progress, + settings_summary=settings_summary, + log_lines=current_logs, + ) + _save(records) + return dict(active) + + initial_started_at = runtime.get("latest_start_time") or _now() + record = { + "run_id": uuid4().hex, + "status": "starting", + "message": message, + "created_at": _now(), + "updated_at": _now(), + "started_at": initial_started_at, + "completed_at": "", + "runtime": runtime, + "progress": progress, + "settings_summary": settings_summary, + "phase_label": "", + "phase_detail": "", + "phase_history": [], + "logs": [], + } _sync_record( - active, - status=active.get("status", "starting"), + record, + status="starting", message=message, runtime=runtime, progress=progress, settings_summary=settings_summary, log_lines=current_logs, ) + records.insert(0, record) _save(records) - return dict(active) - - initial_started_at = runtime.get("latest_start_time") or _now() - record = { - "run_id": uuid4().hex, - "status": "starting", - "message": message, - "created_at": _now(), - "updated_at": _now(), - "started_at": initial_started_at, - "completed_at": "", - "runtime": runtime, - "progress": progress, - "settings_summary": settings_summary, - "phase_label": "", - "phase_detail": "", - "phase_history": [], - "logs": [], - } - _sync_record( - record, - status="starting", - message=message, - runtime=runtime, - progress=progress, - settings_summary=settings_summary, - log_lines=current_logs, - ) - records.insert(0, record) - _save(records) - return dict(record) + return dict(record) def finalize_detect_run(message: str, runtime: dict, progress: dict, settings_summary: dict, active_job: dict | None = None) -> dict | None: - records = _load() - target = _find_active(records) - if not target: - return None - final_status = "stopped" if target.get("status") == "stopping" else "failed" - _sync_record( - target, - status=final_status, - message=message, - runtime=runtime, - progress=progress, - settings_summary=settings_summary, - log_lines=_capture_worker_logs(), - active_job=active_job, - ) - _save(records) - return dict(target) - - -def sync_detect_runs(runtime: dict, progress: dict, settings_summary: dict, active_job: dict | None = None) -> list[dict]: - records = _load() - active = _find_active(records) - current_logs = _capture_worker_logs() - active_job = active_job or {} - runtime_detecting = bool(runtime.get("detecting", False)) - active_job_status = str(active_job.get("status", "") or "").strip() - active_job_open = active_job_status in {"pending", "running"} - execution_active = runtime_detecting or active_job_open or int((progress or {}).get("running", 0) or 0) > 0 - - if runtime.get("running") and execution_active: - if active and not _same_session(active, runtime): - _sync_record( - active, - status="stopped", - message="检测服务已重启，上一轮会话已归档", - runtime=active.get("runtime") or runtime, - progress=active.get("progress") or progress, - settings_summary=active.get("settings_summary") or settings_summary, - log_lines=current_logs, - active_job=active.get("active_job") or active_job, - ) - active = None - if active: - next_status = "running" if active.get("status") != "stopping" else "stopping" - _sync_record( - active, - status=next_status, - message=runtime.get("message") or active.get("message") or "检测服务运行中", - runtime=runtime, - progress=progress, - settings_summary=settings_summary, - log_lines=current_logs, - active_job=active_job, - ) - else: - started_at = runtime.get("latest_start_time") or _now() - record = { - "run_id": uuid4().hex, - "status": "running", - "message": runtime.get("message") or "检测服务运行中", - "created_at": _now(), - "updated_at": _now(), - "started_at": started_at, - "completed_at": "", - "runtime": runtime, - "progress": progress, - "settings_summary": settings_summary, - "phase_label": "", - "phase_detail": "", - "phase_history": [], - "logs": [], - "active_job": active_job, - } - _sync_record( - record, - status="running", - message=record["message"], - runtime=runtime, - progress=progress, - settings_summary=settings_summary, - log_lines=current_logs, - active_job=active_job, - ) - records.insert(0, record) - elif active: - if runtime.get("running") and not execution_active: - if active.get("status") == "stopping": - final_status = "stopped" - final_message = runtime.get("message") or "检测任务已停止，Worker 保持待命" - elif active_job_status == "partial_failed": - final_status = "partial_failed" - final_message = "检测任务已结束，存在部分失败项" - elif active_job_status == "failed": - final_status = "failed" - final_message = "检测任务已结束，任务结果为失败" - else: - final_status = "completed" - final_message = "检测任务已自然完成，Worker 保持待命" - else: - final_status = "stopped" if active.get("status") == "stopping" else "failed" - final_message = runtime.get("message") or ("检测服务已停止" if final_status == "stopped" else "检测服务异常退出") + with _DETECT_RUNS_LOCK: + records = _load() + target = _find_active(records) + if not target: + return None + final_status = "stopped" if target.get("status") == "stopping" else "failed" _sync_record( - active, + target, status=final_status, - message=final_message, + message=message, runtime=runtime, progress=progress, settings_summary=settings_summary, - log_lines=current_logs, + log_lines=_capture_worker_logs(), active_job=active_job, ) + _save(records) + return dict(target) - if records: - records[0]["logs"] = _merge_logs( - records[0].get("logs"), - _filter_logs_since(current_logs, records[0].get("started_at")), - ) - _save(records) - return records +def sync_detect_runs(runtime: dict, progress: dict, settings_summary: dict, active_job: dict | None = None) -> list[dict]: + with _DETECT_RUNS_LOCK: + records = _load() + active = _find_active(records) + current_logs = _capture_worker_logs() + active_job = active_job or {} + runtime_detecting = bool(runtime.get("detecting", False)) + active_job_status = str(active_job.get("status", "") or "").strip() + active_job_open = active_job_status in {"pending", "running"} + execution_active = runtime_detecting or active_job_open or int((progress or {}).get("running", 0) or 0) > 0 + + if runtime.get("running") and execution_active: + if active and not _same_session(active, runtime): + _sync_record( + active, + status="stopped", + message="检测服务已重启，上一轮会话已归档", + runtime=active.get("runtime") or runtime, + progress=active.get("progress") or progress, + settings_summary=active.get("settings_summary") or settings_summary, + log_lines=current_logs, + active_job=active.get("active_job") or active_job, + ) + active = None + if active: + next_status = "running" if active.get("status") != "stopping" else "stopping" + _sync_record( + active, + status=next_status, + message=runtime.get("message") or active.get("message") or "检测服务运行中", + runtime=runtime, + progress=progress, + settings_summary=settings_summary, + log_lines=current_logs, + active_job=active_job, + ) + else: + started_at = runtime.get("latest_start_time") or _now() + record = { + "run_id": uuid4().hex, + "status": "running", + "message": runtime.get("message") or "检测服务运行中", + "created_at": _now(), + "updated_at": _now(), + "started_at": started_at, + "completed_at": "", + "runtime": runtime, + "progress": progress, + "settings_summary": settings_summary, + "phase_label": "", + "phase_detail": "", + "phase_history": [], + "logs": [], + "active_job": active_job, + } + _sync_record( + record, + status="running", + message=record["message"], + runtime=runtime, + progress=progress, + settings_summary=settings_summary, + log_lines=current_logs, + active_job=active_job, + ) + records.insert(0, record) + elif active: + if runtime.get("running") and not execution_active: + if active.get("status") == "stopping": + final_status = "stopped" + final_message = runtime.get("message") or "检测任务已停止，Worker 保持待命" + elif active_job_status == "partial_failed": + final_status = "partial_failed" + final_message = "检测任务已结束，存在部分失败项" + elif active_job_status == "failed": + final_status = "failed" + final_message = "检测任务已结束，任务结果为失败" + else: + final_status = "completed" + final_message = "检测任务已自然完成，Worker 保持待命" + else: + final_status = "stopped" if active.get("status") == "stopping" else "failed" + final_message = runtime.get("message") or ("检测服务已停止" if final_status == "stopped" else "检测服务异常退出") + _sync_record( + active, + status=final_status, + message=final_message, + runtime=runtime, + progress=progress, + settings_summary=settings_summary, + log_lines=current_logs, + active_job=active_job, + ) + + if records: + records[0]["logs"] = _merge_logs( + records[0].get("logs"), + _filter_logs_since(current_logs, records[0].get("started_at")), + ) + + _save(records) + return records def mark_detect_run_stopping(message: str, runtime: dict, progress: dict, settings_summary: dict) -> dict | None: - records = _load() - target = _find_active(records) - if not target: - return None - _sync_record( - target, - status="stopping", - message=message, - runtime=runtime, - progress=progress, - settings_summary=settings_summary, - log_lines=_capture_worker_logs(), - active_job=target.get("active_job") or {}, - ) - _save(records) - return dict(target) + with _DETECT_RUNS_LOCK: + records = _load() + target = _find_active(records) + if not target: + return None + _sync_record( + target, + status="stopping", + message=message, + runtime=runtime, + progress=progress, + settings_summary=settings_summary, + log_lines=_capture_worker_logs(), + active_job=target.get("active_job") or {}, + ) + _save(records) + return dict(target) diff --git a/domain-api/app/services/detect_service.py b/domain-api/app/services/detect_service.py index 84ebafd..a83d5d3 100644 --- a/domain-api/app/services/detect_service.py +++ b/domain-api/app/services/detect_service.py @@ -3,6 +3,8 @@ from __future__ import annotations import json import re import subprocess +import threading +import time from datetime import datetime, timedelta, timezone from app.core.config import settings from app.core.db import get_db @@ -12,13 +14,16 @@ from app.services.debug_event_service import list_debug_events from app.services.cluster_runtime_service import ensure_runtime_schema from app.services.runtime_settings_service import get_runtime_settings from app.services.detect_run_service import sync_detect_runs -from app.services.detect_job_service import get_active_detect_job_summary -from app.services.settings_service import get_settings_payload, resolve_thread_count +from app.services.detect_job_service import get_active_detect_job_summary, get_detect_queue_health +from app.services.settings_service import get_settings_payload, resolve_process_count, resolve_thread_count from app.services.sync_record_service import append_detect_result_projection_if_changed from app.services.worker_control_service import detect_worker_runtime _PROXY_COUNT_RE = re.compile(r"当前可用代理数[:：]\s*(\d+)") +_PROXY_REFRESH_COUNT_RE = re.compile(r"代理池刷新完成，共\s*(\d+)\s*个可用代理") +_PROXY_CACHE_COUNT_RE = re.compile(r"继续沿用缓存\s*(\d+)\s*个") +_PROXY_SHARED_SNAPSHOT_COUNT_RE = re.compile(r"(?:复用共享代理快照|共享代理快照)\s*(\d+)\s*个") _THREAD_COUNT_RE = re.compile(r"当前实际线程数量[:：]\s*(\d+)\s*/\s*(\d+)") _STEP_TRACE_DOMAIN_RE = re.compile(r"domain=([^\s|]+)") _REGISTER_DOMAIN_RE = re.compile(r"检测注册状态[:：]\s*([^\s]+)") @@ -38,6 +43,89 @@ _REMOTE_DEBUG_EVENT_TYPES = { "task_pull_failed", "queue_overdue_leases", } +_DETECT_STATUS_CACHE_LOCK = threading.Lock() +_DETECT_STATUS_CACHE_TTL_SECONDS = 3.0 +_DETECT_STATUS_CACHE_VALUE: dict | None = None +_DETECT_STATUS_CACHE_EXPIRES_AT = 0.0 +_AGGREGATE_RUNTIME_NODE_STALE_AFTER = timedelta(seconds=90) +_DISABLED_MANAGED_NODE_CACHE: set[str] = set() +_DISABLED_MANAGED_NODE_CACHE_EXPIRES_AT = 0.0 + + +def _clone_detect_status_payload(value: dict | None) -> dict: + try: + return json.loads(json.dumps(dict(value or {}), ensure_ascii=False)) + except Exception: + return dict(value or {}) + + +def _extract_debug_event_job_identity(payload: dict | None) -> dict: + normalized_payload = dict(payload or {}) if isinstance(payload, dict) else {} + nested_job = normalized_payload.get("job") if isinstance(normalized_payload.get("job"), dict) else {} + + raw_job_id = normalized_payload.get("job_id") + if raw_job_id in (None, "", 0, "0"): + raw_job_id = normalized_payload.get("target_job_id") + if raw_job_id in (None, "", 0, "0"): + raw_job_id = nested_job.get("job_id") + try: + job_id = int(raw_job_id or 0) + except Exception: + job_id = 0 + + return { + "job_id": job_id, + "job_code": str( + normalized_payload.get("job_code") + or normalized_payload.get("target_job_code") + or nested_job.get("job_code") + or "" + ).strip(), + "cycle_token": str(normalized_payload.get("cycle_token") or nested_job.get("cycle_token") or "").strip(), + "has_identity": bool(job_id > 0 or str( + normalized_payload.get("job_code") + or normalized_payload.get("target_job_code") + or nested_job.get("job_code") + or "" + ).strip()), + } + + +def _debug_event_matches_active_job(record: dict, active_job: dict | None) -> bool: + normalized_active_job = dict(active_job or {}) + active_job_id = int(normalized_active_job.get("job_id") or 0) + active_job_code = str( + normalized_active_job.get("runtime_job_code") + or normalized_active_job.get("job_code") + or "" + ).strip() + active_cycle_token = str(normalized_active_job.get("current_cycle_token") or "").strip() + + if active_job_id <= 0 and not active_job_code and not active_cycle_token: + return True + + identity = _extract_debug_event_job_identity(record.get("payload")) + if not identity["has_identity"]: + return False + + event_job_id = int(identity["job_id"] or 0) + event_job_code = str(identity["job_code"] or "").strip() + event_cycle_token = str(identity["cycle_token"] or "").strip() + + if active_cycle_token and event_cycle_token and event_cycle_token != active_cycle_token: + return False + if active_job_id > 0 and event_job_id > 0 and event_job_id != active_job_id: + return False + if active_job_code and event_job_code and event_job_code != active_job_code: + return False + + if active_job_id > 0 and event_job_id == active_job_id: + return True + if active_job_code and event_job_code and event_job_code == active_job_code: + return True + if active_cycle_token and event_cycle_token and event_cycle_token == active_cycle_token: + return True + return False def _extract_remote_log_node_code(line: str) -> str: @@ -95,6 +183,225 @@ def _runtime_state_key(node_code: str | None = None) -> str: return f"{_RUNTIME_STATE_KEY}:{normalized_node_code}" +def _local_worker_expected_on_this_node() -> bool: + return not ( + str(settings.node_region or "").strip() == "overseas" + and str(settings.node_role or "").strip() == "control" + ) + + +def _int_value(value: object) -> int: + try: + return int(value or 0) + except Exception: + return 0 + + +def _max_runtime_metric(*values: object) -> int: + return max((_int_value(value) for value in values), default=0) + + +def _resolve_capacity_node_code(node_code: str, settings_payload: dict) -> tuple[str, bool]: + normalized_node_code = str(node_code or "").strip() + if not normalized_node_code: + return "", False + + node_thread_counts = dict(settings_payload.get("node_thread_counts") or {}) + node_process_counts = dict(settings_payload.get("node_process_counts") or {}) + parent_node_code, separator, suffix = normalized_node_code.rpartition("-") + if ( + separator + and parent_node_code + and suffix.isalpha() + and len(suffix) <= 3 + and ( + parent_node_code in node_thread_counts + or parent_node_code in node_process_counts + or bool(re.search(r"\d$", parent_node_code)) + ) + ): + return parent_node_code, True + + if normalized_node_code in node_thread_counts or normalized_node_code in node_process_counts: + return normalized_node_code, False + return normalized_node_code, False + + +def _is_current_participant_bucket(item: dict | None) -> bool: + payload = dict(item or {}) + node_code = str(payload.get("node_code") or "").strip() + if not node_code or node_code == "unassigned": + return False + return any( + _int_value(payload.get(field)) > 0 + for field in ("items_claimed", "items_running", "display_running", "current_load", "active_threads") + ) + + +def _aggregate_runtime_node_is_live(item: dict | None) -> bool: + payload = dict(item or {}) + node_code = str(payload.get("node_code") or "").strip() + if not node_code: + return False + if node_code == "unassigned": + return True + status = str(payload.get("status") or "").strip().lower() + if status in {"stale", "offline"}: + return False + last_heartbeat_at = _parse_time(payload.get("last_heartbeat_at")) + if last_heartbeat_at is None: + return True + reference_now = datetime.now(last_heartbeat_at.tzinfo) if last_heartbeat_at.tzinfo else datetime.now() + return (reference_now - last_heartbeat_at) <= _AGGREGATE_RUNTIME_NODE_STALE_AFTER + + +def _filter_live_aggregate_runtime_nodes(node_rows: list[dict] | None) -> list[dict]: + disabled_node_codes = _load_disabled_managed_node_codes( + [ + str(item.get("node_code") or "").strip() + for item in list(node_rows or []) + if isinstance(item, dict) + ] + ) + return [ + dict(item) + for item in list(node_rows or []) + if isinstance(item, dict) + and str(item.get("node_code") or "").strip() not in disabled_node_codes + and _aggregate_runtime_node_is_live(item) + ] + + +def _build_aggregate_detect_capacity(*, active_job: dict | None, settings_payload: dict) -> dict: + node_rows = _filter_live_aggregate_runtime_nodes( + list((active_job or {}).get("distributed_node_stats") or (active_job or {}).get("node_stats") or []) + ) + node_thread_counts = dict(settings_payload.get("node_thread_counts") or {}) + node_process_counts = dict(settings_payload.get("node_process_counts") or {}) + participant_node_codes: list[str] = [] + process_count_total = 0 + max_threads_total = 0 + representative_thread_count = 0 + child_parent_codes: set[str] = set() + + for raw_item in node_rows: + if not isinstance(raw_item, dict) or not _is_current_participant_bucket(raw_item): + continue + node_code = str(raw_item.get("node_code") or "").strip() + capacity_node_code, is_child_instance = _resolve_capacity_node_code(node_code, settings_payload) + if is_child_instance and capacity_node_code: + child_parent_codes.add(capacity_node_code) + + for raw_item in node_rows: + if not isinstance(raw_item, dict) or not _is_current_participant_bucket(raw_item): + continue + + node_code = str(raw_item.get("node_code") or "").strip() + capacity_node_code, is_child_instance = _resolve_capacity_node_code(node_code, settings_payload) + if not capacity_node_code: + continue + if not is_child_instance and node_code in child_parent_codes: + continue + if node_code not in participant_node_codes: + participant_node_codes.append(node_code) + + thread_resolution_node_code = node_code if node_code in node_thread_counts else capacity_node_code + thread_resolution = resolve_thread_count(node_code=thread_resolution_node_code, settings_payload=settings_payload) + per_process_thread_count = max(1, int(thread_resolution["effective_thread_count"] or 1)) + if representative_thread_count <= 0: + representative_thread_count = per_process_thread_count + + if is_child_instance: + process_count = 1 + else: + if node_code in node_process_counts: + process_resolution = resolve_process_count(node_code=node_code, settings_payload=settings_payload) + process_count = max(1, int(process_resolution["effective_process_count"] or 1)) + else: + process_count = 1 + + process_count_total += process_count + max_threads_total += process_count * per_process_thread_count + + return { + "participant_node_codes": participant_node_codes, + "participant_node_count": len(participant_node_codes), + "process_count": process_count_total, + "max_threads": max_threads_total, + "per_process_thread_count": representative_thread_count, + } + + +def _merge_aggregate_active_job_with_queue_health(active_job: dict | None, queue_health: dict | None) -> dict | None: + normalized_active_job = dict(active_job or {}) + normalized_queue_health = dict(queue_health or {}) + if not normalized_queue_health.get("has_active_job"): + return normalized_active_job or active_job + + queue_payload = dict(normalized_queue_health.get("queue") or {}) + queue_job = dict(normalized_queue_health.get("job") or {}) + raw_queue_nodes = [dict(item) for item in list(normalized_queue_health.get("nodes") or []) if isinstance(item, dict)] + queue_nodes = _filter_live_aggregate_runtime_nodes(raw_queue_nodes) + if not queue_payload and not queue_nodes and not queue_job: + return normalized_active_job or active_job + + queue_display_running = sum( + _max_runtime_metric( + item.get("display_running"), + item.get("current_load"), + item.get("active_threads"), + item.get("items_running"), + ) + for item in queue_nodes + if str(item.get("node_code") or "").strip() and str(item.get("node_code") or "").strip() != "unassigned" + ) + if raw_queue_nodes: + display_max_threads = sum(_int_value(item.get("max_threads")) for item in queue_nodes) + display_items_running = queue_display_running + display_active_threads = queue_display_running + else: + display_max_threads = _max_runtime_metric( + sum(_int_value(item.get("max_threads")) for item in queue_nodes), + normalized_active_job.get("display_max_threads"), + ) + display_items_running = _max_runtime_metric( + queue_display_running, + queue_payload.get("display_running"), + normalized_active_job.get("display_items_running"), + normalized_active_job.get("display_active_threads"), + ) + display_active_threads = _max_runtime_metric( + queue_display_running, + queue_payload.get("display_running"), + normalized_active_job.get("display_active_threads"), + normalized_active_job.get("display_items_running"), + ) + + merged = dict(normalized_active_job) + merged.update( + { + "job_id": queue_job.get("job_id", merged.get("job_id")), + "job_code": queue_job.get("job_code", merged.get("job_code")), + "status": queue_job.get("status", merged.get("status")), + "progress_percent": queue_job.get("progress_percent", merged.get("progress_percent", 0)), + "items_total": _int_value(queue_payload.get("items_total", merged.get("items_total"))), + "items_pending": _int_value(queue_payload.get("pending", merged.get("items_pending"))), + "items_claimed": _int_value(queue_payload.get("claimed", merged.get("items_claimed"))), + "items_running": _int_value(queue_payload.get("running", merged.get("items_running"))), + "items_completed": _int_value(queue_payload.get("completed", merged.get("items_completed"))), + "items_blacklisted": _int_value(queue_payload.get("blacklisted", merged.get("items_blacklisted"))), + "items_failed": _int_value(queue_payload.get("failed", merged.get("items_failed"))), + "display_items_running": display_items_running, + "display_active_threads": display_active_threads, + "display_max_threads": display_max_threads, + } + ) + if raw_queue_nodes: + merged["node_stats"] = list(queue_nodes) + merged["distributed_node_stats"] = list(queue_nodes) + return merged + + def _extract_dependency_alerts(lines: list[str]) -> list[dict]: alerts: list[dict] = [] recent_lines = lines[-120:] if lines else [] @@ -140,9 +447,29 @@ def _extract_dependency_alerts(lines: list[str]) -> list[dict]: def _extract_available_proxy_count(lines: list[str]) -> int: for line in reversed(lines): - match = _PROXY_COUNT_RE.search(line) - if match: - return int(match.group(1)) + count = _extract_available_proxy_count_from_text(line) + if count > 0: + return count + return 0 + + +def _extract_available_proxy_count_from_text(text: str) -> int: + normalized_text = str(text or "").strip() + if not normalized_text: + return 0 + for pattern in ( + _PROXY_COUNT_RE, + _PROXY_REFRESH_COUNT_RE, + _PROXY_CACHE_COUNT_RE, + _PROXY_SHARED_SNAPSHOT_COUNT_RE, + ): + match = pattern.search(normalized_text) + if not match: + continue + try: + return int(match.group(1) or 0) + except Exception: + continue return 0 @@ -492,6 +819,8 @@ def _build_remote_log_snapshot_from_debug_events( node_code = str(record.get("node_code") or "").strip() or "unknown" if participating_node_codes and node_code not in participating_node_codes: continue + if not _debug_event_matches_active_job(record, active_job): + continue message = str(record.get("message") or "").strip() if not message: continue @@ -665,6 +994,55 @@ def _load_runtime_state() -> dict: return {} +def _load_disabled_managed_node_codes(node_codes: list[str] | None = None) -> set[str]: + global _DISABLED_MANAGED_NODE_CACHE, _DISABLED_MANAGED_NODE_CACHE_EXPIRES_AT + normalized_codes = [ + str(item or "").strip() + for item in list(node_codes or []) + if str(item or "").strip() + ] + now_ts = time.time() + if not normalized_codes and now_ts < _DISABLED_MANAGED_NODE_CACHE_EXPIRES_AT: + return set(_DISABLED_MANAGED_NODE_CACHE) + try: + with get_db() as conn: + with conn.cursor() as cur: + if normalized_codes: + cur.execute( + """ + SELECT node_code + FROM ops_managed_nodes + WHERE is_enabled = FALSE + AND node_code = ANY(%s) + """, + (normalized_codes,), + ) + else: + cur.execute( + """ + SELECT node_code + FROM ops_managed_nodes + WHERE is_enabled = FALSE + """ + ) + rows = list(cur.fetchall() or []) + except Exception: + if normalized_codes: + return {code for code in normalized_codes if code in _DISABLED_MANAGED_NODE_CACHE} + return set(_DISABLED_MANAGED_NODE_CACHE) + + disabled_codes = { + str(row[0] or "").strip() + for row in rows + if str(row[0] or "").strip() + } + if normalized_codes: + return disabled_codes + _DISABLED_MANAGED_NODE_CACHE = disabled_codes + _DISABLED_MANAGED_NODE_CACHE_EXPIRES_AT = now_ts + 5.0 + return set(disabled_codes) + + def _load_runtime_state_from_cluster_node() -> dict: try: with get_db() as conn: @@ -706,6 +1084,359 @@ def _load_runtime_state_from_cluster_node() -> dict: return {} +def _load_runtime_states_from_cluster_nodes(node_codes: list[str] | tuple[str, ...]) -> dict[str, dict]: + normalized_node_codes = [ + str(item or "").strip() + for item in list(node_codes or []) + if str(item or "").strip() + ] + if not normalized_node_codes: + return {} + try: + with get_db() as conn: + with conn.cursor() as cur: + cur.execute( + """ + SELECT node_code, current_load, metadata_json, last_heartbeat_at + FROM detect_worker_nodes + WHERE node_code = ANY(%s) + """, + (normalized_node_codes,), + ) + rows = list(cur.fetchall() or []) + except Exception: + return {} + + payload: dict[str, dict] = {} + for row in rows: + node_code = str(row[0] or "").strip() + if not node_code: + continue + current_load = int(row[1] or 0) + metadata_json = row[2] + last_heartbeat_at = row[3] + metadata = metadata_json if isinstance(metadata_json, dict) else {} + payload[node_code] = { + "node_code": node_code, + "current_load": current_load, + "last_heartbeat_at": ( + last_heartbeat_at.isoformat(sep=" ", timespec="seconds") + if hasattr(last_heartbeat_at, "isoformat") + else str(last_heartbeat_at or "").strip() + ), + "available_proxy_count": int( + metadata.get("available_proxy_count", metadata.get("proxy_last_available_count", 0)) or 0 + ), + "proxy_runtime_label": str(metadata.get("proxy_runtime_label") or "").strip(), + "proxy_runtime_reason": str(metadata.get("proxy_runtime_reason") or "").strip(), + "proxy_last_refresh_status": str(metadata.get("proxy_last_refresh_status") or "").strip(), + "proxy_last_refresh_time": str(metadata.get("proxy_last_refresh_time") or "").strip(), + "proxy_last_refresh_source_count": int(metadata.get("proxy_last_refresh_source_count", 0) or 0), + "proxy_last_refresh_total_items": int(metadata.get("proxy_last_refresh_total_items", 0) or 0), + "proxy_last_validated_count": int(metadata.get("proxy_last_validated_count", 0) or 0), + "active_threads": int(metadata.get("active_threads", 0) or 0), + "max_threads": int(metadata.get("max_threads", 0) or 0), + "detect_participating": bool(metadata.get("detect_participating", False) or current_load > 0), + } + return payload + + +def _load_recent_proxy_debug_events(node_codes: list[str] | tuple[str, ...], *, window_minutes: int = 20) -> list[dict]: + normalized_node_codes = [ + str(item or "").strip() + for item in list(node_codes or []) + if str(item or "").strip() and str(item or "").strip() != "unassigned" + ] + if not normalized_node_codes: + return [] + + safe_window_minutes = max(5, min(int(window_minutes or 20), 120)) + created_after = datetime.now() - timedelta(minutes=safe_window_minutes) + safe_limit = max(80, min(len(normalized_node_codes) * 20, 800)) + try: + with get_db() as conn: + with conn.cursor() as cur: + cur.execute( + """ + SELECT node_code, message, payload_json, created_at + FROM detect_debug_events + WHERE event_type = 'worker_log' + AND node_code = ANY(%s) + AND created_at >= %s + ORDER BY created_at DESC, id DESC + LIMIT %s + """, + (normalized_node_codes, created_after, safe_limit), + ) + rows = list(cur.fetchall() or []) + except Exception: + return [] + + return [ + { + "event_type": "worker_log", + "node_code": str(row[0] or "").strip(), + "message": str(row[1] or "").strip(), + "payload": row[2] if isinstance(row[2], dict) else {}, + "created_at": ( + row[3].isoformat(sep=" ", timespec="seconds") + if hasattr(row[3], "isoformat") + else str(row[3] or "").strip() + ), + } + for row in rows + if str(row[0] or "").strip() and str(row[1] or "").strip() + ] + + +def _is_proxy_runtime_message(message: str) -> bool: + normalized_message = str(message or "").strip() + if not normalized_message: + return False + if _extract_available_proxy_count_from_text(normalized_message) > 0: + return True + lowered_message = normalized_message.lower() + return any( + keyword in normalized_message or keyword in lowered_message + for keyword in ( + "代理", + "proxy", + "cooldown", + "rate limited", + ) + ) + + +def _infer_proxy_runtime_label_from_message(message: str, *, available_proxy_count: int) -> tuple[str, str]: + normalized_message = str(message or "").strip() + lowered_message = normalized_message.lower() + if available_proxy_count > 0: + return "代理正常", "aggregate_log_healthy" + if "rate limited" in lowered_message or "cooldown" in lowered_message or "冷却" in normalized_message: + return "代理源暂时冷却中", "aggregate_log_cooldown" + if any(keyword in normalized_message for keyword in ("未取到新代理", "未取到可用代理数据", "无可用代理", "未返回可用代理数据")): + return "代理待补货", "aggregate_log_empty" + if "等待首刷" in normalized_message: + return "等待首刷", "aggregate_log_waiting" + return "", "" + + +def _build_aggregate_proxy_runtime_rows_from_events( + *, + active_job: dict | None, + settings_payload: dict, + participant_node_codes: list[str], + participant_server_codes: list[str], +) -> dict[str, dict]: + normalized_active_job = dict(active_job or {}) + events = list(normalized_active_job.get("current_cycle_events") or normalized_active_job.get("recent_events") or []) + events.extend(_load_recent_proxy_debug_events(participant_node_codes)) + if not events: + return {} + + current_cycle_token = str(normalized_active_job.get("current_cycle_token") or "").strip() + allowed_server_codes = {str(item or "").strip() for item in list(participant_server_codes or []) if str(item or "").strip()} + rows: dict[str, dict] = {} + + for event in events: + if not isinstance(event, dict): + continue + if str(event.get("event_type") or "").strip() != "worker_log": + continue + + payload = event.get("payload") if isinstance(event.get("payload"), dict) else {} + event_cycle_token = str(payload.get("cycle_token") or "").strip() + if current_cycle_token and event_cycle_token and event_cycle_token != current_cycle_token: + continue + + node_code = str(event.get("node_code") or "").strip() + if not node_code or node_code == "unassigned": + continue + capacity_node_code, _ = _resolve_capacity_node_code(node_code, settings_payload) + server_code = str(capacity_node_code or node_code or "").strip() + if not server_code or (allowed_server_codes and server_code not in allowed_server_codes): + continue + + message = str(event.get("message") or "").strip() + if not _is_proxy_runtime_message(message): + continue + + created_at = str(event.get("created_at") or "").strip() + available_proxy_count = _extract_available_proxy_count_from_text(message) + label, reason = _infer_proxy_runtime_label_from_message( + message, + available_proxy_count=available_proxy_count, + ) + + row = rows.setdefault( + server_code, + { + "node_code": server_code, + "available_proxy_count": 0, + "proxy_runtime_label": "", + "proxy_runtime_reason": "", + "proxy_last_refresh_status": "", + "proxy_last_refresh_time": "", + "proxy_last_refresh_source_count": 0, + "proxy_last_refresh_total_items": 0, + "proxy_last_validated_count": 0, + "_count_seen_at": "", + "_status_seen_at": "", + }, + ) + + if available_proxy_count > 0 and created_at >= str(row.get("_count_seen_at") or ""): + row["available_proxy_count"] = int(available_proxy_count or 0) + row["_count_seen_at"] = created_at + + if created_at >= str(row.get("_status_seen_at") or ""): + row["proxy_last_refresh_status"] = message + row["proxy_last_refresh_time"] = created_at + row["_status_seen_at"] = created_at + if label: + row["proxy_runtime_label"] = label + if reason: + row["proxy_runtime_reason"] = reason + + return { + server_code: { + key: value + for key, value in row.items() + if not str(key).startswith("_") + } + for server_code, row in rows.items() + } + + +def _build_aggregate_proxy_runtime( + *, + active_job: dict | None, + settings_payload: dict, + fallback_available_proxy_count: int, + fallback_proxy_runtime: dict, +) -> tuple[int, dict]: + node_rows = list((active_job or {}).get("distributed_node_stats") or (active_job or {}).get("node_stats") or []) + participant_node_codes: list[str] = [] + participant_server_codes: list[str] = [] + for raw_item in node_rows: + if not isinstance(raw_item, dict) or not _is_current_participant_bucket(raw_item): + continue + node_code = str(raw_item.get("node_code") or "").strip() + if node_code and node_code != "unassigned" and node_code not in participant_node_codes: + participant_node_codes.append(node_code) + capacity_node_code, _ = _resolve_capacity_node_code(node_code, settings_payload) + normalized_server_code = str(capacity_node_code or node_code or "").strip() + if not normalized_server_code or normalized_server_code == "unassigned": + continue + if normalized_server_code not in participant_server_codes: + participant_server_codes.append(normalized_server_code) + + if not participant_server_codes: + return fallback_available_proxy_count, fallback_proxy_runtime + + event_runtime = _build_aggregate_proxy_runtime_rows_from_events( + active_job=active_job, + settings_payload=settings_payload, + participant_node_codes=participant_node_codes, + participant_server_codes=participant_server_codes, + ) + cluster_runtime = _load_runtime_states_from_cluster_nodes(participant_server_codes) + rows: list[dict] = [] + for code in participant_server_codes: + runtime_row = dict(cluster_runtime.get(code) or {}) + event_row = dict(event_runtime.get(code) or {}) + if not runtime_row and not event_row: + continue + merged_row = { + "node_code": code, + "available_proxy_count": int(runtime_row.get("available_proxy_count", 0) or 0), + "proxy_runtime_label": str(runtime_row.get("proxy_runtime_label") or "").strip(), + "proxy_runtime_reason": str(runtime_row.get("proxy_runtime_reason") or "").strip(), + "proxy_last_refresh_status": str(runtime_row.get("proxy_last_refresh_status") or "").strip(), + "proxy_last_refresh_time": str(runtime_row.get("proxy_last_refresh_time") or runtime_row.get("last_heartbeat_at") or "").strip(), + "proxy_last_refresh_source_count": int(runtime_row.get("proxy_last_refresh_source_count", 0) or 0), + "proxy_last_refresh_total_items": int(runtime_row.get("proxy_last_refresh_total_items", 0) or 0), + "proxy_last_validated_count": int(runtime_row.get("proxy_last_validated_count", 0) or 0), + } + if int(merged_row.get("available_proxy_count", 0) or 0) <= 0 and int(event_row.get("available_proxy_count", 0) or 0) > 0: + merged_row["available_proxy_count"] = int(event_row.get("available_proxy_count", 0) or 0) + if not str(merged_row.get("proxy_runtime_label") or "").strip(): + merged_row["proxy_runtime_label"] = str(event_row.get("proxy_runtime_label") or "").strip() + if not str(merged_row.get("proxy_runtime_reason") or "").strip(): + merged_row["proxy_runtime_reason"] = str(event_row.get("proxy_runtime_reason") or "").strip() + if not str(merged_row.get("proxy_last_refresh_status") or "").strip(): + merged_row["proxy_last_refresh_status"] = str(event_row.get("proxy_last_refresh_status") or "").strip() + if not str(merged_row.get("proxy_last_refresh_time") or "").strip(): + merged_row["proxy_last_refresh_time"] = str(event_row.get("proxy_last_refresh_time") or "").strip() + if int(merged_row.get("proxy_last_refresh_source_count", 0) or 0) <= 0: + merged_row["proxy_last_refresh_source_count"] = int(event_row.get("proxy_last_refresh_source_count", 0) or 0) + if int(merged_row.get("proxy_last_refresh_total_items", 0) or 0) <= 0: + merged_row["proxy_last_refresh_total_items"] = int(event_row.get("proxy_last_refresh_total_items", 0) or 0) + if int(merged_row.get("proxy_last_validated_count", 0) or 0) <= 0: + merged_row["proxy_last_validated_count"] = int(event_row.get("proxy_last_validated_count", 0) or 0) + rows.append(merged_row) + if not rows: + return fallback_available_proxy_count, fallback_proxy_runtime + + total_available_proxy_count = sum(max(0, int(item.get("available_proxy_count", 0) or 0)) for item in rows) + latest_refresh_time = max((str(item.get("proxy_last_refresh_time") or "") for item in rows), default="") + source_count = sum(int(item.get("proxy_last_refresh_source_count", 0) or 0) for item in rows) + raw_items = sum(int(item.get("proxy_last_refresh_total_items", 0) or 0) for item in rows) + validated_count = sum(int(item.get("proxy_last_validated_count", 0) or 0) for item in rows) + refresh_status_parts = [ + f"{str(item.get('node_code') or '')}:{str(item.get('proxy_last_refresh_status') or '').strip()}" + for item in rows + if str(item.get("proxy_last_refresh_status") or "").strip() + ] + refresh_status = "；".join(refresh_status_parts[:6]) + if len(refresh_status_parts) > 6: + refresh_status = f"{refresh_status} 等 {len(refresh_status_parts)} 台" + + if total_available_proxy_count > 0: + return total_available_proxy_count, { + "state": "healthy", + "label": "集群代理正常", + "detail": ( + f"参与服务器 {len(rows)} 台，共可用 {total_available_proxy_count} 个代理" + + (f"；最近状态：{refresh_status}" if refresh_status else "") + ), + "direct_fallback_active": False, + "reason": "aggregate_healthy", + "last_refresh_status": refresh_status, + "last_refresh_time": latest_refresh_time, + "source_count": source_count, + "raw_items": raw_items, + "validated_count": validated_count, + "available_count": total_available_proxy_count, + "source_stats": [], + "supplier_empty": False, + } + + fallback_label = next((str(item.get("proxy_runtime_label") or "").strip() for item in rows if str(item.get("proxy_runtime_label") or "").strip()), "") + fallback_reason = next((str(item.get("proxy_runtime_reason") or "").strip() for item in rows if str(item.get("proxy_runtime_reason") or "").strip()), "") + if fallback_label: + return 0, { + "state": "warming_up", + "label": fallback_label, + "detail": ( + f"参与服务器 {len(rows)} 台，当前尚未汇总到可用代理" + + (f"；最近状态：{refresh_status}" if refresh_status else "") + ), + "direct_fallback_active": bool(fallback_proxy_runtime.get("direct_fallback_active", False)), + "reason": fallback_reason or "aggregate_proxy_unavailable", + "last_refresh_status": refresh_status, + "last_refresh_time": latest_refresh_time, + "source_count": source_count, + "raw_items": raw_items, + "validated_count": validated_count, + "available_count": 0, + "source_stats": [], + "supplier_empty": bool(fallback_proxy_runtime.get("supplier_empty", False)), + } + + return fallback_available_proxy_count, fallback_proxy_runtime + + def _normalize_recent_warning(runtime_state: dict, recent_lines: list[str], available_proxy_count: int) -> str: runtime_warning = str(runtime_state.get("recent_warning", "") or "").strip() if runtime_warning: @@ -869,6 +1600,13 @@ def _build_proxy_runtime_snapshot(settings_payload: dict, runtime_state: dict, a def get_detect_status() -> dict: + global _DETECT_STATUS_CACHE_EXPIRES_AT, _DETECT_STATUS_CACHE_VALUE + + now_ts = time.monotonic() + with _DETECT_STATUS_CACHE_LOCK: + if _DETECT_STATUS_CACHE_VALUE is not None and now_ts < _DETECT_STATUS_CACHE_EXPIRES_AT: + return _clone_detect_status_payload(_DETECT_STATUS_CACHE_VALUE) + try: ensure_runtime_schema() except Exception: @@ -901,12 +1639,18 @@ def get_detect_status() -> dict: pass settings_payload = get_settings_payload() + worker_expected_on_this_node = _local_worker_expected_on_this_node() runtime_settings = get_runtime_settings() worker_online, last_log_time, recent_lines = _load_recent_worker_lines(runtime_settings, max_lines=160) runtime = detect_worker_runtime() runtime_state = _load_runtime_state() if not runtime_state: runtime_state = _load_runtime_state_from_cluster_node() + if not worker_expected_on_this_node: + worker_online = False + last_log_time = "" + recent_lines = [] + runtime_state = {} runtime_started_at = runtime.get("latest_start_time", "") recent_lines = _filter_lines_since(recent_lines, runtime_started_at) available_proxy_count = _extract_available_proxy_count(recent_lines) @@ -961,13 +1705,49 @@ def get_detect_status() -> dict: active_job = get_active_detect_job_summary(event_limit=240) except Exception: active_job = None - if settings.node_region == "overseas" and settings.node_role == "control" and active_job: + aggregate_detect_view = bool(settings.node_region == "overseas" and settings.node_role == "control" and active_job) + aggregate_queue_health = {} + if aggregate_detect_view: + try: + aggregate_queue_health = get_detect_queue_health(window_minutes=15) + except Exception: + aggregate_queue_health = {} + active_job = _merge_aggregate_active_job_with_queue_health(active_job, aggregate_queue_health) + aggregate_capacity = ( + _build_aggregate_detect_capacity(active_job=active_job, settings_payload=settings_payload) + if aggregate_detect_view + else { + "participant_node_codes": [], + "participant_node_count": 0, + "process_count": 0, + "max_threads": 0, + "per_process_thread_count": 0, + } + ) + if aggregate_detect_view: + raw_aggregate_node_rows = list(active_job.get("distributed_node_stats") or active_job.get("node_stats") or []) + aggregate_node_rows = _filter_live_aggregate_runtime_nodes(raw_aggregate_node_rows) + aggregate_display_running = sum( + _max_runtime_metric( + item.get("display_running"), + item.get("current_load"), + item.get("active_threads"), + item.get("items_running"), + ) + for item in aggregate_node_rows + if str(item.get("node_code") or "").strip() and str(item.get("node_code") or "").strip() != "unassigned" + ) + if raw_aggregate_node_rows: + display_running = aggregate_display_running + else: + display_running = _max_runtime_metric( + active_job.get("display_active_threads"), + active_job.get("display_items_running"), + (aggregate_queue_health.get("queue") or {}).get("display_running"), + ) progress = { "pending": int(active_job.get("items_pending", 0) or 0), - "running": int( - active_job.get("display_active_threads", active_job.get("display_items_running", active_job.get("items_running", 0))) - or 0 - ), + "running": display_running, "completed": int(active_job.get("items_completed", 0) or 0), "failed": int(active_job.get("items_failed", 0) or 0), "blacklisted": int(active_job.get("items_blacklisted", 0) or 0), @@ -989,8 +1769,20 @@ def get_detect_status() -> dict: active_thread_snapshot["active"] = local_runtime_load if active_thread_snapshot["max"] <= 0: active_thread_snapshot["max"] = local_runtime_max_threads or effective_thread_count - if settings.node_region == "overseas" and settings.node_role == "control" and active_job: - distributed_node_stats = list(active_job.get("distributed_node_stats") or active_job.get("node_stats") or []) + if aggregate_detect_view: + raw_distributed_node_stats = list(active_job.get("distributed_node_stats") or active_job.get("node_stats") or []) + distributed_node_stats = _filter_live_aggregate_runtime_nodes(raw_distributed_node_stats) + raw_participant_count = sum( + 1 + for item in raw_distributed_node_stats + if str(item.get("node_code") or "").strip() and str(item.get("node_code") or "").strip() != "unassigned" + ) + live_participant_count = sum( + 1 + for item in distributed_node_stats + if str(item.get("node_code") or "").strip() and str(item.get("node_code") or "").strip() != "unassigned" + ) + dropped_aggregate_node_count = max(0, raw_participant_count - live_participant_count) aggregated_active_threads = 0 aggregated_max_threads = 0 for item in distributed_node_stats: @@ -1005,8 +1797,38 @@ def get_detect_status() -> dict: aggregated_max_threads += int(item.get("max_threads", 0) or 0) if aggregated_active_threads > 0: active_thread_snapshot["active"] = aggregated_active_threads + elif dropped_aggregate_node_count <= 0 and _max_runtime_metric( + active_job.get("display_active_threads"), + active_job.get("display_items_running"), + progress.get("running"), + ) > 0: + active_thread_snapshot["active"] = _max_runtime_metric( + active_job.get("display_active_threads"), + active_job.get("display_items_running"), + progress.get("running"), + ) if aggregated_max_threads > 0: active_thread_snapshot["max"] = aggregated_max_threads + elif dropped_aggregate_node_count <= 0 and _max_runtime_metric( + active_job.get("display_max_threads"), + aggregate_capacity.get("max_threads"), + ) > 0: + active_thread_snapshot["max"] = _max_runtime_metric( + active_job.get("display_max_threads"), + aggregate_capacity.get("max_threads"), + ) + configured_max_threads = int(aggregate_capacity.get("max_threads", 0) or 0) + if configured_max_threads > 0: + active_thread_snapshot["max"] = max(active_thread_snapshot["max"], configured_max_threads) + available_proxy_count, proxy_runtime = _build_aggregate_proxy_runtime( + active_job=active_job, + settings_payload=settings_payload, + fallback_available_proxy_count=available_proxy_count, + fallback_proxy_runtime=proxy_runtime, + ) + display_worker_process_count = int(runtime.get("process_count", 0) or 0) + if aggregate_detect_view and int(aggregate_capacity.get("process_count", 0) or 0) > 0: + display_worker_process_count = int(aggregate_capacity.get("process_count", 0) or 0) runtime_snapshot = { **runtime, "detecting": inferred_detecting, @@ -1029,21 +1851,22 @@ def get_detect_status() -> dict: ) remote_log_lines = list(remote_log_snapshot.get("lines") or []) dependency_alerts = _extract_dependency_alerts(recent_lines) - append_detect_result_projection_if_changed( - detect={ - "active_job": active_job, - "progress": progress, - "phase_label": runtime_state.get("phase", ""), - "phase_detail": runtime_state.get("detail", ""), - } - ) + if _local_worker_expected_on_this_node(): + append_detect_result_projection_if_changed( + detect={ + "active_job": active_job, + "progress": progress, + "phase_label": runtime_state.get("phase", ""), + "phase_detail": runtime_state.get("detail", ""), + } + ) - return { + result = { "worker_online": worker_online, "worker_mode": runtime.get("mode", "windows-local"), "worker_service_name": runtime_settings.get("worker_service_name", ""), "api_service_name": runtime_settings.get("api_service_name", ""), - "worker_process_count": runtime.get("process_count", 0), + "worker_process_count": display_worker_process_count, "worker_latest_start_time": runtime.get("latest_start_time", ""), "worker_runtime_message": runtime.get("message", ""), "runtime_state": runtime_state, @@ -1057,6 +1880,11 @@ def get_detect_status() -> dict: "thread_count_node_code": str(thread_count_resolution["node_code"]), "active_thread_count": active_thread_snapshot["active"], "max_thread_count": active_thread_snapshot["max"] or effective_thread_count, + "aggregate_process_count": int(aggregate_capacity.get("process_count", 0) or 0), + "aggregate_participating_node_count": int(aggregate_capacity.get("participant_node_count", 0) or 0), + "aggregate_participating_node_codes": list(aggregate_capacity.get("participant_node_codes") or []), + "aggregate_max_thread_count": int(aggregate_capacity.get("max_threads", 0) or 0), + "aggregate_thread_count_per_process": int(aggregate_capacity.get("per_process_thread_count", 0) or 0), "proxy_enable": settings_payload["proxy_config"].get("proxy_enable", False), "allow_direct": settings_payload["proxy_config"].get("allow_direct", False), "proxy_pool_count": len(settings_payload["proxy_config"].get("proxy_urls", [])), @@ -1080,7 +1908,7 @@ def get_detect_status() -> dict: "progress_percent": progress_percent, "recent_event": runtime_state.get("detail") or _recent_event(recent_lines), "recent_warning": recent_proxy_warning, - "aggregate_detect_view": bool(settings.node_region == "overseas" and settings.node_role == "control" and active_job), + "aggregate_detect_view": aggregate_detect_view, "log_lines": recent_lines, "remote_log_lines": remote_log_lines, "remote_log_line_count": int(remote_log_snapshot.get("line_count", 0) or 0), @@ -1094,3 +1922,7 @@ def get_detect_status() -> dict: "worker_log_sync_enabled": worker_log_sync_enabled, "worker_log_sync_mode": worker_log_sync_mode, } + with _DETECT_STATUS_CACHE_LOCK: + _DETECT_STATUS_CACHE_VALUE = _clone_detect_status_payload(result) + _DETECT_STATUS_CACHE_EXPIRES_AT = time.monotonic() + _DETECT_STATUS_CACHE_TTL_SECONDS + return result diff --git a/domain-api/app/services/domains_service.py b/domain-api/app/services/domains_service.py index ef016a1..8d5285d 100644 --- a/domain-api/app/services/domains_service.py +++ b/domain-api/app/services/domains_service.py @@ -1,5 +1,6 @@ from __future__ import annotations +from datetime import datetime from math import ceil from app.core.db import get_db @@ -113,6 +114,8 @@ def _build_step_details(row: tuple) -> list[dict]: _normalize_step_detail("时光机", row[21]), _normalize_step_detail("站长之家", row[22]), _normalize_step_detail("爱站网", row[23]), + _normalize_step_detail("桔子SEO", row[24]), + _normalize_step_detail("聚查", row[25]), ] @@ -152,6 +155,28 @@ def _normalize_detection_update(value) -> bool | None: raise ValueError("检测结果字段仅支持“是”或“否”") +def _now_text() -> str: + return datetime.now().isoformat(sep=" ", timespec="seconds") + + +def _merge_detection_step_update(field: str, value: bool, existing_payload) -> dict | bool: + if field == "is_chinese_title": + return bool(value) + + existing = dict(existing_payload or {}) if isinstance(existing_payload, dict) else {} + previous_status = existing.get("status") if isinstance(existing.get("status"), bool) else None + merged = dict(existing) + merged["status"] = bool(value) + merged["state"] = "passed" if bool(value) else "failed" + merged["step"] = str(merged.get("step") or field) + if previous_status != bool(value) or not str(merged.get("message") or "").strip(): + merged["message"] = "人工批量更新" + if previous_status != bool(value) or not str(merged.get("checked_at") or "").strip(): + merged["checked_at"] = _now_text() + merged["manual_override"] = True + return merged + + def _build_domain_query_parts(filters: dict | None = None) -> tuple[str, str, list[object]]: filters = filters or {} conditions: list[str] = [] @@ -190,8 +215,9 @@ def _build_domain_query_parts(filters: dict | None = None) -> tuple[str, str, li if filters.get("source_type") is not None: conditions.append("d.source_type = %s") params.append(int(filters["source_type"])) - if filters.get("backlink_gt_10"): - conditions.append("coalesce(dd.backlink_count_gt_10, false) = true") + if filters.get("backlink_gt_10") is not None: + conditions.append("coalesce(dd.backlink_count_gt_10, false) = %s") + params.append(bool(filters["backlink_gt_10"])) from_clause = """ from domains d @@ -264,7 +290,9 @@ def fetch_domains( dd.google_site, dd.wayback_info, dd.chinaz_info, - dd.aizhan_info + dd.aizhan_info, + dd.juziseo_info, + dd.jucha_info {from_clause} {where_clause} order by d.id desc @@ -360,26 +388,7 @@ def fetch_domain_detail(domain_id: int) -> dict | None: if not row: return None - step_details = [ - _normalize_step_detail("百度历史收录", row[16]), - _normalize_step_detail("百度Site收录", row[17]), - { - "label": "标题为中文", - "state": "passed" if bool(row[18]) else "", - "status": bool(row[18]), - "message": "标题含中文" if bool(row[18]) else "", - "checked_at": "", - "step": "中文标题", - "raw": row[18], - }, - _normalize_step_detail("360 Site收录", row[19]), - _normalize_step_detail("Google Site收录", row[20]), - _normalize_step_detail("时光机", row[21]), - _normalize_step_detail("站长之家", row[22]), - _normalize_step_detail("爱站网", row[23]), - _normalize_step_detail("桔子SEO", row[24]), - _normalize_step_detail("聚查", row[25]), - ] + step_details = _build_step_details(row) step_summary = _summarize_step_details(step_details) return { "id": row[0], @@ -523,6 +532,34 @@ def batch_update_domains(domain_ids: list[int], updates: dict) -> dict: tuple(params), ) + existing_detection_payloads: dict[str, object] = {} + existing_detection_row = None + if "backlink_count" in payload or detection_fields.intersection(payload.keys()): + cur.execute( + """ + select + id, + baidu_history, + baidu_site, + is_chinese_title, + qihu360_site, + google_site, + backlink_count_gt_10 + from domain_detections + where domain_id = %s + """, + (domain_id,), + ) + existing_detection_row = cur.fetchone() + if existing_detection_row: + existing_detection_payloads = { + "baidu_history": existing_detection_row[1], + "baidu_site": existing_detection_row[2], + "is_chinese_title": existing_detection_row[3], + "qihu360_site": existing_detection_row[4], + "google_site": existing_detection_row[5], + } + detection_payload: dict[str, object] = {} for field in detection_fields: if field not in payload: @@ -530,15 +567,15 @@ def batch_update_domains(domain_ids: list[int], updates: dict) -> dict: normalized = _normalize_detection_update(payload[field]) if normalized is None: continue - if field == "is_chinese_title": - detection_payload[field] = normalized - else: - detection_payload[field] = {"status": normalized} + detection_payload[field] = _merge_detection_step_update( + field, + normalized, + existing_detection_payloads.get(field), + ) if "backlink_count" in payload: backlink_gt_10 = int(payload["backlink_count"]) > 10 - cur.execute("select id from domain_detections where domain_id = %s", (domain_id,)) - if cur.fetchone(): + if existing_detection_row: cur.execute( "update domain_detections set backlink_count_gt_10 = %s, update_time = now() where domain_id = %s", (backlink_gt_10, domain_id), @@ -553,9 +590,7 @@ def batch_update_domains(domain_ids: list[int], updates: dict) -> dict: ) if detection_payload: - cur.execute("select id from domain_detections where domain_id = %s", (domain_id,)) - existing_detection = cur.fetchone() - if existing_detection: + if existing_detection_row: detection_set_parts: list[str] = [] detection_params: list[object] = [] for field, value in detection_payload.items(): diff --git a/domain-api/app/services/import_task_service.py b/domain-api/app/services/import_task_service.py index 98e1a5f..ef4e1b1 100644 --- a/domain-api/app/services/import_task_service.py +++ b/domain-api/app/services/import_task_service.py @@ -10,6 +10,7 @@ from app.services.import_worker_service import import_domains_from_path _IMPORT_TASK_LOCK = threading.Lock() +_IMPORT_EXECUTION_LOCK = threading.Lock() _SOURCE_TYPE_LABELS = { 6: "手工录入", 7: "TXT 导入", @@ -77,63 +78,64 @@ def _source_label(source_type: int) -> str: def _run_import_task(task_id: str, file_path: str, source_type: int = 7) -> None: - _update_task_with_log( - task_id, - f"导入任务开始执行，来源类型：{_source_label(source_type)}", - status="running", - started_at=_now(), - message=f"导入任务开始执行，来源类型：{_source_label(source_type)}", - phase="reading", - phase_label=_phase_label("reading"), - ) - try: - path = Path(file_path) + with _IMPORT_EXECUTION_LOCK: _update_task_with_log( task_id, - f"开始读取文件：{path.name}", + f"导入任务开始执行，来源类型：{_source_label(source_type)}", + status="running", + started_at=_now(), + message=f"导入任务开始执行，来源类型：{_source_label(source_type)}", phase="reading", phase_label=_phase_label("reading"), ) - raw_lines = path.read_text(encoding="utf-8", errors="replace").splitlines() - total_lines = len(raw_lines) - non_empty = sum(1 for line in raw_lines if line.strip()) - _update_task_with_log( - task_id, - f"文件读取完成，共 {total_lines} 行，非空 {non_empty} 行", - phase="normalizing", - phase_label=_phase_label("normalizing"), - message=f"文件读取完成，准备清洗 {non_empty} 条域名", - ) + try: + path = Path(file_path) + _update_task_with_log( + task_id, + f"开始读取文件：{path.name}", + phase="reading", + phase_label=_phase_label("reading"), + ) + raw_lines = path.read_text(encoding="utf-8", errors="replace").splitlines() + total_lines = len(raw_lines) + non_empty = sum(1 for line in raw_lines if line.strip()) + _update_task_with_log( + task_id, + f"文件读取完成，共 {total_lines} 行，非空 {non_empty} 行", + phase="normalizing", + phase_label=_phase_label("normalizing"), + message=f"文件读取完成，准备清洗 {non_empty} 条域名", + ) - result = import_domains_from_path(path, source_type=source_type) - stats = result.get("stats", {}) - _update_task_with_log( - task_id, - ( - f"导入完成：总数 {stats.get('total', 0)}，有效 {stats.get('valid', 0)}，" - f"新增 {stats.get('added', 0)}，已存在 {stats.get('exists', 0)}，无效 {stats.get('invalid', 0)}，" - f"来源类型 {result.get('source_label') or _source_label(source_type)}" - ), - status="completed", - completed_at=_now(), - result=result, - message=( - f"导入完成：总数 {stats.get('total', 0)}，有效 {stats.get('valid', 0)}，" - f"新增 {stats.get('added', 0)}，已存在 {stats.get('exists', 0)}，无效 {stats.get('invalid', 0)}" - ), - phase="completed", - phase_label=_phase_label("completed"), - ) - except Exception as exc: - _update_task_with_log( - task_id, - f"导入失败：{exc}", - status="failed", - completed_at=_now(), - message=f"导入失败：{exc}", - phase="failed", - phase_label=_phase_label("failed"), - ) + result = import_domains_from_path(path, source_type=source_type) + stats = result.get("stats", {}) + _update_task_with_log( + task_id, + ( + f"导入完成：总数 {stats.get('total', 0)}，有效 {stats.get('valid', 0)}，" + f"新增 {stats.get('added', 0)}，已存在 {stats.get('exists', 0)}，无效 {stats.get('invalid', 0)}，" + f"来源类型 {result.get('source_label') or _source_label(source_type)}" + ), + status="completed", + completed_at=_now(), + result=result, + message=( + f"导入完成：总数 {stats.get('total', 0)}，有效 {stats.get('valid', 0)}，" + f"新增 {stats.get('added', 0)}，已存在 {stats.get('exists', 0)}，无效 {stats.get('invalid', 0)}" + ), + phase="completed", + phase_label=_phase_label("completed"), + ) + except Exception as exc: + _update_task_with_log( + task_id, + f"导入失败：{exc}", + status="failed", + completed_at=_now(), + message=f"导入失败：{exc}", + phase="failed", + phase_label=_phase_label("failed"), + ) def create_import_task(content: bytes, filename: str, source_type: int = 7) -> dict: diff --git a/domain-api/app/services/import_worker_service.py b/domain-api/app/services/import_worker_service.py index 2846974..2f2f3ea 100644 --- a/domain-api/app/services/import_worker_service.py +++ b/domain-api/app/services/import_worker_service.py @@ -47,6 +47,8 @@ def import_domains_from_path(file_path: Path, source_type: int = 7) -> dict: domains = [row[0] for row in normalized_rows] existing_set: set[str] = set() inserted = 0 + exists = 0 + seen_in_batch: set[str] = set() with get_db() as conn: with conn.cursor() as cur: @@ -55,7 +57,12 @@ def import_domains_from_path(file_path: Path, source_type: int = 7) -> dict: existing_set = {row[0] for row in cur.fetchall()} for domain, tld in normalized_rows: + if domain in seen_in_batch: + exists += 1 + continue + seen_in_batch.add(domain) if domain in existing_set: + exists += 1 continue cur.execute( """ @@ -70,11 +77,17 @@ def import_domains_from_path(file_path: Path, source_type: int = 7) -> dict: null, now(), now(), 0, null, 0, 0, 0 ) + on conflict (domain) do nothing returning id """, (domain, tld, source_type), ) - domain_id = cur.fetchone()[0] + inserted_row = cur.fetchone() + if not inserted_row: + existing_set.add(domain) + exists += 1 + continue + domain_id = inserted_row[0] cur.execute( """ insert into detect_tasks (domain_id, task_type, status, priority, retry_count, create_time, update_time) @@ -82,10 +95,10 @@ def import_domains_from_path(file_path: Path, source_type: int = 7) -> dict: """, (domain_id,), ) + existing_set.add(domain) inserted += 1 conn.commit() - exists = len(existing_set) valid = len(normalized_rows) stats = { "total": total, diff --git a/domain-api/app/services/ops_action_executor_core.py b/domain-api/app/services/ops_action_executor_core.py index 13316e9..29be625 100644 --- a/domain-api/app/services/ops_action_executor_core.py +++ b/domain-api/app/services/ops_action_executor_core.py @@ -6,10 +6,10 @@ from pathlib import Path from typing import Protocol import psycopg2 -import redis from app.core.config import settings from app.core.files import runtime_root as api_runtime_root +from app.core.redis_client import get_redis STRUCTURED_ACTIONS = { @@ -272,24 +272,10 @@ def _truncate_detect_runtime_tables(*, include_domains: bool) -> dict: def _flush_runtime_redis() -> dict: - client = redis.Redis( - host=settings.redis_host, - port=settings.redis_port, - password=settings.redis_password or None, - db=settings.redis_db, - decode_responses=True, - socket_connect_timeout=5, - socket_timeout=5, - ) - try: - size_before = int(client.dbsize() or 0) - client.flushdb() - size_after = int(client.dbsize() or 0) - finally: - try: - client.close() - except Exception: - pass + client = get_redis() + size_before = int(client.dbsize() or 0) + client.flushdb() + size_after = int(client.dbsize() or 0) return {"db": int(settings.redis_db), "size_before": size_before, "size_after": size_after} diff --git a/domain-api/app/services/ops_agent_service.py b/domain-api/app/services/ops_agent_service.py index 6a827b5..2620bbf 100644 --- a/domain-api/app/services/ops_agent_service.py +++ b/domain-api/app/services/ops_agent_service.py @@ -7,6 +7,8 @@ import shlex import threading from datetime import datetime, timedelta +from psycopg2 import errors + from app.core.config import settings from app.core.db import get_db from app.services.ops_command_service import build_bash_command @@ -80,6 +82,32 @@ ALTER TABLE ops_jobs ADD COLUMN IF NOT EXISTS last_agent_complete_request_id VAR _OPS_AGENT_SCHEMA_LOCK = threading.Lock() _OPS_AGENT_SCHEMA_READY = False _OPS_AGENT_SCHEMA_ADVISORY_LOCK_KEY = 90421802 +_OPS_AGENT_REQUIRED_TABLES = ("ops_node_tokens", "ops_job_events") +_OPS_AGENT_REQUIRED_COLUMNS = { + "ops_node_tokens": ( + "node_code", + "purpose", + "issued_by", + "is_enabled", + "expires_at", + "last_used_at", + "metadata_json", + "created_at", + "updated_at", + ), + "ops_job_events": ( + "job_id", + "step_id", + "node_code", + "client_event_id", + "event_type", + "level", + "message", + "payload_json", + "created_at", + ), + "ops_jobs": ("last_agent_complete_request_id",), +} def ensure_ops_agent_schema() -> None: @@ -91,14 +119,52 @@ def ensure_ops_agent_schema() -> None: if _OPS_AGENT_SCHEMA_READY: return with get_db() as conn: - conn.autocommit = False with conn.cursor() as cur: - cur.execute("SELECT pg_advisory_xact_lock(%s)", (_OPS_AGENT_SCHEMA_ADVISORY_LOCK_KEY,)) - cur.execute(_AGENT_SCHEMA_SQL) - conn.commit() + if _ops_agent_schema_basics_present(cur): + _OPS_AGENT_SCHEMA_READY = True + return + conn.autocommit = False + try: + with conn.cursor() as cur: + cur.execute("SELECT pg_advisory_xact_lock(%s)", (_OPS_AGENT_SCHEMA_ADVISORY_LOCK_KEY,)) + cur.execute(_AGENT_SCHEMA_SQL) + conn.commit() + except Exception as exc: + recoverable = isinstance(exc, (errors.DeadlockDetected, errors.LockNotAvailable)) + try: + conn.rollback() + except Exception: + pass + if not recoverable: + raise + with conn.cursor() as cur: + if not _ops_agent_schema_basics_present(cur): + raise _OPS_AGENT_SCHEMA_READY = True +def _ops_agent_schema_basics_present(cur) -> bool: + for table_name in _OPS_AGENT_REQUIRED_TABLES: + cur.execute("SELECT to_regclass(%s)", (f"public.{table_name}",)) + row = cur.fetchone() + if not row or not row[0]: + return False + + for table_name, required_columns in _OPS_AGENT_REQUIRED_COLUMNS.items(): + cur.execute( + """ + SELECT column_name + FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = %s + """, + (table_name,), + ) + existing_columns = {str(row[0] or "").strip() for row in list(cur.fetchall() or [])} + if not set(required_columns).issubset(existing_columns): + return False + return True + + def _hash_token(token: str) -> str: return hashlib.sha256(str(token or "").encode("utf-8")).hexdigest() @@ -152,12 +218,21 @@ def _merge_detect_runtime_snapshot(cluster_metadata: dict, metadata: dict, curre "phase_label": str(cluster_metadata.get("phase_label", metadata.get("phase_label", "")) or "").strip(), "phase_detail": str(cluster_metadata.get("phase_detail", metadata.get("phase_detail", "")) or "").strip(), "recent_warning": str(cluster_metadata.get("recent_warning", metadata.get("recent_warning", "")) or "").strip(), + "available_proxy_count": int( + cluster_metadata.get("available_proxy_count", metadata.get("available_proxy_count", 0)) or 0 + ), "proxy_runtime_label": str( cluster_metadata.get("proxy_runtime_label", metadata.get("proxy_runtime_label", "")) or "" ).strip(), "proxy_runtime_reason": str( cluster_metadata.get("proxy_runtime_reason", metadata.get("proxy_runtime_reason", "")) or "" ).strip(), + "proxy_last_refresh_status": str( + cluster_metadata.get("proxy_last_refresh_status", metadata.get("proxy_last_refresh_status", "")) or "" + ).strip(), + "proxy_last_refresh_time": str( + cluster_metadata.get("proxy_last_refresh_time", metadata.get("proxy_last_refresh_time", "")) or "" + ).strip(), "updated_at": str(cluster_metadata.get("updated_at", metadata.get("updated_at", "")) or "").strip(), } @@ -2714,6 +2789,15 @@ def _upsert_agent_detect_runtime(node_code: str, payload: dict) -> None: "phase_label": phase_label, "phase_detail": phase_detail, "recent_warning": recent_warning, + "available_proxy_count": max(0, int(detect_runtime.get("available_proxy_count") or 0)), + "proxy_runtime_label": str(detect_runtime.get("proxy_runtime_label") or "").strip(), + "proxy_runtime_reason": str(detect_runtime.get("proxy_runtime_reason") or "").strip(), + "proxy_last_refresh_status": str(detect_runtime.get("proxy_last_refresh_status") or "").strip(), + "proxy_last_refresh_time": str(detect_runtime.get("proxy_last_refresh_time") or "").strip(), + "proxy_last_refresh_source_count": max(0, int(detect_runtime.get("proxy_last_refresh_source_count") or 0)), + "proxy_last_refresh_total_items": max(0, int(detect_runtime.get("proxy_last_refresh_total_items") or 0)), + "proxy_last_validated_count": max(0, int(detect_runtime.get("proxy_last_validated_count") or 0)), + "proxy_last_available_count": max(0, int(detect_runtime.get("proxy_last_available_count") or 0)), "updated_at": str(detect_runtime.get("updated_at") or "").strip(), "agent_heartbeat_at": datetime.now().isoformat(timespec="seconds"), } @@ -2778,8 +2862,10 @@ def _build_agent_runtime_config_bundle(node_code: str) -> dict: "node_code": str(node_code or "").strip(), "detect_options": dict(settings_payload.get("detect_options") or {}), "proxy_config": dict(settings_payload.get("proxy_config") or {}), - "thread_count": int(settings_payload.get("thread_count", 2) or 2), + "thread_count": int(settings_payload.get("thread_count", 1000) or 1000), "node_thread_counts": dict(settings_payload.get("node_thread_counts") or {}), + "process_count": int(settings_payload.get("process_count", 80) or 80), + "node_process_counts": dict(settings_payload.get("node_process_counts") or {}), "runtime_settings": dict(runtime_settings or {}), "sensitive_words": { "text": str(sensitive_words_payload.get("text") or ""), diff --git a/domain-api/app/services/ops_execution_capability_service.py b/domain-api/app/services/ops_execution_capability_service.py index ecb7f14..8ce091d 100644 --- a/domain-api/app/services/ops_execution_capability_service.py +++ b/domain-api/app/services/ops_execution_capability_service.py @@ -19,6 +19,7 @@ _SSH_ACTIONS = set(STRUCTURED_ACTIONS) | {"deploy.release"} _REMOTE_AGENT_ACTIONS = set(STRUCTURED_ACTIONS) | _REMOTE_AGENT_ONLY_ACTIONS | {"deploy.release"} _CONTROL_PLANE_ACTIONS = { "node.bootstrap", + "migration.execute", } diff --git a/domain-api/app/services/ops_job_service.py b/domain-api/app/services/ops_job_service.py index 9017f4b..6531467 100644 --- a/domain-api/app/services/ops_job_service.py +++ b/domain-api/app/services/ops_job_service.py @@ -5,6 +5,8 @@ import threading from datetime import datetime from uuid import uuid4 +from psycopg2 import errors + from app.core.config import settings from app.core.db import get_db from app.services.ops_execution_capability_service import ( @@ -116,6 +118,28 @@ _LOCAL_RUNTIME_ACTIONS = { _OPS_SCHEMA_LOCK = threading.Lock() _OPS_SCHEMA_READY = False _OPS_SCHEMA_ADVISORY_LOCK_KEY = 90421801 +_OPS_REQUIRED_TABLES = ( + "ops_managed_nodes", + "ops_managed_node_secrets", + "ops_jobs", + "ops_job_steps", +) +_OPS_REQUIRED_COLUMNS = { + "ops_jobs": ( + "risk_level", + "approval_required", + "approval_status", + "approved_by", + "approved_at", + "blocked_reason", + "cancellation_reason", + "dispatched_at", + "target_selector_json", + "policy_json", + "rollout_id", + ), + "ops_job_steps": ("stdout_text", "stderr_text", "result_json"), +} def ensure_ops_schema() -> None: @@ -126,14 +150,52 @@ def ensure_ops_schema() -> None: if _OPS_SCHEMA_READY: return with get_db() as conn: - conn.autocommit = False with conn.cursor() as cur: - cur.execute("SELECT pg_advisory_xact_lock(%s)", (_OPS_SCHEMA_ADVISORY_LOCK_KEY,)) - cur.execute(_OPS_SCHEMA_SQL) - conn.commit() + if _ops_schema_basics_present(cur): + _OPS_SCHEMA_READY = True + return + conn.autocommit = False + try: + with conn.cursor() as cur: + cur.execute("SELECT pg_advisory_xact_lock(%s)", (_OPS_SCHEMA_ADVISORY_LOCK_KEY,)) + cur.execute(_OPS_SCHEMA_SQL) + conn.commit() + except Exception as exc: + recoverable = isinstance(exc, (errors.DeadlockDetected, errors.LockNotAvailable)) + try: + conn.rollback() + except Exception: + pass + if not recoverable: + raise + with conn.cursor() as cur: + if not _ops_schema_basics_present(cur): + raise _OPS_SCHEMA_READY = True +def _ops_schema_basics_present(cur) -> bool: + for table_name in _OPS_REQUIRED_TABLES: + cur.execute("SELECT to_regclass(%s)", (f"public.{table_name}",)) + row = cur.fetchone() + if not row or not row[0]: + return False + + for table_name, required_columns in _OPS_REQUIRED_COLUMNS.items(): + cur.execute( + """ + SELECT column_name + FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = %s + """, + (table_name,), + ) + existing_columns = {str(row[0] or "").strip() for row in list(cur.fetchall() or [])} + if not set(required_columns).issubset(existing_columns): + return False + return True + + def _decode_json(value: object) -> dict: if isinstance(value, dict): return value @@ -819,6 +881,7 @@ def _resolve_bootstrap_target_defaults(target_node_code: str) -> dict: def _execute_control_plane_job( action: str, *, + job_id: int = 0, target_node_code: str, payload: dict | None = None, requested_by: str = "api", @@ -864,6 +927,17 @@ def _execute_control_plane_job( } return False, message, data + if action == "migration.execute": + from app.services.ops_migration_service import execute_ops_migration_job + + return execute_ops_migration_job( + job_id=int(job_id or 0), + target_node_code=str(target_node_code or "").strip(), + payload=dict(normalized_payload or {}), + requested_by=str(requested_by or "api").strip() or "api", + metadata=dict(normalized_metadata or {}), + ) + return False, f"当前未实现控制面执行动作: {action}", {} @@ -1006,6 +1080,7 @@ def _execute_control_plane_job_record(job_id: int) -> tuple[bool, str, dict]: ok, message, result = _execute_control_plane_job( str(job.get("action") or ""), + job_id=int(job_id), target_node_code=str(job.get("target_node_code") or ""), payload=dict(job.get("payload") or {}), requested_by=str(job.get("requested_by") or "api"), @@ -1339,6 +1414,7 @@ def create_ops_job(payload: dict) -> tuple[bool, str, dict]: if execution_mode == "control-plane": ok, message, result = _execute_control_plane_job( action, + job_id=job_id, target_node_code=target_node_code, payload=dict(input_payload or {}), requested_by=requested_by, diff --git a/domain-api/app/services/ops_migration_service.py b/domain-api/app/services/ops_migration_service.py new file mode 100644 index 0000000..2533a10 --- /dev/null +++ b/domain-api/app/services/ops_migration_service.py @@ -0,0 +1,1559 @@ +from __future__ import annotations + +import base64 +import hashlib +import hmac +import json +import os +import shlex +import subprocess +import tempfile +import threading +import time +from io import StringIO +from pathlib import Path + +try: + import paramiko +except ImportError: # pragma: no cover + paramiko = None + +from app.core.config import settings +from app.services.ops_agent_service import append_ops_job_event +from app.services.ops_job_service import create_ops_job, dispatch_ops_job, list_managed_nodes + + +_MIGRATION_TIMEOUT_SECONDS = 180 +_SSH_CONNECT_TIMEOUT_SECONDS = 12 +_MIGRATION_CONFIRMATION_TTL_SECONDS = 1800 +_MIGRATION_ACTION = "migration.execute" +_MIGRATION_JOB_CONTEXTS: dict[int, dict] = {} +_MIGRATION_JOB_CONTEXTS_LOCK = threading.Lock() + + +def get_ops_migration_source_profile() -> dict: + config_files = { + "domain_env": str(Path(settings.domain_root) / ".env"), + "api_env": "/etc/default/domaincheck-api", + "worker_env": "/etc/default/domaincheck-worker", + "ops_center_env": "/etc/default/domaincheck-ops-center", + "web_env_production": str(Path(settings.domain_root).resolve().parent / "domain-web" / ".env.production"), + "api_service": "/etc/systemd/system/domaincheck-api.service", + "worker_service": "/etc/systemd/system/domaincheck-worker.service", + "sync_agent_service": "/etc/systemd/system/domaincheck-sync-agent.service", + "node_agent_service": "/etc/systemd/system/domaincheck-node-agent.service", + "postgresql_service": "/etc/systemd/system/postgresql.service", + } + file_status = { + key: { + "path": path, + "exists": Path(path).exists(), + } + for key, path in config_files.items() + } + return { + "source_node": { + "node_code": settings.node_code, + "region": settings.node_region, + "role": settings.node_role, + "worker_mode": settings.worker_mode, + }, + "workspace": { + "repo_path": str(Path(settings.domain_root).resolve().parent), + "domain_root": str(Path(settings.domain_root).resolve()), + "api_root": str((Path(settings.domain_root).resolve().parent / "domain-api").resolve()), + "web_root": str((Path(settings.domain_root).resolve().parent / "domain-web").resolve()), + }, + "database": { + "host": settings.db_host, + "port": int(settings.db_port), + "database": settings.db_database, + "user": settings.db_user, + "password_configured": bool(str(settings.db_password or "").strip()), + }, + "redis": { + "host": settings.redis_host, + "port": int(settings.redis_port), + "db": int(settings.redis_db), + "password_configured": bool(str(settings.redis_password or "").strip()), + }, + "config_files": file_status, + } + + +def preview_ops_migration(payload: dict | None = None) -> tuple[bool, str, dict]: + normalized = _normalize_payload(payload or {}) + ok, message, context = _build_migration_context(normalized, include_remote_file_reads=True) + if not ok: + return False, message, _sanitize_migration_context(context) + return True, "迁移预检查完成", _sanitize_migration_context(context) + + +def execute_ops_migration(payload: dict | None = None) -> tuple[bool, str, dict]: + normalized = _normalize_payload(payload or {}) + ok, message, context = _build_migration_context(normalized, include_remote_file_reads=True) + if not ok: + return False, message, _sanitize_migration_context(context) + confirmation_ok, confirmation_message, confirmation_data = _validate_execution_confirmation(normalized, context) + if not confirmation_ok: + return False, confirmation_message, _sanitize_migration_context({**context, **confirmation_data}) + + job_payload = _build_migration_job_payload(normalized) + metadata = { + "job_kind": "ops_migration", + "source_node_code": str(context.get("source_profile", {}).get("source_node", {}).get("node_code") or ""), + "plan_steps": list(context.get("plan_steps") or []), + "target_paths": dict(context.get("target_paths") or {}), + "target_db": { + "host": str(context.get("target_db_config", {}).get("host") or ""), + "port": int(context.get("target_db_config", {}).get("port") or 0), + "database": str(context.get("target_db_config", {}).get("database") or ""), + "user": str(context.get("target_db_config", {}).get("user") or ""), + "password_configured": bool(context.get("target_db_config", {}).get("password_configured", False)), + }, + "execution_guard": { + "context_hash": str(context.get("execution_guard", {}).get("context_hash") or ""), + "expires_at": str(context.get("execution_guard", {}).get("expires_at") or ""), + "requires_confirmation": bool(context.get("execution_guard", {}).get("requires_confirmation", False)), + }, + } + create_ok, create_message, create_data = create_ops_job( + { + "action": _MIGRATION_ACTION, + "target_type": "node", + "target_node_code": str(normalized.get("target_node_code") or ""), + "requested_by": "migration-ui", + "execution_mode": "control-plane", + "auto_approve": True, + "run_now": False, + "payload": job_payload, + "metadata": metadata, + } + ) + if not create_ok: + return False, create_message, _sanitize_migration_context({**context, **dict(create_data or {})}) + + job = dict(create_data.get("job") or {}) + job_id = int(job.get("id") or 0) + if job_id <= 0: + return False, "迁移任务创建成功，但缺少 job_id。", _sanitize_migration_context({**context, "job": job}) + + _remember_migration_job_context(job_id, normalized, context) + _start_migration_dispatch_thread(job_id) + return True, "迁移任务已创建，后台开始执行。", { + **_sanitize_migration_context(context), + "job": job, + "job_id": job_id, + "executed_immediately": False, + "execution_steps": [], + } + + +def execute_ops_migration_job( + *, + job_id: int, + target_node_code: str, + payload: dict | None = None, + requested_by: str = "api", + metadata: dict | None = None, +) -> tuple[bool, str, dict]: + cached = _pop_migration_job_context(job_id) + normalized = dict(cached.get("payload") or {}) + if not normalized: + normalized = _normalize_payload(payload or {}) + if target_node_code and not str(normalized.get("target_node_code") or "").strip(): + normalized["target_node_code"] = str(target_node_code or "").strip() + context = dict(cached.get("context") or {}) + if not context: + ok, message, context = _build_migration_context(normalized, include_remote_file_reads=True) + if not ok: + _append_migration_job_event( + job_id=job_id, + target_node_code=str(normalized.get("target_node_code") or ""), + event_type="migration_prepare_failed", + level="error", + message=message, + payload={"blocking_reasons": list(context.get("blocking_reasons") or [])}, + ) + return False, message, _sanitize_migration_context(context) + return _run_migration_plan( + normalized, + context, + job_id=job_id, + requested_by=requested_by, + metadata=dict(metadata or {}), + ) + + +def _normalize_payload(payload: dict) -> dict: + repo_path = str(payload.get("target_repo_path") or "").strip() or "/www/wwwroot/getDomain" + domain_root = str(payload.get("target_domain_root") or "").strip() or "/opt/domaincheck/domainCheck" + api_root = str(payload.get("target_api_root") or "").strip() or "/opt/domaincheck/domain-api" + web_root = str(payload.get("target_web_root") or "").strip() or "/opt/domaincheck/domain-web" + return { + "target_node_code": str(payload.get("target_node_code") or "").strip(), + "target_repo_path": repo_path, + "target_domain_root": domain_root, + "target_api_root": api_root, + "target_web_root": web_root, + "sync_env_files": bool(payload.get("sync_env_files", True)), + "sync_systemd_units": bool(payload.get("sync_systemd_units", True)), + "build_frontend": bool(payload.get("build_frontend", True)), + "overwrite_database": bool(payload.get("overwrite_database", False)), + "backup_target_database": bool(payload.get("backup_target_database", True)), + "restart_services": bool(payload.get("restart_services", True)), + "target_db_host": str(payload.get("target_db_host") or "").strip(), + "target_db_port": int(payload.get("target_db_port") or 0), + "target_db_name": str(payload.get("target_db_name") or "").strip(), + "target_db_user": str(payload.get("target_db_user") or "").strip(), + "target_db_password": str(payload.get("target_db_password") or ""), + "execute_confirmation_token": str(payload.get("execute_confirmation_token") or "").strip(), + "execute_confirmation_text": str(payload.get("execute_confirmation_text") or "").strip(), + } + + +def _build_migration_context(payload: dict, *, include_remote_file_reads: bool) -> tuple[bool, str, dict]: + target_node_code = str(payload.get("target_node_code") or "").strip() + if not target_node_code: + return False, "缺少目标节点编码", {"blocking_reasons": ["target_node_code 不能为空"]} + + node = _find_managed_node(target_node_code) + if not node: + return False, "目标节点不存在或未纳管", {"blocking_reasons": [f"未找到托管节点 {target_node_code}"]} + + ssh_ready = bool(str(node.get("ssh_host") or "").strip() and str(node.get("ssh_user") or "").strip()) + if not ssh_ready: + return False, "目标节点 SSH 信息不完整", { + "target_node": _serialize_target_node(node), + "blocking_reasons": ["目标节点缺少 SSH Host 或 SSH User"], + } + + remote_checks = _collect_remote_checks(node, payload, include_file_reads=include_remote_file_reads) + source_profile = get_ops_migration_source_profile() + remote_db_config = dict(remote_checks.get("remote_db_config") or {}) + + target_db_config = { + "host": str(payload.get("target_db_host") or remote_db_config.get("DB_HOST") or "127.0.0.1").strip() or "127.0.0.1", + "port": int(payload.get("target_db_port") or remote_db_config.get("DB_PORT") or 5432), + "database": str(payload.get("target_db_name") or remote_db_config.get("DB_DATABASE") or "").strip(), + "user": str(payload.get("target_db_user") or remote_db_config.get("DB_USER") or "").strip(), + "password": str(payload.get("target_db_password") or remote_db_config.get("DB_PASSWORD") or ""), + } + target_db_config["password_configured"] = bool(str(target_db_config.get("password") or "").strip()) + + blocking_reasons = list(remote_checks.get("blocking_reasons") or []) + warnings = list(remote_checks.get("warnings") or []) + target_db_inspection = _inspect_target_database(node, target_db_config, remote_checks) + if bool(payload.get("overwrite_database", False)): + if not target_db_config["database"] or not target_db_config["user"]: + blocking_reasons.append("已勾选覆盖数据库，但目标机尚未识别到有效 DB_DATABASE / DB_USER。") + if not bool(remote_checks.get("tools", {}).get("psql", False)): + blocking_reasons.append("已勾选覆盖数据库，但目标机缺少 psql。") + if not bool(remote_checks.get("tools", {}).get("pg_dump", False)) and bool(payload.get("backup_target_database", True)): + blocking_reasons.append("已勾选“覆盖前先备份目标库”，但目标机缺少 pg_dump。") + elif not bool(remote_checks.get("tools", {}).get("pg_dump", False)): + warnings.append("目标机缺少 pg_dump，将无法在覆盖前生成远端数据库备份。") + if bool(target_db_inspection.get("available", False)) and bool(target_db_inspection.get("has_business_data", False)): + warnings.append("目标数据库已存在业务表或统计行数，执行覆盖前必须做二次确认。") + elif not bool(target_db_inspection.get("available", False)): + warnings.append("暂时无法确认目标数据库是否为空，请先检查目标机数据库配置与连通性。") + + context = { + "payload": dict(payload or {}), + "source_profile": source_profile, + "target_node": _serialize_target_node(node), + "target_paths": { + "repo_path": str(payload.get("target_repo_path") or ""), + "domain_root": str(payload.get("target_domain_root") or ""), + "api_root": str(payload.get("target_api_root") or ""), + "web_root": str(payload.get("target_web_root") or ""), + }, + "remote_checks": remote_checks, + "target_db_config": target_db_config, + "target_db_inspection": target_db_inspection, + "plan_steps": _build_plan_steps(payload), + "blocking_reasons": blocking_reasons, + "warnings": warnings, + } + context["execution_guard"] = _build_execution_guard(context) + if blocking_reasons: + return False, "迁移预检查未通过", context + return True, "ok", context + + +def _build_plan_steps(payload: dict) -> list[dict]: + steps = [] + if bool(payload.get("sync_env_files", True)): + steps.append({"key": "sync_env_files", "title": "同步 .env / /etc/default 配置"}) + if bool(payload.get("sync_systemd_units", True)): + steps.append({"key": "sync_systemd_units", "title": "同步 systemd 单元文件"}) + if bool(payload.get("overwrite_database", False)): + steps.append({"key": "overwrite_database", "title": "备份并覆盖目标 PostgreSQL"}) + if bool(payload.get("build_frontend", True)): + steps.append({"key": "build_frontend", "title": "在目标机重新构建前端"}) + if bool(payload.get("restart_services", True)): + steps.append({"key": "restart_services", "title": "daemon-reload 并重启 domaincheck-* 服务"}) + steps.append({"key": "health_check", "title": "校验目标机 /health"}) + return steps + + +def _collect_remote_checks(node: dict, payload: dict, *, include_file_reads: bool) -> dict: + repo_path = str(payload.get("target_repo_path") or "") + domain_root = str(payload.get("target_domain_root") or "") + api_root = str(payload.get("target_api_root") or "") + web_root = str(payload.get("target_web_root") or "") + python_script = f""" +import json +import os +import subprocess +from pathlib import Path + +repo_path = Path({json.dumps(repo_path)}) +domain_root = Path({json.dumps(domain_root)}) +api_root = Path({json.dumps(api_root)}) +web_root = Path({json.dumps(web_root)}) + +def command_exists(name): + completed = subprocess.run(["bash", "-lc", f"command -v {{name}} >/dev/null 2>&1"], capture_output=True, text=True) + return completed.returncode == 0 + +def read_env(path): + values = {{}} + if not path.exists(): + return values + for line in path.read_text(encoding="utf-8", errors="ignore").splitlines(): + stripped = line.strip() + if not stripped or stripped.startswith("#") or "=" not in stripped: + continue + key, value = stripped.split("=", 1) + values[key.strip()] = value.strip().strip("'").strip('"') + return values + +git_commit = "" +if repo_path.exists(): + completed = subprocess.run( + ["bash", "-lc", f"cd {{repo_path}} && git rev-parse HEAD"], + capture_output=True, + text=True, + ) + if completed.returncode == 0: + git_commit = str(completed.stdout or "").strip() + +payload = {{ + "paths": {{ + "repo_exists": repo_path.exists(), + "repo_git": (repo_path / ".git").exists(), + "domain_root_exists": domain_root.exists(), + "api_root_exists": api_root.exists(), + "web_root_exists": web_root.exists(), + }}, + "git_commit": git_commit, + "tools": {{ + "python3": command_exists("python3"), + "python": command_exists("python"), + "node": command_exists("node"), + "npm": command_exists("npm"), + "systemctl": command_exists("systemctl"), + "psql": command_exists("psql"), + "pg_dump": command_exists("pg_dump"), + "curl": command_exists("curl"), + }}, + "env": read_env(domain_root / ".env"), +}} +print(json.dumps(payload, ensure_ascii=False)) +""" + ok, stdout, stderr, _meta = _run_remote_python(node, python_script, timeout=60) + if not ok: + return { + "ok": False, + "blocking_reasons": [f"远端预检查失败: {stderr or stdout or 'unknown error'}"], + "stderr": stderr, + "stdout": stdout, + } + try: + parsed = json.loads(stdout or "{}") + except Exception: + parsed = {} + + paths = dict(parsed.get("paths") or {}) + tools = dict(parsed.get("tools") or {}) + remote_db_config = dict(parsed.get("env") or {}) + blocking_reasons: list[str] = [] + warnings: list[str] = [] + if not bool(paths.get("repo_exists", False)): + blocking_reasons.append(f"目标机项目路径不存在: {repo_path}") + if not bool(paths.get("repo_git", False)): + blocking_reasons.append("目标机项目路径存在，但不是 Git 仓库。") + for tool_name in ("python3", "node", "npm", "systemctl", "curl"): + if not bool(tools.get(tool_name, False)): + blocking_reasons.append(f"目标机缺少 {tool_name}。") + if not bool(paths.get("domain_root_exists", False)): + warnings.append(f"目标 domain_root 不存在: {domain_root}") + if not bool(paths.get("api_root_exists", False)): + warnings.append(f"目标 api_root 不存在: {api_root}") + if not bool(paths.get("web_root_exists", False)): + warnings.append(f"目标 web_root 不存在: {web_root}") + env_files = {} + if include_file_reads: + env_files = _collect_remote_file_presence(node) + return { + "ok": len(blocking_reasons) == 0, + "git_commit": str(parsed.get("git_commit") or ""), + "paths": paths, + "tools": tools, + "remote_db_config": remote_db_config, + "remote_file_presence": env_files, + "blocking_reasons": blocking_reasons, + "warnings": warnings, + } + + +def _inspect_target_database(node: dict, target_db_config: dict, remote_checks: dict) -> dict: + tools = dict(remote_checks.get("tools") or {}) + if not bool(tools.get("psql", False)): + return { + "available": False, + "reason": "target missing psql", + } + if not str(target_db_config.get("database") or "").strip() or not str(target_db_config.get("user") or "").strip(): + return { + "available": False, + "reason": "target database identity incomplete", + } + password_export = "" + if str(target_db_config.get("password") or "").strip(): + password_export = f"export PGPASSWORD={shlex.quote(str(target_db_config.get('password') or ''))}\n" + sql = """ +SELECT json_build_object( + 'database', current_database(), + 'public_table_count', COALESCE(( + SELECT count(*) + FROM information_schema.tables + WHERE table_schema = 'public' + ), 0), + 'business_table_count', COALESCE(( + SELECT count(*) + FROM information_schema.tables + WHERE table_schema = 'public' + AND table_name = ANY(ARRAY['domains','detect_jobs','detect_job_items','detect_worker_nodes','ops_jobs']) + ), 0), + 'approx_total_rows', COALESCE(( + SELECT sum(GREATEST(s.n_live_tup::bigint, 0)) + FROM pg_stat_user_tables s + ), 0), + 'approx_domains_rows', COALESCE(( + SELECT GREATEST(s.n_live_tup::bigint, 0) + FROM pg_stat_user_tables s + WHERE s.relname = 'domains' + LIMIT 1 + ), 0), + 'has_domains_table', EXISTS( + SELECT 1 FROM information_schema.tables WHERE table_schema = 'public' AND table_name = 'domains' + ), + 'has_detect_jobs_table', EXISTS( + SELECT 1 FROM information_schema.tables WHERE table_schema = 'public' AND table_name = 'detect_jobs' + ) +); +""".strip() + script = "\n".join( + [ + "set -euo pipefail", + password_export.rstrip("\n"), + "psql " + f"-h {shlex.quote(str(target_db_config.get('host') or '127.0.0.1'))} " + f"-p {shlex.quote(str(target_db_config.get('port') or 5432))} " + f"-U {shlex.quote(str(target_db_config.get('user') or ''))} " + f"-d {shlex.quote(str(target_db_config.get('database') or ''))} " + "-At " + f"-c {shlex.quote(sql)}", + ] + ) + ok, stdout, stderr, _meta = _run_remote_shell(node, script, timeout=90) + if not ok: + return { + "available": False, + "reason": "query_failed", + "stderr": str(stderr or "").strip(), + } + try: + payload = json.loads(str(stdout or "").strip() or "{}") + except Exception: + return { + "available": False, + "reason": "invalid_json", + "stdout": str(stdout or "").strip(), + } + public_table_count = int(payload.get("public_table_count", 0) or 0) + business_table_count = int(payload.get("business_table_count", 0) or 0) + approx_total_rows = int(payload.get("approx_total_rows", 0) or 0) + approx_domains_rows = int(payload.get("approx_domains_rows", 0) or 0) + has_business_data = business_table_count > 0 or approx_total_rows > 0 or approx_domains_rows > 0 + return { + "available": True, + "database": str(payload.get("database") or ""), + "public_table_count": public_table_count, + "business_table_count": business_table_count, + "approx_total_rows": approx_total_rows, + "approx_domains_rows": approx_domains_rows, + "has_domains_table": bool(payload.get("has_domains_table", False)), + "has_detect_jobs_table": bool(payload.get("has_detect_jobs_table", False)), + "has_business_data": has_business_data, + } + + +def _build_execution_guard(context: dict) -> dict: + payload = dict(context.get("payload") or {}) + target_node = dict(context.get("target_node") or {}) + target_db_config = dict(context.get("target_db_config") or {}) + target_db_inspection = dict(context.get("target_db_inspection") or {}) + context_hash = _build_execution_context_hash(context) + issued_at = int(time.time()) + expires_at = issued_at + _MIGRATION_CONFIRMATION_TTL_SECONDS + required_confirmation_text = "" + if bool(payload.get("overwrite_database", False)) and bool(target_db_inspection.get("has_business_data", False)): + db_name = str(target_db_config.get("database") or "target-db").strip() or "target-db" + required_confirmation_text = f"OVERWRITE {db_name}" + token_payload = { + "target_node_code": str(target_node.get("node_code") or ""), + "context_hash": context_hash, + "issued_at": issued_at, + "expires_at": expires_at, + "required_confirmation_text": required_confirmation_text, + } + encoded_payload = _urlsafe_b64encode(json.dumps(token_payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")) + signature = hmac.new(_migration_confirmation_secret(), encoded_payload.encode("utf-8"), hashlib.sha256).hexdigest() + return { + "token": f"{encoded_payload}.{signature}", + "context_hash": context_hash, + "issued_at": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(issued_at)), + "expires_at": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(expires_at)), + "required_confirmation_text": required_confirmation_text, + "requires_confirmation": bool(required_confirmation_text), + } + + +def _validate_execution_confirmation(payload: dict, context: dict) -> tuple[bool, str, dict]: + guard = dict(context.get("execution_guard") or {}) + token = str(payload.get("execute_confirmation_token") or "").strip() + if not token: + return False, "执行前必须先跑一次预检查并携带确认令牌。", { + "blocking_reasons": ["missing execute_confirmation_token"], + "execution_guard": guard, + } + encoded_payload, dot, signature = token.partition(".") + if not encoded_payload or not dot or not signature: + return False, "确认令牌格式无效。", { + "blocking_reasons": ["invalid confirmation token format"], + "execution_guard": guard, + } + expected_signature = hmac.new(_migration_confirmation_secret(), encoded_payload.encode("utf-8"), hashlib.sha256).hexdigest() + if not hmac.compare_digest(expected_signature, signature): + return False, "确认令牌签名校验失败，请重新预检。", { + "blocking_reasons": ["invalid confirmation token signature"], + "execution_guard": guard, + } + try: + token_payload = json.loads(_urlsafe_b64decode(encoded_payload).decode("utf-8")) + except Exception: + return False, "确认令牌内容无法解析，请重新预检。", { + "blocking_reasons": ["invalid confirmation token payload"], + "execution_guard": guard, + } + if int(token_payload.get("expires_at", 0) or 0) < int(time.time()): + return False, "确认令牌已过期，请重新预检。", { + "blocking_reasons": ["expired confirmation token"], + "execution_guard": guard, + } + if str(token_payload.get("context_hash") or "").strip() != str(guard.get("context_hash") or "").strip(): + return False, "迁移参数已变化，请重新预检生成新的执行令牌。", { + "blocking_reasons": ["confirmation token context mismatch"], + "execution_guard": guard, + } + required_confirmation_text = str(guard.get("required_confirmation_text") or "").strip() + if required_confirmation_text: + provided_text = str(payload.get("execute_confirmation_text") or "").strip() + if provided_text != required_confirmation_text: + return False, "缺少数据库覆盖确认文案，执行被拒绝。", { + "blocking_reasons": ["missing execute_confirmation_text"], + "execution_guard": guard, + } + return True, "ok", {} + + +def _build_execution_context_hash(context: dict) -> str: + payload = dict(context.get("payload") or {}) + serializable = { + "target_node": str(((context.get("target_node") or {}).get("node_code")) or ""), + "target_paths": dict(context.get("target_paths") or {}), + "target_db": { + "host": str(((context.get("target_db_config") or {}).get("host")) or ""), + "port": int(((context.get("target_db_config") or {}).get("port")) or 0), + "database": str(((context.get("target_db_config") or {}).get("database")) or ""), + "user": str(((context.get("target_db_config") or {}).get("user")) or ""), + }, + "sync_env_files": bool(payload.get("sync_env_files", True)), + "sync_systemd_units": bool(payload.get("sync_systemd_units", True)), + "build_frontend": bool(payload.get("build_frontend", True)), + "restart_services": bool(payload.get("restart_services", True)), + "overwrite_database": bool(payload.get("overwrite_database", False)), + "backup_target_database": bool(payload.get("backup_target_database", True)), + } + raw = json.dumps(serializable, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(raw.encode("utf-8")).hexdigest() + + +def _migration_confirmation_secret() -> bytes: + parts = [ + str(settings.node_code or ""), + str(settings.admin_password or ""), + str(settings.db_password or ""), + str(settings.sync_shared_token or ""), + ] + return "|".join(parts).encode("utf-8") + + +def _collect_remote_file_presence(node: dict) -> dict: + files = [ + "/etc/default/domaincheck-api", + "/etc/default/domaincheck-worker", + "/etc/default/domaincheck-ops-center", + "/etc/systemd/system/domaincheck-api.service", + "/etc/systemd/system/domaincheck-worker.service", + "/etc/systemd/system/domaincheck-sync-agent.service", + "/etc/systemd/system/postgresql.service", + ] + python_script = f""" +import json +from pathlib import Path + +files = {json.dumps(files, ensure_ascii=False)} +print(json.dumps({{item: Path(item).exists() for item in files}}, ensure_ascii=False)) +""" + ok, stdout, _stderr, _meta = _run_remote_python(node, python_script, timeout=30) + if not ok: + return {} + try: + return json.loads(stdout or "{}") + except Exception: + return {} + + +def _serialize_target_node(node: dict) -> dict: + return { + "node_code": str(node.get("node_code") or ""), + "title": str(node.get("title") or ""), + "region": str(node.get("region") or ""), + "role": str(node.get("role") or ""), + "ssh_host": str(node.get("ssh_host") or ""), + "ssh_port": int(node.get("ssh_port") or 22), + "ssh_user": str(node.get("ssh_user") or ""), + "auth_mode": str(node.get("auth_mode") or "key"), + } + + +def _find_managed_node(node_code: str) -> dict | None: + normalized = str(node_code or "").strip() + if not normalized: + return None + for item in list_managed_nodes(): + if str(item.get("node_code") or "").strip() == normalized: + return dict(item) + return None + + +def _build_migration_job_payload(payload: dict) -> dict: + normalized = _normalize_payload(payload or {}) + sanitized = { + **normalized, + "target_db_password": "", + "target_db_password_configured": bool(str(normalized.get("target_db_password") or "").strip()), + "execute_confirmation_token": "", + "execute_confirmation_text": "", + } + return sanitized + + +def _sanitize_migration_context(context: dict) -> dict: + sanitized = dict(context or {}) + target_db_config = dict(sanitized.get("target_db_config") or {}) + if "password" in target_db_config: + target_db_config["password"] = "" + sanitized["target_db_config"] = target_db_config + return sanitized + + +def _remember_migration_job_context(job_id: int, payload: dict, context: dict) -> None: + if int(job_id or 0) <= 0: + return + with _MIGRATION_JOB_CONTEXTS_LOCK: + _MIGRATION_JOB_CONTEXTS[int(job_id)] = { + "payload": dict(payload or {}), + "context": dict(context or {}), + "remembered_at": time.strftime("%Y-%m-%d %H:%M:%S"), + } + + +def _pop_migration_job_context(job_id: int) -> dict: + normalized_job_id = int(job_id or 0) + if normalized_job_id <= 0: + return {} + with _MIGRATION_JOB_CONTEXTS_LOCK: + return dict(_MIGRATION_JOB_CONTEXTS.pop(normalized_job_id, {}) or {}) + + +def _start_migration_dispatch_thread(job_id: int) -> None: + normalized_job_id = int(job_id or 0) + if normalized_job_id <= 0: + return + thread = threading.Thread( + target=_dispatch_migration_job_async, + args=(normalized_job_id,), + name=f"ops-migration-{normalized_job_id}", + daemon=True, + ) + thread.start() + + +def _dispatch_migration_job_async(job_id: int) -> None: + try: + ok, message, data = dispatch_ops_job(int(job_id)) + if not ok: + target_node_code = str((data.get("job") or {}).get("target_node_code") or "") + _append_migration_job_event( + job_id=int(job_id), + target_node_code=target_node_code, + event_type="migration_dispatch_failed", + level="error", + message=message, + payload={"result": dict(data or {})}, + ) + except Exception as exc: # pragma: no cover - safety net + _append_migration_job_event( + job_id=int(job_id), + target_node_code="", + event_type="migration_dispatch_failed", + level="error", + message=f"迁移后台派发异常: {exc}", + payload={"exception": repr(exc)}, + ) + + +def _run_migration_plan( + payload: dict, + context: dict, + *, + job_id: int = 0, + requested_by: str = "api", + metadata: dict | None = None, +) -> tuple[bool, str, dict]: + normalized_payload = dict(payload or {}) + context = dict(context or {}) + node = dict(context.get("target_node") or {}) + execution_results: list[dict] = [] + + _append_migration_job_event( + job_id=job_id, + target_node_code=str(node.get("node_code") or ""), + event_type="migration_started", + message=f"迁移任务开始执行，目标节点 {str(node.get('node_code') or '-')}", + payload={ + "requested_by": str(requested_by or "api").strip() or "api", + "metadata": dict(metadata or {}), + "plan_steps": list(context.get("plan_steps") or []), + "target_paths": dict(context.get("target_paths") or {}), + }, + ) + + def run_step(step_key: str, title: str, fn) -> bool: + started_at = time.strftime("%Y-%m-%d %H:%M:%S") + _append_migration_job_event( + job_id=job_id, + target_node_code=str(node.get("node_code") or ""), + event_type="migration_step_started", + message=f"{title} 开始", + payload={ + "step_key": step_key, + "title": title, + "started_at": started_at, + }, + ) + try: + step_ok, step_message, step_data = fn() + except Exception as exc: # pragma: no cover - safety net + step_ok = False + step_message = f"{title} 失败: {exc}" + step_data = {"exception": repr(exc)} + finished_at = time.strftime("%Y-%m-%d %H:%M:%S") + step_record = { + "step_key": step_key, + "title": title, + "ok": bool(step_ok), + "message": str(step_message or ""), + "data": dict(step_data or {}), + "started_at": started_at, + "finished_at": finished_at, + } + execution_results.append(step_record) + _append_migration_job_event( + job_id=job_id, + target_node_code=str(node.get("node_code") or ""), + event_type="migration_step_completed", + level="success" if step_ok else "error", + message=str(step_message or title), + payload={ + "step_key": step_key, + "title": title, + "started_at": started_at, + "finished_at": finished_at, + "ok": bool(step_ok), + "result": dict(step_data or {}), + }, + ) + return bool(step_ok) + + if bool(normalized_payload.get("sync_env_files", True)): + if not run_step( + "sync_env_files", + "同步环境配置", + lambda: _sync_env_files(node, context), + ): + result = _sanitize_migration_context({**context, "execution_steps": execution_results}) + _append_migration_job_event( + job_id=job_id, + target_node_code=str(node.get("node_code") or ""), + event_type="migration_failed", + level="error", + message="迁移中断：同步环境配置失败", + payload={"execution_steps": execution_results}, + ) + return False, "迁移中断：同步环境配置失败", result + + if bool(normalized_payload.get("sync_systemd_units", True)): + if not run_step( + "sync_systemd_units", + "同步 systemd 单元", + lambda: _sync_systemd_units(node, context), + ): + result = _sanitize_migration_context({**context, "execution_steps": execution_results}) + _append_migration_job_event( + job_id=job_id, + target_node_code=str(node.get("node_code") or ""), + event_type="migration_failed", + level="error", + message="迁移中断：同步 systemd 单元失败", + payload={"execution_steps": execution_results}, + ) + return False, "迁移中断：同步 systemd 单元失败", result + + if bool(normalized_payload.get("overwrite_database", False)): + if not run_step( + "overwrite_database", + "覆盖目标数据库", + lambda: _overwrite_target_database(node, context, job_id=job_id), + ): + result = _sanitize_migration_context({**context, "execution_steps": execution_results}) + _append_migration_job_event( + job_id=job_id, + target_node_code=str(node.get("node_code") or ""), + event_type="migration_failed", + level="error", + message="迁移中断：目标数据库覆盖失败", + payload={"execution_steps": execution_results}, + ) + return False, "迁移中断：目标数据库覆盖失败", result + + if bool(normalized_payload.get("build_frontend", True)): + if not run_step( + "build_frontend", + "构建前端资源", + lambda: _build_remote_frontend(node, context, job_id=job_id), + ): + result = _sanitize_migration_context({**context, "execution_steps": execution_results}) + _append_migration_job_event( + job_id=job_id, + target_node_code=str(node.get("node_code") or ""), + event_type="migration_failed", + level="error", + message="迁移中断：前端构建失败", + payload={"execution_steps": execution_results}, + ) + return False, "迁移中断：前端构建失败", result + + if bool(normalized_payload.get("restart_services", True)): + if not run_step( + "restart_services", + "刷新并重启服务", + lambda: _restart_remote_services(node, job_id=job_id), + ): + result = _sanitize_migration_context({**context, "execution_steps": execution_results}) + _append_migration_job_event( + job_id=job_id, + target_node_code=str(node.get("node_code") or ""), + event_type="migration_failed", + level="error", + message="迁移中断：服务重启失败", + payload={"execution_steps": execution_results}, + ) + return False, "迁移中断：服务重启失败", result + + if not run_step( + "health_check", + "执行远端健康检查", + lambda: _run_remote_health_check(node, job_id=job_id), + ): + result = _sanitize_migration_context({**context, "execution_steps": execution_results}) + _append_migration_job_event( + job_id=job_id, + target_node_code=str(node.get("node_code") or ""), + event_type="migration_failed", + level="error", + message="迁移完成，但健康检查失败", + payload={"execution_steps": execution_results}, + ) + return False, "迁移完成，但健康检查失败", result + + result = _sanitize_migration_context({**context, "execution_steps": execution_results}) + _append_migration_job_event( + job_id=job_id, + target_node_code=str(node.get("node_code") or ""), + event_type="migration_finished", + level="success", + message="迁移执行完成", + payload={"execution_steps": execution_results}, + ) + return True, "迁移执行完成", result + + +def _append_migration_job_event( + *, + job_id: int, + target_node_code: str, + event_type: str, + message: str, + level: str = "info", + payload: dict | None = None, +) -> None: + if int(job_id or 0) <= 0: + return + append_ops_job_event( + job_id=int(job_id), + node_code=str(target_node_code or "").strip(), + event_type=str(event_type or "migration_event").strip() or "migration_event", + level=str(level or "info").strip() or "info", + message=str(message or "").strip()[:2000], + payload={ + **dict(payload or {}), + "occurred_at": time.strftime("%Y-%m-%d %H:%M:%S"), + "summary_text": str(message or "").strip()[:2000], + }, + ) + + +def _run_remote_python(node: dict, script: str, *, timeout: int = 60) -> tuple[bool, str, str, dict]: + remote_command = "\n".join( + [ + "set -euo pipefail", + "if command -v python3 >/dev/null 2>&1; then PYTHON_BIN=python3; elif command -v python >/dev/null 2>&1; then PYTHON_BIN=python; else echo 'python missing' >&2; exit 127; fi", + '"${PYTHON_BIN}" - <<\'PY\'', + script.rstrip("\n"), + "PY", + ] + ) + return _run_remote_shell(node, remote_command, timeout=timeout) + + +def _run_remote_shell( + node: dict, + remote_script: str, + *, + timeout: int = _MIGRATION_TIMEOUT_SECONDS, + stdin_data: bytes | None = None, +) -> tuple[bool, str, str, dict]: + ssh_host = str(node.get("ssh_host") or "").strip() + ssh_user = str(node.get("ssh_user") or "").strip() + ssh_port = int(node.get("ssh_port") or 22) + auth_mode = str(node.get("auth_mode") or "key").strip() or "key" + secret = _load_node_secret(node) + ssh_password = str(secret.get("ssh_password") or "") + ssh_private_key = str(secret.get("ssh_private_key") or "") + + if paramiko is not None and ((auth_mode == "password" and ssh_password) or ssh_private_key): + completed = _run_paramiko_command( + ssh_host=ssh_host, + ssh_port=ssh_port, + ssh_user=ssh_user, + remote_command=remote_script, + timeout_seconds=timeout, + ssh_password=ssh_password if auth_mode == "password" else "", + ssh_private_key=ssh_private_key if auth_mode == "key" else "", + stdin_data=stdin_data, + ) + return completed.returncode == 0, completed.stdout, completed.stderr, { + "executor": "paramiko", + "returncode": int(completed.returncode or 0), + } + + wrapped_command = f"bash -lc {shlex.quote(remote_script)}" + completed = subprocess.run( + [ + "ssh", + "-o", + "BatchMode=yes", + "-o", + "StrictHostKeyChecking=accept-new", + "-o", + f"ConnectTimeout={_SSH_CONNECT_TIMEOUT_SECONDS}", + "-p", + str(ssh_port), + f"{ssh_user}@{ssh_host}", + wrapped_command, + ], + input=stdin_data, + capture_output=True, + timeout=timeout, + ) + return completed.returncode == 0, completed.stdout.decode("utf-8", errors="replace"), completed.stderr.decode("utf-8", errors="replace"), { + "executor": "ssh", + "returncode": int(completed.returncode or 0), + } + + +def _upload_text(node: dict, remote_path: str, content: str, *, mode: int = 0o600) -> tuple[bool, str, dict]: + payload = base64.b64encode(str(content or "").encode("utf-8")).decode("ascii") + python_script = f""" +import base64 +import os +from pathlib import Path + +target = Path({json.dumps(remote_path)}) +target.parent.mkdir(parents=True, exist_ok=True) +target.write_bytes(base64.b64decode({json.dumps(payload)})) +os.chmod(target, {int(mode)}) +print("ok") +""" + ok, stdout, stderr, meta = _run_remote_python(node, python_script, timeout=60) + return ok, stderr or stdout or ("ok" if ok else "upload failed"), meta + + +def _upload_file(node: dict, local_path: str, remote_path: str) -> tuple[bool, str]: + local = Path(local_path) + if not local.exists(): + return False, f"本地文件不存在: {local_path}" + ssh_host = str(node.get("ssh_host") or "").strip() + ssh_user = str(node.get("ssh_user") or "").strip() + ssh_port = int(node.get("ssh_port") or 22) + auth_mode = str(node.get("auth_mode") or "key").strip() or "key" + secret = _load_node_secret(node) + ssh_password = str(secret.get("ssh_password") or "") + ssh_private_key = str(secret.get("ssh_private_key") or "") + + if paramiko is not None and ((auth_mode == "password" and ssh_password) or ssh_private_key): + client = _open_paramiko_client( + ssh_host=ssh_host, + ssh_port=ssh_port, + ssh_user=ssh_user, + ssh_password=ssh_password if auth_mode == "password" else "", + ssh_private_key=ssh_private_key if auth_mode == "key" else "", + ) + try: + sftp = client.open_sftp() + remote_parent = str(Path(remote_path).parent) + _ensure_sftp_dir(sftp, remote_parent) + sftp.put(str(local), remote_path) + sftp.close() + finally: + client.close() + return True, "ok" + + mkdir_script = f"mkdir -p {shlex.quote(str(Path(remote_path).parent))}" + ok, _stdout, stderr, _meta = _run_remote_shell(node, mkdir_script, timeout=30) + if not ok: + return False, stderr or "无法创建远端目录" + completed = subprocess.run( + [ + "scp", + "-P", + str(ssh_port), + str(local), + f"{ssh_user}@{ssh_host}:{remote_path}", + ], + capture_output=True, + text=True, + timeout=_MIGRATION_TIMEOUT_SECONDS, + ) + if completed.returncode != 0: + return False, str(completed.stderr or completed.stdout or "scp failed").strip() + return True, "ok" + + +def _truncate_output(value: object, limit: int = 4000) -> str: + text = str(value or "").strip() + if len(text) <= limit: + return text + return text[-limit:] + + +def _sync_env_files(node: dict, context: dict) -> tuple[bool, str, dict]: + target_paths = dict(context.get("target_paths") or {}) + local_domain_root = Path(context.get("source_profile", {}).get("workspace", {}).get("domain_root", settings.domain_root)) + local_web_root = Path(context.get("source_profile", {}).get("workspace", {}).get("web_root", local_domain_root.parent / "domain-web")) + files = [ + (str(local_domain_root / ".env"), f"{target_paths.get('domain_root')}/.env", 0o600), + ("/etc/default/domaincheck-api", "/etc/default/domaincheck-api", 0o644), + ("/etc/default/domaincheck-worker", "/etc/default/domaincheck-worker", 0o644), + ("/etc/default/domaincheck-ops-center", "/etc/default/domaincheck-ops-center", 0o644), + (str(local_web_root / ".env.production"), f"{target_paths.get('web_root')}/.env.production", 0o644), + ] + uploaded: list[dict] = [] + for local_path, remote_path, mode in files: + path = Path(local_path) + if not path.exists(): + continue + ok, message, _meta = _upload_text(node, remote_path, path.read_text(encoding="utf-8", errors="ignore"), mode=mode) + uploaded.append({"local_path": local_path, "remote_path": remote_path, "ok": ok, "message": message}) + if not ok: + return False, f"同步文件失败: {remote_path}", {"uploaded": uploaded} + return True, "环境配置已同步", {"uploaded": uploaded} + + +def _sync_systemd_units(node: dict, _context: dict) -> tuple[bool, str, dict]: + files = [ + "/etc/systemd/system/domaincheck-api.service", + "/etc/systemd/system/domaincheck-worker.service", + "/etc/systemd/system/domaincheck-sync-agent.service", + "/etc/systemd/system/domaincheck-node-agent.service", + "/etc/systemd/system/postgresql.service", + ] + uploaded: list[dict] = [] + for file_path in files: + path = Path(file_path) + if not path.exists(): + continue + ok, message, _meta = _upload_text(node, file_path, path.read_text(encoding="utf-8", errors="ignore"), mode=0o644) + uploaded.append({"path": file_path, "ok": ok, "message": message}) + if not ok: + return False, f"同步 systemd 单元失败: {file_path}", {"uploaded": uploaded} + return True, "systemd 单元已同步", {"uploaded": uploaded} + + +def _overwrite_target_database(node: dict, context: dict, *, job_id: int = 0) -> tuple[bool, str, dict]: + source_db = dict(context.get("source_profile", {}).get("database") or {}) + dump_dir = Path(tempfile.mkdtemp(prefix="domaincheck-migration-")) + dump_path = dump_dir / "source.sql" + _append_migration_job_event( + job_id=job_id, + target_node_code=str(node.get("node_code") or ""), + event_type="migration_db_dump_started", + message="开始导出源 PostgreSQL", + payload={ + "database": str(source_db.get("database") or ""), + "host": str(source_db.get("host") or ""), + "port": int(source_db.get("port") or 5432), + "user": str(source_db.get("user") or ""), + }, + ) + dump_command = [ + "pg_dump", + "-h", + str(source_db.get("host") or "127.0.0.1"), + "-p", + str(source_db.get("port") or 5432), + "-U", + str(source_db.get("user") or ""), + "--clean", + "--if-exists", + "--no-owner", + "--no-privileges", + "-d", + str(source_db.get("database") or ""), + "-f", + str(dump_path), + ] + env = os.environ.copy() + if str(settings.db_password or "").strip(): + env["PGPASSWORD"] = str(settings.db_password or "") + dump_completed = subprocess.run(dump_command, capture_output=True, text=True, env=env, timeout=3600) + if dump_completed.returncode != 0: + _append_migration_job_event( + job_id=job_id, + target_node_code=str(node.get("node_code") or ""), + event_type="migration_db_dump_failed", + level="error", + message="源数据库导出失败", + payload={ + "stdout": _truncate_output(dump_completed.stdout), + "stderr": _truncate_output(dump_completed.stderr), + }, + ) + return False, "源数据库导出失败", { + "stderr": str(dump_completed.stderr or "").strip(), + "stdout": str(dump_completed.stdout or "").strip(), + } + _append_migration_job_event( + job_id=job_id, + target_node_code=str(node.get("node_code") or ""), + event_type="migration_db_dump_completed", + level="success", + message="源 PostgreSQL 导出完成", + payload={ + "dump_path": str(dump_path), + "stdout": _truncate_output(dump_completed.stdout), + "stderr": _truncate_output(dump_completed.stderr), + }, + ) + + remote_dump_path = f"/tmp/domaincheck-migration-{int(time.time())}.sql" + _append_migration_job_event( + job_id=job_id, + target_node_code=str(node.get("node_code") or ""), + event_type="migration_db_upload_started", + message="开始上传数据库 dump 到目标机", + payload={"remote_dump_path": remote_dump_path}, + ) + upload_ok, upload_message = _upload_file(node, str(dump_path), remote_dump_path) + if not upload_ok: + _append_migration_job_event( + job_id=job_id, + target_node_code=str(node.get("node_code") or ""), + event_type="migration_db_upload_failed", + level="error", + message="数据库 dump 上传失败", + payload={"remote_dump_path": remote_dump_path, "message": upload_message}, + ) + return False, "数据库 dump 上传失败", {"message": upload_message} + _append_migration_job_event( + job_id=job_id, + target_node_code=str(node.get("node_code") or ""), + event_type="migration_db_upload_completed", + level="success", + message="数据库 dump 已上传到目标机", + payload={"remote_dump_path": remote_dump_path}, + ) + + backup_result = {} + target_db = dict(context.get("target_db_config") or {}) + can_backup_target = bool(target_db.get("database")) and bool(target_db.get("user")) and bool(target_db.get("host")) + wants_backup = bool(context.get("payload", {}).get("backup_target_database", True)) + if wants_backup and can_backup_target: + backup_result = _backup_remote_database(node, context) + if not bool(backup_result.get("ok", True)): + _append_migration_job_event( + job_id=job_id, + target_node_code=str(node.get("node_code") or ""), + event_type="migration_target_backup_failed", + level="error", + message="目标数据库备份失败", + payload=dict(backup_result or {}), + ) + return False, "目标数据库备份失败", backup_result + _append_migration_job_event( + job_id=job_id, + target_node_code=str(node.get("node_code") or ""), + event_type="migration_target_backup_completed", + level="success", + message="目标数据库备份完成", + payload=dict(backup_result or {}), + ) + + restore_result = _restore_remote_database(node, context, remote_dump_path) + if not bool(restore_result.get("ok", False)): + _append_migration_job_event( + job_id=job_id, + target_node_code=str(node.get("node_code") or ""), + event_type="migration_target_restore_failed", + level="error", + message="目标数据库恢复失败", + payload=dict(restore_result or {}), + ) + return False, "目标数据库恢复失败", restore_result + _append_migration_job_event( + job_id=job_id, + target_node_code=str(node.get("node_code") or ""), + event_type="migration_target_restore_completed", + level="success", + message="目标数据库恢复完成", + payload=dict(restore_result or {}), + ) + return True, "目标数据库已覆盖", { + "remote_dump_path": remote_dump_path, + "backup": backup_result, + "restore": restore_result, + } + + +def _backup_remote_database(node: dict, context: dict) -> dict: + target_db = dict(context.get("target_db_config") or {}) + backup_path = f"/tmp/domaincheck-target-backup-{int(time.time())}.sql" + password_export = "" + if str(target_db.get("password") or "").strip(): + password_export = f"export PGPASSWORD={shlex.quote(str(target_db.get('password') or ''))}\n" + script = "\n".join( + [ + "set -euo pipefail", + password_export.rstrip("\n"), + "mkdir -p /tmp", + "pg_dump " + f"-h {shlex.quote(str(target_db.get('host') or '127.0.0.1'))} " + f"-p {shlex.quote(str(target_db.get('port') or 5432))} " + f"-U {shlex.quote(str(target_db.get('user') or ''))} " + f"-d {shlex.quote(str(target_db.get('database') or ''))} " + f"-f {shlex.quote(backup_path)}", + f"echo {shlex.quote(backup_path)}", + ] + ) + ok, stdout, stderr, _meta = _run_remote_shell(node, script, timeout=1800) + return { + "ok": ok, + "backup_path": str(stdout or "").strip() if ok else "", + "stderr": str(stderr or "").strip(), + } + + +def _restore_remote_database(node: dict, context: dict, remote_dump_path: str) -> dict: + target_db = dict(context.get("target_db_config") or {}) + password_export = "" + if str(target_db.get("password") or "").strip(): + password_export = f"export PGPASSWORD={shlex.quote(str(target_db.get('password') or ''))}\n" + script = "\n".join( + [ + "set -euo pipefail", + password_export.rstrip("\n"), + f"cat {shlex.quote(remote_dump_path)} | psql " + f"-h {shlex.quote(str(target_db.get('host') or '127.0.0.1'))} " + f"-p {shlex.quote(str(target_db.get('port') or 5432))} " + f"-U {shlex.quote(str(target_db.get('user') or ''))} " + f"-d {shlex.quote(str(target_db.get('database') or ''))}", + "echo restored", + ] + ) + ok, stdout, stderr, _meta = _run_remote_shell(node, script, timeout=3600) + return { + "ok": ok, + "stdout": str(stdout or "").strip(), + "stderr": str(stderr or "").strip(), + } + + +def _build_remote_frontend(node: dict, context: dict, *, job_id: int = 0) -> tuple[bool, str, dict]: + repo_path = str(context.get("target_paths", {}).get("repo_path") or "") + install_script = "\n".join( + [ + "set -euo pipefail", + f"cd {shlex.quote(repo_path)}", + "if command -v npm >/dev/null 2>&1; then", + " cd domain-web", + " npm install", + " npm run build", + "else", + " echo 'npm missing' >&2", + " exit 127", + "fi", + ] + ) + _append_migration_job_event( + job_id=job_id, + target_node_code=str(node.get("node_code") or ""), + event_type="migration_frontend_build_started", + message="开始执行目标机前端构建", + payload={"repo_path": repo_path}, + ) + ok, stdout, stderr, _meta = _run_remote_shell(node, install_script, timeout=3600) + _append_migration_job_event( + job_id=job_id, + target_node_code=str(node.get("node_code") or ""), + event_type="migration_frontend_build_completed" if ok else "migration_frontend_build_failed", + level="success" if ok else "error", + message="前端构建完成" if ok else "前端构建失败", + payload={ + "repo_path": repo_path, + "stdout": _truncate_output(stdout), + "stderr": _truncate_output(stderr), + }, + ) + return ok, "前端构建完成" if ok else "前端构建失败", { + "stdout": _truncate_output(stdout), + "stderr": _truncate_output(stderr), + } + + +def _restart_remote_services(node: dict, *, job_id: int = 0) -> tuple[bool, str, dict]: + script = "\n".join( + [ + "set -euo pipefail", + "systemctl daemon-reload", + "systemctl enable postgresql domaincheck-api domaincheck-worker domaincheck-sync-agent >/dev/null 2>&1 || true", + "systemctl restart postgresql", + "systemctl restart domaincheck-api domaincheck-worker domaincheck-sync-agent", + "systemctl --no-pager --full status domaincheck-api domaincheck-worker domaincheck-sync-agent | sed -n '1,120p'", + ] + ) + _append_migration_job_event( + job_id=job_id, + target_node_code=str(node.get("node_code") or ""), + event_type="migration_service_restart_started", + message="开始刷新并重启目标机服务", + payload={}, + ) + ok, stdout, stderr, _meta = _run_remote_shell(node, script, timeout=300) + _append_migration_job_event( + job_id=job_id, + target_node_code=str(node.get("node_code") or ""), + event_type="migration_service_restart_completed" if ok else "migration_service_restart_failed", + level="success" if ok else "error", + message="远端服务已重启" if ok else "远端服务重启失败", + payload={ + "stdout": _truncate_output(stdout), + "stderr": _truncate_output(stderr), + }, + ) + return ok, "远端服务已重启" if ok else "远端服务重启失败", { + "stdout": _truncate_output(stdout), + "stderr": _truncate_output(stderr), + } + + +def _run_remote_health_check(node: dict, *, job_id: int = 0) -> tuple[bool, str, dict]: + script = "curl -fsS http://127.0.0.1:8100/health" + _append_migration_job_event( + job_id=job_id, + target_node_code=str(node.get("node_code") or ""), + event_type="migration_health_check_started", + message="开始执行目标机健康检查", + payload={}, + ) + ok, stdout, stderr, _meta = _run_remote_shell(node, script, timeout=60) + data = {} + try: + data = json.loads(stdout or "{}") + except Exception: + data = {"raw": str(stdout or "").strip()} + _append_migration_job_event( + job_id=job_id, + target_node_code=str(node.get("node_code") or ""), + event_type="migration_health_check_completed" if ok else "migration_health_check_failed", + level="success" if ok else "error", + message="远端健康检查通过" if ok else "远端健康检查失败", + payload={ + "response": data, + "stderr": _truncate_output(stderr), + }, + ) + return ok, "远端健康检查通过" if ok else "远端健康检查失败", { + "response": data, + "stderr": str(stderr or "").strip(), + } + + +def _load_node_secret(node: dict) -> dict: + from app.services.ops_runtime_executor_service import _load_ssh_secret # noqa: PLC0415 + + return dict(_load_ssh_secret(str(node.get("node_code") or "").strip()) or {}) + + +def _open_paramiko_client( + *, + ssh_host: str, + ssh_port: int, + ssh_user: str, + ssh_password: str = "", + ssh_private_key: str = "", +): + if paramiko is None: + raise RuntimeError("paramiko is not installed") + client = paramiko.SSHClient() + client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + connect_kwargs = { + "hostname": ssh_host, + "port": int(ssh_port), + "username": ssh_user, + "timeout": _SSH_CONNECT_TIMEOUT_SECONDS, + "banner_timeout": _SSH_CONNECT_TIMEOUT_SECONDS, + "auth_timeout": _SSH_CONNECT_TIMEOUT_SECONDS, + "look_for_keys": False, + "allow_agent": False, + } + if ssh_password: + connect_kwargs["password"] = ssh_password + elif ssh_private_key: + connect_kwargs["pkey"] = _load_private_key(ssh_private_key) + else: + connect_kwargs["look_for_keys"] = True + connect_kwargs["allow_agent"] = True + client.connect(**connect_kwargs) + return client + + +def _run_paramiko_command( + *, + ssh_host: str, + ssh_port: int, + ssh_user: str, + remote_command: str, + timeout_seconds: int, + ssh_password: str = "", + ssh_private_key: str = "", + stdin_data: bytes | None = None, +) -> subprocess.CompletedProcess: + client = _open_paramiko_client( + ssh_host=ssh_host, + ssh_port=ssh_port, + ssh_user=ssh_user, + ssh_password=ssh_password, + ssh_private_key=ssh_private_key, + ) + try: + transport = client.get_transport() + channel = transport.open_session(timeout=timeout_seconds) + channel.settimeout(timeout_seconds) + channel.exec_command(f"bash -lc {shlex.quote(remote_command)}") + if stdin_data: + channel.sendall(stdin_data) + channel.shutdown_write() + stdout = channel.makefile("rb").read().decode("utf-8", errors="replace") + stderr = channel.makefile_stderr("rb").read().decode("utf-8", errors="replace") + return subprocess.CompletedProcess( + args=["paramiko", f"{ssh_user}@{ssh_host}"], + returncode=int(channel.recv_exit_status()), + stdout=stdout, + stderr=stderr, + ) + finally: + client.close() + + +def _load_private_key(private_key_text: str): + if paramiko is None: + raise RuntimeError("paramiko is not installed") + key_text = str(private_key_text or "") + for key_cls in (paramiko.Ed25519Key, paramiko.RSAKey, paramiko.ECDSAKey, paramiko.DSSKey): + try: + return key_cls.from_private_key(StringIO(key_text)) + except Exception: + continue + raise ValueError("无法识别 SSH 私钥格式") + + +def _ensure_sftp_dir(sftp, remote_directory: str) -> None: + normalized = str(remote_directory or "").strip() + if not normalized or normalized == "/": + return + parts = [] + current = Path(normalized) + while str(current) not in {"", ".", "/"}: + parts.append(str(current)) + current = current.parent + for directory in reversed(parts): + try: + sftp.stat(directory) + except IOError: + sftp.mkdir(directory) + + +def _urlsafe_b64encode(value: bytes) -> str: + return base64.urlsafe_b64encode(value).decode("ascii").rstrip("=") + + +def _urlsafe_b64decode(value: str) -> bytes: + normalized = str(value or "").strip() + padding = "=" * (-len(normalized) % 4) + return base64.urlsafe_b64decode(normalized + padding) diff --git a/domain-api/app/services/ops_policy_service.py b/domain-api/app/services/ops_policy_service.py index 07e657a..9995520 100644 --- a/domain-api/app/services/ops_policy_service.py +++ b/domain-api/app/services/ops_policy_service.py @@ -41,6 +41,7 @@ _HIGH_RISK_ACTIONS = { _CRITICAL_RISK_ACTIONS = { "deploy.rollback", "node.bootstrap", + "migration.execute", "cluster.reconfigure", "runtime.reset_lab_state", } @@ -114,6 +115,13 @@ def _preview_action_payload_guardrails( if target_nodes_total > 1: recommendations.append("接管动作建议按单节点节奏推进，先确认首台节点接入成功后再继续放量。") + if action == "migration.execute": + if execution_mode != "control-plane": + blocking_reasons.append("migration.execute 仅支持 control-plane 执行方式。") + if bool(payload.get("overwrite_database", False)): + approval_reasons.append("迁移任务包含数据库覆盖，正式环境必须显式确认后再执行。") + recommendations.append("迁移属于长任务，建议通过后台任务窗口持续观察日志与健康检查结果。") + if action in _CRITICAL_RISK_ACTIONS: approval_reasons.append("该动作属于 critical 风险动作，正式环境必须审批。") elif action in _HIGH_RISK_ACTIONS: diff --git a/domain-api/app/services/ops_release_executor_core.py b/domain-api/app/services/ops_release_executor_core.py index 56c1730..0cf6cbe 100644 --- a/domain-api/app/services/ops_release_executor_core.py +++ b/domain-api/app/services/ops_release_executor_core.py @@ -11,6 +11,7 @@ import tarfile import textwrap import time import urllib.error +import urllib.parse import urllib.request from datetime import datetime from pathlib import Path @@ -23,6 +24,9 @@ _SYSTEMD_TEMPLATE_SPECS = { "domaincheck-worker": { "template": Path("domain-api/deploy/systemd/domain-worker.service"), }, + "domaincheck-worker@": { + "template": Path("domain-api/deploy/systemd/domain-worker@.service"), + }, "domaincheck-sync-agent": { "template": Path("domain-api/deploy/systemd/domain-sync-agent.service"), }, @@ -71,6 +75,78 @@ def normalize_release_health_check_services( return normalize_text_list(health_check_service_source) +def normalize_release_health_check_urls(raw_value: object) -> list[str]: + normalized_urls: list[str] = [] + for raw_item in normalize_text_list(raw_value): + normalized_item = str(raw_item or "").strip() + if not normalized_item: + continue + try: + parsed = urllib.parse.urlsplit(normalized_item) + except Exception: + normalized_urls.append(normalized_item) + continue + normalized_path = str(parsed.path or "").rstrip("/") + if normalized_path in {"/api/v1/runtime/status", "/runtime/status"}: + parsed = parsed._replace(path="/health", query="", fragment="") + normalized_item = urllib.parse.urlunsplit(parsed) + normalized_urls.append(normalized_item) + return normalized_urls + + +def _dedupe_service_names(service_names: list[str]) -> list[str]: + deduped: list[str] = [] + seen: set[str] = set() + for item in service_names: + normalized = str(item or "").strip() + if not normalized or normalized in seen: + continue + seen.add(normalized) + deduped.append(normalized) + return deduped + + +def _list_worker_instance_units(run_command, base_service_name: str) -> list[str]: + normalized_base = str(base_service_name or "").strip() + if normalized_base != "domaincheck-worker": + return [] + code, stdout, stderr = run_command( + [ + "systemctl", + "list-units", + "--type=service", + "--all", + "domaincheck-worker@*", + "--no-legend", + "--plain", + ], + timeout=30, + ) + raw_output = stdout if stdout.strip() else stderr + if int(code or 0) != 0 and not str(raw_output or "").strip(): + return [] + units: list[str] = [] + for line in str(raw_output or "").splitlines(): + parts = line.strip().split() + if not parts: + continue + unit_name = str(parts[0] or "").strip() + if unit_name: + units.append(unit_name) + return _dedupe_service_names(units) + + +def expand_release_service_units(run_command, service_names: list[str]) -> list[str]: + expanded: list[str] = [] + for item in list(service_names or []): + normalized = str(item or "").strip() + if not normalized: + continue + expanded.append(normalized) + expanded.extend(_list_worker_instance_units(run_command, normalized)) + return _dedupe_service_names(expanded) + + def collect_service_state(run_command, service_name: str) -> dict: code, stdout, stderr = run_command(["systemctl", "is-active", service_name], timeout=15) state = stdout or stderr @@ -295,7 +371,7 @@ def _systemd_dropin_content(service_name: str, install_root: str) -> str: "[Service]", f"WorkingDirectory={normalized_install_root}/current/domain-api", "ExecStart=", - f"ExecStart={normalized_install_root}/domainCheck/.venv/bin/python -m uvicorn app.main:app --host 0.0.0.0 --port 8100", + f"ExecStart={normalized_install_root}/domainCheck/.venv/bin/python -m uvicorn app.main:app --host 0.0.0.0 --port 8100 --timeout-graceful-shutdown 15", ] ) if normalized_service_name == "domaincheck-worker": @@ -411,7 +487,7 @@ def execute_release_action( install_root = Path(str(normalized_payload.get("install_root") or "/opt/domaincheck")).resolve() switch_current = coerce_bool(normalized_payload.get("switch_current", True), default=True) restart_services = normalize_text_list(normalized_payload.get("restart_services")) - health_check_urls = normalize_text_list(normalized_payload.get("health_check_urls")) + health_check_urls = normalize_release_health_check_urls(normalized_payload.get("health_check_urls")) health_check_services = normalize_release_health_check_services( normalized_payload, restart_services, @@ -457,14 +533,17 @@ def execute_release_action( "prepared_dirs": prepared_dirs, } + expanded_restart_services = expand_release_service_units(run_command, restart_services) + expanded_health_check_services = expand_release_service_units(run_command, health_check_services) + service_execstarts = [ collect_service_execstart(run_command, service_name) - for service_name in restart_services + for service_name in expanded_restart_services if str(service_name or "").strip() ] service_identities = [ collect_service_identity(run_command, service_name) - for service_name in restart_services + for service_name in expanded_restart_services if str(service_name or "").strip() ] current_link_text = str(current_link) @@ -645,7 +724,7 @@ def execute_release_action( ) restarted: list[dict] = [] - for service_name in restart_services: + for service_name in expanded_restart_services: normalized_service_name = str(service_name or "").strip() if not normalized_service_name: continue @@ -681,7 +760,7 @@ def execute_release_action( health_ok, health_result = run_release_health_checks( urls=health_check_urls, - services=health_check_services, + services=expanded_health_check_services, timeout=health_check_timeout_seconds, retries=health_check_retries, interval_seconds=health_check_interval_seconds, @@ -723,7 +802,7 @@ def execute_release_action( ) rollback_result["post_rollback_health"] = run_release_health_checks( urls=health_check_urls, - services=health_check_services, + services=expanded_health_check_services, timeout=health_check_timeout_seconds, retries=0, interval_seconds=0, @@ -766,6 +845,8 @@ def execute_release_action( "current_link": str(current_link), "previous_current_target": previous_current_target, "prepared_dirs": prepared_dirs, + "expanded_restart_services": expanded_restart_services, + "expanded_health_check_services": expanded_health_check_services, "execstart_alignment": execstart_alignment, "systemd_sync": systemd_sync_result, "daemon_reload": daemon_reload_result, @@ -784,6 +865,7 @@ def build_remote_release_action_script( normalize_text_list, coerce_bool, normalize_release_health_check_services, + normalize_release_health_check_urls, collect_service_state, check_health_url, run_release_health_checks, @@ -793,6 +875,9 @@ def build_remote_release_action_script( _pick_release_owner_group, apply_release_permissions, collect_service_execstart, + _dedupe_service_names, + _list_worker_instance_units, + expand_release_service_units, _write_text_file, _systemd_dropin_content, _sync_release_systemd_units, @@ -816,8 +901,11 @@ def build_remote_release_action_script( ) return f"""from __future__ import annotations +import grp import hashlib import json +import os +import pwd import shutil import tarfile import time diff --git a/domain-api/app/services/ops_release_service.py b/domain-api/app/services/ops_release_service.py index 49b5eb1..6fe82c6 100644 --- a/domain-api/app/services/ops_release_service.py +++ b/domain-api/app/services/ops_release_service.py @@ -12,6 +12,8 @@ from threading import Lock from urllib.parse import urlsplit, urlunsplit from uuid import uuid4 +from psycopg2 import errors + from app.core.db import get_db from app.services.build_info_service import get_runtime_build_info from app.services.ops_command_service import build_bash_command @@ -74,6 +76,19 @@ _ROLLOUT_INSPECTION_ACTION_KEYS = ("health.snapshot", "logs.collect", "diagnosti _SAFE_RELEASE_PACKAGE_NAME_RE = re.compile(r"^[A-Za-z0-9._-]+$") _RELEASE_SCHEMA_LOCK = Lock() _RELEASE_SCHEMA_READY = False +_RELEASE_SCHEMA_ADVISORY_LOCK_KEY = 90421803 +_RELEASE_REQUIRED_TABLES = ("ops_releases", "ops_release_rollouts") +_RELEASE_REQUIRED_COLUMNS = { + "ops_release_rollouts": ( + "rollout_code", + "target_nodes_json", + "batch_cursor", + "batches_total", + "jobs_total", + "jobs_created", + "result_summary_json", + ), +} def ensure_ops_release_schema() -> None: @@ -86,13 +101,52 @@ def ensure_ops_release_schema() -> None: if _RELEASE_SCHEMA_READY: return with get_db() as conn: - conn.autocommit = False with conn.cursor() as cur: - cur.execute(_RELEASE_SCHEMA_SQL) - conn.commit() + if _ops_release_schema_basics_present(cur): + _RELEASE_SCHEMA_READY = True + return + conn.autocommit = False + try: + with conn.cursor() as cur: + cur.execute("SELECT pg_advisory_xact_lock(%s)", (_RELEASE_SCHEMA_ADVISORY_LOCK_KEY,)) + cur.execute(_RELEASE_SCHEMA_SQL) + conn.commit() + except Exception as exc: + recoverable = isinstance(exc, (errors.DeadlockDetected, errors.LockNotAvailable)) + try: + conn.rollback() + except Exception: + pass + if not recoverable: + raise + with conn.cursor() as cur: + if not _ops_release_schema_basics_present(cur): + raise _RELEASE_SCHEMA_READY = True +def _ops_release_schema_basics_present(cur) -> bool: + for table_name in _RELEASE_REQUIRED_TABLES: + cur.execute("SELECT to_regclass(%s)", (f"public.{table_name}",)) + row = cur.fetchone() + if not row or not row[0]: + return False + + for table_name, required_columns in _RELEASE_REQUIRED_COLUMNS.items(): + cur.execute( + """ + SELECT column_name + FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = %s + """, + (table_name,), + ) + existing_columns = {str(row[0] or "").strip() for row in list(cur.fetchall() or [])} + if not set(required_columns).issubset(existing_columns): + return False + return True + + def _decode_json(value: object) -> dict: if isinstance(value, dict): return value @@ -1091,7 +1145,11 @@ def build_rollout_target_operational_readiness( desired_release: dict | None = None, ) -> dict: from app.services.cluster_runtime_service import get_cluster_snapshot - from app.services.ops_agent_service import ensure_ops_agent_schema, get_managed_node_onboarding + from app.services.ops_agent_service import ( + ensure_ops_agent_schema, + get_managed_node_onboarding, + list_managed_nodes_with_agent_state, + ) from app.services.ops_job_service import list_managed_nodes ensure_ops_agent_schema() @@ -1132,6 +1190,7 @@ def build_rollout_target_operational_readiness( for item in list(cluster_snapshot.get("nodes") or []) if str(item.get("node_code") or "").strip() } + managed_nodes_payload = list_managed_nodes_with_agent_state() managed_nodes = list_managed_nodes() managed_map = { str(item.get("node_code") or "").strip(): item @@ -1239,7 +1298,11 @@ def build_rollout_target_operational_readiness( last_seen_at = str(managed.get("last_seen_at") or "").strip() or str(metadata.get("last_seen_at") or "").strip() cluster_status = str(cluster_node.get("status") or target.get("status") or "").strip() current_load = int(cluster_node.get("current_load", target.get("current_load", 0)) or 0) - onboarding = get_managed_node_onboarding(node_code) if node_code else {} + onboarding = ( + get_managed_node_onboarding(node_code, nodes_payload=managed_nodes_payload) + if node_code + else {} + ) onboarding_stage = dict(onboarding.get("onboarding_stage") or {}) recovery_decision = dict(onboarding.get("recovery_decision") or {}) onboarding_stage_code = str(onboarding_stage.get("code") or "").strip() diff --git a/domain-api/app/services/ops_service.py b/domain-api/app/services/ops_service.py index f7a97cc..b922a11 100644 --- a/domain-api/app/services/ops_service.py +++ b/domain-api/app/services/ops_service.py @@ -4180,6 +4180,8 @@ def get_ops_activity_stream( status: str = "", execution_mode: str = "", query: str = "", + runtime_status: dict | None = None, + managed_nodes_payload: dict | None = None, ) -> dict: safe_limit = min(max(int(limit or _OPS_ACTIVITY_FETCH_LIMIT), 1), 100) safe_scan_limit = min(max(int(scan_limit or (safe_limit * 4)), safe_limit), 400) @@ -4187,12 +4189,18 @@ def get_ops_activity_stream( normalized_status = str(status or "").strip() normalized_execution_mode = str(execution_mode or "").strip() normalized_query = str(query or "").strip() - runtime_status = get_runtime_status() - execution_scene = _build_ops_execution_scene(runtime_status.get("detect") or {}) - managed_nodes_payload = list_managed_nodes_with_agent_state() + resolved_runtime_status = dict(runtime_status or {}) + if not resolved_runtime_status: + resolved_runtime_status = get_runtime_status() + execution_scene = _build_ops_execution_scene(resolved_runtime_status.get("detect") or {}) + resolved_managed_nodes_payload = dict(managed_nodes_payload or {}) + if not resolved_managed_nodes_payload: + resolved_managed_nodes_payload = list_managed_nodes_with_agent_state( + participation_payload=resolved_runtime_status.get("detect") or {} + ) managed_node_map = { str(item.get("node_code") or "").strip(): dict(item or {}) - for item in list(managed_nodes_payload.get("nodes") or []) + for item in list(resolved_managed_nodes_payload.get("nodes") or []) if str(item.get("node_code") or "").strip() } @@ -4219,7 +4227,10 @@ def get_ops_activity_stream( rollouts = list_release_rollouts(limit=safe_scan_limit) rollout_items = [_build_rollout_activity(rollout) for rollout in rollouts if int(rollout.get("id") or 0) > 0] - runbook = get_ops_runbook() + runbook = get_ops_runbook( + runtime_status=resolved_runtime_status, + managed_nodes_payload=resolved_managed_nodes_payload, + ) runbook_items = [ _build_runbook_sequence_activity(sequence) for sequence in list(runbook.get("control_sequences") or []) @@ -8099,7 +8110,11 @@ def get_ops_overview() -> dict: managed_nodes = list(managed_nodes_payload.get("nodes") or []) managed_nodes_summary = managed_nodes_payload.get("summary") or {} inspection_overview = get_ops_inspection_overview(managed_nodes=managed_nodes) - activity_stream = get_ops_activity_stream(limit=8) + activity_stream = get_ops_activity_stream( + limit=8, + runtime_status=runtime, + managed_nodes_payload=managed_nodes_payload, + ) release_summary = get_release_summary() preferred_release = _preferred_release_for_ops() release_launchpad = get_release_launchpad() @@ -11555,17 +11570,30 @@ def get_ops_blueprint() -> dict: } -def get_ops_runbook() -> dict: - runtime = get_runtime_status() +def get_ops_runbook( + *, + runtime_status: dict | None = None, + managed_nodes_payload: dict | None = None, + release_launchpad: dict | None = None, +) -> dict: + runtime = dict(runtime_status or {}) + if not runtime: + runtime = get_runtime_status() readiness = runtime.get("readiness") or {} worker_runtime = runtime.get("worker") or {} sync_agent_runtime = runtime.get("sync_agent") or {} - managed_nodes_payload = list_managed_nodes_with_agent_state(participation_payload=runtime.get("detect") or {}) - release_launchpad = get_release_launchpad() + resolved_managed_nodes_payload = dict(managed_nodes_payload or {}) + if not resolved_managed_nodes_payload: + resolved_managed_nodes_payload = list_managed_nodes_with_agent_state( + participation_payload=runtime.get("detect") or {} + ) + resolved_release_launchpad = dict(release_launchpad or {}) + if not resolved_release_launchpad: + resolved_release_launchpad = get_release_launchpad() control_sequences = _attach_ops_runbook_sequence_resolutions( _build_ops_runbook_control_sequences( - managed_nodes_payload=managed_nodes_payload, - release_launchpad=release_launchpad, + managed_nodes_payload=resolved_managed_nodes_payload, + release_launchpad=resolved_release_launchpad, ), requested_by="api/runbook", ) @@ -11606,6 +11634,6 @@ def get_ops_runbook() -> dict: "status": str(readiness.get("status") or ""), "summary": str(readiness.get("summary") or ""), }, - "release_launchpad": release_launchpad, + "release_launchpad": resolved_release_launchpad, "control_sequences": control_sequences, } diff --git a/domain-api/app/services/runtime_control_service.py b/domain-api/app/services/runtime_control_service.py index 1fc8de7..557eafd 100644 --- a/domain-api/app/services/runtime_control_service.py +++ b/domain-api/app/services/runtime_control_service.py @@ -211,14 +211,21 @@ def runtime_action(action: str, payload: dict | None = None) -> tuple[bool, str, _emit_runtime_action_event(normalized_action, stage="finished", ok=command_ok, message=command_message, data=result) return command_ok, command_message, result if normalized_action == "stop_detection": - command_ok, command_message = send_worker_command("stop_detection") + command_ok, command_message = send_worker_command( + "stop_detection", + payload={ + key: value + for key, value in normalized_payload.items() + if value not in (None, "") + }, + ) result = _build_runtime_action_result( action=normalized_action, poll_after_seconds=2, refresh_runtime=True, ok=command_ok, message=command_message, - data={}, + data={"payload": normalized_payload}, ) _emit_runtime_action_event(normalized_action, stage="finished", ok=command_ok, message=command_message, data=result) return command_ok, command_message, result diff --git a/domain-api/app/services/runtime_settings_service.py b/domain-api/app/services/runtime_settings_service.py index cec4b46..f473875 100644 --- a/domain-api/app/services/runtime_settings_service.py +++ b/domain-api/app/services/runtime_settings_service.py @@ -1,7 +1,10 @@ from __future__ import annotations +import json + from app.core.config import settings from app.core.files import read_runtime_json, write_runtime_json +from app.core.redis_client import get_redis DEFAULT_RUNTIME_SETTINGS = { @@ -11,13 +14,50 @@ DEFAULT_RUNTIME_SETTINGS = { "sync_agent_service_name": settings.sync_agent_service_name, "worker_log_sync_enabled": False, "worker_log_sync_mode": "key", + "control_node_autoresume_enabled": False, + "claim_recent_jobs_first": False, + "claim_recent_jobs_limit": 0, + "claim_recent_jobs_window_hours": 0, + "claim_batch_floor": 0, + "claim_batch_ceil": 0, + "submit_backlog_floor": 0, + "submit_backlog_ceil": 0, + "dispatch_cap_multiplier": 1, + "pending_buffer_cap_multiplier": 1, } +RUNTIME_SETTINGS_REDIS_KEY = "domain_tool:runtime_settings" +CONFIG_UPDATE_CHANNEL = "domain_tool:config_update" + def _normalize_worker_log_sync_mode(value: object) -> str: return "full" if str(value or "").strip().lower() == "full" else "key" +def _normalize_bool(value: object, default: bool = False) -> bool: + if value is None: + return bool(default) + if isinstance(value, bool): + return value + return str(value or "").strip().lower() not in {"", "0", "false", "no", "off"} + + +def _normalize_non_negative_int(value: object, default: int = 0) -> int: + try: + normalized = int(value) + except (TypeError, ValueError): + normalized = int(default) + return max(0, normalized) + + +def _normalize_positive_int(value: object, default: int = 1) -> int: + try: + normalized = int(value) + except (TypeError, ValueError): + normalized = int(default) + return max(1, normalized) + + def normalize_runtime_settings(payload: dict | None) -> dict: merged = dict(DEFAULT_RUNTIME_SETTINGS) if isinstance(payload, dict): @@ -33,8 +73,28 @@ def normalize_runtime_settings(payload: dict | None) -> dict: value = str(merged.get(key) or "").strip() merged[key] = value or DEFAULT_RUNTIME_SETTINGS[key] - merged["worker_log_sync_enabled"] = bool(merged.get("worker_log_sync_enabled", False)) + merged["worker_log_sync_enabled"] = _normalize_bool(merged.get("worker_log_sync_enabled", False), default=False) merged["worker_log_sync_mode"] = _normalize_worker_log_sync_mode(merged.get("worker_log_sync_mode")) + merged["control_node_autoresume_enabled"] = _normalize_bool( + merged.get("control_node_autoresume_enabled", False), + default=False, + ) + merged["claim_recent_jobs_first"] = _normalize_bool( + merged.get("claim_recent_jobs_first", False), + default=False, + ) + for key in ("claim_batch_floor", "claim_batch_ceil", "submit_backlog_floor", "submit_backlog_ceil"): + merged[key] = _normalize_non_negative_int(merged.get(key), DEFAULT_RUNTIME_SETTINGS[key]) + for key in ( + "claim_recent_jobs_limit", + "claim_recent_jobs_window_hours", + "dispatch_cap_multiplier", + "pending_buffer_cap_multiplier", + ): + if key in {"claim_recent_jobs_limit", "claim_recent_jobs_window_hours"}: + merged[key] = _normalize_non_negative_int(merged.get(key), DEFAULT_RUNTIME_SETTINGS[key]) + continue + merged[key] = _normalize_positive_int(merged.get(key), DEFAULT_RUNTIME_SETTINGS[key]) return merged @@ -43,7 +103,17 @@ def get_runtime_settings() -> dict: return normalize_runtime_settings(stored) +def _sync_runtime_settings_update(runtime_settings: dict) -> None: + try: + redis_client = get_redis() + redis_client.set(RUNTIME_SETTINGS_REDIS_KEY, json.dumps(runtime_settings, ensure_ascii=False)) + redis_client.publish(CONFIG_UPDATE_CHANNEL, "runtime_settings") + except Exception: + pass + + def update_runtime_settings(payload: dict) -> dict: merged = normalize_runtime_settings({**get_runtime_settings(), **(payload or {})}) write_runtime_json("runtime_settings.json", merged) + _sync_runtime_settings_update(merged) return merged diff --git a/domain-api/app/services/runtime_status_service.py b/domain-api/app/services/runtime_status_service.py index 3a3699a..bf65e9e 100644 --- a/domain-api/app/services/runtime_status_service.py +++ b/domain-api/app/services/runtime_status_service.py @@ -1,6 +1,10 @@ from __future__ import annotations +import json import os +import threading +import time +from datetime import datetime, timedelta from pathlib import Path from app.core.config import settings @@ -14,11 +18,48 @@ from app.services.detect_job_service import ( _load_latest_runtime_active_job_snapshot, get_detect_capacity_plan, get_detect_queue_health, + get_active_detect_job_summary, +) +from app.services.sync_record_service import ( + _pick_latest_projection_row, + append_runtime_projection_if_changed, + get_sync_summary, ) -from app.services.sync_record_service import append_runtime_projection_if_changed, get_sync_summary from app.services.runtime_settings_service import get_runtime_settings from app.services.worker_control_service import detect_sync_agent_runtime, detect_worker_runtime +_DOMAIN_INVENTORY_CACHE_LOCK = threading.Lock() +_DOMAIN_INVENTORY_CACHE_TTL_SECONDS = 30.0 +_DOMAIN_INVENTORY_CACHE_VALUE: dict | None = None +_DOMAIN_INVENTORY_CACHE_EXPIRES_AT = 0.0 +_RUNTIME_STATUS_CACHE_LOCK = threading.Lock() +_RUNTIME_STATUS_CACHE_TTL_SECONDS = 3.0 +_RUNTIME_STATUS_CACHE_VALUE: dict | None = None +_RUNTIME_STATUS_CACHE_EXPIRES_AT = 0.0 +_RUNTIME_STATUS_CACHE_SIGNATURE: tuple[object, ...] = () + + +def _clone_runtime_status_payload(value: dict | None) -> dict: + try: + return json.loads(json.dumps(dict(value or {}), ensure_ascii=False)) + except Exception: + return dict(value or {}) + + +def _runtime_status_cache_signature() -> tuple[object, ...]: + return ( + str(settings.node_code or ""), + str(settings.node_region or ""), + str(settings.node_role or ""), + int(settings.api_port or 0), + id(_build_runtime_detect_context), + id(detect_sync_agent_runtime), + id(append_runtime_projection_if_changed), + id(get_sync_summary), + id(_build_multi_region_readiness), + id(get_runtime_build_info), + ) + def _align_queue_health_with_backlog(queue_health: dict | None, backlog_snapshot: dict | None) -> dict: normalized = dict(queue_health or {}) @@ -70,9 +111,132 @@ def _decode_projection_payload(value: object) -> dict: return {} +def _build_backlog_snapshot_from_active_job(active_job: dict | None) -> dict: + normalized_job = dict(active_job or {}) + if not normalized_job: + return {} + + pending_total = max(0, int(normalized_job.get("items_pending", 0) or 0)) + claimed_total = max( + max( + int(normalized_job.get("items_claimed", 0) or 0), + int(normalized_job.get("display_items_claimed", 0) or 0), + ), + 0, + ) + running_total = max( + max( + int(normalized_job.get("items_running", 0) or 0), + int(normalized_job.get("display_items_running", 0) or 0), + ), + int(normalized_job.get("display_active_threads", 0) or 0), + 0, + ) + completed_total = max(0, int(normalized_job.get("items_completed", 0) or 0)) + blacklisted_total = max(0, int(normalized_job.get("items_blacklisted", 0) or 0)) + failed_total = max(0, int(normalized_job.get("items_failed", 0) or 0)) + + register_pending = 0 + downstream_pending = 0 + for raw_step in list(normalized_job.get("step_stats") or normalized_job.get("raw_step_stats") or []): + if not isinstance(raw_step, dict): + continue + step_code = str(raw_step.get("step_code") or raw_step.get("code") or "").strip() + step_pending = max( + int(raw_step.get("items_pending", raw_step.get("pending", 0)) or 0), + 0, + ) + if step_pending <= 0: + continue + if step_code == "detect_register": + register_pending += step_pending + else: + downstream_pending += step_pending + + if register_pending <= 0 and downstream_pending <= 0 and pending_total > 0: + downstream_pending = pending_total + + if ( + pending_total <= 0 + and claimed_total <= 0 + and running_total <= 0 + and completed_total <= 0 + and blacklisted_total <= 0 + and failed_total <= 0 + ): + return {} + + return { + "pending_total": pending_total, + "claimed_total": claimed_total, + "running_total": running_total, + "completed_total": completed_total, + "blacklisted_total": blacklisted_total, + "failed_total": failed_total, + "register_pending": register_pending, + "downstream_pending": downstream_pending, + } + + def _load_detect_backlog_snapshot() -> dict: + active_job_snapshot = _build_backlog_snapshot_from_active_job( + get_active_detect_job_summary(event_limit=1) + ) + if active_job_snapshot: + return active_job_snapshot + with get_db() as conn: with conn.cursor() as cur: + cur.execute( + """ + SELECT id, status, COALESCE(started_at, created_at) AS activity_at + FROM detect_jobs + WHERE status IN ('pending', 'running') + ORDER BY + CASE WHEN status = 'running' THEN 0 ELSE 1 END, + COALESCE(started_at, created_at) DESC, + id DESC + LIMIT %s + """, + (32,), + ) + job_rows = list(cur.fetchall() or []) + selected_job_ids: list[int] = [] + fallback_job_id = 0 + for raw_job_id, raw_status, raw_activity_at in job_rows: + try: + job_id = int(raw_job_id or 0) + except (TypeError, ValueError): + continue + if job_id <= 0: + continue + if fallback_job_id <= 0: + fallback_job_id = job_id + if job_id in selected_job_ids: + continue + status = str(raw_status or "").strip().lower() + keep = status == "running" + if not keep and raw_activity_at is not None: + now = datetime.now(raw_activity_at.tzinfo) if getattr(raw_activity_at, "tzinfo", None) else datetime.now() + keep = now - raw_activity_at <= timedelta(hours=6) + if not keep: + continue + selected_job_ids.append(job_id) + if len(selected_job_ids) >= 4: + break + if not selected_job_ids and fallback_job_id > 0: + selected_job_ids.append(fallback_job_id) + if not selected_job_ids: + return { + "pending_total": 0, + "claimed_total": 0, + "running_total": 0, + "completed_total": 0, + "blacklisted_total": 0, + "failed_total": 0, + "register_pending": 0, + "downstream_pending": 0, + } cur.execute( """ SELECT @@ -85,9 +249,9 @@ def _load_detect_backlog_snapshot() -> dict: COUNT(*) FILTER (WHERE item.status = 'pending' AND item.step_code = 'detect_register') AS register_pending, COUNT(*) FILTER (WHERE item.status = 'pending' AND item.step_code <> 'detect_register') AS downstream_pending FROM detect_job_items item - JOIN detect_jobs job ON job.id = item.job_id - WHERE job.status IN ('pending', 'running') - """ + WHERE item.job_id = ANY(%s) + """, + (selected_job_ids,), ) row = cur.fetchone() or (0, 0, 0, 0, 0, 0, 0, 0) return { @@ -109,17 +273,20 @@ def _load_latest_remote_runtime_projection_backlog() -> dict: with conn.cursor() as cur: cur.execute( """ - SELECT payload_json + SELECT payload_json, COALESCE(updated_at, created_at) FROM detect_sync_records - WHERE sync_type = 'runtime_projection' + WHERE sync_type IN ('runtime_ingest', 'runtime_projection') AND source_region = 'mainland' AND target_region = 'overseas' - AND status IN ('projected', 'pushing', 'synced') - ORDER BY updated_at DESC, id DESC - LIMIT 1 + AND status IN ('received', 'projected', 'pushing', 'synced') + ORDER BY + CASE WHEN sync_type = 'runtime_ingest' THEN 0 ELSE 1 END ASC, + COALESCE(updated_at, created_at) DESC, + id DESC + LIMIT 50 """ ) - row = cur.fetchone() + row = _pick_latest_projection_row(list(cur.fetchall() or []), created_at_index=1) if not row: return {} payload = _decode_projection_payload(row[0]) @@ -155,6 +322,458 @@ def _merge_backlog_snapshots(primary: dict, secondary: dict) -> dict: return merged +def _safe_int_value(value: object, default: int = 0) -> int: + try: + return int(value or 0) + except Exception: + return int(default) + + +def _safe_float_value(value: object, default: float = 0.0) -> float: + try: + return float(value or 0.0) + except Exception: + return float(default) + + +def _safe_text_value(value: object) -> str: + return str(value or "").strip() + + +def _load_domain_inventory_summary() -> dict: + global _DOMAIN_INVENTORY_CACHE_VALUE, _DOMAIN_INVENTORY_CACHE_EXPIRES_AT + + now_ts = time.monotonic() + with _DOMAIN_INVENTORY_CACHE_LOCK: + if _DOMAIN_INVENTORY_CACHE_VALUE is not None and now_ts < _DOMAIN_INVENTORY_CACHE_EXPIRES_AT: + return dict(_DOMAIN_INVENTORY_CACHE_VALUE) + + authoritative = bool(settings.node_region == "overseas" and settings.node_role == "control") + summary = { + "scope_label": "海外主库总盘子" if authoritative else "当前节点本地库存", + "authoritative": authoritative, + "domains_total": 0, + "pending_total": 0, + "completed_total": 0, + "running_total": 0, + "blacklist_total": 0, + "failed_total": 0, + "processed_total": 0, + "remaining_total": 0, + } + try: + with get_db() as conn: + with conn.cursor() as cur: + cur.execute( + """ + SELECT + COUNT(*) AS domains_total, + COUNT(*) FILTER (WHERE detect_status = 0) AS pending_total, + COUNT(*) FILTER (WHERE detect_status = 1) AS completed_total, + COUNT(*) FILTER (WHERE detect_status = 2) AS running_total, + COUNT(*) FILTER (WHERE detect_status = 3) AS blacklist_total, + COUNT(*) FILTER (WHERE detect_status = 4) AS failed_total + FROM domains + """ + ) + row = cur.fetchone() or (0, 0, 0, 0, 0, 0) + summary.update( + { + "domains_total": int(row[0] or 0), + "pending_total": int(row[1] or 0), + "completed_total": int(row[2] or 0), + "running_total": int(row[3] or 0), + "blacklist_total": int(row[4] or 0), + "failed_total": int(row[5] or 0), + } + ) + summary["processed_total"] = ( + int(summary["completed_total"]) + + int(summary["blacklist_total"]) + + int(summary["failed_total"]) + ) + summary["remaining_total"] = int(summary["pending_total"]) + int(summary["running_total"]) + except Exception as exc: + summary["error"] = str(exc) + + with _DOMAIN_INVENTORY_CACHE_LOCK: + _DOMAIN_INVENTORY_CACHE_VALUE = dict(summary) + _DOMAIN_INVENTORY_CACHE_EXPIRES_AT = time.monotonic() + _DOMAIN_INVENTORY_CACHE_TTL_SECONDS + return dict(summary) + + +def _build_detect_observation_top_nodes(participating_nodes: list[dict] | None) -> list[dict]: + rows: list[dict] = [] + for raw_item in list(participating_nodes or []): + item = dict(raw_item or {}) + node_code = _safe_text_value(item.get("node_code")) + if not node_code: + continue + current_load = max( + _safe_int_value(item.get("current_load")), + _safe_int_value(item.get("active_threads")), + _safe_int_value(item.get("items_running")), + _safe_int_value(item.get("items_claimed")), + ) + processed_recent = _safe_int_value(item.get("processed_recent")) + if current_load <= 0 and processed_recent <= 0 and not bool(item.get("is_current_participant", False)): + continue + rows.append( + { + "node_code": node_code, + "participation_label": _safe_text_value(item.get("participation_label")) or "参与中", + "items_claimed": _safe_int_value(item.get("items_claimed")), + "items_running": _safe_int_value(item.get("items_running")), + "active_threads": _safe_int_value(item.get("active_threads")), + "max_threads": _safe_int_value(item.get("max_threads")), + "processed_recent": processed_recent, + "processed_per_minute": round(_safe_float_value(item.get("processed_per_minute")), 2), + "current_load": current_load, + } + ) + rows.sort( + key=lambda item: ( + -int(item.get("current_load", 0) or 0), + -int(item.get("processed_recent", 0) or 0), + -int(item.get("items_claimed", 0) or 0), + str(item.get("node_code") or ""), + ) + ) + return rows[:6] + + +def _build_detect_observation_top_steps(detect_payload: dict) -> list[dict]: + queue_steps = list(((detect_payload.get("queue_health") or {}).get("steps") or [])) + if not queue_steps: + queue_steps = list(((detect_payload.get("active_job") or {}).get("step_stats") or [])) + rows: list[dict] = [] + for raw_item in queue_steps: + item = dict(raw_item or {}) + step_code = _safe_text_value(item.get("step_code") or item.get("code")) + if not step_code: + continue + rows.append( + { + "step_code": step_code, + "step_name": _safe_text_value(item.get("step_name")) or step_code, + "pending": max( + _safe_int_value(item.get("items_pending")), + _safe_int_value(item.get("pending")), + ), + "running": max( + _safe_int_value(item.get("items_running")), + _safe_int_value(item.get("running")), + ), + "processed_recent": _safe_int_value(item.get("processed_recent")), + "failed_recent": _safe_int_value(item.get("failed_recent")), + "blacklisted_recent": _safe_int_value(item.get("blacklisted_recent")), + } + ) + rows.sort( + key=lambda item: ( + -int(item.get("pending", 0) or 0), + -int(item.get("running", 0) or 0), + -int(item.get("processed_recent", 0) or 0), + str(item.get("step_name") or ""), + ) + ) + return [item for item in rows if item["pending"] > 0 or item["running"] > 0 or item["processed_recent"] > 0][:6] + + +def _build_detect_observation_summary( + *, + detect_payload: dict, + cluster_snapshot: dict, + inventory_summary: dict | None = None, +) -> dict: + normalized_detect = dict(detect_payload or {}) + active_job = dict(normalized_detect.get("active_job") or {}) + queue_health = dict(normalized_detect.get("queue_health") or {}) + queue = dict(queue_health.get("queue") or {}) + throughput = dict(queue_health.get("throughput") or {}) + participation_summary = dict(normalized_detect.get("participation_summary") or {}) + source_inventory = dict(inventory_summary or {}) + participating_nodes = [ + dict(item) + for item in list(normalized_detect.get("participating_nodes") or []) + if isinstance(item, dict) + ] + top_active_nodes = _build_detect_observation_top_nodes(participating_nodes) + top_steps = _build_detect_observation_top_steps(normalized_detect) + + job_id = _safe_int_value(active_job.get("job_id")) + job_code = _safe_text_value(active_job.get("job_code")) + job_status = _safe_text_value(active_job.get("status")) or "-" + progress_percent = round( + _safe_float_value(active_job.get("progress_percent"), _safe_float_value(normalized_detect.get("progress_percent"))), + 2, + ) + + pending = max( + _safe_int_value(active_job.get("items_pending")), + _safe_int_value(queue.get("pending")), + _safe_int_value((normalized_detect.get("backlog") or {}).get("pending_total")), + ) + claimed = max( + _safe_int_value(active_job.get("display_items_claimed")), + _safe_int_value(active_job.get("items_claimed")), + _safe_int_value(queue.get("display_claimed")), + _safe_int_value(queue.get("claimed")), + _safe_int_value((normalized_detect.get("backlog") or {}).get("claimed_total")), + ) + running = max( + _safe_int_value(active_job.get("display_items_running")), + _safe_int_value(active_job.get("display_active_threads")), + _safe_int_value(active_job.get("items_running")), + _safe_int_value(queue.get("display_running")), + _safe_int_value(queue.get("running")), + _safe_int_value((normalized_detect.get("backlog") or {}).get("running_total")), + ) + completed = max( + _safe_int_value(active_job.get("items_completed")), + _safe_int_value(queue.get("completed")), + _safe_int_value((normalized_detect.get("backlog") or {}).get("completed_total")), + ) + failed = max( + _safe_int_value(active_job.get("items_failed")), + _safe_int_value(queue.get("failed")), + _safe_int_value((normalized_detect.get("backlog") or {}).get("failed_total")), + ) + blacklisted = max( + _safe_int_value(active_job.get("items_blacklisted")), + _safe_int_value(queue.get("blacklisted")), + _safe_int_value((normalized_detect.get("backlog") or {}).get("blacklisted_total")), + ) + effective_items_total = max( + _safe_int_value(active_job.get("items_total")), + _safe_int_value(queue.get("items_total")), + pending + claimed + running + completed + failed + blacklisted, + ) + raw_items_total = max( + _safe_int_value(active_job.get("raw_items_total")), + effective_items_total, + ) + raw_pending = max(_safe_int_value(active_job.get("raw_items_pending")), pending) + raw_claimed = max(_safe_int_value(active_job.get("raw_items_claimed")), claimed) + raw_running = max(_safe_int_value(active_job.get("raw_items_running")), running) + raw_completed = max(_safe_int_value(active_job.get("raw_items_completed")), completed) + raw_failed = max(_safe_int_value(active_job.get("raw_items_failed")), failed) + raw_blacklisted = max(_safe_int_value(active_job.get("raw_items_blacklisted")), blacklisted) + + derived_active_processes = sum( + 1 + for item in participating_nodes + if bool(item.get("is_dispatch_active", False)) + or _safe_int_value(item.get("items_running")) > 0 + or _safe_int_value(item.get("items_claimed")) > 0 + or _safe_int_value(item.get("active_threads")) > 0 + ) + if derived_active_processes <= 0: + derived_active_processes = max( + _safe_int_value(participation_summary.get("dispatch_active_nodes")), + _safe_int_value(normalized_detect.get("aggregate_process_count")), + ) + active_threads = max( + _safe_int_value(normalized_detect.get("active_thread_count")), + _safe_int_value(active_job.get("display_active_threads")), + _safe_int_value(active_job.get("display_items_running")), + _safe_int_value(queue.get("display_running")), + _safe_int_value(queue.get("running")), + ) + active_max_threads = max( + _safe_int_value(normalized_detect.get("max_thread_count")), + _safe_int_value(active_job.get("display_max_threads")), + sum(_safe_int_value(item.get("max_threads")) for item in top_active_nodes if _safe_int_value(item.get("max_threads")) > 0), + ) + thread_utilization_percent = round((active_threads / active_max_threads) * 100, 2) if active_max_threads > 0 else 0.0 + + processed_recent = max( + _safe_int_value(active_job.get("processed_recent")), + _safe_int_value(throughput.get("processed_recent")), + ) + processed_per_minute = max( + round(_safe_float_value(active_job.get("processed_per_minute")), 2), + round(_safe_float_value(throughput.get("processed_per_minute")), 2), + ) + completed_recent = max( + _safe_int_value(active_job.get("completed_recent")), + _safe_int_value(throughput.get("completed_recent")), + ) + failed_recent = max( + _safe_int_value(active_job.get("failed_recent")), + _safe_int_value(throughput.get("failed_recent")), + ) + blacklisted_recent = max( + _safe_int_value(active_job.get("blacklisted_recent")), + _safe_int_value(throughput.get("blacklisted_recent")), + ) + + online_worker_nodes = _safe_int_value(((cluster_snapshot.get("summary") or {}).get("online_worker_nodes"))) + participating_node_count = max( + _safe_int_value(participation_summary.get("participating_nodes")), + len(participating_nodes), + ) + dispatch_active_nodes = max( + _safe_int_value(participation_summary.get("dispatch_active_nodes")), + derived_active_processes, + ) + source_domains_total = _safe_int_value(source_inventory.get("domains_total")) + source_pending_total = _safe_int_value(source_inventory.get("pending_total")) + source_completed_total = _safe_int_value(source_inventory.get("completed_total")) + source_running_total = _safe_int_value(source_inventory.get("running_total")) + source_blacklist_total = _safe_int_value(source_inventory.get("blacklist_total")) + source_failed_total = _safe_int_value(source_inventory.get("failed_total")) + source_remaining_total = _safe_int_value(source_inventory.get("remaining_total")) + + if job_id <= 0 and not job_code and pending <= 0 and claimed <= 0 and running <= 0 and active_threads <= 0: + state = "idle" + state_label = "当前空闲" + state_reason = "当前没有活跃检测任务，执行面处于待命或暂时没有可观察样本。" + elif derived_active_processes > 0 and active_threads > 0 and processed_recent > 0: + state = "running" + state_label = "真跑中" + state_reason = "已经看到真实执行进程、活跃线程和近窗吞吐，不是只剩日志残影。" + elif derived_active_processes > 0 and active_threads > 0 and max(running, claimed) > 0: + state = "slow" + state_label = "在跑但偏慢" + state_reason = "执行面已经起来了，但最近吞吐还没完全拉起来，瓶颈更像外部步骤或领取效率。" + elif pending > 0 and derived_active_processes <= 0 and active_threads <= 0 and processed_recent <= 0: + state = "not_running" + state_label = "没跑起来" + state_reason = "队列里还有积压，但当前没看到真实执行进程和线程在持续消化。" + elif pending > 0 and processed_recent <= 0: + state = "stale" + state_label = "疑似残影" + state_reason = "还能看到积压或运行中计数，但最近没有吞吐增量，需要继续查执行链或状态回传。" + else: + state = "watching" + state_label = "正在观察" + state_reason = "当前已有部分运行信号，但还需要继续观察吞吐和结果产出是否稳定。" + + if source_domains_total > 0 and raw_items_total > 0: + scope_hint = ( + f"{_safe_text_value(source_inventory.get('scope_label')) or '当前总盘子'} {source_domains_total}；" + f"当前活跃批次原始 {raw_items_total} 项，展示口径 {effective_items_total} 项，不等于全盘累计。" + ) + elif source_domains_total > 0: + scope_hint = ( + f"{_safe_text_value(source_inventory.get('scope_label')) or '当前总盘子'} {source_domains_total}；" + "当前还没有可对齐的活跃批次。" + ) + else: + scope_hint = "当前总盘子摘要暂时不可用。" + + if failed_recent > max(completed_recent, blacklisted_recent) and failed_recent > 0: + focus_hint = ( + f"最近失败 {failed_recent} 明显高于完成 {completed_recent}，更像外部步骤超时、代理/RDAP 或链路异常。" + ) + elif blacklisted_recent > 0 and blacklisted_recent >= failed_recent: + focus_hint = f"最近黑名单命中 {blacklisted_recent}，说明当前推进里有一部分是被规则直接拦截。" + elif pending > 0 and active_threads <= 0: + focus_hint = "队列还有积压，但当前看不到真实执行线程，先别盯日志，先盯任务领取和 worker 存活。" + elif active_max_threads > 0 and active_threads > 0 and thread_utilization_percent < 10: + focus_hint = f"线程利用率只有 {thread_utilization_percent}% ，执行面没有吃满，瓶颈更像外部链路或任务领取。" + elif processed_recent > 0: + focus_hint = f"近 15 分钟已处理 {processed_recent} 项，约 {processed_per_minute} 项/分钟。" + else: + focus_hint = "当前还没有明显的近窗吞吐样本，继续盯任务是否持续出结果。" + + summary_lines = [ + ( + f"{_safe_text_value(source_inventory.get('scope_label')) or '总盘子'} {source_domains_total}，" + f"库存待检测 {source_pending_total} / 已通过 {source_completed_total} / 运行中 {source_running_total} / " + f"失败 {source_failed_total} / 黑名单 {source_blacklist_total}。" + ), + ( + f"当前任务 {job_code or '-'}，展示 {effective_items_total} 项 / 原始 {raw_items_total} 项；" + f"待 {pending} / 领 {claimed} / 跑 {running} / 完 {completed} / 失败 {failed} / 黑名单 {blacklisted}。" + ), + ( + f"真实执行面 {derived_active_processes} 个进程，活跃线程 {active_threads} / {active_max_threads}，" + f"线程利用率 {thread_utilization_percent}%。" + ), + ( + f"近 15 分钟处理 {processed_recent} 项，约 {processed_per_minute} 项/分钟，" + f"其中完成 {completed_recent}、失败 {failed_recent}、黑名单 {blacklisted_recent}。" + ), + ] + + return { + "state": state, + "state_label": state_label, + "state_reason": state_reason, + "scope_hint": scope_hint, + "focus_hint": focus_hint, + "source_inventory": { + "scope_label": _safe_text_value(source_inventory.get("scope_label")) or "当前总盘子", + "authoritative": bool(source_inventory.get("authoritative", False)), + "domains_total": source_domains_total, + "pending_total": source_pending_total, + "completed_total": source_completed_total, + "running_total": source_running_total, + "blacklist_total": source_blacklist_total, + "failed_total": source_failed_total, + "processed_total": _safe_int_value(source_inventory.get("processed_total")), + "remaining_total": source_remaining_total, + "error": _safe_text_value(source_inventory.get("error")), + }, + "job": { + "job_id": job_id, + "job_code": job_code, + "status": job_status, + "progress_percent": progress_percent, + }, + "active_batch": { + "job_id": job_id, + "job_code": job_code, + "status": job_status, + "started_at": _safe_text_value(active_job.get("started_at")), + "progress_percent": progress_percent, + "effective_items_total": effective_items_total, + "raw_items_total": raw_items_total, + "pending": pending, + "claimed": claimed, + "running": running, + "completed": completed, + "failed": failed, + "blacklisted": blacklisted, + "raw_pending": raw_pending, + "raw_claimed": raw_claimed, + "raw_running": raw_running, + "raw_completed": raw_completed, + "raw_failed": raw_failed, + "raw_blacklisted": raw_blacklisted, + }, + "backlog": { + "pending": pending, + "claimed": claimed, + "running": running, + "completed": completed, + "failed": failed, + "blacklisted": blacklisted, + }, + "execution": { + "active_processes": derived_active_processes, + "dispatch_active_nodes": dispatch_active_nodes, + "participating_nodes": participating_node_count, + "online_worker_nodes": online_worker_nodes, + "active_threads": active_threads, + "max_threads": active_max_threads, + "thread_utilization_percent": thread_utilization_percent, + }, + "throughput": { + "processed_recent": processed_recent, + "processed_per_minute": processed_per_minute, + "completed_recent": completed_recent, + "failed_recent": failed_recent, + "blacklisted_recent": blacklisted_recent, + }, + "top_active_nodes": top_active_nodes, + "top_steps": top_steps, + "summary_lines": summary_lines, + } + + def _runtime_log_path(filename: str) -> str: path = Path(__file__).resolve().parents[2] / "runtime" / "logs" / filename return str(path) @@ -192,6 +811,7 @@ def _build_multi_region_readiness( summary = cluster_snapshot.get("summary") or {} batch_summary = (sync_summary.get("detect_result_batches") or {}) batch_states = batch_summary.get("state_counts") or {} + batch_applicable = bool(batch_summary.get("applicable", True)) online_control_nodes = int(summary.get("online_control_nodes", 0) or 0) online_worker_nodes = int(summary.get("online_worker_nodes", 0) or 0) @@ -274,14 +894,16 @@ def _build_multi_region_readiness( projected_batches = int(batch_states.get("projected", 0) or 0) pushing_batches = int(batch_states.get("pushing", 0) or 0) synced_batches = int(batch_states.get("synced", 0) or 0) - if failed_batches > 0: + if failed_batches > 0 and batch_applicable: warning_issues.append(f"存在 {failed_batches} 个结果批次同步失败，需要检查 sync-agent 或目标接收面。") - if projected_batches > 0: + if projected_batches > 0 and batch_applicable: warning_issues.append(f"存在 {projected_batches} 个结果批次仍待推送。") - if pushing_batches > 0: + if pushing_batches > 0 and batch_applicable: info_items.append(f"当前有 {pushing_batches} 个结果批次正在推送。") - if synced_batches > 0: + if synced_batches > 0 and batch_applicable: info_items.append(f"最近已接收 {synced_batches} 个结果批次。") + if not batch_applicable: + info_items.append(str(batch_summary.get("reason") or "当前节点不承载本地结果批次推送。")) if worker_runtime.get("running", False): info_items.append("当前节点本机 Worker 进程在线。") @@ -317,8 +939,10 @@ def _build_multi_region_readiness( }, "sync": { "enabled": bool(sync_summary.get("enabled", False)), + "push_expected_on_this_node": bool(sync_summary.get("push_expected_on_this_node", False)), "source_region": sync_summary.get("source_region", ""), "target_region": sync_summary.get("target_region", ""), + "applicable": batch_applicable, "projected_batches": projected_batches, "pushing_batches": pushing_batches, "failed_batches": failed_batches, @@ -647,23 +1271,67 @@ def _build_detect_participation_summary( } -def get_runtime_status() -> dict: +def _merge_detect_payload_with_queue_health(detect_payload: dict, queue_health: dict) -> dict: + normalized_detect = dict(detect_payload or {}) + normalized_queue_health = dict(queue_health or {}) + if not normalized_queue_health.get("has_active_job"): + return normalized_detect + + queue_payload = dict(normalized_queue_health.get("queue") or {}) + queue_job = dict(normalized_queue_health.get("job") or {}) + queue_nodes = [dict(item) for item in list(normalized_queue_health.get("nodes") or []) if isinstance(item, dict)] + active_nodes = [ + item + for item in queue_nodes + if int(item.get("display_running", item.get("items_running", 0)) or 0) > 0 + or int(item.get("items_claimed", 0) or 0) > 0 + or int(item.get("active_threads", 0) or 0) > 0 + ] + display_max_threads = sum(int(item.get("max_threads", 0) or 0) for item in active_nodes or queue_nodes) + normalized_detect["active_job"] = { + **dict(normalized_detect.get("active_job") or {}), + "job_id": queue_job.get("job_id"), + "job_code": queue_job.get("job_code", ""), + "status": queue_job.get("status", ""), + "progress_percent": float(queue_job.get("progress_percent", 0) or 0), + "items_total": int(queue_payload.get("items_total", 0) or 0), + "items_pending": int(queue_payload.get("pending", 0) or 0), + "items_claimed": int(queue_payload.get("claimed", 0) or 0), + "items_running": int(queue_payload.get("running", 0) or 0), + "items_completed": int(queue_payload.get("completed", 0) or 0), + "items_blacklisted": int(queue_payload.get("blacklisted", 0) or 0), + "items_failed": int(queue_payload.get("failed", 0) or 0), + "display_items_running": int(queue_payload.get("display_running", queue_payload.get("running", 0)) or 0), + "display_active_threads": int(queue_payload.get("display_running", queue_payload.get("running", 0)) or 0), + "display_max_threads": int(display_max_threads or 0), + "node_stats": list(queue_nodes), + "distributed_node_stats": list(queue_nodes), + } + normalized_detect["active_thread_count"] = int(queue_payload.get("display_running", queue_payload.get("running", 0)) or 0) + normalized_detect["max_thread_count"] = int(display_max_threads or normalized_detect.get("max_thread_count", 0) or 0) + normalized_detect["aggregate_participating_node_count"] = len(active_nodes) + normalized_detect["aggregate_participating_node_codes"] = [str(item.get("node_code") or "") for item in active_nodes] + normalized_detect["aggregate_process_count"] = len(active_nodes) + normalized_detect["queue_health"] = normalized_queue_health + return normalized_detect + + +def _build_runtime_detect_context(*, window_minutes: int = 15) -> dict: runtime_settings = get_runtime_settings() worker_runtime = detect_worker_runtime() - sync_agent_runtime = detect_sync_agent_runtime() worker_expected_on_this_node = not (settings.node_region == "overseas" and settings.node_role == "control") - api_pid = os.getpid() + inventory_summary = _load_domain_inventory_summary() detect_snapshot = get_detect_status() latest_run = (detect_snapshot.get("runs") or [None])[0] or {} cluster_snapshot = get_cluster_snapshot() - queue_health = get_detect_queue_health(window_minutes=15) + queue_health = get_detect_queue_health(window_minutes=window_minutes) effective_online_worker_nodes = int((cluster_snapshot.get("summary") or {}).get("online_worker_nodes", 0) or 0) if effective_online_worker_nodes <= 0 and worker_runtime.get("running", False): effective_online_worker_nodes = max(1, worker_runtime.get("process_count", 1) or 1) backlog_snapshot = _load_detect_backlog_snapshot() remote_backlog_snapshot = _load_latest_remote_runtime_projection_backlog() - runtime_snapshot_backlog = dict(_load_latest_runtime_active_job_snapshot(15).get("backlog") or {}) + runtime_snapshot_backlog = dict(_load_latest_runtime_active_job_snapshot(window_minutes).get("backlog") or {}) backlog_snapshot = _merge_backlog_snapshots(backlog_snapshot, remote_backlog_snapshot) backlog_snapshot = _merge_backlog_snapshots(backlog_snapshot, runtime_snapshot_backlog) capacity_plan = get_detect_capacity_plan( @@ -680,6 +1348,11 @@ def get_runtime_status() -> dict: "progress": detect_snapshot.get("progress", {}), "active_thread_count": detect_snapshot.get("active_thread_count", 0), "max_thread_count": detect_snapshot.get("max_thread_count", 0), + "aggregate_process_count": detect_snapshot.get("aggregate_process_count", 0), + "aggregate_participating_node_count": detect_snapshot.get("aggregate_participating_node_count", 0), + "aggregate_participating_node_codes": detect_snapshot.get("aggregate_participating_node_codes", []), + "aggregate_max_thread_count": detect_snapshot.get("aggregate_max_thread_count", 0), + "aggregate_thread_count_per_process": detect_snapshot.get("aggregate_thread_count_per_process", 0), "available_proxy_count": detect_snapshot.get("available_proxy_count", 0), "proxy_pool_count": detect_snapshot.get("proxy_pool_count", 0), "proxy_runtime_label": detect_snapshot.get("proxy_runtime_label", ""), @@ -698,6 +1371,7 @@ def get_runtime_status() -> dict: "runs_count": len(detect_snapshot.get("runs") or []), "worker_online": worker_runtime.get("running", False), "worker_mode": worker_runtime.get("mode", runtime_settings.get("worker_mode", "windows-local")), + "aggregate_detect_view": bool(detect_snapshot.get("aggregate_detect_view", False)), "queue_health": queue_health, "backlog": backlog_snapshot, "capacity_plan": capacity_plan, @@ -712,17 +1386,28 @@ def get_runtime_status() -> dict: "last_line": str(detect_snapshot.get("remote_log_last_line") or ""), "preview_lines": list(detect_snapshot.get("remote_log_lines") or [])[-20:], }, + "_detect_snapshot": detect_snapshot, } + if settings.node_region == "overseas" and settings.node_role == "control": + detect_payload = _merge_detect_payload_with_queue_health(detect_payload, queue_health) + aggregate_remote_detect = bool(detect_payload.get("aggregate_detect_view") and detect_payload.get("active_job")) if not worker_expected_on_this_node: detect_payload.update( { - "phase_label": "当前节点不承载", - "phase_detail": "当前节点为海外控制面，仅承载 API 控制与同步接收，不执行本机检测任务。", - "recent_event": "", + "phase_label": ( + detect_snapshot.get("phase_label") + or ("集群执行中" if aggregate_remote_detect else "当前节点不承载") + ), + "phase_detail": ( + "当前节点为海外控制面，仅承载 API 控制与同步接收，不执行本机检测任务。" + + ( + f" 当前汇总执行由 {int(detect_payload.get('aggregate_participating_node_count', 0) or 0)} 台参与节点、" + f"{int(detect_payload.get('aggregate_process_count', 0) or 0)} 个检测进程承担。" + if aggregate_remote_detect + else "" + ) + ), "recent_warning": "", - "progress_percent": 0, - "active_thread_count": 0, - "max_thread_count": 0, "available_proxy_count": 0, "proxy_pool_count": 0, "proxy_runtime_label": "不适用", @@ -737,11 +1422,20 @@ def get_runtime_status() -> dict: "proxy_last_available_count": 0, "proxy_source_stats": [], "dependency_alerts": [], - "active_job": None, - "runs_count": 0, "worker_online": False, } ) + if not aggregate_remote_detect: + detect_payload.update( + { + "recent_event": "", + "progress_percent": 0, + "active_thread_count": 0, + "max_thread_count": 0, + "active_job": None, + "runs_count": 0, + } + ) detect_payload["participating_nodes"] = _build_participating_detect_nodes( cluster_snapshot=cluster_snapshot, detect_snapshot=detect_payload, @@ -757,6 +1451,183 @@ def get_runtime_status() -> dict: standby_nodes=detect_payload["non_participating_nodes"], cluster_snapshot=cluster_snapshot, ) + detect_payload["observation_summary"] = _build_detect_observation_summary( + detect_payload=detect_payload, + cluster_snapshot=cluster_snapshot, + inventory_summary=inventory_summary, + ) + return { + "runtime_settings": runtime_settings, + "worker_runtime": worker_runtime, + "worker_expected_on_this_node": worker_expected_on_this_node, + "detect_payload": detect_payload, + "cluster_snapshot": cluster_snapshot, + } + + +def _build_lightweight_runtime_projection_context() -> dict: + runtime_settings = get_runtime_settings() + worker_runtime = detect_worker_runtime() + active_job = dict(get_active_detect_job_summary(event_limit=10) or {}) + cluster_snapshot = get_cluster_snapshot() + + backlog_snapshot = _load_detect_backlog_snapshot() + active_job_backlog = _build_backlog_snapshot_from_active_job(active_job) + if active_job_backlog: + backlog_snapshot = _merge_backlog_snapshots(backlog_snapshot, active_job_backlog) + + active_job_payload: dict | None = None + progress_payload = { + "pending": int(backlog_snapshot.get("pending_total", 0) or 0), + "running": int(backlog_snapshot.get("running_total", 0) or 0), + "completed": int(backlog_snapshot.get("completed_total", 0) or 0), + "blacklisted": int(backlog_snapshot.get("blacklisted_total", 0) or 0), + "failed": int(backlog_snapshot.get("failed_total", 0) or 0), + } + queue_payload = { + "items_total": sum(progress_payload.values()), + "pending": progress_payload["pending"], + "claimed": int(backlog_snapshot.get("claimed_total", 0) or 0), + "running": progress_payload["running"], + "display_claimed": int(backlog_snapshot.get("claimed_total", 0) or 0), + "display_running": progress_payload["running"], + "completed": progress_payload["completed"], + "blacklisted": progress_payload["blacklisted"], + "failed": progress_payload["failed"], + "terminal": progress_payload["completed"] + progress_payload["blacklisted"] + progress_payload["failed"], + } + + display_running = progress_payload["running"] + display_claimed = queue_payload["claimed"] + display_max_threads = int(worker_runtime.get("thread_count", 0) or runtime_settings.get("thread_count", 0) or 0) + if active_job: + display_running = max( + int(active_job.get("display_items_running", active_job.get("items_running", 0)) or 0), + int(active_job.get("display_active_threads", 0) or 0), + progress_payload["running"], + ) + display_claimed = max( + int(active_job.get("display_items_claimed", active_job.get("items_claimed", 0)) or 0), + queue_payload["claimed"], + ) + display_max_threads = max( + int(active_job.get("display_max_threads", 0) or 0), + display_max_threads, + ) + active_job_payload = { + "job_id": active_job.get("job_id"), + "job_code": active_job.get("job_code", ""), + "status": active_job.get("status", ""), + "progress_percent": active_job.get("progress_percent", 0), + "items_total": int(active_job.get("items_total", queue_payload["items_total"]) or queue_payload["items_total"]), + "items_terminal": int(active_job.get("items_terminal", queue_payload["terminal"]) or queue_payload["terminal"]), + "items_pending": int(active_job.get("items_pending", queue_payload["pending"]) or queue_payload["pending"]), + "items_claimed": int(active_job.get("items_claimed", queue_payload["claimed"]) or queue_payload["claimed"]), + "items_running": int(active_job.get("items_running", queue_payload["running"]) or queue_payload["running"]), + "items_completed": int(active_job.get("items_completed", queue_payload["completed"]) or queue_payload["completed"]), + "items_blacklisted": int(active_job.get("items_blacklisted", queue_payload["blacklisted"]) or queue_payload["blacklisted"]), + "items_failed": int(active_job.get("items_failed", queue_payload["failed"]) or queue_payload["failed"]), + "display_items_claimed": display_claimed, + "display_items_running": display_running, + "display_active_threads": display_running, + "display_max_threads": display_max_threads, + "node_stats": list(active_job.get("node_stats") or []), + "distributed_node_stats": list(active_job.get("distributed_node_stats") or active_job.get("node_stats") or []), + "step_stats": list(active_job.get("step_stats") or []), + "raw_step_stats": list(active_job.get("raw_step_stats") or []), + } + + phase_label = str((active_job_payload or {}).get("status") or "").strip() or ( + "running" if worker_runtime.get("running", False) else "idle" + ) + phase_detail = "" + if active_job_payload: + phase_detail = f"active job {active_job_payload.get('job_code', '')}" + elif worker_runtime.get("running", False): + phase_detail = str(worker_runtime.get("message") or "").strip() or "worker running" + + detect_payload = { + "phase_label": phase_label, + "phase_detail": phase_detail, + "recent_event": "", + "recent_warning": "", + "progress_percent": float((active_job_payload or {}).get("progress_percent", 0) or 0), + "progress": progress_payload, + "active_thread_count": display_running, + "max_thread_count": display_max_threads, + "aggregate_process_count": int(worker_runtime.get("process_count", 0) or 0), + "aggregate_participating_node_count": int((cluster_snapshot.get("summary") or {}).get("online_worker_nodes", 0) or 0), + "aggregate_participating_node_codes": [], + "aggregate_max_thread_count": int(worker_runtime.get("max_threads", 0) or display_max_threads or 0), + "aggregate_thread_count_per_process": int(runtime_settings.get("thread_count", 0) or 0), + "available_proxy_count": 0, + "proxy_pool_count": 0, + "proxy_runtime_label": "", + "proxy_runtime_detail": "", + "proxy_runtime_reason": "", + "proxy_supplier_empty": False, + "proxy_last_refresh_status": "", + "proxy_last_refresh_time": "", + "proxy_last_refresh_source_count": 0, + "proxy_last_refresh_total_items": 0, + "proxy_last_validated_count": 0, + "proxy_last_available_count": 0, + "proxy_source_stats": [], + "dependency_alerts": [], + "active_job": active_job_payload, + "runs_count": 1 if active_job_payload else 0, + "worker_online": bool(worker_runtime.get("running", False)), + "worker_mode": worker_runtime.get("mode", runtime_settings.get("worker_mode", "linux-systemd")), + "aggregate_detect_view": False, + "queue_health": {"queue": queue_payload}, + "backlog": backlog_snapshot, + "capacity_plan": {}, + "log_sync": {"enabled": False, "mode": "", "line_count": 0, "source_node_count": 0, "source_nodes": [], "source_node_summaries": [], "last_at": "", "last_line": "", "preview_lines": []}, + } + return { + "detect_payload": detect_payload, + "cluster_snapshot": cluster_snapshot, + } + + +def refresh_runtime_projection_snapshot(*, window_minutes: int = 15) -> dict: + context = _build_lightweight_runtime_projection_context() + record_id = append_runtime_projection_if_changed( + detect=context["detect_payload"], + cluster=context["cluster_snapshot"], + ) + return { + "record_id": record_id, + "active_thread_count": int((context["detect_payload"] or {}).get("active_thread_count", 0) or 0), + "max_thread_count": int((context["detect_payload"] or {}).get("max_thread_count", 0) or 0), + "queue_display_running": int( + ((((context["detect_payload"] or {}).get("queue_health") or {}).get("queue") or {}).get("display_running", 0) or 0) + ), + } + + +def get_runtime_status() -> dict: + global _RUNTIME_STATUS_CACHE_EXPIRES_AT, _RUNTIME_STATUS_CACHE_SIGNATURE, _RUNTIME_STATUS_CACHE_VALUE + + now_ts = time.monotonic() + cache_signature = _runtime_status_cache_signature() + with _RUNTIME_STATUS_CACHE_LOCK: + if ( + _RUNTIME_STATUS_CACHE_VALUE is not None + and now_ts < _RUNTIME_STATUS_CACHE_EXPIRES_AT + and _RUNTIME_STATUS_CACHE_SIGNATURE == cache_signature + ): + return _clone_runtime_status_payload(_RUNTIME_STATUS_CACHE_VALUE) + + context = _build_runtime_detect_context(window_minutes=15) + runtime_settings = context["runtime_settings"] + worker_runtime = context["worker_runtime"] + worker_expected_on_this_node = bool(context["worker_expected_on_this_node"]) + detect_payload = dict(context["detect_payload"] or {}) + cluster_snapshot = dict(context["cluster_snapshot"] or {}) + sync_agent_runtime = detect_sync_agent_runtime() + api_pid = os.getpid() + detect_snapshot = dict(detect_payload.pop("_detect_snapshot", {}) or {}) append_runtime_projection_if_changed(detect=detect_payload, cluster=cluster_snapshot) sync_summary = get_sync_summary(record_limit=5) readiness = _build_multi_region_readiness( @@ -836,7 +1707,7 @@ def get_runtime_status() -> dict: "cluster_summary": cluster_snapshot.get("summary") or {}, } - return { + result = { **compatibility_payload, "api": api_payload, "node": node_payload, @@ -847,6 +1718,11 @@ def get_runtime_status() -> dict: "sync": sync_summary, "readiness": readiness, } + with _RUNTIME_STATUS_CACHE_LOCK: + _RUNTIME_STATUS_CACHE_SIGNATURE = cache_signature + _RUNTIME_STATUS_CACHE_VALUE = _clone_runtime_status_payload(result) + _RUNTIME_STATUS_CACHE_EXPIRES_AT = time.monotonic() + _RUNTIME_STATUS_CACHE_TTL_SECONDS + return result def get_runtime_preflight() -> dict: diff --git a/domain-api/app/services/settings_service.py b/domain-api/app/services/settings_service.py index edd4ba3..56ce41a 100644 --- a/domain-api/app/services/settings_service.py +++ b/domain-api/app/services/settings_service.py @@ -14,6 +14,8 @@ REDIS_KEYS = { "proxy_config": "domain_tool:proxy_config", "thread_count": "domain_tool:thread_count", "node_thread_counts": "domain_tool:node_thread_counts", + "process_count": "domain_tool:process_count", + "node_process_counts": "domain_tool:node_process_counts", "credentials": "domain_tool:credentials", "runtime_settings": "domain_tool:runtime_settings", } @@ -40,6 +42,16 @@ def _normalize_thread_count(value: object, *, field_name: str = "thread_count") return thread_count +def _normalize_process_count(value: object, *, field_name: str = "process_count") -> int: + try: + process_count = int(value) + except Exception as exc: + raise ValueError(f"{field_name} must be an integer") from exc + if process_count < 1: + raise ValueError(f"{field_name} must be >= 1") + return process_count + + def _normalize_node_thread_counts(payload: object) -> dict[str, int]: if payload in (None, ""): return {} @@ -55,14 +67,32 @@ def _normalize_node_thread_counts(payload: object) -> dict[str, int]: return normalized +def _normalize_node_process_counts(payload: object) -> dict[str, int]: + if payload in (None, ""): + return {} + if not isinstance(payload, dict): + raise ValueError("node_process_counts must be an object") + + normalized: dict[str, int] = {} + for raw_node_code, raw_process_count in payload.items(): + node_code = str(raw_node_code or "").strip() + if not node_code: + raise ValueError("node_process_counts contains empty node code") + normalized[node_code] = _normalize_process_count( + raw_process_count, + field_name=f"node_process_counts.{node_code}", + ) + return normalized + + def _load_thread_count_config() -> tuple[int, dict[str, int]]: - thread_count_payload = read_json("thread_count.json", default={"thread_count": "2"}) + thread_count_payload = read_json("thread_count.json", default={"thread_count": "1000"}) node_thread_counts_payload = read_json("node_thread_counts.json", default={}) try: - default_thread_count = _normalize_thread_count(thread_count_payload.get("thread_count", 2)) + default_thread_count = _normalize_thread_count(thread_count_payload.get("thread_count", 1000)) except ValueError: - default_thread_count = 2 + default_thread_count = 1000 try: node_thread_counts = _normalize_node_thread_counts(node_thread_counts_payload) except ValueError: @@ -86,9 +116,40 @@ def _load_thread_count_config() -> tuple[int, dict[str, int]]: return default_thread_count, node_thread_counts +def _load_process_count_config() -> tuple[int, dict[str, int]]: + process_count_payload = read_json("process_count.json", default={"process_count": "80"}) + node_process_counts_payload = read_json("node_process_counts.json", default={}) + + try: + default_process_count = _normalize_process_count(process_count_payload.get("process_count", 80)) + except ValueError: + default_process_count = 80 + try: + node_process_counts = _normalize_node_process_counts(node_process_counts_payload) + except ValueError: + node_process_counts = {} + + redis_client = get_redis() + try: + if redis_process_count := redis_client.get(REDIS_KEYS["process_count"]): + try: + default_process_count = _normalize_process_count(redis_process_count) + except ValueError: + pass + if redis_node_process_counts := redis_client.get(REDIS_KEYS["node_process_counts"]): + try: + node_process_counts = _normalize_node_process_counts(json.loads(redis_node_process_counts)) + except ValueError: + pass + except Exception: + pass + + return default_process_count, node_process_counts + + def resolve_thread_count(node_code: str | None = None, settings_payload: dict | None = None) -> dict: payload = settings_payload or get_settings_payload() - default_thread_count = int(payload.get("thread_count", 2)) + default_thread_count = int(payload.get("thread_count", 1000)) node_thread_counts = _normalize_node_thread_counts(payload.get("node_thread_counts", {})) normalized_node_code = str(node_code or app_settings.node_code or "").strip() @@ -110,10 +171,35 @@ def resolve_thread_count(node_code: str | None = None, settings_payload: dict | } +def resolve_process_count(node_code: str | None = None, settings_payload: dict | None = None) -> dict: + payload = settings_payload or get_settings_payload() + default_process_count = int(payload.get("process_count", 80)) + node_process_counts = _normalize_node_process_counts(payload.get("node_process_counts", {})) + normalized_node_code = str(node_code or app_settings.node_code or "").strip() + + override_process_count = None + source = "default" + effective_process_count = default_process_count + if normalized_node_code and normalized_node_code in node_process_counts: + override_process_count = node_process_counts[normalized_node_code] + effective_process_count = override_process_count + source = "node_override" + + return { + "node_code": normalized_node_code, + "default_process_count": default_process_count, + "effective_process_count": effective_process_count, + "override_process_count": override_process_count, + "source": source, + "node_process_counts": node_process_counts, + } + + def get_settings_payload() -> dict: detect_options = read_json("detect_options.json", default={}) proxy_config = read_json("proxy_config.json", default={}) thread_count, node_thread_counts = _load_thread_count_config() + process_count, node_process_counts = _load_process_count_config() redis_client = get_redis() try: @@ -129,6 +215,8 @@ def get_settings_payload() -> dict: "proxy_config": proxy_config, "thread_count": thread_count, "node_thread_counts": node_thread_counts, + "process_count": process_count, + "node_process_counts": node_process_counts, "current_node_code": app_settings.node_code, "runtime_settings": get_runtime_settings(), } @@ -193,12 +281,18 @@ def update_settings_payload(payload: dict) -> dict: proxy_config = payload.get("proxy_config", current["proxy_config"]) thread_count = _normalize_thread_count(payload.get("thread_count", current["thread_count"])) node_thread_counts = _normalize_node_thread_counts(payload.get("node_thread_counts", current.get("node_thread_counts", {}))) + process_count = _normalize_process_count(payload.get("process_count", current.get("process_count", 80))) + node_process_counts = _normalize_node_process_counts( + payload.get("node_process_counts", current.get("node_process_counts", {})) + ) runtime_settings = update_runtime_settings(payload.get("runtime_settings", current["runtime_settings"])) write_json("detect_options.json", detect_options) write_json("proxy_config.json", proxy_config) write_json("thread_count.json", {"thread_count": str(thread_count)}) write_json("node_thread_counts.json", node_thread_counts) + write_json("process_count.json", {"process_count": str(process_count)}) + write_json("node_process_counts.json", node_process_counts) redis_client = get_redis() try: redis_client.set(REDIS_KEYS["detect_options"], json.dumps(detect_options, ensure_ascii=False)) @@ -209,12 +303,16 @@ def update_settings_payload(payload: dict) -> dict: redis_client.publish("domain_tool:thread_count:update", str(thread_count)) redis_client.set(REDIS_KEYS["node_thread_counts"], json.dumps(node_thread_counts, ensure_ascii=False)) redis_client.publish("domain_tool:node_thread_counts:update", json.dumps(node_thread_counts, ensure_ascii=False)) - redis_client.set(REDIS_KEYS["runtime_settings"], json.dumps(runtime_settings, ensure_ascii=False)) - redis_client.publish("domain_tool:config_update", "runtime_settings") + redis_client.set(REDIS_KEYS["process_count"], process_count) + redis_client.publish("domain_tool:process_count:update", str(process_count)) + redis_client.set(REDIS_KEYS["node_process_counts"], json.dumps(node_process_counts, ensure_ascii=False)) + redis_client.publish("domain_tool:node_process_counts:update", json.dumps(node_process_counts, ensure_ascii=False)) redis_client.publish("domain_tool:config_update", "node_thread_counts") + redis_client.publish("domain_tool:config_update", "node_process_counts") redis_client.publish("domain_tool:config_update", "detect_options") redis_client.publish("domain_tool:config_update", "proxy_config") redis_client.publish("domain_tool:config_update", "thread_count") + redis_client.publish("domain_tool:config_update", "process_count") except Exception: pass @@ -223,6 +321,8 @@ def update_settings_payload(payload: dict) -> dict: "proxy_config": proxy_config, "thread_count": thread_count, "node_thread_counts": node_thread_counts, + "process_count": process_count, + "node_process_counts": node_process_counts, "current_node_code": app_settings.node_code, "runtime_settings": runtime_settings, } @@ -254,6 +354,12 @@ def validate_settings_payload(payload: dict) -> None: if "node_thread_counts" in payload: _normalize_node_thread_counts(payload["node_thread_counts"]) + if "process_count" in payload: + _normalize_process_count(payload["process_count"]) + + if "node_process_counts" in payload: + _normalize_node_process_counts(payload["node_process_counts"]) + if "detect_options" in payload: detect_options = payload["detect_options"] if not isinstance(detect_options, dict): diff --git a/domain-api/app/services/sync_push_service.py b/domain-api/app/services/sync_push_service.py index 6df4820..a86228c 100644 --- a/domain-api/app/services/sync_push_service.py +++ b/domain-api/app/services/sync_push_service.py @@ -2,6 +2,7 @@ from __future__ import annotations import hashlib import json +import os import socket import urllib.error import urllib.parse @@ -11,17 +12,30 @@ from uuid import uuid4 from app.core.config import settings from app.core.db import get_db +from app.core.redis_client import get_redis from app.services.cluster_runtime_service import ( cleanup_imported_runtime_nodes, cleanup_imported_runtime_nodes_many, + get_cluster_snapshot, register_node_heartbeat, ) from app.services.detect_job_service import ( _load_domain_pipeline_snapshot, + get_active_detect_job_summary, resolve_initial_domain_pipeline_item, ) -from app.services.settings_service import get_settings_payload -from app.services.sync_record_service import _decode_json, _normalize_region +from app.services.settings_service import ( + get_settings_payload, + resolve_process_count, + resolve_thread_count, +) +from app.services.sync_record_service import ( + _RUNTIME_PROJECTION_FUTURE_SKEW_GRACE, + _decode_json, + _normalize_region, + _pick_latest_projection_row, + append_runtime_projection_if_changed, +) _DETECT_RESULT_EVENT_TYPES = { @@ -30,6 +44,222 @@ _DETECT_RESULT_EVENT_TYPES = { "domain_failed", "domain_blacklisted", } +_LOCAL_BACKLOG_PENDING_FRESHNESS_HOURS = 6 +_LOCAL_BACKLOG_MAX_JOBS = 4 +_SYNC_PULL_WORKER_WAKE_TTL_SECONDS = 20 +_SYNC_PULL_WORKER_WAKE_KEY_PREFIX = "domain_tool:sync_pull_worker_wake" + + +def _flag_enabled(raw_value: object, *, default: bool = False) -> bool: + if raw_value is None: + return bool(default) + if isinstance(raw_value, bool): + return raw_value + return str(raw_value or "").strip().lower() not in {"", "0", "false", "no", "off"} + + +def _fast_runtime_projection_enabled() -> bool: + return _flag_enabled( + os.getenv("DOMAINCHECK_SYNC_RUNTIME_FAST_PROJECTION"), + default=False, + ) + + +def _local_projection_node_code(node_code: str) -> bool: + normalized_node_code = str(node_code or "").strip() + local_node_code = str(settings.node_code or "").strip() + if not normalized_node_code or not local_node_code: + return False + return normalized_node_code == local_node_code or normalized_node_code.startswith(f"{local_node_code}-") + + +def _append_fast_runtime_projection_snapshot() -> int | None: + local_node_code = str(settings.node_code or "").strip() + if not local_node_code: + return None + + with get_db() as conn: + with conn.cursor() as cur: + cur.execute( + """ + SELECT node_code, region, role, status, current_load, metadata_json + FROM detect_worker_nodes + WHERE node_code = %s OR node_code LIKE %s + ORDER BY node_code ASC + """, + (local_node_code, f"{local_node_code}-%"), + ) + raw_rows = list(cur.fetchall() or []) + + if not raw_rows: + return None + + cluster_nodes: list[dict] = [] + queue_nodes: list[dict] = [] + busy_nodes: list[str] = [] + stale_nodes: list[str] = [] + offline_nodes: list[str] = [] + online_worker_nodes = 0 + dedicated_online_worker_nodes = 0 + online_control_nodes = 0 + display_running = 0 + display_max_threads = 0 + controller_metadata: dict = {} + + for node_code, region, role, status, current_load, metadata_json in raw_rows: + metadata = dict(metadata_json or {}) + normalized_node_code = str(node_code or "").strip() + normalized_role = str(role or metadata.get("source_role") or "").strip() or "worker" + normalized_status = str(status or metadata.get("source_status") or "").strip() or "unknown" + normalized_region = _normalize_region(region, settings.node_region) + normalized_current_load = int(current_load or 0) + active_threads = int(metadata.get("active_threads", 0) or 0) + max_threads = int(metadata.get("max_threads", 0) or 0) + detect_participating = bool(metadata.get("detect_participating", False) or normalized_current_load > 0 or active_threads > 0) + runtime_running = max(active_threads, normalized_current_load) + + if normalized_status == "busy": + busy_nodes.append(normalized_node_code) + elif normalized_status == "stale": + stale_nodes.append(normalized_node_code) + elif normalized_status == "offline": + offline_nodes.append(normalized_node_code) + + if normalized_status not in {"stale", "offline"}: + if normalized_role == "worker": + online_worker_nodes += 1 + if normalized_node_code != local_node_code: + dedicated_online_worker_nodes += 1 + elif normalized_role == "control": + online_control_nodes += 1 + + cluster_nodes.append( + { + "node_code": normalized_node_code, + "role": normalized_role, + "status": normalized_status, + "current_load": normalized_current_load, + "active_threads": active_threads, + "max_threads": max_threads, + "detect_participating": detect_participating, + } + ) + + queue_nodes.append( + { + "node_code": normalized_node_code, + "items_total": 0, + "items_pending": 0, + "items_claimed": 0, + "items_running": runtime_running, + "items_completed": 0, + "items_blacklisted": 0, + "items_failed": 0, + "display_running": runtime_running, + "display_claimed": 0, + "current_load": normalized_current_load, + "active_threads": active_threads, + "max_threads": max_threads, + "role": normalized_role, + "status": normalized_status, + "detect_participating": detect_participating, + } + ) + + display_running += runtime_running + display_max_threads += max_threads + + if normalized_node_code == local_node_code: + controller_metadata = metadata + + controller_job_code = str(controller_metadata.get("active_job_code") or "").strip() + controller_job_status = str(controller_metadata.get("active_job_status") or "").strip() + detect_payload = { + "worker_online": True, + "worker_mode": str(controller_metadata.get("worker_mode") or "linux-systemd").strip() or "linux-systemd", + "phase_label": str(controller_metadata.get("phase_label") or "集群执行中").strip() or "集群执行中", + "phase_detail": str(controller_metadata.get("phase_detail") or "").strip(), + "proxy_runtime_label": str(controller_metadata.get("proxy_runtime_label") or "").strip(), + "proxy_runtime_reason": str(controller_metadata.get("proxy_runtime_reason") or "").strip(), + "detect_participating": bool(display_running > 0), + "progress": { + "pending": 0, + "running": display_running, + "completed": 0, + "blacklisted": 0, + "failed": 0, + }, + "queue_health": { + "queue": { + "items_total": 0, + "pending": 0, + "claimed": 0, + "running": display_running, + "completed": 0, + "blacklisted": 0, + "failed": 0, + "terminal": 0, + "display_claimed": 0, + "display_running": display_running, + "display_max_threads": display_max_threads, + }, + "nodes": list(queue_nodes), + }, + "active_job": { + "job_id": None, + "job_code": controller_job_code, + "status": controller_job_status, + "progress_percent": 0, + "items_total": 0, + "items_terminal": 0, + "items_pending": 0, + "items_claimed": 0, + "items_running": display_running, + "items_failed": 0, + "items_completed": 0, + "display_items_claimed": 0, + "display_items_running": display_running, + "display_active_threads": display_running, + "display_max_threads": display_max_threads, + "node_stats": list(queue_nodes), + "distributed_node_stats": list(queue_nodes), + }, + "backlog": {}, + "dependency_alerts": [], + } + cluster_payload = { + "nodes": [ + { + "node_code": item["node_code"], + "role": item["role"], + "status": item["status"], + "current_load": item["current_load"], + "metadata": { + "active_threads": item["active_threads"], + "max_threads": item["max_threads"], + "detect_participating": item["detect_participating"], + "source_role": item["role"], + "source_status": item["status"], + }, + } + for item in cluster_nodes + ], + "nodes_total": len(cluster_nodes), + "summary": { + "busy_nodes": busy_nodes, + "stale_nodes": stale_nodes, + "offline_nodes": offline_nodes, + "online_worker_nodes": online_worker_nodes, + "dedicated_online_worker_nodes": dedicated_online_worker_nodes, + "online_control_nodes": online_control_nodes, + }, + } + return append_runtime_projection_if_changed( + detect=detect_payload, + cluster=cluster_payload, + source_region=_normalize_region(settings.sync_source_region, settings.node_region), + target_region=_normalize_region(settings.sync_target_region, "overseas"), + ) def _format_time(value: datetime | None) -> str: @@ -69,6 +299,42 @@ def _task_ack_url(base_url: str) -> str: return f"{text}/api/v1/runtime/task-ack" +def _build_sync_pull_worker_wake_key( + *, + projection_job_code: str = "", + projection_cycle_token: str = "", + target_job_code: str = "", + source_record_id: int = 0, +) -> str: + scope = ( + str(projection_cycle_token or "").strip() + or str(projection_job_code or "").strip() + or str(target_job_code or "").strip() + or f"record-{int(source_record_id or 0)}" + ) + return f"{_SYNC_PULL_WORKER_WAKE_KEY_PREFIX}:{scope}" + + +def _acquire_sync_pull_worker_wake_guard(key: str, ttl_seconds: int = _SYNC_PULL_WORKER_WAKE_TTL_SECONDS) -> bool: + normalized_key = str(key or "").strip() + if not normalized_key: + return True + try: + redis_client = get_redis() + return bool( + redis_client.set( + normalized_key, + datetime.now().isoformat(timespec="seconds"), + ex=max(1, int(ttl_seconds or 1)), + nx=True, + ) + ) + except Exception: + # Wake dedupe is a throughput optimization; fall back to legacy behavior + # if Redis is temporarily unavailable. + return True + + def _projection_ingest_type(sync_type: str) -> str: if sync_type == "runtime_projection": return "runtime_ingest" @@ -126,21 +392,82 @@ def _refresh_remote_runtime_node(*, source_region: str, projection: dict, receiv cleanup_imported_runtime_nodes(region=region, role=role, keep_node_code=node_code) active_job = projection.get("active_job") or {} - worker_node_codes: list[str] = [] + worker_rows_by_code: dict[str, dict] = {} + for cluster_node in list(projection.get("cluster_nodes") or []): + if not isinstance(cluster_node, dict): + continue + worker_node_code = str(cluster_node.get("node_code") or "").strip() + if not worker_node_code or worker_node_code == node_code: + continue + worker_rows_by_code[worker_node_code] = { + "node_code": worker_node_code, + "role": str(cluster_node.get("role") or "worker").strip() or "worker", + "status": str(cluster_node.get("status") or "").strip(), + "current_load": int(cluster_node.get("current_load", 0) or 0), + "active_threads": int(cluster_node.get("active_threads", 0) or 0), + "max_threads": int(cluster_node.get("max_threads", 0) or 0), + "detect_participating": bool(cluster_node.get("detect_participating", False)), + "items_total": 0, + "items_running": 0, + "items_claimed": 0, + "items_completed": 0, + "items_failed": 0, + "items_blacklisted": 0, + "metrics_source": "runtime", + } + for node_stat in list(active_job.get("node_stats") or []): worker_node_code = str(node_stat.get("node_code") or "").strip() - if not worker_node_code or worker_node_code == "unassigned": + if not worker_node_code or worker_node_code == "unassigned" or worker_node_code == node_code: continue - if worker_node_code == node_code: - continue - items_running = int(node_stat.get("items_running", 0) or 0) - items_claimed = int(node_stat.get("items_claimed", 0) or 0) - items_total = int(node_stat.get("items_total", 0) or 0) - worker_runtime_load = int(node_stat.get("current_load", 0) or 0) - worker_active_threads = int(node_stat.get("active_threads", worker_runtime_load) or 0) - worker_max_threads = int(node_stat.get("max_threads", 0) or 0) - worker_load = max(worker_active_threads, items_running, 0) - worker_status = "busy" if worker_load > 0 else "online" + worker_row = worker_rows_by_code.setdefault( + worker_node_code, + { + "node_code": worker_node_code, + "role": str(node_stat.get("role") or "worker").strip() or "worker", + "status": str(node_stat.get("status") or "").strip(), + "current_load": int(node_stat.get("current_load", 0) or 0), + "active_threads": int(node_stat.get("active_threads", 0) or 0), + "max_threads": int(node_stat.get("max_threads", 0) or 0), + "detect_participating": False, + "items_total": 0, + "items_running": 0, + "items_claimed": 0, + "items_completed": 0, + "items_failed": 0, + "items_blacklisted": 0, + "metrics_source": "runtime", + }, + ) + worker_row["role"] = str(node_stat.get("role") or worker_row.get("role") or "worker").strip() or "worker" + worker_row["status"] = str(node_stat.get("status") or worker_row.get("status") or "").strip() + worker_row["current_load"] = max(int(worker_row.get("current_load", 0) or 0), int(node_stat.get("current_load", 0) or 0)) + worker_row["active_threads"] = max(int(worker_row.get("active_threads", 0) or 0), int(node_stat.get("active_threads", 0) or 0)) + worker_row["max_threads"] = max(int(worker_row.get("max_threads", 0) or 0), int(node_stat.get("max_threads", 0) or 0)) + worker_row["detect_participating"] = bool( + worker_row.get("detect_participating", False) + or int(node_stat.get("items_running", 0) or 0) > 0 + or int(node_stat.get("items_claimed", 0) or 0) > 0 + or int(node_stat.get("active_threads", 0) or 0) > 0 + ) + worker_row["items_total"] = int(node_stat.get("items_total", worker_row.get("items_total", 0)) or 0) + worker_row["items_running"] = int(node_stat.get("items_running", worker_row.get("items_running", 0)) or 0) + worker_row["items_claimed"] = int(node_stat.get("items_claimed", worker_row.get("items_claimed", 0)) or 0) + worker_row["items_completed"] = int(node_stat.get("items_completed", worker_row.get("items_completed", 0)) or 0) + worker_row["items_failed"] = int(node_stat.get("items_failed", worker_row.get("items_failed", 0)) or 0) + worker_row["items_blacklisted"] = int(node_stat.get("items_blacklisted", worker_row.get("items_blacklisted", 0)) or 0) + worker_row["metrics_source"] = str(node_stat.get("metrics_source") or worker_row.get("metrics_source") or "runtime").strip() or "runtime" + + worker_node_codes: list[str] = [] + for worker_node_code, worker_row in sorted(worker_rows_by_code.items()): + items_running = int(worker_row.get("items_running", 0) or 0) + items_claimed = int(worker_row.get("items_claimed", 0) or 0) + items_total = int(worker_row.get("items_total", 0) or 0) + worker_runtime_load = int(worker_row.get("current_load", 0) or 0) + worker_active_threads = int(worker_row.get("active_threads", worker_runtime_load) or 0) + worker_max_threads = int(worker_row.get("max_threads", 0) or 0) + worker_load = max(worker_active_threads, items_running, worker_runtime_load, 0) + worker_status = str(worker_row.get("status") or "").strip() or ("busy" if worker_load > 0 else "online") worker_metadata = { "service": "runtime-ingest", "projection_source_region": source_region, @@ -155,12 +482,13 @@ def _refresh_remote_runtime_node(*, source_region: str, projection: dict, receiv "job_items_total": items_total, "job_items_running": items_running, "job_items_claimed": items_claimed, - "job_items_completed": int(node_stat.get("items_completed", 0) or 0), - "job_items_failed": int(node_stat.get("items_failed", 0) or 0), - "job_items_blacklisted": int(node_stat.get("items_blacklisted", 0) or 0), - "metrics_source": str(node_stat.get("metrics_source") or "runtime").strip() or "runtime", - "source_status": str(node_stat.get("status") or "").strip(), - "source_role": str(node_stat.get("role") or "worker").strip() or "worker", + "job_items_completed": int(worker_row.get("items_completed", 0) or 0), + "job_items_failed": int(worker_row.get("items_failed", 0) or 0), + "job_items_blacklisted": int(worker_row.get("items_blacklisted", 0) or 0), + "metrics_source": str(worker_row.get("metrics_source") or "runtime").strip() or "runtime", + "source_status": str(worker_row.get("status") or "").strip(), + "source_role": str(worker_row.get("role") or "worker").strip() or "worker", + "detect_participating": bool(worker_row.get("detect_participating", False)), "derived_from": node_code, } register_node_heartbeat( @@ -181,6 +509,7 @@ def _refresh_remote_runtime_node(*, source_region: str, projection: dict, receiv def _load_latest_projection(sync_type: str) -> dict | None: source_region = _normalize_region(settings.sync_source_region, settings.node_region) target_region = _normalize_region(settings.sync_target_region, "overseas") + future_cutoff = datetime.now() + _RUNTIME_PROJECTION_FUTURE_SKEW_GRACE with get_db() as conn: with conn.cursor() as cur: cur.execute( @@ -190,12 +519,15 @@ def _load_latest_projection(sync_type: str) -> dict | None: WHERE sync_type = %s AND source_region = %s AND target_region = %s - ORDER BY created_at DESC, id DESC - LIMIT 1 + ORDER BY + CASE WHEN created_at <= %s THEN 0 ELSE 1 END ASC, + created_at DESC, + id DESC + LIMIT 200 """, - (sync_type, source_region, target_region), + (sync_type, source_region, target_region, future_cutoff), ) - row = cur.fetchone() + row = _pick_latest_projection_row(list(cur.fetchall() or []), created_at_index=5) if not row: return None return { @@ -222,7 +554,7 @@ def _load_pushable_projections(sync_type: str, limit: int) -> list[dict]: WHERE sync_type = %s AND source_region = %s AND target_region = %s - ORDER BY created_at ASC, id ASC + ORDER BY created_at DESC, id DESC LIMIT %s """, (sync_type, source_region, target_region, safe_limit * 5), @@ -259,20 +591,158 @@ def _load_pushable_projections(sync_type: str, limit: int) -> list[dict]: def _estimate_total_worker_threads(settings_payload: dict | None = None) -> int: payload = settings_payload if isinstance(settings_payload, dict) else get_settings_payload() - default_threads = max(1, int(payload.get("thread_count", 100) or 100)) - node_thread_counts = payload.get("node_thread_counts") if isinstance(payload.get("node_thread_counts"), dict) else {} - total_threads = 0 - for raw_value in node_thread_counts.values(): + thread_info = resolve_thread_count(settings.node_code, settings_payload=payload) + process_info = resolve_process_count(settings.node_code, settings_payload=payload) + effective_threads = max( + 1, + int(thread_info.get("effective_thread_count", payload.get("thread_count", 100)) or 100), + ) + effective_process_count = max( + 1, + int(process_info.get("effective_process_count", payload.get("process_count", 1)) or 1), + ) + return effective_threads * effective_process_count + + +def _sync_task_projection_limit_cap() -> int: + configured_cap = int(os.getenv("DOMAINCHECK_SYNC_TASK_LIMIT_CAP", "200000") or 200000) + return max(10000, configured_cap) + + +def _resolve_task_pull_request_limit(limit: int | None, settings_payload: dict | None = None) -> int: + configured = max(5000, int(settings.sync_batch_size or 200)) + estimated_total_threads = _estimate_total_worker_threads(settings_payload) + cap = _sync_task_projection_limit_cap() + default_limit = max( + configured, + min(cap, max(10000, estimated_total_threads * 2)), + ) + requested = int(limit or default_limit) + return max(1, min(requested, cap)) + + +def _select_relevant_backlog_job_ids_from_rows( + job_rows: list[tuple[object, object, object]] | tuple[tuple[object, object, object], ...], + *, + freshness_hours: int = _LOCAL_BACKLOG_PENDING_FRESHNESS_HOURS, + limit: int = _LOCAL_BACKLOG_MAX_JOBS, +) -> list[int]: + safe_limit = max(1, min(int(limit or _LOCAL_BACKLOG_MAX_JOBS), 16)) + safe_freshness_hours = max(1, min(int(freshness_hours or _LOCAL_BACKLOG_PENDING_FRESHNESS_HOURS), 168)) + + selected: list[int] = [] + fallback_job_id = 0 + for raw_job_id, raw_status, raw_activity_at in list(job_rows or []): try: - total_threads += max(0, int(raw_value or 0)) + job_id = int(raw_job_id or 0) except (TypeError, ValueError): continue - return max(total_threads, default_threads) + if job_id <= 0: + continue + if fallback_job_id <= 0: + fallback_job_id = job_id + if job_id in selected: + continue + status = str(raw_status or "").strip().lower() + keep = status == "running" + if not keep and raw_activity_at is not None: + now = datetime.now(raw_activity_at.tzinfo) if getattr(raw_activity_at, "tzinfo", None) else datetime.now() + keep = now - raw_activity_at <= timedelta(hours=safe_freshness_hours) + if not keep: + continue + selected.append(job_id) + if len(selected) >= safe_limit: + break + if not selected and fallback_job_id > 0: + selected.append(fallback_job_id) + return selected + + +def _build_backlog_snapshot_from_active_job(active_job: dict | None) -> dict: + normalized_job = dict(active_job or {}) + if not normalized_job: + return {} + + pending_total = max(0, int(normalized_job.get("items_pending", 0) or 0)) + claimed_total = max( + max( + int(normalized_job.get("items_claimed", 0) or 0), + int(normalized_job.get("display_items_claimed", 0) or 0), + ), + 0, + ) + running_total = max( + max( + int(normalized_job.get("items_running", 0) or 0), + int(normalized_job.get("display_items_running", 0) or 0), + ), + int(normalized_job.get("display_active_threads", 0) or 0), + 0, + ) + + register_pending = 0 + downstream_pending = 0 + for raw_step in list(normalized_job.get("step_stats") or normalized_job.get("raw_step_stats") or []): + if not isinstance(raw_step, dict): + continue + step_code = str(raw_step.get("step_code") or raw_step.get("code") or "").strip() + step_pending = max( + int(raw_step.get("items_pending", raw_step.get("pending", 0)) or 0), + 0, + ) + if step_pending <= 0: + continue + if step_code == "detect_register": + register_pending += step_pending + else: + downstream_pending += step_pending + + if register_pending <= 0 and downstream_pending <= 0 and pending_total > 0: + downstream_pending = pending_total + + if pending_total <= 0 and claimed_total <= 0 and running_total <= 0: + return {} + + return { + "pending_total": pending_total, + "claimed_total": claimed_total, + "running_total": running_total, + "register_pending": register_pending, + "downstream_pending": downstream_pending, + } def _load_local_detect_backlog_snapshot() -> dict: + active_job_snapshot = _build_backlog_snapshot_from_active_job( + get_active_detect_job_summary(event_limit=1) + ) + if active_job_snapshot: + return active_job_snapshot + with get_db() as conn: with conn.cursor() as cur: + cur.execute( + """ + SELECT id, status, COALESCE(started_at, created_at) AS activity_at + FROM detect_jobs + WHERE status IN ('pending', 'running') + ORDER BY + CASE WHEN status = 'running' THEN 0 ELSE 1 END, + COALESCE(started_at, created_at) DESC, + id DESC + LIMIT %s + """, + (_LOCAL_BACKLOG_MAX_JOBS * 8,), + ) + selected_job_ids = _select_relevant_backlog_job_ids_from_rows(cur.fetchall()) + if not selected_job_ids: + return { + "pending_total": 0, + "claimed_total": 0, + "running_total": 0, + "register_pending": 0, + "downstream_pending": 0, + } cur.execute( """ SELECT @@ -282,9 +752,9 @@ def _load_local_detect_backlog_snapshot() -> dict: COUNT(*) FILTER (WHERE item.status = 'pending' AND item.step_code = 'detect_register') AS register_pending, COUNT(*) FILTER (WHERE item.status = 'pending' AND item.step_code <> 'detect_register') AS downstream_pending FROM detect_job_items item - JOIN detect_jobs job ON job.id = item.job_id - WHERE job.status IN ('pending', 'running') - """ + WHERE item.job_id = ANY(%s) + """, + (selected_job_ids,), ) row = cur.fetchone() or (0, 0, 0, 0, 0) return { @@ -422,9 +892,9 @@ def _task_selection_sql() -> str: def _task_projection_limit(limit: int | None) -> int: - requested = max(1, int(limit or 5000)) + requested = max(1, int(limit or max(5000, int(settings.sync_batch_size or 200)))) configured = max(5000, int(settings.sync_batch_size or 200)) - cap = max(10000, configured, 5000) + cap = max(configured, _sync_task_projection_limit_cap()) return max(1, min(requested, cap)) @@ -1422,15 +1892,21 @@ def ingest_runtime_projection(payload: dict, *, shared_token: str | None = None) def _push_projection_now(sync_type: str, ingest_url: str) -> tuple[bool, str, dict]: if sync_type == "runtime_projection": - # Regenerate the runtime snapshot before every push so the sync agent - # does not keep replaying a stale projection record while the worker - # thread count / phase is still changing. - from app.services.runtime_status_service import get_runtime_status + if _fast_runtime_projection_enabled(): + try: + _append_fast_runtime_projection_snapshot() + except Exception as exc: + return False, f"快速刷新 runtime_projection 失败: {exc}", {"action": "push_sync", "sync_type": sync_type} + else: + # Refresh only the lightweight runtime projection snapshot before every + # push so the sync agent does not keep replaying a stale record while + # avoiding the full runtime/status assembly cost. + from app.services.runtime_status_service import refresh_runtime_projection_snapshot - try: - get_runtime_status() - except Exception as exc: - return False, f"刷新 runtime_projection 失败: {exc}", {"action": "push_sync", "sync_type": sync_type} + try: + refresh_runtime_projection_snapshot(window_minutes=15) + except Exception as exc: + return False, f"刷新 runtime_projection 失败: {exc}", {"action": "push_sync", "sync_type": sync_type} source_record = _load_latest_projection(sync_type) if not source_record: return False, f"当前没有可推送的{sync_type}", {"action": "push_sync", "sync_type": sync_type} @@ -1667,12 +2143,10 @@ def pull_detect_task_batch_now(limit: int | None = None) -> tuple[bool, str, dic if not export_url or not ack_url: return False, "未配置任务拉取目标地址", {"action": "pull_tasks", "pull_state": "misconfigured", "ui_level": "warning", "poll_schedule_seconds": []} - configured_limit = max(5000, int(settings.sync_batch_size or 200)) - requested_limit = int(limit or configured_limit) - safe_limit = max(1, min(requested_limit, max(10000, configured_limit))) settings_payload = get_settings_payload() + safe_limit = _resolve_task_pull_request_limit(limit, settings_payload=settings_payload) backlog_snapshot = _load_local_detect_backlog_snapshot() - backlog_limits = _build_task_pull_backlog_limits(configured_limit, settings_payload=settings_payload) + backlog_limits = _build_task_pull_backlog_limits(safe_limit, settings_payload=settings_payload) should_throttle, throttle_reason = _should_throttle_task_pull(backlog_snapshot, backlog_limits) if should_throttle: return True, "本地待处理积压较高，暂停拉取新批次", { @@ -1804,17 +2278,48 @@ def pull_detect_task_batch_now(limit: int | None = None) -> tuple[bool, str, dic try: from app.services.worker_control_service import send_worker_command - start_ok, start_message = send_worker_command( - "start_detection", - payload={ - "source": "sync-pull", - "source_record_id": source_record_id, - "target_job_id": int(result.get("target_job_id", 0) or 0), - "target_job_code": str(result.get("target_job_code") or "").strip(), - }, + projection_active_job = dict(projection.get("active_job") or {}) + projection_job_id = int(projection_active_job.get("job_id", 0) or 0) + projection_job_code = str(projection_active_job.get("job_code") or "").strip() + projection_cycle_token = str( + projection_active_job.get("current_cycle_token") + or projection_active_job.get("cycle_token") + or "" + ).strip() + start_payload = { + "source": "sync-pull", + "source_record_id": source_record_id, + "target_job_id": int(result.get("target_job_id", 0) or 0), + "target_job_code": str(result.get("target_job_code") or "").strip(), + } + # Mainland ingest creates a local target_job_* for queue ownership, but + # worker runtime/log identity should still follow the upstream active + # detect job so cluster aggregation keeps controller activity attached + # to the real pipeline job instead of the local sync-pull surrogate. + if projection_job_id > 0: + start_payload["job_id"] = projection_job_id + if projection_job_code: + start_payload["job_code"] = projection_job_code + if projection_cycle_token: + start_payload["cycle_token"] = projection_cycle_token + wake_guard_key = _build_sync_pull_worker_wake_key( + projection_job_code=projection_job_code, + projection_cycle_token=projection_cycle_token, + target_job_code=str(result.get("target_job_code") or "").strip(), + source_record_id=source_record_id, ) - result["worker_start_ok"] = bool(start_ok) - result["worker_start_message"] = str(start_message or "").strip() + if _acquire_sync_pull_worker_wake_guard(wake_guard_key): + start_ok, start_message = send_worker_command( + "start_detection", + payload=start_payload, + ) + result["worker_start_ok"] = bool(start_ok) + result["worker_start_message"] = str(start_message or "").strip() + result["worker_start_skipped"] = False + else: + result["worker_start_ok"] = True + result["worker_start_skipped"] = True + result["worker_start_message"] = "已跳过同任务短窗内重复 Worker 唤起" except Exception as exc: result["worker_start_ok"] = False result["worker_start_message"] = f"同步入库后自动唤起 Worker 失败: {exc}" diff --git a/domain-api/app/services/sync_record_service.py b/domain-api/app/services/sync_record_service.py index 7b7910b..1d197fc 100644 --- a/domain-api/app/services/sync_record_service.py +++ b/domain-api/app/services/sync_record_service.py @@ -51,6 +51,80 @@ _TERMINAL_DETECT_RESULT_EVENT_TYPES = { "domain_blacklisted", } +_RUNTIME_PROJECTION_HEARTBEAT_INTERVAL = timedelta(seconds=45) +_RUNTIME_PROJECTION_FUTURE_SKEW_GRACE = timedelta(minutes=5) + + +def _runtime_projection_activity_signature(projection: dict) -> dict: + normalized_projection = dict(projection or {}) + active_job = dict(normalized_projection.get("active_job") or {}) + normalized_nodes: list[tuple] = [] + for raw_item in list(active_job.get("distributed_node_stats") or active_job.get("node_stats") or []): + if not isinstance(raw_item, dict): + continue + node_code = str(raw_item.get("node_code") or "").strip() + if not node_code: + continue + normalized_nodes.append( + ( + node_code, + int(raw_item.get("display_running", raw_item.get("current_load", 0)) or 0), + int(raw_item.get("active_threads", 0) or 0), + int(raw_item.get("max_threads", 0) or 0), + int(raw_item.get("items_claimed", 0) or 0), + int(raw_item.get("items_running", 0) or 0), + int(raw_item.get("items_total", 0) or 0), + str(raw_item.get("status") or "").strip(), + ) + ) + normalized_cluster_nodes: list[tuple] = [] + for raw_item in list(normalized_projection.get("cluster_nodes") or []): + if not isinstance(raw_item, dict): + continue + node_code = str(raw_item.get("node_code") or "").strip() + if not node_code: + continue + normalized_cluster_nodes.append( + ( + node_code, + str(raw_item.get("role") or "").strip(), + str(raw_item.get("status") or "").strip(), + int(raw_item.get("current_load", 0) or 0), + int(raw_item.get("active_threads", 0) or 0), + int(raw_item.get("max_threads", 0) or 0), + bool(raw_item.get("detect_participating", False)), + ) + ) + return { + "active_thread_count": int(normalized_projection.get("active_thread_count", 0) or 0), + "max_thread_count": int(normalized_projection.get("max_thread_count", 0) or 0), + "job_display_running": int(active_job.get("display_items_running", 0) or 0), + "job_display_claimed": int(active_job.get("display_items_claimed", 0) or 0), + "job_display_max_threads": int(active_job.get("display_max_threads", 0) or 0), + "job_items_total": int(active_job.get("items_total", 0) or 0), + "job_items_running": int(active_job.get("items_running", 0) or 0), + "job_items_claimed": int(active_job.get("items_claimed", 0) or 0), + "node_stats": normalized_nodes, + "cluster_nodes": normalized_cluster_nodes, + } + + +def _pick_latest_projection_row(rows: list[tuple], *, created_at_index: int) -> tuple | None: + candidates = list(rows or []) + if not candidates: + return None + fallback = candidates[0] + for row in candidates: + if len(row) <= int(created_at_index): + return row + created_at = row[created_at_index] + if not isinstance(created_at, datetime): + return row + now = datetime.now(created_at.tzinfo) if created_at.tzinfo else datetime.now() + if created_at <= now + _RUNTIME_PROJECTION_FUTURE_SKEW_GRACE: + return row + return fallback + def _collect_recent_domain_events(active_job: dict, limit: int = 30) -> list[dict]: safe_limit = max(1, int(limit or 30)) @@ -214,10 +288,15 @@ def _should_append_runtime_projection(previous_payload: dict, current_projection if previous_alerts != current_alerts: return True + if _runtime_projection_activity_signature(previous_projection) != _runtime_projection_activity_signature(current_projection): + return True + if not previous_created_at: return True now = datetime.now(previous_created_at.tzinfo) if previous_created_at.tzinfo else datetime.now() - return now - previous_created_at >= timedelta(seconds=45) + if previous_created_at > now + _RUNTIME_PROJECTION_FUTURE_SKEW_GRACE: + return True + return now - previous_created_at >= _RUNTIME_PROJECTION_HEARTBEAT_INTERVAL @db_read_retry() @@ -293,6 +372,25 @@ def get_detect_result_sync_batches(limit: int = 5) -> dict: safe_limit = max(1, min(int(limit or 5), 20)) source_region = _normalize_region(settings.sync_source_region, settings.node_region) target_region = _normalize_region(settings.sync_target_region, "overseas") + local_worker_expected = _local_node_expected_to_execute_worker() + if not local_worker_expected: + return { + "applicable": False, + "local_worker_expected": False, + "reason": "当前节点不承载本地检测执行，结果批次推送概览不适用。", + "source_region": source_region, + "target_region": target_region, + "jobs_total": 0, + "state_counts": { + "synced": 0, + "delivered": 0, + "pushing": 0, + "projected": 0, + "failed": 0, + "unsynced": 0, + }, + "batches": [], + } batches: list[dict] = [] with get_db() as conn: @@ -424,6 +522,9 @@ def get_detect_result_sync_batches(limit: int = 5) -> dict: state_counts[state] = state_counts.get(state, 0) + 1 return { + "applicable": True, + "local_worker_expected": local_worker_expected, + "reason": "", "source_region": source_region, "target_region": target_region, "jobs_total": len(batches), @@ -436,6 +537,11 @@ def get_detect_result_sync_batches(limit: int = 5) -> dict: def get_sync_summary(record_limit: int = 10) -> dict: source_region = _normalize_region(settings.sync_source_region, settings.node_region) target_region = _normalize_region(settings.sync_target_region, "overseas") + local_worker_expected = _local_node_expected_to_execute_worker() + push_expected_on_this_node = bool( + str(settings.node_region or "").strip() == "mainland" + and str(settings.node_role or "").strip() == "control" + ) with get_db() as conn: with conn.cursor() as cur: cur.execute( @@ -486,6 +592,8 @@ def get_sync_summary(record_limit: int = 10) -> dict: return { "enabled": bool(settings.sync_push_enabled), + "push_expected_on_this_node": push_expected_on_this_node, + "local_worker_expected_on_this_node": local_worker_expected, "source_region": source_region, "target_region": target_region, "target_api_base_url": settings.sync_target_api_base_url, @@ -540,6 +648,107 @@ def _local_node_expected_to_execute_worker() -> bool: return node_role == "worker" or (node_region == "mainland" and node_role == "control") +def _projection_node_rows(*, detect: dict, active_job: dict) -> list[dict]: + queue_health = dict(detect.get("queue_health") or {}) + queue_nodes = [ + dict(item) + for item in list(queue_health.get("nodes") or []) + if isinstance(item, dict) and str(item.get("node_code") or "").strip() + ] + if queue_nodes: + return queue_nodes + return [ + dict(item) + for item in list(active_job.get("distributed_node_stats") or active_job.get("node_stats") or []) + if isinstance(item, dict) and str(item.get("node_code") or "").strip() + ] + + +def _projection_cluster_node_rows(*, cluster: dict) -> list[dict]: + normalized_rows: list[dict] = [] + for raw_item in list(cluster.get("nodes") or []): + if not isinstance(raw_item, dict): + continue + node_code = str(raw_item.get("node_code") or "").strip() + if not _is_local_projection_node(node_code): + continue + metadata = dict(raw_item.get("metadata") or {}) + normalized_rows.append( + { + "node_code": node_code, + "role": str(raw_item.get("role") or metadata.get("source_role") or "").strip(), + "status": str(raw_item.get("status") or metadata.get("source_status") or "").strip(), + "current_load": int(raw_item.get("current_load", 0) or 0), + "active_threads": int(metadata.get("active_threads", 0) or 0), + "max_threads": int(metadata.get("max_threads", raw_item.get("max_threads", 0)) or 0), + "detect_participating": bool( + raw_item.get("detect_participating", metadata.get("detect_participating", False)) + ), + } + ) + normalized_rows.sort(key=lambda item: str(item.get("node_code") or "")) + return normalized_rows + + +def _projection_display_summary(*, detect: dict, active_job: dict, node_rows: list[dict]) -> dict: + queue_payload = dict((detect.get("queue_health") or {}).get("queue") or {}) + display_running = 0 + display_max_threads = 0 + display_claimed = 0 + running_items = 0 + + for raw_item in list(node_rows or []): + item = dict(raw_item or {}) + running_items += int(item.get("items_running", 0) or 0) + display_running += max( + int(item.get("display_running", 0) or 0), + int(item.get("current_load", 0) or 0), + int(item.get("active_threads", 0) or 0), + int(item.get("items_running", 0) or 0), + ) + display_max_threads += max(0, int(item.get("max_threads", 0) or 0)) + display_claimed += max( + int(item.get("items_claimed", 0) or 0), + int(item.get("display_claimed", 0) or 0), + ) + + display_running = max( + display_running, + int(queue_payload.get("display_running", queue_payload.get("running", 0)) or 0), + int(active_job.get("display_active_threads", active_job.get("display_items_running", active_job.get("items_running", 0))) or 0), + ) + running_items = max( + running_items, + int(queue_payload.get("running", 0) or 0), + int(active_job.get("items_running", 0) or 0), + ) + display_claimed = max( + display_claimed, + int(queue_payload.get("display_claimed", queue_payload.get("claimed", 0)) or 0), + int(active_job.get("display_items_claimed", active_job.get("items_claimed", 0)) or 0), + ) + display_max_threads = max( + display_max_threads, + int(active_job.get("display_max_threads", 0) or 0), + int(detect.get("aggregate_max_thread_count", 0) or 0), + int(detect.get("max_thread_count", 0) or 0), + ) + return { + "items_running": running_items, + "display_running": display_running, + "display_claimed": display_claimed, + "display_max_threads": display_max_threads, + } + + +def _is_local_projection_node(node_code: str) -> bool: + normalized_node_code = str(node_code or "").strip() + local_node_code = str(settings.node_code or "").strip() + if not normalized_node_code or not local_node_code: + return False + return normalized_node_code == local_node_code or normalized_node_code.startswith(f"{local_node_code}-") + + def _build_runtime_projection_payload( *, detect: dict, @@ -549,6 +758,19 @@ def _build_runtime_projection_payload( ) -> dict: active_job = detect.get("active_job") or {} local_worker_expected = _local_node_expected_to_execute_worker() + projection_node_rows = _projection_node_rows(detect=detect, active_job=active_job) if local_worker_expected else [] + projection_cluster_rows = _projection_cluster_node_rows(cluster=cluster) if local_worker_expected else [] + display_summary = ( + _projection_display_summary(detect=detect, active_job=active_job, node_rows=projection_node_rows) + if local_worker_expected + else { + "items_running": 0, + "display_running": 0, + "display_claimed": 0, + "display_max_threads": 0, + } + ) + queue_payload = dict((detect.get("queue_health") or {}).get("queue") or {}) if local_worker_expected else {} local_participating = False for node in list(cluster.get("nodes") or []): if str(node.get("node_code") or "").strip() != settings.node_code: @@ -558,15 +780,22 @@ def _build_runtime_projection_payload( break local_job_bucket = {} if local_worker_expected: - for item in list(active_job.get("node_stats") or []): - if str(item.get("node_code") or "").strip() != settings.node_code: + for item in projection_node_rows: + if not _is_local_projection_node(str(item.get("node_code") or "").strip()): continue local_job_bucket = item + local_participating = local_participating or bool( + int(item.get("display_running", 0) or 0) > 0 + or int(item.get("active_threads", 0) or 0) > 0 + or int(item.get("items_running", 0) or 0) > 0 + or int(item.get("items_claimed", 0) or 0) > 0 + ) break if local_worker_expected and not local_participating: local_participating = bool( int(local_job_bucket.get("items_running", 0) or 0) > 0 or int(local_job_bucket.get("items_claimed", 0) or 0) > 0 + or int(display_summary.get("display_running", 0) or 0) > 0 ) projection_active_job = ( @@ -575,12 +804,18 @@ def _build_runtime_projection_payload( "job_code": active_job.get("job_code", ""), "status": active_job.get("status", ""), "progress_percent": active_job.get("progress_percent", 0), - "items_total": active_job.get("items_total", 0), - "items_terminal": active_job.get("items_terminal", 0), - "items_pending": active_job.get("items_pending", 0), - "items_running": active_job.get("items_running", 0), - "items_failed": active_job.get("items_failed", 0), - "node_stats": list(active_job.get("node_stats") or []), + "items_total": int(queue_payload.get("items_total", active_job.get("items_total", 0)) or 0), + "items_terminal": int(queue_payload.get("terminal", active_job.get("items_terminal", 0)) or 0), + "items_pending": int(queue_payload.get("pending", active_job.get("items_pending", 0)) or 0), + "items_claimed": int(queue_payload.get("claimed", active_job.get("items_claimed", 0)) or 0), + "items_running": int(display_summary.get("items_running", 0) or 0), + "items_failed": int(queue_payload.get("failed", active_job.get("items_failed", 0)) or 0), + "display_items_claimed": int(display_summary.get("display_claimed", 0) or 0), + "display_items_running": int(display_summary.get("display_running", 0) or 0), + "display_active_threads": int(display_summary.get("display_running", 0) or 0), + "display_max_threads": int(display_summary.get("display_max_threads", 0) or 0), + "node_stats": list(projection_node_rows), + "distributed_node_stats": list(projection_node_rows), } if local_worker_expected else { @@ -591,18 +826,24 @@ def _build_runtime_projection_payload( "items_total": 0, "items_terminal": 0, "items_pending": 0, + "items_claimed": 0, "items_running": 0, "items_failed": 0, + "display_items_claimed": 0, + "display_items_running": 0, + "display_active_threads": 0, + "display_max_threads": 0, "node_stats": [], + "distributed_node_stats": [], } ) progress_payload = ( { - "pending": int((detect.get("progress") or {}).get("pending", 0) or 0), - "running": int((detect.get("progress") or {}).get("running", 0) or 0), - "completed": int((detect.get("progress") or {}).get("completed", 0) or 0), - "blacklisted": int((detect.get("progress") or {}).get("blacklisted", 0) or 0), - "failed": int((detect.get("progress") or {}).get("failed", 0) or 0), + "pending": int(queue_payload.get("pending", (detect.get("progress") or {}).get("pending", 0)) or 0), + "running": int(display_summary.get("display_running", 0) or 0), + "completed": int(queue_payload.get("completed", (detect.get("progress") or {}).get("completed", 0)) or 0), + "blacklisted": int(queue_payload.get("blacklisted", (detect.get("progress") or {}).get("blacklisted", 0)) or 0), + "failed": int(queue_payload.get("failed", (detect.get("progress") or {}).get("failed", 0)) or 0), } if local_worker_expected else { @@ -624,8 +865,8 @@ def _build_runtime_projection_payload( "worker_online": bool(detect.get("worker_online", False)) if local_worker_expected else False, "detect_participating": local_participating if local_worker_expected else False, "worker_mode": detect.get("worker_mode", ""), - "active_thread_count": int(detect.get("active_thread_count", 0) or 0) if local_worker_expected else 0, - "max_thread_count": int(detect.get("max_thread_count", 0) or 0) if local_worker_expected else 0, + "active_thread_count": int(display_summary.get("display_running", 0) or 0) if local_worker_expected else 0, + "max_thread_count": int(display_summary.get("display_max_threads", 0) or 0) if local_worker_expected else 0, "phase_label": detect.get("phase_label", ""), "phase_detail": detect.get("phase_detail", ""), "proxy_runtime_label": detect.get("proxy_runtime_label", ""), @@ -633,6 +874,7 @@ def _build_runtime_projection_payload( "progress": progress_payload, "backlog": dict(detect.get("backlog") or {}) if local_worker_expected else {}, "active_job": projection_active_job, + "cluster_nodes": list(projection_cluster_rows), "cluster_summary": { "nodes_total": int(cluster.get("nodes_total", 0) or 0), "online_worker_nodes": int((cluster.get("summary") or {}).get("online_worker_nodes", 0) or 0), @@ -670,6 +912,7 @@ def append_runtime_projection_if_changed( ) -> int | None: normalized_source_region = _normalize_region(source_region, _normalize_region(settings.sync_source_region, settings.node_region)) normalized_target_region = _normalize_region(target_region, _normalize_region(settings.sync_target_region, "overseas")) + future_cutoff = datetime.now() + _RUNTIME_PROJECTION_FUTURE_SKEW_GRACE payload = _build_runtime_projection_payload( detect=detect, cluster=cluster, @@ -686,16 +929,17 @@ def append_runtime_projection_if_changed( WHERE sync_type = 'runtime_projection' AND source_region = %s AND target_region = %s - ORDER BY created_at DESC, id DESC - LIMIT 1 + ORDER BY + CASE WHEN created_at <= %s THEN 0 ELSE 1 END ASC, + created_at DESC, + id DESC + LIMIT 200 """, - (normalized_source_region, normalized_target_region), + (normalized_source_region, normalized_target_region, future_cutoff), ) - latest = cur.fetchone() + latest = _pick_latest_projection_row(list(cur.fetchall() or []), created_at_index=1) latest_payload = _decode_json(latest[0]) if latest else {} latest_created_at = latest[1] if latest else None - if latest_payload.get("projection_hash") == payload["projection_hash"]: - return None if not _should_append_runtime_projection(latest_payload, payload["projection"], latest_created_at): return None cur.execute( diff --git a/domain-api/app/services/worker_control_service.py b/domain-api/app/services/worker_control_service.py index fe388b7..fdd7a43 100644 --- a/domain-api/app/services/worker_control_service.py +++ b/domain-api/app/services/worker_control_service.py @@ -7,6 +7,8 @@ from datetime import datetime from pathlib import Path from uuid import uuid4 +import redis + from app.core.config import settings from app.core.redis_client import get_redis from app.services.runtime_settings_service import get_runtime_settings @@ -16,6 +18,101 @@ WORKER_CONTROL_CHANNEL = "domain_tool:worker_control" WORKER_PENDING_COMMAND_KEY = "domain_tool:worker_pending_command" +def _normalize_target_node_codes(payload: dict | None) -> list[str]: + if not isinstance(payload, dict): + return [] + + normalized_targets: list[str] = [] + + def append_target(raw_value: object) -> None: + normalized_value = str(raw_value or "").strip() + if normalized_value and normalized_value not in normalized_targets: + normalized_targets.append(normalized_value) + + for key in ("target_node_codes", "node_codes"): + raw_value = payload.get(key) + if isinstance(raw_value, (list, tuple, set)): + for item in raw_value: + append_target(item) + elif isinstance(raw_value, str) and raw_value.strip(): + for item in raw_value.split(","): + append_target(item) + if normalized_targets: + return normalized_targets + + for key in ("target_node_code", "node_code"): + raw_value = payload.get(key) + if raw_value not in (None, ""): + append_target(raw_value) + if normalized_targets: + return normalized_targets + + return normalized_targets + + +def _pending_command_keys(command_payload: dict) -> list[str]: + target_node_codes = _normalize_target_node_codes(command_payload) + if not target_node_codes: + return [WORKER_PENDING_COMMAND_KEY] + return [f"{WORKER_PENDING_COMMAND_KEY}:{node_code}" for node_code in target_node_codes] + + +def _dedupe_target_node_codes(node_codes: list[str]) -> list[str]: + deduped: list[str] = [] + seen: set[str] = set() + for raw_value in list(node_codes or []): + normalized_value = str(raw_value or "").strip() + if not normalized_value or normalized_value in seen: + continue + seen.add(normalized_value) + deduped.append(normalized_value) + return deduped + + +def _expand_local_linux_worker_target_node_codes(service_name: str) -> list[str]: + base_node_code = str(settings.node_code or "").strip() + normalized_service_name = str(service_name or "").strip() + if not base_node_code or not normalized_service_name: + return [] + + target_node_codes = [base_node_code] + for unit in _expand_linux_worker_control_units(normalized_service_name): + normalized_unit = str(unit or "").strip() + if not normalized_unit: + continue + if normalized_unit.endswith(".service"): + normalized_unit = normalized_unit[:-8] + if normalized_unit == normalized_service_name: + continue + template_prefix = f"{normalized_service_name}@" + if not normalized_unit.startswith(template_prefix): + continue + instance_suffix = str(normalized_unit.split("@", 1)[1] or "").strip() + if instance_suffix: + target_node_codes.append(f"{base_node_code}-{instance_suffix}") + return _dedupe_target_node_codes(target_node_codes) + + +def _publish_worker_command(redis_client, *, serialized: str, pending_keys: list[str]) -> None: + for key in pending_keys: + redis_client.set(key, serialized, ex=120) + redis_client.publish(WORKER_CONTROL_CHANNEL, serialized) + + +def _build_direct_redis_client() -> redis.Redis: + return redis.Redis( + host=settings.redis_host, + port=settings.redis_port, + password=settings.redis_password or None, + db=settings.redis_db, + decode_responses=True, + socket_connect_timeout=5, + socket_timeout=5, + retry_on_timeout=True, + client_name=f"domain-api-workerctl:{settings.node_code}:{os.getpid()}", + ) + + def _domain_root() -> Path: return Path(settings.domain_root) @@ -37,6 +134,125 @@ def _run_shell(command: list[str], timeout: int = 20) -> subprocess.CompletedPro return subprocess.run(command, capture_output=True, text=True, timeout=timeout) +def _parse_process_count_output(result: subprocess.CompletedProcess[str]) -> int | None: + raw_output = (result.stdout or "").strip() + if raw_output.isdigit(): + return max(0, int(raw_output or 0)) + return None + + +def _probe_linux_worker_process_count() -> int: + probe_commands = ( + (["bash", "-lc", "pgrep -fc '[d]etect_worker.py' || true"], 8), + (["bash", "-lc", "ps -eo args= | grep '[d]etect_worker.py' | wc -l"], 12), + ) + for command, timeout in probe_commands: + try: + result = _run_shell(command, timeout=timeout) + except Exception: + continue + parsed_count = _parse_process_count_output(result) + if parsed_count is not None: + return parsed_count + return 0 + + +def _probe_linux_worker_instance_count(service_name: str) -> int: + normalized_service_name = str(service_name or "").strip() + if not normalized_service_name: + return 0 + template_prefix = normalized_service_name[:-8] if normalized_service_name.endswith(".service") else normalized_service_name + try: + result = _run_systemctl( + [ + "list-units", + f"{template_prefix}@*", + "--type=service", + "--all", + "--no-legend", + "--plain", + ], + timeout=12, + require_sudo=False, + ) + except Exception: + return 0 + if result.returncode != 0: + return 0 + count = 0 + for raw_line in (result.stdout or "").splitlines(): + line = str(raw_line or "").strip() + if not line: + continue + parts = line.split() + if len(parts) < 4: + continue + if parts[2] != "active" or parts[3] != "running": + continue + count += 1 + return count + + +def _dedupe_units(units: list[str]) -> list[str]: + deduped: list[str] = [] + seen: set[str] = set() + for item in units: + normalized = str(item or "").strip() + if not normalized or normalized in seen: + continue + seen.add(normalized) + deduped.append(normalized) + return deduped + + +def _list_linux_worker_instance_units(service_name: str) -> list[str]: + normalized_service_name = str(service_name or "").strip() + if not normalized_service_name: + return [] + template_prefix = normalized_service_name[:-8] if normalized_service_name.endswith(".service") else normalized_service_name + units: list[str] = [] + try: + result = _run_systemctl( + [ + "list-units", + f"{template_prefix}@*", + "--type=service", + "--all", + "--no-legend", + "--plain", + ], + timeout=12, + require_sudo=False, + ) + for raw_line in (result.stdout or "").splitlines(): + parts = str(raw_line or "").strip().split() + if parts: + units.append(str(parts[0] or "").strip()) + except Exception: + pass + + managed_prefix = f"{normalized_service_name}-" + try: + for candidate in Path("/etc/default").iterdir(): + if not candidate.is_file(): + continue + if not candidate.name.startswith(managed_prefix): + continue + suffix = str(candidate.name[len(managed_prefix):] or "").strip() + if suffix: + units.append(f"{normalized_service_name}@{suffix}") + except Exception: + pass + return _dedupe_units(units) + + +def _expand_linux_worker_control_units(service_name: str) -> list[str]: + normalized_service_name = str(service_name or "").strip() + if not normalized_service_name: + return [] + return _dedupe_units([normalized_service_name, *_list_linux_worker_instance_units(normalized_service_name)]) + + def _run_systemctl( command: list[str], timeout: int = 20, @@ -192,7 +408,36 @@ def _windows_runtime() -> dict: def _linux_runtime() -> dict: runtime = _runtime_config() service_name = runtime["worker_service_name"] - return probe_systemd_service(service_name, mode="linux-systemd") + service_probe = probe_systemd_service(service_name, mode="linux-systemd") + instance_count = _probe_linux_worker_instance_count(service_name) + process_count = _probe_linux_worker_process_count() + + if process_count <= 0 and instance_count <= 0: + return service_probe + + latest_start_time = str(service_probe.get("latest_start_time") or "").strip() + if service_probe.get("running", False) or instance_count > 0: + message = service_probe.get("message") or f"multi-instance active ({process_count})" + if instance_count > 0 and not service_probe.get("running", False): + message = f"template instances active ({instance_count})" + return { + **service_probe, + "running": True, + "process_count": process_count, + "latest_start_time": latest_start_time, + "message": message, + } + message = str(service_probe.get("message") or "").strip() + if message: + message = f"{message}; detected {process_count} unmanaged worker processes" + else: + message = f"detected {process_count} unmanaged worker processes" + return { + **service_probe, + "process_count": process_count, + "latest_start_time": latest_start_time, + "message": message, + } def detect_sync_agent_runtime() -> dict: @@ -232,9 +477,13 @@ def start_worker() -> tuple[bool, str]: worker_mode = runtime["worker_mode"] service_name = runtime["worker_service_name"] if worker_mode == "linux-systemd": - result = _run_systemctl(["start", service_name], timeout=30) + units = _expand_linux_worker_control_units(service_name) + result = _run_systemctl(["start", *units], timeout=max(30, 15 * max(1, len(units)))) if result.returncode != 0: return False, normalize_systemctl_error(result.stderr or result.stdout or "启动 Linux Worker 失败", service_name=service_name) + extra_units = max(0, len(units) - 1) + if extra_units > 0: + return True, f"Linux Worker 启动命令已发送: {service_name}，附带 {extra_units} 个实例" return True, f"Linux Worker 启动命令已发送: {service_name}" if os.name != "nt": @@ -260,9 +509,13 @@ def stop_worker() -> tuple[bool, str]: worker_mode = runtime["worker_mode"] service_name = runtime["worker_service_name"] if worker_mode == "linux-systemd": - result = _run_systemctl(["stop", service_name], timeout=30) + units = _expand_linux_worker_control_units(service_name) + result = _run_systemctl(["stop", *units], timeout=max(30, 15 * max(1, len(units)))) if result.returncode != 0: return False, normalize_systemctl_error(result.stderr or result.stdout or "停止 Linux Worker 失败", service_name=service_name) + extra_units = max(0, len(units) - 1) + if extra_units > 0: + return True, f"Linux Worker 停止命令已发送: {service_name}，附带 {extra_units} 个实例" return True, f"Linux Worker 停止命令已发送: {service_name}" if os.name != "nt": @@ -289,15 +542,42 @@ def stop_worker() -> tuple[bool, str]: def send_worker_command(action: str, payload: dict | None = None) -> tuple[bool, str]: + command_payload = {"action": action} + if payload: + command_payload.update(payload) + runtime = _runtime_config() + explicit_targets = _normalize_target_node_codes(command_payload) + if not explicit_targets and str(runtime.get("worker_mode") or "").strip() == "linux-systemd": + expanded_targets = _expand_local_linux_worker_target_node_codes( + str(runtime.get("worker_service_name") or "").strip() + ) + if expanded_targets: + command_payload["target_node_codes"] = expanded_targets + command_payload["request_id"] = str(command_payload.get("request_id") or f"workerctl-{uuid4().hex[:12]}") + serialized = json.dumps(command_payload, ensure_ascii=False) + pending_keys = _pending_command_keys(command_payload) + target_node_codes = _normalize_target_node_codes(command_payload) + + direct_client = None try: - redis_client = get_redis() - command_payload = {"action": action} - if payload: - command_payload.update(payload) - command_payload["request_id"] = str(command_payload.get("request_id") or f"workerctl-{uuid4().hex[:12]}") - serialized = json.dumps(command_payload, ensure_ascii=False) - redis_client.set(WORKER_PENDING_COMMAND_KEY, serialized, ex=120) - redis_client.publish(WORKER_CONTROL_CHANNEL, serialized) - return True, f"已发送 Worker 控制指令: {action}" + try: + _publish_worker_command(get_redis(), serialized=serialized, pending_keys=pending_keys) + except Exception: + direct_client = _build_direct_redis_client() + _publish_worker_command(direct_client, serialized=serialized, pending_keys=pending_keys) + + if pending_keys == [WORKER_PENDING_COMMAND_KEY]: + return True, f"已发送 Worker 控制指令: {action}" + if len(target_node_codes) <= 4: + target_summary = ",".join(target_node_codes) + else: + target_summary = f"{len(target_node_codes)} targets" + return True, f"已发送 Worker 控制指令: {action} -> {target_summary}" except Exception as exc: return False, f"发送 Worker 控制指令失败: {exc}" + finally: + if direct_client is not None: + try: + direct_client.close() + except Exception: + pass diff --git a/domain-api/app/sync_agent.py b/domain-api/app/sync_agent.py index 95e6396..1cdfe2e 100644 --- a/domain-api/app/sync_agent.py +++ b/domain-api/app/sync_agent.py @@ -1,8 +1,10 @@ from __future__ import annotations import logging +import os import time +from app.core.db import db_read_retry, get_db from app.core.config import settings from app.services.debug_event_service import push_debug_event from app.services.detect_job_service import ( @@ -19,6 +21,10 @@ from app.services.sync_push_service import ( pull_detect_task_batch_now, push_runtime_projection_now, ) +from app.services.worker_control_service import ( + _expand_local_linux_worker_target_node_codes, + send_worker_command, +) logger = logging.getLogger("domaincheck.sync_agent") @@ -33,6 +39,9 @@ _IDLE_SYNC_KEYWORDS = ( "暂停拉取", ) +_LAST_OVERLAP_JOB_ID = 0 +_LAST_OVERLAP_TRIGGERED_AT = 0.0 + def _append_detect_result_projection_snapshot(active_job: dict) -> None: if not active_job: @@ -278,6 +287,196 @@ def _emit_sync_result_breakdown(data: dict | None) -> None: ) +def _bool_env(name: str, default: bool) -> bool: + raw = str(os.getenv(name, "1" if default else "0") or ("1" if default else "0")).strip().lower() + return raw not in {"0", "false", "off", "no"} + + +def _int_env(name: str, default: int, *, minimum: int = 0, maximum: int | None = None) -> int: + try: + value = int(os.getenv(name, str(default)) or default) + except (TypeError, ValueError): + value = int(default) + value = max(minimum, value) + if maximum is not None: + value = min(maximum, value) + return value + + +@db_read_retry(attempts=3, initial_delay_seconds=0.05, backoff=2.0) +def _select_overlap_start_candidate() -> dict | None: + if settings.node_region != "mainland" or settings.node_role != "control": + return None + if not _bool_env("DOMAINCHECK_OVERLAP_HANDOFF_ENABLED", True): + return None + + min_running_jobs = _int_env("DOMAINCHECK_OVERLAP_HANDOFF_MIN_RUNNING_JOBS", 1, minimum=1, maximum=16) + running_age_seconds = _int_env("DOMAINCHECK_OVERLAP_HANDOFF_MIN_RUNNING_AGE_SECONDS", 300, minimum=30, maximum=7200) + with get_db() as conn: + with conn.cursor() as cur: + cur.execute( + """ + WITH running_state AS ( + SELECT + COUNT(*) AS running_jobs, + COALESCE( + MAX( + EXTRACT( + EPOCH FROM (CURRENT_TIMESTAMP - COALESCE(job.started_at, job.created_at)) + ) + ), + 0 + ) AS max_running_age_seconds + FROM detect_jobs AS job + WHERE job.status = 'running' + ), + pending_candidate AS ( + SELECT + job.id, + job.job_code, + job.task_mode, + COALESCE(job.started_at, job.created_at) AS activity_at + FROM detect_jobs AS job + WHERE job.status = 'pending' + ORDER BY + COALESCE(job.started_at, job.created_at) DESC, + job.id DESC + LIMIT 1 + ) + SELECT + pending.id, + pending.job_code, + pending.task_mode, + running_state.running_jobs, + running_state.max_running_age_seconds, + 'overlap_tail_handoff' AS selection_reason + FROM pending_candidate AS pending + CROSS JOIN running_state + WHERE running_state.running_jobs >= %s + AND running_state.max_running_age_seconds >= %s + """, + ( + min_running_jobs, + running_age_seconds, + ), + ) + row = cur.fetchone() + if not row: + return None + return { + "job_id": int(row[0] or 0), + "job_code": str(row[1] or "").strip(), + "task_mode": str(row[2] or "domain_pipeline").strip() or "domain_pipeline", + "running_jobs": int(row[3] or 0), + "max_running_age_seconds": int(float(row[4] or 0) or 0), + "selection_reason": str(row[5] or "overlap_tail_handoff").strip() or "overlap_tail_handoff", + } + + +@db_read_retry(attempts=3, initial_delay_seconds=0.05, backoff=2.0) +def _select_overlap_target_node_codes() -> list[str]: + if settings.node_region != "mainland" or settings.node_role != "control": + return [] + + target_limit = _int_env("DOMAINCHECK_OVERLAP_HANDOFF_TARGETS", 12, minimum=1, maximum=32) + target_scan_limit = _int_env( + "DOMAINCHECK_OVERLAP_HANDOFF_TARGET_SCAN_LIMIT", + max(target_limit * 4, 16), + minimum=target_limit, + maximum=256, + ) + max_current_load = _int_env("DOMAINCHECK_OVERLAP_HANDOFF_TARGET_MAX_CURRENT_LOAD", 0, minimum=0, maximum=4096) + + expanded_targets = _expand_local_linux_worker_target_node_codes(str(settings.worker_service_name or "").strip()) + if not expanded_targets: + return [] + + with get_db() as conn: + with conn.cursor() as cur: + cur.execute( + """ + SELECT node_code, current_load + FROM detect_worker_nodes + WHERE node_code = ANY(%s) + ORDER BY + CASE WHEN COALESCE(current_load, 0) <= %s THEN 0 ELSE 1 END ASC, + COALESCE(current_load, 0) ASC, + COALESCE(update_time, last_heartbeat_at) DESC, + node_code ASC + LIMIT %s + """, + ( + expanded_targets, + max_current_load, + target_scan_limit, + ), + ) + rows = list(cur.fetchall() or []) + + preferred: list[str] = [] + fallback: list[str] = [] + seen: set[str] = set() + for raw_node_code, raw_current_load in rows: + node_code = str(raw_node_code or "").strip() + if not node_code or node_code in seen: + continue + seen.add(node_code) + fallback.append(node_code) + if int(raw_current_load or 0) <= max_current_load: + preferred.append(node_code) + + ordered = preferred + [node_code for node_code in fallback if node_code not in set(preferred)] + if not ordered: + ordered = list(expanded_targets or []) + return ordered[:target_limit] + + +def _maybe_trigger_overlap_start() -> tuple[bool, str, dict]: + global _LAST_OVERLAP_JOB_ID, _LAST_OVERLAP_TRIGGERED_AT + + candidate = _select_overlap_start_candidate() + if not candidate: + return False, "当前没有满足尾盘接棒条件的 pending job", {} + + cooldown_seconds = _int_env("DOMAINCHECK_OVERLAP_HANDOFF_COOLDOWN_SECONDS", 120, minimum=15, maximum=1800) + job_id = int(candidate.get("job_id") or 0) + now_ts = time.time() + if ( + job_id > 0 + and job_id == _LAST_OVERLAP_JOB_ID + and now_ts - float(_LAST_OVERLAP_TRIGGERED_AT or 0.0) < cooldown_seconds + ): + return False, f"overlap handoff 冷却中: job_id={job_id}", candidate + + target_node_codes = _select_overlap_target_node_codes() + if not target_node_codes: + return False, f"overlap handoff 未找到可接棒 worker: job_id={job_id}", candidate + + payload = { + "job_id": job_id, + "job_code": str(candidate.get("job_code") or "").strip(), + "target_job_id": job_id, + "target_job_code": str(candidate.get("job_code") or "").strip(), + "task_mode": str(candidate.get("task_mode") or "domain_pipeline").strip() or "domain_pipeline", + "source": "overlap-handoff", + "selection_reason": str(candidate.get("selection_reason") or "overlap_tail_handoff").strip() or "overlap_tail_handoff", + "tail_handoff_candidate": True, + "target_node_codes": target_node_codes, + } + ok, message = send_worker_command("start_detection", payload=payload) + if ok and job_id > 0: + _LAST_OVERLAP_JOB_ID = job_id + _LAST_OVERLAP_TRIGGERED_AT = now_ts + push_debug_event( + service="sync-agent", + event_type="overlap_handoff_started", + level="warning", + message=message, + payload={**candidate, "cooldown_seconds": cooldown_seconds, "target_node_codes": target_node_codes}, + ) + return ok, message, candidate + + def _run_pipeline_stage_processor() -> tuple[bool, str, dict]: process_limit = max(500, min(int(settings.sync_pipeline_process_limit or 5000), 5000)) ok, message, data = process_detect_pipeline_now(limit=process_limit) @@ -295,6 +494,94 @@ def _run_pipeline_stage_processor() -> tuple[bool, str, dict]: return ok, message, data +def _emit_runtime_debug_snapshots() -> None: + active_job = get_active_detect_job_summary(event_limit=10) + for projection_job in _select_projection_job_snapshots(): + _append_detect_result_projection_snapshot(projection_job) + + if not active_job: + return + + queue_health = _build_aligned_queue_health_snapshot( + active_job, + get_detect_queue_health(window_minutes=15), + ) + recent_events = _filter_runtime_events_for_job( + list_recent_detect_run_events(limit=24), + job_code=str(active_job.get("runtime_job_code") or active_job.get("job_code") or "").strip(), + job_id=int(active_job.get("job_id", 0) or 0), + limit=8, + ) + push_debug_event( + service="detect-runtime", + event_type="active_job_snapshot", + level="info", + message=f"active job {active_job.get('job_code', '')} status={active_job.get('status', '')}", + payload={ + "job": { + "job_id": active_job.get("job_id"), + "job_code": active_job.get("job_code", ""), + "status": active_job.get("status", ""), + "items_total": active_job.get("items_total", 0), + "items_pending": active_job.get("items_pending", 0), + "items_claimed": active_job.get("items_claimed", 0), + "items_running": active_job.get("items_running", 0), + "items_completed": active_job.get("items_completed", 0), + "items_failed": active_job.get("items_failed", 0), + "progress_percent": active_job.get("progress_percent", 0), + "node_stats": list(active_job.get("node_stats") or []), + }, + "queue_health": queue_health, + "backlog": _load_local_detect_backlog_snapshot(), + "recent_events": recent_events, + }, + ) + if queue_health.get("queue", {}).get("overdue_leases", 0): + push_debug_event( + service="detect-runtime", + event_type="queue_overdue_leases", + level="warning", + message=f"检测队列存在过期租约 {queue_health.get('queue', {}).get('overdue_leases', 0)} 个", + payload=queue_health, + ) + for event in recent_events: + event_type = str(event.get("event_type") or "").strip() + if event_type not in {"domain_started", "domain_completed", "domain_failed", "domain_blacklisted"}: + continue + push_debug_event( + service="worker-event", + event_type=event_type, + level=str(event.get("level") or "info"), + message=str(event.get("message") or "").strip(), + payload={ + "job_id": event.get("job_id"), + "node_code": event.get("node_code", ""), + "created_at": event.get("created_at", ""), + **(event.get("payload") or {}), + }, + ) + + +def _run_sync_tick_once() -> dict: + overlap_ok, overlap_message, overlap_data = _maybe_trigger_overlap_start() + sync_ok, sync_message, sync_data = push_runtime_projection_now() + pull_ok, pull_message, pull_data = pull_detect_task_batch_now() + pipeline_ok, pipeline_message, pipeline_data = _run_pipeline_stage_processor() + try: + _emit_runtime_debug_snapshots() + debug_snapshot_error = "" + except Exception as debug_exc: # pragma: no cover - logged by caller path + logger.warning("runtime debug snapshot skipped: %s", debug_exc) + debug_snapshot_error = str(debug_exc) + return { + "sync": {"ok": sync_ok, "message": sync_message, "data": sync_data}, + "pull": {"ok": pull_ok, "message": pull_message, "data": pull_data}, + "pipeline": {"ok": pipeline_ok, "message": pipeline_message, "data": pipeline_data}, + "overlap": {"ok": overlap_ok, "message": overlap_message, "data": overlap_data}, + "debug_snapshot_error": debug_snapshot_error, + } + + def main() -> None: logging.basicConfig( level=logging.INFO, @@ -313,28 +600,24 @@ def main() -> None: ) while True: try: - pipeline_ok, pipeline_message, pipeline_data = _run_pipeline_stage_processor() - logger.info( - "pipeline tick: ok=%s message=%s data=%s", - pipeline_ok, - pipeline_message, - pipeline_data, - ) - active_job = get_active_detect_job_summary(event_limit=10) - for projection_job in _select_projection_job_snapshots(): - _append_detect_result_projection_snapshot(projection_job) - ok, message, data = push_runtime_projection_now() - logger.info("sync tick: ok=%s message=%s data=%s", ok, message, data) + tick = _run_sync_tick_once() + sync_ok = bool((tick.get("sync") or {}).get("ok")) + sync_message = str((tick.get("sync") or {}).get("message") or "") + sync_data = (tick.get("sync") or {}).get("data") or {} + logger.info("sync tick: ok=%s message=%s data=%s", sync_ok, sync_message, sync_data) push_debug_event( service="sync-agent", event_type="sync_tick", - level="info" if ok else "warning", - message=message, - payload={"ok": ok, "data": data}, + level="info" if sync_ok else "warning", + message=sync_message, + payload={"ok": sync_ok, "data": sync_data}, ) - _emit_structured_tick(base_event_type="sync_push", ok=ok, message=message, data=data) - _emit_sync_result_breakdown(data) - pull_ok, pull_message, pull_data = pull_detect_task_batch_now() + _emit_structured_tick(base_event_type="sync_push", ok=sync_ok, message=sync_message, data=sync_data) + _emit_sync_result_breakdown(sync_data) + + pull_ok = bool((tick.get("pull") or {}).get("ok")) + pull_message = str((tick.get("pull") or {}).get("message") or "") + pull_data = (tick.get("pull") or {}).get("data") or {} logger.info("task pull tick: ok=%s message=%s data=%s", pull_ok, pull_message, pull_data) push_debug_event( service="sync-agent", @@ -344,65 +627,33 @@ def main() -> None: payload={"ok": pull_ok, "data": pull_data}, ) _emit_structured_tick(base_event_type="task_pull", ok=pull_ok, message=pull_message, data=pull_data) - if active_job: - queue_health = _build_aligned_queue_health_snapshot( - active_job, - get_detect_queue_health(window_minutes=15), - ) - recent_events = _filter_runtime_events_for_job( - list_recent_detect_run_events(limit=24), - job_code=str(active_job.get("runtime_job_code") or active_job.get("job_code") or "").strip(), - job_id=int(active_job.get("job_id", 0) or 0), - limit=8, - ) - push_debug_event( - service="detect-runtime", - event_type="active_job_snapshot", - level="info", - message=f"active job {active_job.get('job_code', '')} status={active_job.get('status', '')}", - payload={ - "job": { - "job_id": active_job.get("job_id"), - "job_code": active_job.get("job_code", ""), - "status": active_job.get("status", ""), - "items_total": active_job.get("items_total", 0), - "items_pending": active_job.get("items_pending", 0), - "items_claimed": active_job.get("items_claimed", 0), - "items_running": active_job.get("items_running", 0), - "items_completed": active_job.get("items_completed", 0), - "items_failed": active_job.get("items_failed", 0), - "progress_percent": active_job.get("progress_percent", 0), - "node_stats": list(active_job.get("node_stats") or []), - }, - "queue_health": queue_health, - "backlog": _load_local_detect_backlog_snapshot(), - "recent_events": recent_events, - }, - ) - if queue_health.get("queue", {}).get("overdue_leases", 0): - push_debug_event( - service="detect-runtime", - event_type="queue_overdue_leases", - level="warning", - message=f"检测队列存在过期租约 {queue_health.get('queue', {}).get('overdue_leases', 0)} 个", - payload=queue_health, - ) - for event in recent_events: - event_type = str(event.get("event_type") or "").strip() - if event_type not in {"domain_started", "domain_completed", "domain_failed", "domain_blacklisted"}: - continue - push_debug_event( - service="worker-event", - event_type=event_type, - level=str(event.get("level") or "info"), - message=str(event.get("message") or "").strip(), - payload={ - "job_id": event.get("job_id"), - "node_code": event.get("node_code", ""), - "created_at": event.get("created_at", ""), - **(event.get("payload") or {}), - }, - ) + + pipeline_ok = bool((tick.get("pipeline") or {}).get("ok")) + pipeline_message = str((tick.get("pipeline") or {}).get("message") or "") + pipeline_data = (tick.get("pipeline") or {}).get("data") or {} + logger.info( + "pipeline tick: ok=%s message=%s data=%s", + pipeline_ok, + pipeline_message, + pipeline_data, + ) + + overlap_ok = bool((tick.get("overlap") or {}).get("ok")) + overlap_message = str((tick.get("overlap") or {}).get("message") or "") + overlap_data = (tick.get("overlap") or {}).get("data") or {} + logger.info( + "overlap tick: ok=%s message=%s data=%s", + overlap_ok, + overlap_message, + overlap_data, + ) + push_debug_event( + service="sync-agent", + event_type="overlap_tick", + level="info" if overlap_ok else "warning", + message=overlap_message, + payload={"ok": overlap_ok, "data": overlap_data}, + ) except Exception as exc: logger.exception("sync tick failed: %s", exc) push_debug_event( diff --git a/domain-api/deploy/linux/smoke_test.py b/domain-api/deploy/linux/smoke_test.py index a010497..731d785 100644 --- a/domain-api/deploy/linux/smoke_test.py +++ b/domain-api/deploy/linux/smoke_test.py @@ -8,7 +8,7 @@ from datetime import datetime import requests -def _fetch_json(session: requests.Session, url: str, timeout: int = 15) -> tuple[bool, str, dict | None]: +def _fetch_json(session: requests.Session, url: str, timeout: int = 25) -> tuple[bool, str, dict | None]: try: response = session.get(url, timeout=timeout) response.raise_for_status() @@ -45,20 +45,22 @@ def main() -> int: checks: list[dict[str, object]] = [] endpoints = [ - ("health", f"{base_url}/health"), - ("runtime_status", f"{base_url}/api/v1/runtime/status"), - ("runtime_preflight", f"{base_url}/api/v1/runtime/preflight"), - ("dashboard_overview", f"{base_url}/api/v1/dashboard/overview"), - ("settings_export", f"{base_url}/api/v1/settings/export"), - ("settings_backups", f"{base_url}/api/v1/settings/backups"), - ("detect_status", f"{base_url}/api/v1/detect/status"), - ("imports_summary", f"{base_url}/api/v1/imports/summary"), - ("exports", f"{base_url}/api/v1/exports"), - ("logs_latest", f"{base_url}/api/v1/logs/latest"), + ("health", f"{base_url}/health", 25), + ("runtime_status", f"{base_url}/api/v1/runtime/status", 25), + ("runtime_preflight", f"{base_url}/api/v1/runtime/preflight", 25), + # overview merges multiple heavy runtime aggregates and can legitimately + # take longer than lightweight health/readiness endpoints on live load. + ("dashboard_overview", f"{base_url}/api/v1/dashboard/overview", 25), + ("settings_export", f"{base_url}/api/v1/settings/export", 25), + ("settings_backups", f"{base_url}/api/v1/settings/backups", 25), + ("detect_status", f"{base_url}/api/v1/detect/status", 25), + ("imports_summary", f"{base_url}/api/v1/imports/summary", 25), + ("exports", f"{base_url}/api/v1/exports", 25), + ("logs_latest", f"{base_url}/api/v1/logs/latest", 25), ] - for name, url in endpoints: - ok, message, payload = _fetch_json(session, url) + for name, url, timeout in endpoints: + ok, message, payload = _fetch_json(session, url, timeout=timeout) checks.append( { "name": name, diff --git a/domain-api/deploy/multi-region/README.md b/domain-api/deploy/multi-region/README.md index 61cce35..dea4992 100644 --- a/domain-api/deploy/multi-region/README.md +++ b/domain-api/deploy/multi-region/README.md @@ -39,6 +39,7 @@ - `domain-api/deploy/multi-region/drive_ops_action.sh` - `domain-api/deploy/multi-region/templates/domaincheck-node-agent.env.example` - `domain-api/deploy/multi-region/templates/domaincheck-ops-center.env.example` +- `domain-api/deploy/multi-region/templates/domaincheck-worker.instance.env.example` - `domain-api/deploy/multi-region/check_cluster.sh` - `domain-api/deploy/multi-region/check_node_agent.sh` - `domain-api/deploy/multi-region/check_ops_center_stack.sh` @@ -218,6 +219,32 @@ bash domain-api/deploy/multi-region/drive_ops_center.sh go-live-recover - `runtime/build-info` - 一份最终 `status / headline / missing_items / tooling_items` 收口摘要 +## 一点六、单机多进程 Worker 起步方式 + +如果大陆执行机是高核大内存机器，而单个 `detect_worker.py` 进程还吃不满机器，可以先不要继续堆单进程线程数，优先改成“同机多实例 Worker”： + +```bash +# 1. 复制实例模板，按实例编号准备独立 env +cp domain-api/deploy/multi-region/templates/domaincheck-worker.instance.env.example /etc/default/domaincheck-worker-a +cp domain-api/deploy/multi-region/templates/domaincheck-worker.instance.env.example /etc/default/domaincheck-worker-b + +# 2. 为每个实例设置不同的 NODE_CODE +# mainland-controller-01-a +# mainland-controller-01-b + +# 3. 启用 systemd 模板实例 +cp domain-api/deploy/systemd/domain-worker@.service /etc/systemd/system/domaincheck-worker@.service +systemctl daemon-reload +systemctl enable --now domaincheck-worker@a +systemctl enable --now domaincheck-worker@b +``` + +注意： + +- 同一台机器上的每个实例必须使用不同的 `NODE_CODE` +- 每个实例建议先用中等线程数压测，不要直接把单进程线程数拉到极限 +- 这版控制指令已经支持按 `NODE_CODE` 定向，不同实例不会再一起响应同一条本地 Worker 指令 + 说明： - `missing_items` 只保留真正影响部署或运行面的缺口 diff --git a/domain-api/deploy/multi-region/templates/domaincheck-worker.controller.env.example b/domain-api/deploy/multi-region/templates/domaincheck-worker.controller.env.example index b1fc4f0..e031560 100644 --- a/domain-api/deploy/multi-region/templates/domaincheck-worker.controller.env.example +++ b/domain-api/deploy/multi-region/templates/domaincheck-worker.controller.env.example @@ -1,5 +1,8 @@ WORKER_MODE=linux-systemd QT_QPA_PLATFORM=offscreen + +# 单实例控制节点参考。若同一台大陆大机需要起多个 Worker 进程， +# 请改用 domaincheck-worker.instance.env.example + domain-worker@.service。 NODE_CODE=mainland-controller-01 NODE_REGION=mainland NODE_ROLE=control diff --git a/domain-api/deploy/multi-region/templates/domaincheck-worker.instance.env.example b/domain-api/deploy/multi-region/templates/domaincheck-worker.instance.env.example new file mode 100644 index 0000000..83ae2ac --- /dev/null +++ b/domain-api/deploy/multi-region/templates/domaincheck-worker.instance.env.example @@ -0,0 +1,40 @@ +WORKER_MODE=linux-systemd +QT_QPA_PLATFORM=offscreen + +# 对应 systemd 模板: +# cp domaincheck-worker.instance.env.example /etc/default/domaincheck-worker-a +# cp domain-api/deploy/systemd/domain-worker@.service /etc/systemd/system/domaincheck-worker@.service +# systemctl daemon-reload +# systemctl enable --now domaincheck-worker@a +# +# 同一台机器上起多个 worker 实例时，每个实例必须使用不同的 NODE_CODE， +# 否则运行态、控制指令和集群心跳会互相覆盖。 +NODE_CODE=mainland-controller-01-a +WORKER_PARENT_NODE_CODE=mainland-controller-01 +NODE_REGION=mainland +NODE_ROLE=control + +# DB +# DB_HOST=127.0.0.1 +# DB_PORT=5432 +# DB_DATABASE=domain +# DB_USER=postgres +# DB_PASSWORD=postgres + +# REDIS +# REDIS_HOST=127.0.0.1 +# REDIS_PORT=6379 +# REDIS_PASSWORD= +# REDIS_DB=0 + +# SYNC +SYNC_PUSH_ENABLED=true +SYNC_SOURCE_REGION=mainland +SYNC_TARGET_REGION=overseas +SYNC_TARGET_API_BASE_URL=http://海外控制面IP:8100/api/v1 +SYNC_SHARED_TOKEN= +SYNC_BATCH_SIZE=5000 +SYNC_POLL_INTERVAL_SECONDS=2 + +# 第一版建议每实例先用中等线程数压测，不要直接把单进程线程拉到极限。 +# THREAD_COUNT=1000 diff --git a/domain-api/deploy/systemd/domain-api.service b/domain-api/deploy/systemd/domain-api.service index 50973e5..fbc7442 100644 --- a/domain-api/deploy/systemd/domain-api.service +++ b/domain-api/deploy/systemd/domain-api.service @@ -11,9 +11,10 @@ Environment="WORKER_MODE=linux-systemd" Environment="API_HOST=0.0.0.0" Environment="API_PORT=8100" Environment="DOMAIN_ROOT=/opt/domaincheck/domainCheck" -ExecStart=/opt/domaincheck/domainCheck/.venv/bin/python -m uvicorn app.main:app --host 0.0.0.0 --port 8100 +ExecStart=/opt/domaincheck/domainCheck/.venv/bin/python -m uvicorn app.main:app --host 0.0.0.0 --port 8100 --timeout-graceful-shutdown 15 Restart=always RestartSec=5 +TimeoutStopSec=20 User=www Group=www diff --git a/domain-api/deploy/systemd/domain-worker.service b/domain-api/deploy/systemd/domain-worker.service index c55c1cd..31b5449 100644 --- a/domain-api/deploy/systemd/domain-worker.service +++ b/domain-api/deploy/systemd/domain-worker.service @@ -4,13 +4,15 @@ After=network.target redis.service postgresql.service [Service] Type=simple -WorkingDirectory=/opt/domaincheck/domainCheck +WorkingDirectory=/opt/domaincheck/current/domainCheck EnvironmentFile=-/etc/default/domaincheck-worker Environment="WORKER_MODE=linux-systemd" Environment="QT_QPA_PLATFORM=offscreen" -ExecStart=/opt/domaincheck/domainCheck/.venv/bin/python /opt/domaincheck/domainCheck/detect_worker.py +ExecStart=/opt/domaincheck/domainCheck/.venv/bin/python /opt/domaincheck/current/domainCheck/detect_worker.py Restart=always RestartSec=5 +LimitNOFILE=65535 +TasksMax=infinity User=www Group=www diff --git a/domain-api/deploy/systemd/domain-worker@.service b/domain-api/deploy/systemd/domain-worker@.service new file mode 100644 index 0000000..ce38781 --- /dev/null +++ b/domain-api/deploy/systemd/domain-worker@.service @@ -0,0 +1,21 @@ +[Unit] +Description=domainCheck Worker Instance %i +After=network.target redis.service postgresql.service + +[Service] +Type=simple +WorkingDirectory=/opt/domaincheck/current/domainCheck +EnvironmentFile=-/etc/default/domaincheck-worker-%i +Environment="WORKER_MODE=linux-systemd" +Environment="QT_QPA_PLATFORM=offscreen" +ExecStart=/opt/domaincheck/domainCheck/.venv/bin/python /opt/domaincheck/current/domainCheck/detect_worker.py +Restart=always +RestartSec=5 +LimitNOFILE=65535 +TasksMax=infinity +SyslogIdentifier=domaincheck-worker@%i +User=www +Group=www + +[Install] +WantedBy=multi-user.target diff --git a/domain-api/tests/test_build_info_service.py b/domain-api/tests/test_build_info_service.py index 78f8fdd..3945795 100644 --- a/domain-api/tests/test_build_info_service.py +++ b/domain-api/tests/test_build_info_service.py @@ -81,6 +81,9 @@ class BuildInfoServiceTests(unittest.TestCase): self.assertEqual("env-sha256", payload["checksum"]) self.assertFalse(payload["route_surface"]["surface_complete"]) self.assertIn("ops_stack_diagnosis", payload["route_surface"]["missing_keys"]) + self.assertIn("ops_migration_source_profile", payload["route_surface"]["missing_keys"]) + self.assertIn("ops_migration_preview", payload["route_surface"]["missing_keys"]) + self.assertIn("ops_migration_execute", payload["route_surface"]["missing_keys"]) self.assertIn("ops_node_onboarding_bootstrap_preview", payload["route_surface"]["missing_keys"]) self.assertIn("ops_node_onboarding_bootstrap_execute", payload["route_surface"]["missing_keys"]) self.assertIn("ops_node_onboarding_acceptance_preview", payload["route_surface"]["missing_keys"]) diff --git a/domain-api/tests/test_cluster_runtime_service.py b/domain-api/tests/test_cluster_runtime_service.py index 009c8b9..073862c 100644 --- a/domain-api/tests/test_cluster_runtime_service.py +++ b/domain-api/tests/test_cluster_runtime_service.py @@ -1,12 +1,65 @@ from __future__ import annotations import unittest +from datetime import datetime, timedelta from unittest.mock import MagicMock, patch +from psycopg2 import errors + from app.services import cluster_runtime_service class RuntimeSchemaGuardTests(unittest.TestCase): + def test_ensure_runtime_schema_does_not_toggle_autocommit_after_queries(self) -> None: + class _Connection: + def __init__(self) -> None: + self._autocommit = False + self.touched = False + self.committed = False + + @property + def autocommit(self): + return self._autocommit + + @autocommit.setter + def autocommit(self, value): + if self.touched: + raise AssertionError("autocommit should not be reassigned after queries start") + self._autocommit = value + + def cursor(self): + conn = self + + class _CursorContext: + def __enter__(self_inner): + conn.touched = True + return cursor + + def __exit__(self_inner, exc_type, exc, tb): + return False + + return _CursorContext() + + def commit(self): + self.committed = True + + cursor = MagicMock() + conn = _Connection() + db_cm = MagicMock() + db_cm.__enter__.return_value = conn + + with patch.object(cluster_runtime_service, "_RUNTIME_SCHEMA_READY", False): + with patch.object(cluster_runtime_service, "get_db", return_value=db_cm): + with patch.object(cluster_runtime_service, "_runtime_schema_basics_present", return_value=False): + cluster_runtime_service.ensure_runtime_schema() + + cursor.execute.assert_any_call( + "SELECT pg_advisory_xact_lock(%s)", + (cluster_runtime_service._RUNTIME_SCHEMA_ADVISORY_LOCK_ID,), + ) + cursor.execute.assert_any_call(cluster_runtime_service._RUNTIME_SCHEMA_SQL) + self.assertTrue(conn.committed) + def test_ensure_runtime_schema_executes_only_once_per_process(self) -> None: conn = MagicMock() cursor_cm = MagicMock() @@ -18,11 +71,11 @@ class RuntimeSchemaGuardTests(unittest.TestCase): with patch.object(cluster_runtime_service, "_RUNTIME_SCHEMA_READY", False): with patch.object(cluster_runtime_service, "get_db", return_value=db_cm) as mocked_get_db: - cluster_runtime_service.ensure_runtime_schema() - cluster_runtime_service.ensure_runtime_schema() + with patch.object(cluster_runtime_service, "_runtime_schema_basics_present", return_value=False): + cluster_runtime_service.ensure_runtime_schema() + cluster_runtime_service.ensure_runtime_schema() mocked_get_db.assert_called_once() - self.assertEqual(cursor.execute.call_count, 2) cursor.execute.assert_any_call( "SELECT pg_advisory_xact_lock(%s)", (cluster_runtime_service._RUNTIME_SCHEMA_ADVISORY_LOCK_ID,), @@ -30,6 +83,118 @@ class RuntimeSchemaGuardTests(unittest.TestCase): cursor.execute.assert_any_call(cluster_runtime_service._RUNTIME_SCHEMA_SQL) conn.commit.assert_called_once() + def test_ensure_runtime_schema_skips_ddl_when_required_schema_already_exists(self) -> None: + conn = MagicMock() + cursor_cm = MagicMock() + cursor = MagicMock() + conn.cursor.return_value = cursor_cm + cursor_cm.__enter__.return_value = cursor + db_cm = MagicMock() + db_cm.__enter__.return_value = conn + + with patch.object(cluster_runtime_service, "_RUNTIME_SCHEMA_READY", False): + with patch.object(cluster_runtime_service, "get_db", return_value=db_cm): + with patch.object(cluster_runtime_service, "_runtime_schema_basics_present", return_value=True): + with patch.object(cluster_runtime_service, "_runtime_missing_indexes", return_value=iter(())): + cluster_runtime_service.ensure_runtime_schema() + + self.assertFalse( + any(sql == cluster_runtime_service._RUNTIME_SCHEMA_SQL for sql, _params in cursor.execute.call_args_list) + ) + conn.commit.assert_not_called() + + def test_ensure_runtime_schema_accepts_deadlock_when_required_schema_already_exists(self) -> None: + class _Cursor: + def __init__(self, *, raise_on_schema=False, fetchone_values=None, fetchall_values=None) -> None: + self.raise_on_schema = raise_on_schema + self.fetchone_values = list(fetchone_values or []) + self.fetchall_values = list(fetchall_values or []) + self.executed = [] + + def execute(self, sql, params=None): + self.executed.append((sql, params)) + if self.raise_on_schema and sql == cluster_runtime_service._RUNTIME_SCHEMA_SQL: + raise errors.DeadlockDetected() + + def fetchone(self): + if self.fetchone_values: + return self.fetchone_values.pop(0) + return None + + def fetchall(self): + if self.fetchall_values: + return self.fetchall_values.pop(0) + return [] + + class _CursorContext: + def __init__(self, cursor) -> None: + self.cursor = cursor + + def __enter__(self): + return self.cursor + + def __exit__(self, exc_type, exc, tb): + return False + + conn = MagicMock() + first_cursor = _Cursor() + second_cursor = _Cursor(raise_on_schema=True) + third_cursor = _Cursor() + conn.cursor.side_effect = [ + _CursorContext(first_cursor), + _CursorContext(second_cursor), + _CursorContext(third_cursor), + ] + db_cm = MagicMock() + db_cm.__enter__.return_value = conn + + with patch.object(cluster_runtime_service, "_RUNTIME_SCHEMA_READY", False): + with patch.object(cluster_runtime_service, "get_db", return_value=db_cm): + with patch.object(cluster_runtime_service, "_runtime_schema_basics_present", side_effect=[False, True]): + with patch.object(cluster_runtime_service, "_runtime_missing_indexes", return_value=iter(())): + cluster_runtime_service.ensure_runtime_schema() + + conn.rollback.assert_called_once() + conn.commit.assert_not_called() + + def test_ensure_runtime_schema_repairs_missing_indexes_without_full_ddl(self) -> None: + conn = MagicMock() + cursor_cm = MagicMock() + cursor = MagicMock() + conn.cursor.return_value = cursor_cm + cursor_cm.__enter__.return_value = cursor + db_cm = MagicMock() + db_cm.__enter__.return_value = conn + + with patch.object(cluster_runtime_service, "_RUNTIME_SCHEMA_READY", False): + with patch.object(cluster_runtime_service, "get_db", return_value=db_cm): + with patch.object(cluster_runtime_service, "_runtime_schema_basics_present", return_value=True): + with patch.object( + cluster_runtime_service, + "_runtime_missing_indexes", + side_effect=[iter(("idx_detect_job_items_claim_step_ready",)), iter(())], + ): + with patch.object(cluster_runtime_service, "_ensure_runtime_schema_indexes") as mocked_ensure_indexes: + cluster_runtime_service.ensure_runtime_schema() + + mocked_ensure_indexes.assert_called_once() + self.assertFalse( + any(sql == cluster_runtime_service._RUNTIME_SCHEMA_SQL for sql, _params in cursor.execute.call_args_list) + ) + + def test_runtime_missing_indexes_treats_invalid_indexes_as_missing(self) -> None: + cur = MagicMock() + cur.fetchall.return_value = [ + ("idx_detect_job_items_job_domain_step", True, True, True), + ("idx_detect_job_items_claim_step_ready", False, True, True), + ] + + missing = list(cluster_runtime_service._runtime_missing_indexes(cur)) + + self.assertIn("idx_detect_job_items_claim_step_ready", missing) + self.assertIn("idx_detect_job_items_claim_job_step_ready", missing) + self.assertNotIn("idx_detect_job_items_job_domain_step", missing) + def test_control_node_supports_worker_only_on_mainland_with_worker_signals(self) -> None: self.assertFalse( cluster_runtime_service._control_node_supports_worker( @@ -88,6 +253,187 @@ class RuntimeSchemaGuardTests(unittest.TestCase): ) ) + def test_get_cluster_snapshot_counts_all_rows_even_when_display_nodes_are_limited(self) -> None: + now = datetime.now() + display_rows = [ + ( + f"mainland-worker-{index:03d}", + "mainland", + "worker", + f"worker-{index:03d}", + f"10.0.0.{index}", + "online", + "test", + 0, + {}, + now, + now, + ) + for index in range(1, 101) + ] + summary_rows = display_rows + [ + ( + "mainland-worker-101", + "mainland", + "worker", + "worker-101", + "10.0.0.101", + "online", + "test", + 0, + {}, + now, + now, + ) + ] + conn = MagicMock() + cursor_cm = MagicMock() + cursor = MagicMock() + cursor.fetchall.side_effect = [display_rows, summary_rows] + cursor.fetchone.side_effect = [(12,), (34,)] + conn.cursor.return_value = cursor_cm + cursor_cm.__enter__.return_value = cursor + db_cm = MagicMock() + db_cm.__enter__.return_value = conn + + with patch.object(cluster_runtime_service, "get_db", return_value=db_cm): + with patch.object(cluster_runtime_service, "prune_expired_runtime_nodes"): + with patch.object(cluster_runtime_service, "register_local_control_heartbeat"): + with patch.object(cluster_runtime_service, "_load_managed_node_overlays", return_value={}): + with patch.object(cluster_runtime_service, "_load_disabled_managed_node_codes", return_value=set()): + snapshot = cluster_runtime_service.get_cluster_snapshot() + + self.assertEqual(100, len(snapshot["nodes"])) + self.assertEqual(101, snapshot["nodes_total"]) + self.assertEqual(101, snapshot["summary"]["online_worker_nodes"]) + self.assertEqual(12, snapshot["jobs_total"]) + self.assertEqual(34, snapshot["active_job_items"]) + + def test_get_cluster_snapshot_excludes_disabled_managed_nodes(self) -> None: + now = datetime.now() + display_rows = [ + ("mainland-controller-01", "mainland", "control", "controller", "10.0.0.1", "online", "test", 1, {}, now, now), + ("mainland-worker-01", "mainland", "worker", "worker-01", "10.0.0.2", "busy", "test", 1, {}, now, now), + ] + summary_rows = list(display_rows) + conn = MagicMock() + cursor_cm = MagicMock() + cursor = MagicMock() + cursor.fetchall.side_effect = [display_rows, summary_rows] + cursor.fetchone.side_effect = [(1,), (2,)] + conn.cursor.return_value = cursor_cm + cursor_cm.__enter__.return_value = cursor + db_cm = MagicMock() + db_cm.__enter__.return_value = conn + + with patch.object(cluster_runtime_service, "get_db", return_value=db_cm): + with patch.object(cluster_runtime_service, "prune_expired_runtime_nodes"): + with patch.object(cluster_runtime_service, "register_local_control_heartbeat"): + with patch.object(cluster_runtime_service, "_load_managed_node_overlays", return_value={}): + with patch.object(cluster_runtime_service, "_load_disabled_managed_node_codes", return_value={"mainland-worker-01"}): + snapshot = cluster_runtime_service.get_cluster_snapshot() + + self.assertEqual(["mainland-controller-01"], [item["node_code"] for item in snapshot["nodes"]]) + self.assertEqual(1, snapshot["nodes_total"]) + self.assertEqual(1, snapshot["summary"]["online_control_nodes"]) + self.assertEqual(0, snapshot["summary"]["online_worker_nodes"]) + + def test_get_cluster_snapshot_prefers_imported_runtime_update_time_for_freshness(self) -> None: + now = datetime.now() + stale_heartbeat = now - timedelta(minutes=8) + fresh_ingest = now - timedelta(seconds=20) + display_rows = [ + ( + "mainland-controller-01-da", + "mainland", + "worker", + "controller", + "10.0.0.1", + "busy", + "test", + 42, + { + "service": "runtime-ingest", + "updated_at": fresh_ingest.isoformat(timespec="seconds"), + "active_threads": 42, + }, + stale_heartbeat, + fresh_ingest, + ) + ] + summary_rows = list(display_rows) + conn = MagicMock() + cursor_cm = MagicMock() + cursor = MagicMock() + cursor.fetchall.side_effect = [display_rows, summary_rows] + cursor.fetchone.side_effect = [(1,), (1,)] + conn.cursor.return_value = cursor_cm + cursor_cm.__enter__.return_value = cursor + db_cm = MagicMock() + db_cm.__enter__.return_value = conn + + with patch.object(cluster_runtime_service, "get_db", return_value=db_cm): + with patch.object(cluster_runtime_service, "prune_expired_runtime_nodes"): + with patch.object(cluster_runtime_service, "register_local_control_heartbeat"): + with patch.object(cluster_runtime_service, "_load_managed_node_overlays", return_value={}): + with patch.object(cluster_runtime_service, "_load_disabled_managed_node_codes", return_value=set()): + snapshot = cluster_runtime_service.get_cluster_snapshot() + + self.assertEqual(1, snapshot["summary"]["online_worker_nodes"]) + self.assertEqual([], snapshot["summary"]["offline_nodes"]) + self.assertEqual("busy", snapshot["nodes"][0]["status"]) + + def test_register_local_control_heartbeat_uses_detect_status_snapshot(self) -> None: + detect_status = { + "worker_online": True, + "active_thread_count": 17, + "max_thread_count": 320, + "available_proxy_count": 41, + "proxy_runtime_label": "代理可用", + "proxy_runtime_reason": "pool_ready", + "proxy_last_refresh_status": "ok", + "proxy_last_refresh_time": "2026-04-24 01:30:00", + "proxy_last_refresh_source_count": 3, + "proxy_last_refresh_total_items": 120, + "proxy_last_validated_count": 110, + "proxy_last_available_count": 41, + "phase_label": "running", + "phase_detail": "正在执行检测", + "active_job": { + "job_code": "detect-20260424013000-abcd12", + "status": "running", + "node_stats": [ + { + "node_code": "mainland-controller-01", + "items_total": 56, + "items_claimed": 5, + "items_running": 9, + "items_completed": 42, + } + ], + }, + } + + with patch.object(cluster_runtime_service.settings, "node_code", "mainland-controller-01"): + with patch.object(cluster_runtime_service.settings, "node_region", "mainland"): + with patch.object(cluster_runtime_service.settings, "node_role", "control"): + with patch("app.services.detect_service.get_detect_status", return_value=detect_status): + with patch.object(cluster_runtime_service, "register_node_heartbeat") as mock_register: + cluster_runtime_service.register_local_control_heartbeat() + + mock_register.assert_called_once() + payload = mock_register.call_args.kwargs + self.assertEqual("mainland-controller-01", payload["node_code"]) + self.assertEqual("busy", payload["status"]) + self.assertEqual(17, payload["current_load"]) + self.assertTrue(payload["metadata"]["worker_online"]) + self.assertTrue(payload["metadata"]["detect_participating"]) + self.assertEqual("detect-20260424013000-abcd12", payload["metadata"]["active_job_code"]) + self.assertEqual(17, payload["metadata"]["active_threads"]) + self.assertEqual(320, payload["metadata"]["max_threads"]) + self.assertEqual(41, payload["metadata"]["available_proxy_count"]) + self.assertEqual("running", payload["metadata"]["phase_label"]) + if __name__ == "__main__": unittest.main() diff --git a/domain-api/tests/test_dashboard_service.py b/domain-api/tests/test_dashboard_service.py index 0c6c451..f9085e0 100644 --- a/domain-api/tests/test_dashboard_service.py +++ b/domain-api/tests/test_dashboard_service.py @@ -39,20 +39,22 @@ class _FakeConnection: class DashboardServiceTests(unittest.TestCase): + @patch("app.services.dashboard.get_detect_status") @patch("app.services.dashboard._fetch_active_jobs_aggregate") @patch("app.services.dashboard.get_detect_capacity_plan") @patch("app.services.dashboard.get_detect_queue_health") - @patch("app.services.dashboard.get_runtime_status") + @patch("app.services.dashboard._build_dashboard_runtime_summary") @patch("app.services.dashboard.get_active_detect_job_summary") @patch("app.services.dashboard.get_db") def test_fetch_overview_includes_ops_metrics( self, mock_get_db, mock_get_active_detect_job_summary, - mock_get_runtime_status, + mock_build_dashboard_runtime_summary, mock_get_detect_queue_health, mock_get_detect_capacity_plan, mock_fetch_active_jobs_aggregate, + mock_get_detect_status, ) -> None: mock_get_db.return_value = _FakeConnection( responses=[ @@ -122,7 +124,7 @@ class DashboardServiceTests(unittest.TestCase): "items_blacklisted": 0, "items_failed": 0, } - mock_get_runtime_status.return_value = { + mock_build_dashboard_runtime_summary.return_value = { "worker": {"running": True, "mode": "linux-systemd", "expected_on_this_node": True}, "node": {"region": "overseas", "role": "control"}, "cluster": {"summary": {"online_worker_nodes": 2, "dedicated_online_worker_nodes": 1, "online_control_nodes": 1}}, @@ -198,6 +200,15 @@ class DashboardServiceTests(unittest.TestCase): "remaining_items": 193, "recommended_additional_workers": 1, } + mock_get_detect_status.return_value = { + "available_proxy_count": 1011, + "proxy_runtime_label": "集群代理正常", + "proxy_runtime_detail": "参与服务器 2 台，共可用 1011 个代理", + "proxy_last_refresh_status": "mainland-controller-01:201；mainland-worker-01:810", + "aggregate_process_count": 10, + "aggregate_participating_node_count": 10, + "active_thread_count": 165, + } data = fetch_overview() @@ -218,6 +229,10 @@ class DashboardServiceTests(unittest.TestCase): self.assertEqual(165, data["queue_display_running_total"]) self.assertEqual(7, data["queue_completed_total"]) self.assertEqual(9438, data["backlog_pending_total"]) + self.assertEqual(1011, data["cluster_proxy_available_count"]) + self.assertEqual("集群代理正常", data["cluster_proxy_runtime_label"]) + self.assertEqual(10, data["aggregate_process_count"]) + self.assertEqual(1, data["ops_summary"]["active_execution_nodes"]) self.assertEqual(8487, data["backlog_register_pending_total"]) self.assertEqual(951, data["backlog_downstream_pending_total"]) self.assertEqual(12, data["retry_total"]) @@ -228,10 +243,256 @@ class DashboardServiceTests(unittest.TestCase): self.assertEqual(1.5, data["ops_summary"]["estimated_hours_remaining"]) self.assertEqual(1, data["active_execution_nodes"]) self.assertEqual(1, data["ops_summary"]["active_execution_nodes"]) + self.assertEqual(0, data["current_job_blacklisted"]) + self.assertEqual(0, data["recent_blacklisted_total"]) + self.assertEqual(0, data["cumulative_blacklisted_total"]) self.assertEqual(2, len(data["step_queue"])) self.assertTrue(any(item["step_code"] == "detect_360_site" for item in data["step_queue"])) self.assertEqual(1, len(data["node_throughput"])) + @patch("app.services.dashboard._fetch_active_jobs_aggregate") + @patch("app.services.dashboard.get_detect_capacity_plan") + @patch("app.services.dashboard.get_detect_queue_health") + @patch("app.services.dashboard._build_dashboard_runtime_summary") + @patch("app.services.dashboard.get_active_detect_job_summary") + @patch("app.services.dashboard.get_db") + def test_fetch_overview_prefers_active_job_display_running_when_queue_snapshot_is_stale( + self, + mock_get_db, + mock_get_active_detect_job_summary, + mock_build_dashboard_runtime_summary, + mock_get_detect_queue_health, + mock_get_detect_capacity_plan, + mock_fetch_active_jobs_aggregate, + ) -> None: + mock_get_db.return_value = _FakeConnection( + responses=[ + (1000,), + (900,), + (10,), + (5,), + (0,), + (1,), + (430,), + (420,), + (17,), + ] + ) + mock_fetch_active_jobs_aggregate.return_value = { + "active_jobs_total": 1, + "queue": {"items_total": 8402, "pending": 5001, "claimed": 0, "running": 1, "completed": 0, "blacklisted": 0, "failed": 0}, + "throughput": {"processed_recent": 0, "processed_per_minute": 0.0}, + "steps": [], + "nodes": [], + "retry_total": 0, + } + mock_get_active_detect_job_summary.return_value = { + "job_id": 11, + "job_code": "sync-overseas-51", + "status": "running", + "items_total": 8402, + "items_pending": 5001, + "items_claimed": 0, + "items_running": 0, + "items_completed": 0, + "items_blacklisted": 0, + "items_failed": 0, + "display_items_running": 1432, + "display_active_threads": 138, + "display_max_threads": 80000, + "distributed_node_stats": [ + {"node_code": "mainland-controller-01-a", "display_running": 1000, "active_threads": 1000, "max_threads": 1000}, + {"node_code": "mainland-controller-01-b", "display_running": 432, "active_threads": 432, "max_threads": 1000}, + ], + } + mock_build_dashboard_runtime_summary.return_value = { + "worker": {"running": False, "mode": "linux-systemd", "expected_on_this_node": False}, + "node": {"region": "overseas", "role": "control"}, + "cluster": {"summary": {"online_worker_nodes": 1, "dedicated_online_worker_nodes": 0, "online_control_nodes": 2}}, + "detect": { + "backlog": { + "pending_total": 5001, + "claimed_total": 0, + "running_total": 0, + "completed_total": 0, + "blacklisted_total": 0, + "failed_total": 0, + "register_pending": 3495, + "downstream_pending": 1506, + } + }, + } + mock_get_detect_queue_health.return_value = { + "has_active_job": True, + "job": {"job_id": 11, "job_code": "sync-overseas-51", "runtime_job_code": "sync-overseas-51", "status": "running", "progress_percent": 40.49}, + "queue": { + "items_total": 8402, + "pending": 5001, + "claimed": 0, + "display_claimed": 0, + "running": 1, + "display_running": 1, + "completed": 0, + "blacklisted": 0, + "failed": 0, + }, + "throughput": {"processed_recent": 0, "processed_per_minute": 0.0}, + "steps": [], + "runtime_activity": {}, + "nodes": [], + } + mock_get_detect_capacity_plan.return_value = { + "estimated_hours_remaining": 0, + "remaining_items": 5001, + "recommended_additional_workers": 0, + } + + data = fetch_overview() + + self.assertEqual(1432, data["queue_display_running_total"]) + self.assertEqual(1432, data["active_job"]["items_display_running"]) + self.assertEqual(1432, data["active_job"]["display_items_running"]) + self.assertEqual(138, data["active_job"]["display_active_threads"]) + self.assertEqual(80000, data["active_job"]["display_max_threads"]) + self.assertEqual(2, len(data["active_job"]["distributed_node_stats"])) + self.assertEqual(80000, data["queue_display_max_threads"]) + self.assertEqual(0, data["current_job_blacklisted"]) + self.assertEqual(0, data["recent_blacklisted_total"]) + self.assertEqual(0, data["cumulative_blacklisted_total"]) + + @patch("app.services.dashboard._fetch_active_jobs_aggregate") + @patch("app.services.dashboard.get_detect_capacity_plan") + @patch("app.services.dashboard.get_detect_queue_health") + @patch("app.services.dashboard._build_dashboard_runtime_summary") + @patch("app.services.dashboard.get_active_detect_job_summary") + @patch("app.services.dashboard.get_db") + def test_fetch_overview_counts_active_execution_nodes_from_full_node_set( + self, + mock_get_db, + mock_get_active_detect_job_summary, + mock_build_dashboard_runtime_summary, + mock_get_detect_queue_health, + mock_get_detect_capacity_plan, + mock_fetch_active_jobs_aggregate, + ) -> None: + mock_get_db.return_value = _FakeConnection(responses=[(0,)] * 9) + mock_fetch_active_jobs_aggregate.return_value = { + "active_jobs_total": 1, + "queue": {"items_total": 9, "pending": 0, "claimed": 0, "running": 9, "completed": 0, "blacklisted": 0, "failed": 0}, + "throughput": {"processed_recent": 0, "processed_per_minute": 0.0, "completed_recent": 0, "blacklisted_recent": 0, "failed_recent": 0}, + "steps": [], + "nodes": [ + { + "node_code": f"mainland-controller-01-{index:02d}", + "items_running": 1, + "items_claimed": 0, + "processed_recent": 0, + "processed_per_minute": 0.0, + "completed_recent": 0, + "failed_recent": 0, + "blacklisted_recent": 0, + } + for index in range(9) + ], + "retry_total": 0, + } + mock_get_active_detect_job_summary.return_value = {} + mock_build_dashboard_runtime_summary.return_value = { + "worker": {"running": True, "mode": "linux-systemd", "expected_on_this_node": True}, + "node": {"region": "mainland", "role": "worker"}, + "cluster": {"summary": {"online_worker_nodes": 1, "dedicated_online_worker_nodes": 0, "online_control_nodes": 1}}, + "detect": {"backlog": {}}, + } + mock_get_detect_queue_health.return_value = { + "has_active_job": False, + "queue": {}, + "throughput": {"processed_recent": 0, "processed_per_minute": 0.0}, + "steps": [], + "runtime_activity": {}, + "nodes": [], + "runtime_snapshot_backlog": {}, + } + mock_get_detect_capacity_plan.return_value = { + "estimated_hours_remaining": 0, + "remaining_items": 0, + "recommended_additional_workers": 0, + } + + data = fetch_overview() + + self.assertEqual(8, len(data["node_throughput"])) + self.assertEqual(9, data["active_execution_nodes"]) + self.assertEqual(9, data["ops_summary"]["active_execution_nodes"]) + + @patch("app.services.dashboard._fetch_active_jobs_aggregate") + @patch("app.services.dashboard.get_detect_capacity_plan") + @patch("app.services.dashboard.get_detect_queue_health") + @patch("app.services.dashboard._build_dashboard_runtime_summary") + @patch("app.services.dashboard.get_active_detect_job_summary") + @patch("app.services.dashboard.get_db") + def test_fetch_overview_keeps_active_job_summary_when_queue_health_temporarily_empty( + self, + mock_get_db, + mock_get_active_detect_job_summary, + mock_build_dashboard_runtime_summary, + mock_get_detect_queue_health, + mock_get_detect_capacity_plan, + mock_fetch_active_jobs_aggregate, + ) -> None: + mock_get_db.return_value = _FakeConnection(responses=[(0,)] * 9) + mock_fetch_active_jobs_aggregate.return_value = { + "active_jobs_total": 1, + "queue": {"items_total": 8402, "pending": 5001, "claimed": 0, "running": 537, "completed": 0, "blacklisted": 0, "failed": 0}, + "throughput": {"processed_recent": 12, "processed_per_minute": 0.8, "completed_recent": 10, "blacklisted_recent": 1, "failed_recent": 1}, + "steps": [], + "nodes": [], + "retry_total": 0, + } + mock_get_active_detect_job_summary.return_value = { + "job_id": 255, + "job_code": "sync-overseas-255", + "runtime_job_code": "sync-overseas-255", + "status": "running", + "progress_percent": 22.4, + "items_total": 8402, + "items_pending": 5001, + "items_claimed": 0, + "items_running": 537, + "items_completed": 0, + "items_blacklisted": 0, + "items_failed": 0, + "display_items_running": 11799, + "display_active_threads": 11799, + "display_max_threads": 75400, + "distributed_node_stats": [{"node_code": "mainland-controller-01-a", "display_running": 537, "active_threads": 537, "max_threads": 1000}], + } + mock_build_dashboard_runtime_summary.return_value = { + "worker": {"running": False, "mode": "linux-systemd", "expected_on_this_node": False}, + "node": {"region": "overseas", "role": "control"}, + "cluster": {"summary": {"online_worker_nodes": 1, "dedicated_online_worker_nodes": 0, "online_control_nodes": 1}}, + "detect": {"backlog": {}}, + } + mock_get_detect_queue_health.return_value = { + "has_active_job": False, + "queue": {}, + "throughput": {"processed_recent": 0, "processed_per_minute": 0.0}, + "steps": [], + "runtime_activity": {}, + "nodes": [], + "runtime_snapshot_backlog": {}, + } + mock_get_detect_capacity_plan.return_value = { + "estimated_hours_remaining": 0, + "remaining_items": 5001, + "recommended_additional_workers": 0, + } + + data = fetch_overview() + + self.assertEqual("sync-overseas-255", data["active_job"]["job_code"]) + self.assertEqual(11799, data["active_job"]["display_items_running"]) + self.assertEqual(75400, data["active_job"]["display_max_threads"]) + if __name__ == "__main__": unittest.main() diff --git a/domain-api/tests/test_debug_event_service.py b/domain-api/tests/test_debug_event_service.py new file mode 100644 index 0000000..3d194b8 --- /dev/null +++ b/domain-api/tests/test_debug_event_service.py @@ -0,0 +1,83 @@ +from __future__ import annotations + +import unittest +from unittest.mock import patch + +from app.services import debug_event_service + + +class DebugEventServiceTests(unittest.TestCase): + def test_ingest_debug_event_requires_configured_shared_token(self) -> None: + with patch.object(debug_event_service.settings, "sync_shared_token", ""): + ok, message, data = debug_event_service.ingest_debug_event( + { + "source_region": "mainland", + "node_code": "mainland-controller-01", + "service": "worker-event", + "event_type": "worker_log", + "message": "开始检测域名: a.com", + "payload": {"job_id": 1, "job_code": "sync-overseas-1"}, + }, + shared_token=None, + ) + + self.assertFalse(ok) + self.assertIn("token 未配置", message) + self.assertTrue(data["configuration_required"]) + + def test_resolve_target_job_for_debug_event_rejects_missing_identity(self) -> None: + job, reason = debug_event_service._resolve_target_job_for_debug_event({"domain": "a.com"}) + + self.assertIsNone(job) + self.assertEqual("missing_job_identity", reason) + + @patch("app.services.debug_event_service._load_detect_job_summary_by_job_code") + @patch("app.services.detect_job_service.get_detect_job_summary") + @patch("app.services.detect_job_service.get_active_detect_job_summary") + def test_resolve_target_job_for_debug_event_prefers_payload_job_id_over_current_active_job( + self, + mock_get_active_detect_job_summary, + mock_get_detect_job_summary, + mock_load_detect_job_summary_by_job_code, + ) -> None: + mock_get_active_detect_job_summary.return_value = { + "job_id": 12, + "job_code": "sync-overseas-12", + "current_cycle_token": "cycle-12", + } + mock_get_detect_job_summary.return_value = { + "job_id": 11, + "job_code": "sync-overseas-11", + "current_cycle_token": "cycle-11", + } + + job, reason = debug_event_service._resolve_target_job_for_debug_event( + {"job_id": 11, "job_code": "sync-overseas-11", "cycle_token": "cycle-11"} + ) + + self.assertEqual("matched", reason) + self.assertEqual(11, job["job_id"]) + mock_get_detect_job_summary.assert_called_once_with(11, event_limit=1) + mock_load_detect_job_summary_by_job_code.assert_not_called() + + @patch("app.services.detect_job_service.get_active_detect_job_summary") + def test_resolve_target_job_for_debug_event_rejects_cycle_mismatch( + self, + mock_get_active_detect_job_summary, + ) -> None: + mock_get_active_detect_job_summary.return_value = { + "job_id": 12, + "job_code": "sync-overseas-12", + "current_cycle_token": "cycle-current", + } + + job, reason = debug_event_service._resolve_target_job_for_debug_event( + {"job_id": 12, "job_code": "sync-overseas-12", "cycle_token": "cycle-old"} + ) + + self.assertIsNone(job) + self.assertEqual("cycle_mismatch", reason) + + +if __name__ == "__main__": + unittest.main() diff --git a/domain-api/tests/test_detect_api_routes.py b/domain-api/tests/test_detect_api_routes.py index 08102bc..cf1e5f9 100644 --- a/domain-api/tests/test_detect_api_routes.py +++ b/domain-api/tests/test_detect_api_routes.py @@ -7,6 +7,153 @@ from app.api.routes import detect as detect_route class DetectApiRoutesTestCase(unittest.TestCase): + @patch("app.api.routes.detect.create_detect_run_snapshot") + @patch("app.api.routes.detect._dispatch_remote_detect_start") + @patch("app.api.routes.detect.get_settings_payload") + @patch("app.api.routes.detect.get_detect_status") + @patch("app.api.routes.detect.send_worker_command") + @patch("app.api.routes.detect.start_worker") + @patch("app.api.routes.detect.append_detect_job_event") + @patch("app.api.routes.detect.create_detect_job_if_needed") + def test_start_detect_skips_redundant_systemctl_when_worker_already_online( + self, + mock_create_job, + mock_append_event, + mock_start_worker, + mock_send_worker_command, + mock_get_detect_status, + mock_get_settings_payload, + mock_dispatch_remote, + mock_create_snapshot, + ) -> None: + mock_create_job.return_value = { + "job_id": 30, + "job_code": "sync-overseas-30", + "status": "running", + "items_pending": 100, + "items_claimed": 0, + "items_running": 0, + "task_mode": "domain_pipeline", + "step_code": "", + } + mock_get_detect_status.return_value = { + "worker_mode": "linux-systemd", + "worker_online": True, + "worker_process_count": 60, + "worker_latest_start_time": "2026-04-24 23:30:00", + "worker_runtime_message": "running", + "progress": {}, + } + mock_send_worker_command.return_value = (True, "已发送 Worker 控制指令: start_detection") + mock_get_settings_payload.return_value = { + "runtime": {"thread_count": 1000}, + "proxy_config": {"proxy_enable": True, "allow_direct": True, "proxy_urls": ["a"]}, + } + mock_dispatch_remote.return_value = { + "queued_jobs": [], + "queued_total": 0, + "failed_total": 0, + "target_summary": {"controller_nodes": [], "worker_nodes": []}, + } + + with patch.object(detect_route.settings, "node_region", "mainland"), patch.object( + detect_route.settings, "node_role", "control" + ): + response = detect_route.start_detect() + + self.assertEqual(0, response.code) + self.assertIn("检测端已在运行，跳过重复启动", response.message) + mock_start_worker.assert_not_called() + mock_send_worker_command.assert_called_once() + event_types = [call.kwargs.get("event_type") for call in mock_append_event.call_args_list] + self.assertIn("job_dispatch_start_skipped", event_types) + self.assertIn("job_dispatch_sent", event_types) + mock_create_snapshot.assert_called_once() + + @patch("app.api.routes.detect.create_detect_run_snapshot") + @patch("app.api.routes.detect._dispatch_remote_detect_start") + @patch("app.api.routes.detect.get_settings_payload") + @patch("app.api.routes.detect.get_detect_status") + @patch("app.api.routes.detect.send_worker_command") + @patch("app.api.routes.detect.start_worker") + @patch("app.api.routes.detect.append_detect_job_event") + @patch("app.api.routes.detect.create_detect_job_if_needed") + def test_start_detect_falls_back_to_direct_command_when_worker_already_running( + self, + mock_create_job, + mock_append_event, + mock_start_worker, + mock_send_worker_command, + mock_get_detect_status, + mock_get_settings_payload, + mock_dispatch_remote, + mock_create_snapshot, + ) -> None: + mock_create_job.return_value = { + "job_id": 31, + "job_code": "sync-overseas-31", + "status": "running", + "items_pending": 100, + "items_claimed": 0, + "items_running": 0, + "task_mode": "domain_pipeline", + "step_code": "", + } + mock_start_worker.return_value = ( + False, + "domaincheck-worker 控制失败：当前运行用户没有免密 systemctl 权限，请为 API 进程授予对应 sudo/systemd 权限", + ) + mock_send_worker_command.return_value = (True, "已发送 Worker 控制指令: start_detection") + mock_get_detect_status.side_effect = [ + { + "worker_mode": "linux-systemd", + "worker_online": False, + "worker_process_count": 0, + "worker_latest_start_time": "", + "worker_runtime_message": "starting", + "progress": {}, + }, + { + "worker_mode": "linux-systemd", + "worker_online": True, + "worker_process_count": 60, + "worker_latest_start_time": "2026-04-24 23:30:00", + "worker_runtime_message": "running", + "progress": {}, + }, + { + "worker_mode": "linux-systemd", + "worker_online": True, + "worker_process_count": 60, + "worker_latest_start_time": "2026-04-24 23:30:00", + "worker_runtime_message": "running", + "progress": {}, + }, + ] + mock_get_settings_payload.return_value = { + "runtime": {"thread_count": 1000}, + "proxy_config": {"proxy_enable": True, "allow_direct": True, "proxy_urls": ["a"]}, + } + mock_dispatch_remote.return_value = { + "queued_jobs": [], + "queued_total": 0, + "failed_total": 0, + "target_summary": {"controller_nodes": [], "worker_nodes": []}, + } + + with patch.object(detect_route.settings, "node_region", "mainland"), patch.object( + detect_route.settings, "node_role", "control" + ): + response = detect_route.start_detect() + + self.assertEqual(0, response.code) + self.assertIn("检测端已在运行，改为直接发送控制指令", response.message) + mock_send_worker_command.assert_called_once() + event_types = [call.kwargs.get("event_type") for call in mock_append_event.call_args_list] + self.assertIn("job_dispatch_start_degraded", event_types) + self.assertIn("job_dispatch_sent", event_types) + mock_create_snapshot.assert_called_once() + @patch("app.api.routes.detect.create_detect_run_snapshot") @patch("app.api.routes.detect._dispatch_remote_detect_start") @patch("app.api.routes.detect.get_settings_payload") @@ -65,11 +212,94 @@ class DetectApiRoutesTestCase(unittest.TestCase): mock_start_worker.assert_not_called() mock_send_worker_command.assert_not_called() mock_dispatch_remote.assert_called_once() - mock_create_snapshot.assert_called_once() + mock_create_snapshot.assert_not_called() event_types = [call.kwargs.get("event_type") for call in mock_append_event.call_args_list] self.assertIn("job_dispatch_requested", event_types) self.assertIn("job_dispatch_skipped_local", event_types) + @patch("app.api.routes.detect.finalize_detect_run") + @patch("app.api.routes.detect.mark_detect_run_stopping") + @patch("app.api.routes.detect.get_settings_payload") + @patch("app.api.routes.detect.get_detect_status") + @patch("app.api.routes.detect._dispatch_remote_detect_stop") + @patch("app.api.routes.detect.send_worker_command") + @patch("app.api.routes.detect.get_active_detect_job_summary") + def test_stop_detect_forwards_target_payload( + self, + mock_get_active_job, + mock_send_worker_command, + mock_dispatch_remote_stop, + mock_get_detect_status, + mock_get_settings_payload, + mock_mark_detect_run_stopping, + mock_finalize_detect_run, + ) -> None: + mock_get_active_job.return_value = { + "job_id": 29, + "job_code": "sync-overseas-29", + "status": "running", + "current_cycle_token": "cycle-29", + } + mock_send_worker_command.return_value = (True, "已发送 Worker 控制指令") + mock_dispatch_remote_stop.return_value = { + "queued_jobs": [{"node_code": "mainland-controller-01"}], + "queued_total": 1, + "failed_total": 0, + "target_summary": {"controller_nodes": ["mainland-controller-01"], "worker_nodes": []}, + } + mock_get_detect_status.return_value = { + "worker_mode": "linux-systemd", + "worker_online": True, + "worker_process_count": 1, + "worker_latest_start_time": "", + "worker_runtime_message": "running", + "progress": {}, + } + mock_get_settings_payload.return_value = { + "thread_count": 1000, + "node_thread_counts": {}, + "process_count": 80, + "node_process_counts": {}, + "proxy_config": {"proxy_enable": True, "allow_direct": False, "proxy_urls": ["a"]}, + } + + response = detect_route.stop_detect(payload={"target_node_codes": ["mainland-controller-01"]}) + + self.assertEqual(0, response.code) + mock_send_worker_command.assert_called_once_with( + "stop_detection", + payload={"target_node_codes": ["mainland-controller-01"]}, + ) + mock_dispatch_remote_stop.assert_called_once_with( + active_job=mock_get_active_job.return_value, + cycle_token="cycle-29", + payload={"target_node_codes": ["mainland-controller-01"]}, + ) + mock_mark_detect_run_stopping.assert_called_once() + mock_finalize_detect_run.assert_not_called() + + @patch("app.api.routes.detect.create_ops_job") + @patch("app.api.routes.detect.list_managed_nodes") + def test_dispatch_remote_detect_stop_filters_target_nodes(self, mock_list_managed_nodes, mock_create_ops_job) -> None: + mock_list_managed_nodes.return_value = [ + {"node_code": "mainland-controller-01", "region": "mainland", "role": "control", "last_seen_at": "2026-04-23T14:00:00", "is_enabled": True}, + {"node_code": "mainland-worker-01", "region": "mainland", "role": "worker", "last_seen_at": "2026-04-23T14:00:00", "is_enabled": True}, + ] + mock_create_ops_job.return_value = (True, "queued", {}) + + result = detect_route._dispatch_remote_detect_stop( + active_job={"job_id": 29, "job_code": "sync-overseas-29"}, + cycle_token="cycle-29", + payload={"target_node_codes": ["mainland-worker-01"]}, + ) + + self.assertEqual(1, result["queued_total"]) + self.assertEqual(["mainland-controller-01"], result["target_summary"]["controller_nodes"]) + self.assertEqual(["mainland-worker-01"], result["target_summary"]["worker_nodes"]) + mock_create_ops_job.assert_called_once() + create_payload = mock_create_ops_job.call_args.args[0] + self.assertEqual("mainland-worker-01", create_payload["target_node_code"]) + if __name__ == "__main__": unittest.main() diff --git a/domain-api/tests/test_detect_job_service.py b/domain-api/tests/test_detect_job_service.py index 9a92f98..7ef326d 100644 --- a/domain-api/tests/test_detect_job_service.py +++ b/domain-api/tests/test_detect_job_service.py @@ -1,16 +1,22 @@ import unittest +from datetime import datetime, timedelta from unittest.mock import MagicMock, patch from psycopg2 import errors +import app.services.detect_job_service as detect_job_service_module from app.services.detect_job_service import ( append_detect_job_event, + _build_active_job_summary_from_runtime_snapshot, _build_step_payload, _classify_pipeline_item_outcome, _classify_runtime_debug_event, + _build_runtime_snapshot_from_projection, _enrich_active_job_summary_with_runtime, _build_display_summary, _build_runtime_display_bucket, + _load_latest_runtime_active_job_snapshot, + get_active_detect_job_summary, normalize_detect_step_code, get_detect_queue_health, process_detect_pipeline_now, @@ -20,7 +26,253 @@ from app.services.detect_job_service import ( ) +class _SequenceCursor: + def __init__(self, *, fetchone_results=None, fetchall_results=None) -> None: + self.fetchone_results = list(fetchone_results or []) + self.fetchall_results = list(fetchall_results or []) + self.exec_calls = [] + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def execute(self, sql, params=None): + self.exec_calls.append((sql, params)) + + def fetchone(self): + if self.fetchone_results: + return self.fetchone_results.pop(0) + return None + + def fetchall(self): + if self.fetchall_results: + return self.fetchall_results.pop(0) + return [] + + +class _SequenceConn: + def __init__(self, *, fetchone_results=None, fetchall_results=None) -> None: + self.cursor_obj = _SequenceCursor(fetchone_results=fetchone_results, fetchall_results=fetchall_results) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def cursor(self): + return self.cursor_obj + + class DetectJobServiceTests(unittest.TestCase): + def test_load_runtime_display_rows_excludes_disabled_managed_nodes(self) -> None: + cursor = MagicMock() + cursor.fetchall.return_value = [ + ("mainland-controller-01", "mainland", "control", "busy", 1, {}, datetime.now()), + ("mainland-worker-01", "mainland", "worker", "busy", 1, {}, datetime.now()), + ] + + with patch("app.services.detect_job_service._load_disabled_managed_node_codes", return_value={"mainland-worker-01"}): + rows = detect_job_service_module._load_runtime_display_rows(cursor) + + self.assertEqual(["mainland-controller-01"], [row[0] for row in rows]) + + def test_build_active_job_summary_from_runtime_snapshot_keeps_non_zero_display_when_queue_display_is_zero(self) -> None: + with patch("app.services.detect_job_service.settings.node_region", "mainland"): + summary = _build_active_job_summary_from_runtime_snapshot( + { + "job": { + "job_id": 60, + "job_code": "sync-overseas-255", + "status": "running", + "items_total": 8402, + "items_pending": 7865, + "items_claimed": 0, + "items_running": 537, + "items_completed": 0, + "items_blacklisted": 0, + "items_failed": 0, + "items_terminal": 0, + "display_items_running": 0, + "display_active_threads": 0, + "display_max_threads": 0, + "node_stats": [ + { + "node_code": "mainland-controller-01-a", + "items_running": 0, + "display_running": 0, + "active_threads": 0, + "max_threads": 0, + } + ], + }, + "queue_health": { + "queue": { + "items_total": 8402, + "pending": 7865, + "claimed": 0, + "running": 537, + "display_running": 0, + "completed": 0, + "blacklisted": 0, + "failed": 0, + "terminal": 0, + }, + "nodes": [ + { + "node_code": "mainland-controller-01-a", + "items_running": 0, + "display_running": 0, + "active_threads": 0, + "max_threads": 0, + } + ], + "steps": [], + "throughput": {}, + }, + "_snapshot_source": "runtime_ingest", + } + ) + + assert summary is not None + self.assertEqual(537, summary["display_items_running"]) + self.assertEqual(537, summary["display_active_threads"]) + self.assertEqual(537, summary["display_current_load"]) + + def test_build_active_job_summary_from_runtime_snapshot_drops_stale_runtime_nodes_from_live_display(self) -> None: + stale_heartbeat = (datetime.now() - timedelta(minutes=8)).isoformat(sep=" ", timespec="seconds") + snapshot_created_at = datetime.now().isoformat(sep=" ", timespec="seconds") + summary = _build_active_job_summary_from_runtime_snapshot( + { + "job": { + "job_id": 61, + "job_code": "sync-overseas-256", + "status": "running", + "items_total": 8402, + "items_pending": 7865, + "items_claimed": 0, + "items_running": 537, + "items_completed": 0, + "items_blacklisted": 0, + "items_failed": 0, + "items_terminal": 0, + "display_items_running": 537, + "display_active_threads": 537, + "node_stats": [ + { + "node_code": "mainland-controller-01-a", + "items_running": 537, + "display_running": 537, + "active_threads": 537, + "max_threads": 1000, + "status": "busy", + "last_heartbeat_at": stale_heartbeat, + } + ], + }, + "queue_health": { + "queue": { + "items_total": 8402, + "pending": 7865, + "claimed": 0, + "running": 537, + "display_running": 537, + "completed": 0, + "blacklisted": 0, + "failed": 0, + "terminal": 0, + }, + "nodes": [ + { + "node_code": "mainland-controller-01-a", + "items_running": 537, + "display_running": 537, + "active_threads": 537, + "max_threads": 1000, + "status": "busy", + "last_heartbeat_at": stale_heartbeat, + } + ], + "steps": [], + "throughput": {}, + }, + "_snapshot_source": "runtime_ingest", + "_created_at": snapshot_created_at, + } + ) + + assert summary is not None + self.assertEqual(537, summary["items_running"]) + self.assertEqual(0, summary["display_items_running"]) + self.assertEqual(0, summary["display_active_threads"]) + self.assertEqual([], summary["node_stats"]) + + def test_build_active_job_summary_from_runtime_snapshot_does_not_promote_unassigned_runtime_load(self) -> None: + with patch("app.services.detect_job_service.settings.node_region", "mainland"): + summary = _build_active_job_summary_from_runtime_snapshot( + { + "job": { + "job_id": 62, + "job_code": "sync-overseas-257", + "status": "running", + "items_total": 4000, + "items_pending": 0, + "items_claimed": 23, + "items_running": 2100, + "items_completed": 1117, + "items_blacklisted": 0, + "items_failed": 760, + "items_terminal": 1877, + "display_items_running": 0, + "display_active_threads": 0, + "display_max_threads": 0, + "node_stats": [ + { + "node_code": "unassigned", + "items_claimed": 23, + "items_running": 1863, + "display_running": 1863, + "active_threads": 0, + "max_threads": 0, + } + ], + }, + "queue_health": { + "queue": { + "items_total": 4000, + "pending": 0, + "claimed": 23, + "running": 2100, + "display_running": 2100, + "completed": 1117, + "blacklisted": 0, + "failed": 760, + "terminal": 1877, + }, + "nodes": [ + { + "node_code": "unassigned", + "items_claimed": 23, + "items_running": 1863, + "display_running": 1863, + "active_threads": 0, + "max_threads": 0, + } + ], + "steps": [], + "throughput": {}, + }, + "_snapshot_source": "runtime_ingest", + } + ) + + assert summary is not None + self.assertEqual(0, summary["display_items_running"]) + self.assertEqual(0, summary["display_active_threads"]) + self.assertEqual([], summary["display_active_node_codes"]) + def test_append_detect_job_event_skips_missing_job_fk(self) -> None: class FakeCursor: def __init__(self) -> None: @@ -98,6 +350,141 @@ class DetectJobServiceTests(unittest.TestCase): self.assertEqual(2, calls["count"]) mock_sleep.assert_called_once() + def test_build_runtime_snapshot_from_projection_prefers_distributed_projection_nodes(self) -> None: + snapshot = _build_runtime_snapshot_from_projection( + { + "node": { + "node_code": "mainland-controller-01", + "region": "mainland", + "role": "control", + }, + "active_thread_count": 1972, + "max_thread_count": 2000, + "progress": { + "pending": 4578, + "running": 1972, + "completed": 690, + "blacklisted": 0, + "failed": 24, + }, + "active_job": { + "job_id": 55, + "job_code": "sync-overseas-55", + "status": "running", + "progress_percent": 13.49, + "items_total": 5292, + "items_terminal": 714, + "items_pending": 4578, + "items_running": 1972, + "items_failed": 24, + "display_items_claimed": 120, + "display_items_running": 1972, + "display_active_threads": 1972, + "display_max_threads": 2000, + "distributed_node_stats": [ + { + "node_code": "mainland-controller-01-a", + "items_running": 1000, + "display_running": 1000, + "active_threads": 1000, + "max_threads": 1000, + "region": "mainland", + "role": "control", + "status": "busy", + }, + { + "node_code": "mainland-controller-01-b", + "items_running": 972, + "display_running": 972, + "active_threads": 972, + "max_threads": 1000, + "region": "mainland", + "role": "control", + "status": "busy", + }, + ], + }, + } + ) + + queue = snapshot["queue_health"]["queue"] + nodes = snapshot["queue_health"]["nodes"] + + self.assertEqual(1972, queue["running"]) + self.assertEqual(1972, queue["display_running"]) + self.assertEqual(3, len(nodes)) + self.assertEqual("mainland-controller-01-a", nodes[0]["node_code"]) + self.assertEqual(1000, nodes[0]["display_running"]) + self.assertEqual(972, nodes[1]["active_threads"]) + self.assertEqual("unassigned", nodes[2]["node_code"]) + + def test_enrich_active_job_summary_with_runtime_does_not_overwrite_non_zero_display_with_zero_snapshot(self) -> None: + summary = _enrich_active_job_summary_with_runtime( + { + "job_id": 60, + "job_code": "sync-overseas-255", + "status": "running", + "items_total": 8402, + "items_pending": 7865, + "items_claimed": 0, + "items_running": 537, + "items_completed": 0, + "items_blacklisted": 0, + "items_failed": 0, + "items_terminal": 0, + "display_items_running": 537, + "display_active_threads": 537, + "display_current_load": 537, + "display_max_threads": 80000, + "recent_events": [], + }, + runtime_activity={}, + runtime_snapshot={ + "job": { + "job_id": 60, + "job_code": "sync-overseas-255", + "status": "running", + "node_stats": [ + { + "node_code": "mainland-controller-01-a", + "items_running": 0, + "display_running": 0, + "active_threads": 0, + "max_threads": 0, + } + ], + }, + "queue_health": { + "queue": { + "items_total": 8402, + "pending": 7865, + "claimed": 0, + "running": 537, + "display_running": 0, + "completed": 0, + "blacklisted": 0, + "failed": 0, + "terminal": 0, + }, + "nodes": [ + { + "node_code": "mainland-controller-01-a", + "items_running": 0, + "display_running": 0, + "active_threads": 0, + "max_threads": 0, + } + ], + "steps": [], + }, + }, + ) + + assert summary is not None + self.assertEqual(537, summary["display_items_running"]) + self.assertEqual(537, summary["display_active_threads"]) + self.assertEqual(80000, summary["display_max_threads"]) + def test_classify_runtime_debug_event_maps_single_step_finalized_completed(self) -> None: classified = _classify_runtime_debug_event( event_type="worker_log", @@ -178,6 +565,833 @@ class DetectJobServiceTests(unittest.TestCase): self.assertIsNone(bucket) + def test_load_latest_runtime_active_job_snapshot_uses_runtime_ingest_when_debug_missing(self) -> None: + ingest_payload = { + "projection": { + "node": { + "node_code": "mainland-controller-01", + "region": "mainland", + "role": "control", + }, + "active_thread_count": 177, + "max_thread_count": 2000, + "progress": { + "pending": 200, + "running": 177, + "completed": 440, + "blacklisted": 0, + "failed": 4, + }, + "backlog": { + "pending_total": 200, + "running_total": 177, + }, + "active_job": { + "job_id": 551, + "job_code": "sync-overseas-55", + "status": "running", + "items_total": 821, + "items_terminal": 444, + "items_failed": 4, + "progress_percent": 54.08, + "node_stats": [ + { + "node_code": "mainland-controller-01-a", + "items_total": 400, + "items_pending": 0, + "items_claimed": 80, + "items_running": 100, + "items_completed": 220, + "items_blacklisted": 0, + "items_failed": 0, + "active_threads": 100, + "max_threads": 1000, + } + ], + }, + } + } + fake_conn = _SequenceConn( + fetchall_results=[ + [], + [(ingest_payload, datetime(2026, 4, 22, 18, 5, 36))], + ] + ) + + with patch("app.services.detect_job_service.settings.node_region", "overseas"): + with patch("app.services.detect_job_service.settings.node_role", "control"): + with patch("app.services.detect_job_service.get_db", return_value=fake_conn): + with patch("app.services.detect_job_service._load_runtime_activity_snapshot", return_value={}): + snapshot = _load_latest_runtime_active_job_snapshot(15) + + self.assertEqual("runtime_ingest", snapshot["_snapshot_source"]) + self.assertEqual("sync-overseas-55", snapshot["job"]["job_code"]) + self.assertEqual(821, snapshot["queue_health"]["queue"]["items_total"]) + self.assertEqual(177, snapshot["queue_health"]["queue"]["display_running"]) + self.assertEqual(200, snapshot["backlog"]["pending_total"]) + + def test_load_latest_runtime_active_job_snapshot_prefers_fresher_runtime_ingest(self) -> None: + debug_payload = { + "job": { + "job_id": 84, + "job_code": "sync-overseas-366", + "status": "running", + }, + "queue_health": { + "queue": { + "items_total": 5000, + "running": 2, + } + }, + } + ingest_payload = { + "projection": { + "node": { + "node_code": "mainland-controller-01", + "region": "mainland", + "role": "control", + }, + "active_thread_count": 177, + "max_thread_count": 2000, + "progress": { + "pending": 200, + "running": 177, + "completed": 440, + "blacklisted": 0, + "failed": 4, + }, + "active_job": { + "job_id": 551, + "job_code": "sync-overseas-55", + "status": "running", + "items_total": 821, + "items_terminal": 444, + "items_failed": 4, + }, + } + } + fake_conn = _SequenceConn( + fetchall_results=[ + [(debug_payload, datetime(2026, 4, 22, 17, 37, 30))], + [(ingest_payload, datetime(2026, 4, 22, 18, 5, 36))], + ] + ) + + with patch("app.services.detect_job_service.settings.node_region", "overseas"): + with patch("app.services.detect_job_service.settings.node_role", "control"): + with patch("app.services.detect_job_service.get_db", return_value=fake_conn): + with patch("app.services.detect_job_service._load_runtime_activity_snapshot", return_value={}): + snapshot = _load_latest_runtime_active_job_snapshot(15) + + self.assertEqual("runtime_ingest", snapshot["_snapshot_source"]) + self.assertEqual("sync-overseas-55", snapshot["job"]["job_code"]) + + def test_load_latest_runtime_active_job_snapshot_prefers_richer_ingest_over_newer_debug_snapshot(self) -> None: + debug_payload = { + "job": { + "job_id": 376, + "job_code": "sync-overseas-376", + "status": "running", + "display_items_running": 138, + "display_active_threads": 138, + "node_stats": [ + { + "node_code": "mainland-controller-01", + "items_running": 138, + "display_running": 138, + "active_threads": 138, + "max_threads": 2000, + } + ], + }, + "queue_health": { + "queue": { + "items_total": 8402, + "running": 138, + "display_running": 138, + }, + "nodes": [ + { + "node_code": "mainland-controller-01", + "items_running": 138, + "display_running": 138, + "active_threads": 138, + "max_threads": 2000, + } + ], + }, + } + ingest_payload = { + "projection": { + "node": { + "node_code": "mainland-controller-01", + "region": "mainland", + "role": "control", + }, + "active_thread_count": 5909, + "max_thread_count": 80000, + "progress": { + "pending": 2493, + "running": 5909, + "completed": 0, + "blacklisted": 0, + "failed": 0, + }, + "active_job": { + "job_id": 551, + "job_code": "sync-overseas-51", + "status": "running", + "items_total": 8402, + "items_terminal": 0, + "items_failed": 0, + "display_items_running": 5909, + "display_active_threads": 5909, + "distributed_node_stats": [ + { + "node_code": "mainland-controller-01-a", + "items_running": 1000, + "display_running": 1000, + "active_threads": 1000, + "max_threads": 1000, + }, + { + "node_code": "mainland-controller-01-b", + "items_running": 1000, + "display_running": 1000, + "active_threads": 1000, + "max_threads": 1000, + }, + { + "node_code": "mainland-controller-01-c", + "items_running": 3909, + "display_running": 3909, + "active_threads": 3909, + "max_threads": 78000, + }, + ], + }, + } + } + fake_conn = _SequenceConn( + fetchall_results=[ + [(debug_payload, datetime(2026, 4, 23, 2, 45, 0))], + [(ingest_payload, datetime(2026, 4, 23, 2, 38, 55))], + ] + ) + + with patch("app.services.detect_job_service.settings.node_region", "overseas"): + with patch("app.services.detect_job_service.settings.node_role", "control"): + with patch("app.services.detect_job_service.get_db", return_value=fake_conn): + with patch("app.services.detect_job_service._load_runtime_activity_snapshot", return_value={}): + snapshot = _load_latest_runtime_active_job_snapshot(15) + + self.assertEqual("runtime_ingest", snapshot["_snapshot_source"]) + self.assertEqual("sync-overseas-51", snapshot["job"]["job_code"]) + self.assertEqual(5909, snapshot["queue_health"]["queue"]["display_running"]) + self.assertEqual( + [ + "mainland-controller-01-a", + "mainland-controller-01-b", + "mainland-controller-01-c", + "unassigned", + ], + [item["node_code"] for item in snapshot["queue_health"]["nodes"]], + ) + + def test_load_latest_runtime_active_job_snapshot_prefers_focus_job_match_over_newer_debug_snapshot(self) -> None: + debug_payload = { + "job": { + "job_id": 376, + "job_code": "sync-overseas-376", + "status": "running", + }, + "queue_health": { + "queue": { + "items_total": 5000, + "running": 282, + } + }, + } + ingest_payload = { + "projection": { + "node": { + "node_code": "mainland-controller-01", + "region": "mainland", + "role": "control", + }, + "active_thread_count": 177, + "max_thread_count": 2000, + "progress": { + "pending": 7800, + "running": 177, + "completed": 420, + "blacklisted": 0, + "failed": 5, + }, + "active_job": { + "job_id": 551, + "job_code": "sync-overseas-51", + "status": "running", + "items_total": 8402, + "items_terminal": 425, + "items_failed": 5, + }, + } + } + fake_conn = _SequenceConn( + fetchall_results=[ + [(debug_payload, datetime(2026, 4, 22, 23, 17, 47))], + [ + ({"projection": {"active_job": {"job_code": "sync-overseas-55"}}}, datetime(2026, 4, 22, 18, 6, 10)), + (ingest_payload, datetime(2026, 4, 22, 18, 5, 36)), + ], + ] + ) + + with patch("app.services.detect_job_service.settings.node_region", "overseas"): + with patch("app.services.detect_job_service.settings.node_role", "control"): + with patch("app.services.detect_job_service.get_db", return_value=fake_conn): + with patch( + "app.services.detect_job_service._load_runtime_activity_snapshot", + return_value={"focus_job_code": "sync-overseas-51"}, + ): + snapshot = _load_latest_runtime_active_job_snapshot(15) + + self.assertEqual("runtime_ingest", snapshot["_snapshot_source"]) + self.assertEqual("sync-overseas-51", snapshot["job"]["job_code"]) + + def test_load_latest_runtime_ingest_active_job_snapshot_keeps_recent_focus_match_over_marginally_newer_non_focus_snapshot(self) -> None: + rich_non_focus_payload = { + "projection": { + "node": { + "node_code": "mainland-controller-01", + "region": "mainland", + "role": "control", + }, + "active_thread_count": 5909, + "max_thread_count": 80000, + "progress": { + "pending": 2493, + "running": 5909, + "completed": 0, + "blacklisted": 0, + "failed": 0, + }, + "active_job": { + "job_id": 198, + "job_code": "sync-overseas-198", + "status": "running", + "items_total": 8402, + "items_terminal": 0, + "items_failed": 0, + "display_items_running": 5909, + "display_active_threads": 5909, + "distributed_node_stats": [ + { + "node_code": "mainland-controller-01-a", + "items_running": 1000, + "display_running": 1000, + "active_threads": 1000, + "max_threads": 1000, + }, + { + "node_code": "mainland-controller-01-b", + "items_running": 4909, + "display_running": 4909, + "active_threads": 4909, + "max_threads": 79000, + }, + ], + }, + } + } + focus_match_payload = { + "projection": { + "node": { + "node_code": "mainland-controller-01", + "region": "mainland", + "role": "control", + }, + "active_thread_count": 177, + "max_thread_count": 2000, + "progress": { + "pending": 7800, + "running": 177, + "completed": 420, + "blacklisted": 0, + "failed": 5, + }, + "active_job": { + "job_id": 551, + "job_code": "sync-overseas-51", + "status": "running", + "items_total": 8402, + "items_terminal": 425, + "items_failed": 5, + "display_items_running": 177, + "display_active_threads": 177, + "distributed_node_stats": [ + { + "node_code": "mainland-controller-01", + "items_running": 177, + "display_running": 177, + "active_threads": 177, + "max_threads": 2000, + } + ], + }, + } + } + fake_conn = _SequenceConn( + fetchall_results=[ + [ + (rich_non_focus_payload, datetime(2026, 4, 23, 2, 45, 0)), + (focus_match_payload, datetime(2026, 4, 23, 2, 41, 0)), + ] + ] + ) + + with patch("app.services.detect_job_service.settings.node_region", "overseas"): + with patch("app.services.detect_job_service.settings.node_role", "control"): + with patch("app.services.detect_job_service.get_db", return_value=fake_conn): + snapshot = detect_job_service_module._load_latest_runtime_ingest_active_job_snapshot( + 15, + preferred_job_codes=["sync-overseas-51"], + ) + + self.assertEqual("runtime_ingest", snapshot["_snapshot_source"]) + self.assertEqual("sync-overseas-51", snapshot["job"]["job_code"]) + + def test_load_latest_runtime_active_job_snapshot_prefers_newer_richer_ingest_over_stale_focus_job_match(self) -> None: + rich_ingest_payload = { + "projection": { + "node": { + "node_code": "mainland-controller-01", + "region": "mainland", + "role": "control", + }, + "active_thread_count": 5909, + "max_thread_count": 80000, + "progress": { + "pending": 2493, + "running": 5909, + "completed": 0, + "blacklisted": 0, + "failed": 0, + }, + "active_job": { + "job_id": 198, + "job_code": "sync-overseas-198", + "status": "running", + "items_total": 8402, + "items_terminal": 0, + "items_failed": 0, + "display_items_running": 5909, + "display_active_threads": 5909, + "distributed_node_stats": [ + { + "node_code": "mainland-controller-01-a", + "items_running": 1000, + "display_running": 1000, + "active_threads": 1000, + "max_threads": 1000, + }, + { + "node_code": "mainland-controller-01-b", + "items_running": 1000, + "display_running": 1000, + "active_threads": 1000, + "max_threads": 1000, + }, + { + "node_code": "mainland-controller-01-c", + "items_running": 3909, + "display_running": 3909, + "active_threads": 3909, + "max_threads": 78000, + }, + ], + }, + } + } + stale_focus_ingest_payload = { + "projection": { + "node": { + "node_code": "mainland-controller-01", + "region": "mainland", + "role": "control", + }, + "active_thread_count": 138, + "max_thread_count": 2000, + "progress": { + "pending": 5001, + "running": 0, + "completed": 0, + "blacklisted": 0, + "failed": 0, + }, + "active_job": { + "job_id": 51, + "job_code": "sync-overseas-51", + "status": "running", + "items_total": 8402, + "items_terminal": 3402, + "items_failed": 0, + "display_items_running": 138, + "display_active_threads": 138, + "distributed_node_stats": [ + { + "node_code": "mainland-controller-01", + "items_running": 0, + "display_running": 138, + "active_threads": 138, + "max_threads": 2000, + } + ], + }, + } + } + fake_conn = _SequenceConn( + fetchall_results=[ + [], + [ + (rich_ingest_payload, datetime(2026, 4, 23, 2, 38, 55)), + (stale_focus_ingest_payload, datetime(2026, 4, 22, 18, 2, 4)), + ], + ] + ) + + with patch("app.services.detect_job_service.settings.node_region", "overseas"): + with patch("app.services.detect_job_service.settings.node_role", "control"): + with patch("app.services.detect_job_service.get_db", return_value=fake_conn): + with patch( + "app.services.detect_job_service._load_runtime_activity_snapshot", + return_value={"focus_job_code": "sync-overseas-51", "job_codes": ["sync-overseas-51"]}, + ): + snapshot = _load_latest_runtime_active_job_snapshot(15) + + self.assertEqual("runtime_ingest", snapshot["_snapshot_source"]) + self.assertEqual("sync-overseas-198", snapshot["job"]["job_code"]) + self.assertEqual(5909, snapshot["queue_health"]["queue"]["display_running"]) + + def test_get_active_detect_job_summary_falls_back_to_runtime_snapshot(self) -> None: + runtime_snapshot = { + "job": { + "job_id": 551, + "job_code": "sync-overseas-55", + "status": "running", + "progress_percent": 54.08, + "node_stats": [ + { + "node_code": "mainland-controller-01", + "items_total": 821, + "items_pending": 200, + "items_claimed": 0, + "items_running": 177, + "items_completed": 440, + "items_blacklisted": 0, + "items_failed": 4, + "active_threads": 177, + "max_threads": 2000, + } + ], + }, + "queue_health": { + "queue": { + "items_total": 821, + "pending": 200, + "claimed": 0, + "running": 177, + "display_claimed": 0, + "display_running": 177, + "completed": 440, + "blacklisted": 0, + "failed": 4, + "terminal": 444, + }, + "nodes": [ + { + "node_code": "mainland-controller-01", + "items_total": 821, + "items_pending": 200, + "items_claimed": 0, + "items_running": 177, + "items_completed": 440, + "items_blacklisted": 0, + "items_failed": 4, + "active_threads": 177, + "max_threads": 2000, + } + ], + "throughput": { + "processed_recent": 0, + "processed_per_minute": 0, + "completed_recent": 0, + "blacklisted_recent": 0, + "failed_recent": 0, + }, + }, + "_created_at": "2026-04-22 18:05:36", + "_snapshot_source": "runtime_ingest", + } + fake_conn = _SequenceConn(fetchall_results=[[]]) + + with patch("app.services.detect_job_service.get_db", return_value=fake_conn): + with patch("app.services.detect_job_service._load_latest_runtime_active_job_snapshot", return_value=runtime_snapshot): + with patch("app.services.detect_job_service._load_runtime_activity_snapshot", return_value={}): + summary = get_active_detect_job_summary(event_limit=10) + + self.assertIsNotNone(summary) + self.assertEqual("sync-overseas-55", summary["job_code"]) + self.assertEqual(821, summary["items_total"]) + self.assertEqual(177, summary["display_items_running"]) + + def test_recycle_expired_detect_job_items_once_refreshes_touched_jobs(self) -> None: + conn = MagicMock() + main_cursor_cm = MagicMock() + main_cursor = MagicMock() + unlock_cursor_cm = MagicMock() + unlock_cursor = MagicMock() + main_cursor_cm.__enter__.return_value = main_cursor + main_cursor_cm.__exit__.return_value = False + unlock_cursor_cm.__enter__.return_value = unlock_cursor + unlock_cursor_cm.__exit__.return_value = False + conn.cursor.side_effect = [main_cursor_cm, unlock_cursor_cm] + main_cursor.fetchone.side_effect = [ + (True,), + (2, [11, 12]), + ("domain_pipeline", 1, 0, 0, 0, 0), + ("single_step", 0, 0, 1, 0, 0), + ] + db_cm = MagicMock() + db_cm.__enter__.return_value = conn + db_cm.__exit__.return_value = False + + with patch("app.services.detect_job_service.get_db", return_value=db_cm): + with patch("app.services.detect_job_service.push_debug_event") as mock_push: + recycled = detect_job_service_module._recycle_expired_detect_job_items_once() + + self.assertEqual(2, recycled) + self.assertTrue(any("UPDATE detect_job_items" in call.args[0] for call in main_cursor.execute.call_args_list)) + self.assertTrue(any("FOR UPDATE SKIP LOCKED" in call.args[0] for call in main_cursor.execute.call_args_list)) + self.assertTrue( + any( + "UPDATE detect_jobs" in call.args[0] + and call.args[1] == (11,) + for call in main_cursor.execute.call_args_list + ) + ) + self.assertTrue( + any( + "UPDATE detect_jobs" in call.args[0] + and call.args[1] == ("failed", 12) + for call in main_cursor.execute.call_args_list + ) + ) + unlock_cursor.execute.assert_called_once_with( + "SELECT pg_advisory_unlock(%s)", + (detect_job_service_module._DETECT_JOB_ITEM_RECYCLE_ADVISORY_LOCK_KEY,), + ) + self.assertGreaterEqual(conn.commit.call_count, 1) + conn.rollback.assert_not_called() + mock_push.assert_called_once() + + def test_recycle_expired_detect_job_items_once_skips_when_lock_is_busy(self) -> None: + conn = MagicMock() + main_cursor_cm = MagicMock() + main_cursor = MagicMock() + main_cursor_cm.__enter__.return_value = main_cursor + main_cursor_cm.__exit__.return_value = False + conn.cursor.return_value = main_cursor_cm + main_cursor.fetchone.return_value = (False,) + db_cm = MagicMock() + db_cm.__enter__.return_value = conn + db_cm.__exit__.return_value = False + + with patch("app.services.detect_job_service.get_db", return_value=db_cm): + with patch("app.services.detect_job_service.push_debug_event") as mock_push: + recycled = detect_job_service_module._recycle_expired_detect_job_items_once() + + self.assertEqual(0, recycled) + conn.rollback.assert_called_once() + conn.commit.assert_not_called() + mock_push.assert_not_called() + + def test_get_active_detect_job_summary_attempts_expired_item_recycle_before_query(self) -> None: + fake_conn = _SequenceConn(fetchall_results=[[]]) + + with patch("app.services.detect_job_service.get_db", return_value=fake_conn): + with patch("app.services.detect_job_service._maybe_recycle_expired_detect_job_items") as mock_recycle: + with patch("app.services.detect_job_service._load_latest_runtime_active_job_snapshot", return_value={}): + with patch("app.services.detect_job_service._load_runtime_activity_snapshot", return_value={}): + summary = get_active_detect_job_summary(event_limit=10) + + self.assertIsNone(summary) + mock_recycle.assert_called_once_with() + + def test_get_active_detect_job_summary_prefers_runtime_focus_job_code(self) -> None: + rows = [ + ( + 85, + "sync-overseas-376", + "manual", + "domain_pipeline", + "", + "running", + "tester", + datetime(2026, 4, 22, 22, 0, 0), + datetime(2026, 4, 22, 22, 0, 1), + None, + ), + ( + 11, + "sync-overseas-51", + "manual", + "domain_pipeline", + "", + "running", + "tester", + datetime(2026, 4, 22, 17, 0, 0), + datetime(2026, 4, 22, 17, 0, 1), + None, + ), + ] + fake_conn = _SequenceConn(fetchall_results=[rows]) + selected_rows = [] + + def _fake_fetch(_cur, row, event_limit=20): + selected_rows.append((row, event_limit)) + return {"job_id": row[0], "job_code": row[1], "status": row[5]} + + with patch("app.services.detect_job_service.get_db", return_value=fake_conn): + with patch( + "app.services.detect_job_service._load_runtime_activity_snapshot", + return_value={"focus_job_code": "sync-overseas-51", "job_codes": ["sync-overseas-51"]}, + ): + with patch("app.services.detect_job_service._load_latest_runtime_active_job_snapshot", return_value={}): + with patch("app.services.detect_job_service._fetch_job_summary", side_effect=_fake_fetch): + with patch( + "app.services.detect_job_service._enrich_active_job_summary_with_runtime", + side_effect=lambda summary, **kwargs: summary, + ): + summary = get_active_detect_job_summary(event_limit=10) + + self.assertEqual("sync-overseas-51", summary["job_code"]) + self.assertEqual("sync-overseas-51", selected_rows[0][0][1]) + self.assertEqual(10, selected_rows[0][1]) + + def test_get_active_detect_job_summary_prefers_runtime_snapshot_job_code_over_stale_runtime_focus(self) -> None: + rows = [ + ( + 85, + "sync-overseas-376", + "manual", + "domain_pipeline", + "", + "running", + "tester", + datetime(2026, 4, 22, 22, 0, 0), + datetime(2026, 4, 22, 22, 0, 1), + None, + ), + ( + 11, + "sync-overseas-51", + "manual", + "domain_pipeline", + "", + "running", + "tester", + datetime(2026, 4, 22, 17, 0, 0), + datetime(2026, 4, 22, 17, 0, 1), + None, + ), + ] + fake_conn = _SequenceConn(fetchall_results=[rows]) + selected_rows = [] + + def _fake_fetch(_cur, row, event_limit=20): + selected_rows.append((row, event_limit)) + return {"job_id": row[0], "job_code": row[1], "status": row[5]} + + with patch("app.services.detect_job_service.get_db", return_value=fake_conn): + with patch( + "app.services.detect_job_service._load_runtime_activity_snapshot", + return_value={"focus_job_code": "sync-overseas-51", "job_codes": ["sync-overseas-51"]}, + ): + with patch( + "app.services.detect_job_service._load_latest_runtime_active_job_snapshot", + return_value={"job": {"job_code": "sync-overseas-376"}}, + ): + with patch("app.services.detect_job_service._fetch_job_summary", side_effect=_fake_fetch): + with patch( + "app.services.detect_job_service._enrich_active_job_summary_with_runtime", + side_effect=lambda summary, **kwargs: summary, + ): + summary = get_active_detect_job_summary(event_limit=10) + + self.assertEqual("sync-overseas-376", summary["job_code"]) + self.assertEqual("sync-overseas-376", selected_rows[0][0][1]) + self.assertEqual(10, selected_rows[0][1]) + + def test_get_active_detect_job_summary_falls_back_when_detect_jobs_legacy_columns_are_missing(self) -> None: + rows = [ + ( + 85, + "sync-overseas-376", + "manual", + "domain_pipeline", + "", + "running", + "tester", + datetime(2026, 4, 22, 22, 0, 0), + datetime(2026, 4, 22, 22, 0, 1), + None, + ), + ] + + class LegacyCursor(_SequenceCursor): + def execute(self, sql, params=None): + self.exec_calls.append((sql, params)) + normalized_sql = str(sql or "") + if ( + "FROM detect_jobs" in normalized_sql + and "task_mode" in normalized_sql + and " AS task_mode" not in normalized_sql + ): + raise errors.UndefinedColumn() + + class LegacyConn(_SequenceConn): + def __init__(self) -> None: + self.cursor_obj = LegacyCursor(fetchall_results=[rows]) + + fake_conn = LegacyConn() + + with patch.object(detect_job_service_module, "_DETECT_JOBS_LEGACY_SELECT_MODE", None): + with patch("app.services.detect_job_service.get_db", return_value=fake_conn): + with patch("app.services.detect_job_service._load_runtime_activity_snapshot", return_value={}): + with patch("app.services.detect_job_service._load_latest_runtime_active_job_snapshot", return_value={}): + with patch( + "app.services.detect_job_service._fetch_job_summary", + side_effect=lambda _cur, row, event_limit=20: { + "job_id": row[0], + "job_code": row[1], + "task_mode": row[3], + "step_code": row[4], + "status": row[5], + }, + ): + with patch( + "app.services.detect_job_service._enrich_active_job_summary_with_runtime", + side_effect=lambda summary, **kwargs: summary, + ): + summary = get_active_detect_job_summary(event_limit=10) + + self.assertEqual("sync-overseas-376", summary["job_code"]) + self.assertEqual("domain_pipeline", summary["task_mode"]) + self.assertEqual("", summary["step_code"]) + self.assertTrue(any(" AS task_mode" in call[0] for call in fake_conn.cursor_obj.exec_calls)) + def test_build_display_summary_ignores_inflated_raw_current_load(self) -> None: summary = _build_display_summary( [ @@ -378,6 +1592,75 @@ class DetectJobServiceTests(unittest.TestCase): self.assertEqual("worker_log", summary["latest_event"]["event_type"]) self.assertEqual("sync-overseas-31437", summary["recent_events"][0]["payload"]["job_code"]) + def test_enrich_active_job_summary_with_runtime_does_not_promote_unassigned_snapshot_running(self) -> None: + summary = _enrich_active_job_summary_with_runtime( + { + "job_id": 384, + "job_code": "sync-overseas-16654", + "status": "running", + "display_items_running": 331, + "display_current_load": 331, + "display_active_threads": 331, + "display_max_threads": 1000, + "display_active_node_codes": [], + }, + event_limit=10, + window_minutes=15, + runtime_activity={ + "focus_job_code": "sync-overseas-16654", + "job_codes": ["sync-overseas-16654"], + "processed_recent": 0, + "completed_recent": 0, + "failed_recent": 0, + "blacklisted_recent": 0, + }, + runtime_snapshot={ + "_has_runtime_node_rows": True, + "_dropped_runtime_node_count": 0, + "job": { + "job_id": 384, + "job_code": "sync-overseas-16654", + "status": "running", + "node_stats": [ + { + "node_code": "unassigned", + "items_claimed": 23, + "items_running": 1863, + "display_running": 1863, + "active_threads": 0, + "max_threads": 0, + } + ], + }, + "queue_health": { + "queue": { + "items_total": 4000, + "pending": 0, + "claimed": 23, + "running": 2100, + "completed": 1117, + "blacklisted": 0, + "failed": 760, + "display_running": 2100, + }, + "nodes": [ + { + "node_code": "unassigned", + "items_claimed": 23, + "items_running": 1863, + "display_running": 1863, + "active_threads": 0, + "max_threads": 0, + } + ], + }, + }, + ) + + self.assertEqual(0, summary["display_items_running"]) + self.assertEqual(0, summary["display_active_threads"]) + self.assertEqual([], summary["display_active_node_codes"]) + def test_get_detect_queue_health_preserves_runtime_node_throughput_after_snapshot_override(self) -> None: lease_row = (None, None, 0, 0) throughput_rows = [] @@ -579,8 +1862,9 @@ class DetectJobServiceTests(unittest.TestCase): with patch("app.services.detect_job_service.get_active_detect_job_summary", return_value=active_job): with patch("app.services.detect_job_service._load_runtime_activity_snapshot", return_value=runtime_activity): with patch("app.services.detect_job_service._load_latest_runtime_active_job_snapshot", return_value=runtime_snapshot): - with patch("app.services.detect_job_service.get_db", return_value=db_cm): - health = get_detect_queue_health(window_minutes=15) + with patch("app.services.detect_job_service._load_runtime_display_rows", return_value=runtime_display_rows): + with patch("app.services.detect_job_service.get_db", return_value=db_cm): + health = get_detect_queue_health(window_minutes=15) node_map = {item["node_code"]: item for item in health["nodes"]} self.assertEqual(74, node_map["mainland-controller-01"]["processed_recent"]) @@ -594,6 +1878,699 @@ class DetectJobServiceTests(unittest.TestCase): self.assertEqual(189, health["queue"]["display_running"]) self.assertNotIn("overseas-control-01", node_map) + def test_get_detect_queue_health_preserves_snapshot_node_display_metrics_for_child_instances(self) -> None: + lease_row = (None, None, 0, 0) + throughput_rows = [] + step_throughput_rows = [] + runtime_display_rows = [] + + conn = MagicMock() + cursor_cm = MagicMock() + cursor = MagicMock() + conn.cursor.return_value = cursor_cm + cursor_cm.__enter__.return_value = cursor + db_cm = MagicMock() + db_cm.__enter__.return_value = conn + cursor.fetchone.return_value = lease_row + cursor.fetchall.side_effect = [throughput_rows, step_throughput_rows, runtime_display_rows] + + active_job = { + "job_id": 198, + "job_code": "sync-overseas-198", + "status": "running", + "items_total": 10344, + "items_pending": 4945, + "items_claimed": 0, + "items_running": 5909, + "items_completed": 5344, + "items_blacklisted": 0, + "items_failed": 42, + "progress_percent": 52.07, + "display_items_claimed": 13, + "display_items_running": 11799, + "display_active_threads": 11799, + "display_max_threads": 75400, + "node_stats": [], + "distributed_node_stats": [ + { + "node_code": "mainland-controller-01-ae", + "items_total": 947, + "items_pending": 0, + "items_claimed": 0, + "items_running": 947, + "items_completed": 0, + "items_blacklisted": 0, + "items_failed": 0, + }, + { + "node_code": "mainland-controller-01-au", + "items_total": 637, + "items_pending": 0, + "items_claimed": 0, + "items_running": 637, + "items_completed": 0, + "items_blacklisted": 0, + "items_failed": 0, + }, + ], + "step_stats": [], + } + runtime_activity = { + "processed_recent": 0, + "completed_recent": 0, + "failed_recent": 0, + "blacklisted_recent": 0, + "focus_job_code": "sync-overseas-198", + "job_codes": ["sync-overseas-198"], + "step_code": "detect_register", + "nodes": {}, + } + runtime_snapshot = { + "job": { + "job_id": 198, + "job_code": "sync-overseas-198", + "progress_percent": 52.07, + }, + "queue_health": { + "queue": { + "items_total": 10344, + "pending": 4945, + "claimed": 0, + "running": 5909, + "completed": 5344, + "blacklisted": 0, + "failed": 42, + "display_claimed": 13, + "display_running": 5909, + }, + "nodes": [ + { + "node_code": "mainland-controller-01-ae", + "items_total": 947, + "items_pending": 0, + "items_claimed": 0, + "items_running": 947, + "items_completed": 0, + "items_blacklisted": 0, + "items_failed": 0, + "display_running": 947, + "current_load": 947, + "active_threads": 947, + "max_threads": 1000, + }, + { + "node_code": "mainland-controller-01-au", + "items_total": 637, + "items_pending": 0, + "items_claimed": 0, + "items_running": 637, + "items_completed": 0, + "items_blacklisted": 0, + "items_failed": 0, + "display_running": 637, + "current_load": 637, + "active_threads": 637, + "max_threads": 1000, + }, + ], + "steps": [], + }, + } + + with patch("app.services.detect_job_service.settings.node_region", "overseas"): + with patch("app.services.detect_job_service.settings.node_role", "control"): + with patch("app.services.detect_job_service.settings.node_code", "overseas-control-01"): + with patch("app.services.detect_job_service.get_active_detect_job_summary", return_value=active_job): + with patch("app.services.detect_job_service._load_runtime_activity_snapshot", return_value=runtime_activity): + with patch("app.services.detect_job_service._load_latest_runtime_active_job_snapshot", return_value=runtime_snapshot): + with patch("app.services.detect_job_service.get_db", return_value=db_cm): + with patch("app.services.detect_job_service._load_disabled_managed_node_codes", return_value=set()): + health = get_detect_queue_health(window_minutes=15) + + node_map = {item["node_code"]: item for item in health["nodes"]} + self.assertEqual(1584, health["queue"]["display_running"]) + self.assertEqual(947, node_map["mainland-controller-01-ae"]["display_running"]) + self.assertEqual(947, node_map["mainland-controller-01-ae"]["active_threads"]) + self.assertEqual(1000, node_map["mainland-controller-01-ae"]["max_threads"]) + self.assertEqual(637, node_map["mainland-controller-01-au"]["display_running"]) + self.assertEqual(637, node_map["mainland-controller-01-au"]["active_threads"]) + + def test_get_detect_queue_health_drops_stale_runtime_nodes_from_display_running(self) -> None: + lease_row = (None, None, 0, 0) + throughput_rows = [] + step_throughput_rows = [] + runtime_display_rows = [] + + conn = MagicMock() + cursor_cm = MagicMock() + cursor = MagicMock() + conn.cursor.return_value = cursor_cm + cursor_cm.__enter__.return_value = cursor + db_cm = MagicMock() + db_cm.__enter__.return_value = conn + cursor.fetchone.return_value = lease_row + cursor.fetchall.side_effect = [throughput_rows, step_throughput_rows, runtime_display_rows] + + stale_heartbeat = (datetime.now() - timedelta(minutes=8)).isoformat(sep=" ", timespec="seconds") + live_heartbeat = datetime.now().isoformat(sep=" ", timespec="seconds") + active_job = { + "job_id": 199, + "job_code": "sync-overseas-199", + "status": "running", + "items_total": 2600, + "items_pending": 500, + "items_claimed": 0, + "items_running": 2100, + "items_completed": 0, + "items_blacklisted": 0, + "items_failed": 0, + "progress_percent": 80.77, + "display_items_running": 2100, + "display_active_threads": 2100, + "display_max_threads": 2000, + "node_stats": [], + "distributed_node_stats": [ + { + "node_code": "mainland-controller-01-a", + "items_total": 300, + "items_pending": 0, + "items_claimed": 0, + "items_running": 300, + "items_completed": 0, + "items_blacklisted": 0, + "items_failed": 0, + }, + { + "node_code": "mainland-controller-01-b", + "items_total": 1800, + "items_pending": 0, + "items_claimed": 0, + "items_running": 1800, + "items_completed": 0, + "items_blacklisted": 0, + "items_failed": 0, + }, + ], + "step_stats": [], + } + runtime_activity = { + "processed_recent": 0, + "completed_recent": 0, + "failed_recent": 0, + "blacklisted_recent": 0, + "focus_job_code": "sync-overseas-199", + "job_codes": ["sync-overseas-199"], + "step_code": "detect_register", + "nodes": {}, + } + runtime_snapshot = { + "_created_at": datetime.now().isoformat(sep=" ", timespec="seconds"), + "job": { + "job_id": 199, + "job_code": "sync-overseas-199", + "progress_percent": 80.77, + }, + "queue_health": { + "queue": { + "items_total": 2600, + "pending": 500, + "claimed": 0, + "running": 2100, + "completed": 0, + "blacklisted": 0, + "failed": 0, + "display_claimed": 0, + "display_running": 2100, + }, + "nodes": [ + { + "node_code": "mainland-controller-01-a", + "items_total": 300, + "items_pending": 0, + "items_claimed": 0, + "items_running": 300, + "items_completed": 0, + "items_blacklisted": 0, + "items_failed": 0, + "display_running": 300, + "current_load": 300, + "active_threads": 300, + "max_threads": 1000, + "status": "busy", + "last_heartbeat_at": live_heartbeat, + }, + { + "node_code": "mainland-controller-01-b", + "items_total": 1800, + "items_pending": 0, + "items_claimed": 0, + "items_running": 1800, + "items_completed": 0, + "items_blacklisted": 0, + "items_failed": 0, + "display_running": 1800, + "current_load": 1800, + "active_threads": 1800, + "max_threads": 1000, + "status": "stale", + "last_heartbeat_at": stale_heartbeat, + }, + ], + "steps": [], + }, + } + + with patch("app.services.detect_job_service.settings.node_region", "overseas"): + with patch("app.services.detect_job_service.settings.node_role", "control"): + with patch("app.services.detect_job_service.settings.node_code", "overseas-control-01"): + with patch("app.services.detect_job_service.get_active_detect_job_summary", return_value=active_job): + with patch("app.services.detect_job_service._load_runtime_activity_snapshot", return_value=runtime_activity): + with patch("app.services.detect_job_service._load_latest_runtime_active_job_snapshot", return_value=runtime_snapshot): + with patch("app.services.detect_job_service._load_runtime_display_rows", return_value=runtime_display_rows): + with patch("app.services.detect_job_service.get_db", return_value=db_cm): + health = get_detect_queue_health(window_minutes=15) + + self.assertEqual(300, health["queue"]["running"]) + self.assertEqual(300, health["queue"]["display_running"]) + self.assertEqual(["mainland-controller-01-a"], [item["node_code"] for item in health["nodes"]]) + + def test_enrich_active_job_summary_drops_old_runtime_ingest_nodes_without_heartbeat(self) -> None: + stale_snapshot_created_at = (datetime.now() - timedelta(minutes=12)).isoformat(sep=" ", timespec="seconds") + summary = _enrich_active_job_summary_with_runtime( + { + "job_id": 384, + "job_code": "sync-overseas-16654", + "status": "running", + "display_items_running": 0, + "display_current_load": 0, + "display_active_threads": 0, + "display_max_threads": 0, + "display_active_node_codes": [], + }, + event_limit=10, + window_minutes=15, + runtime_activity={ + "focus_job_code": "sync-overseas-16654", + "job_codes": ["sync-overseas-16654"], + "processed_recent": 0, + "completed_recent": 0, + "failed_recent": 0, + "blacklisted_recent": 0, + }, + runtime_snapshot={ + "_snapshot_source": "runtime_ingest", + "_created_at": stale_snapshot_created_at, + "job": { + "job_id": 384, + "job_code": "sync-overseas-16654", + "status": "running", + "display_items_running": 2100, + "display_active_threads": 2100, + "node_stats": [ + { + "node_code": "mainland-controller-01-ah", + "items_running": 62, + "display_running": 62, + }, + { + "node_code": "mainland-controller-01-bn", + "items_running": 61, + "display_running": 61, + }, + { + "node_code": "unassigned", + "items_running": 1863, + "display_running": 1863, + }, + ], + }, + "queue_health": { + "queue": { + "items_total": 4000, + "pending": 0, + "claimed": 23, + "running": 2100, + "completed": 1117, + "blacklisted": 0, + "failed": 760, + "display_running": 2100, + }, + "nodes": [ + { + "node_code": "mainland-controller-01-ah", + "items_running": 62, + "display_running": 62, + }, + { + "node_code": "mainland-controller-01-bn", + "items_running": 61, + "display_running": 61, + }, + { + "node_code": "unassigned", + "items_running": 1863, + "display_running": 1863, + }, + ], + }, + }, + ) + self.assertEqual(0, summary["display_items_running"]) + self.assertEqual(0, summary["display_active_threads"]) + self.assertEqual([], summary["display_active_node_codes"]) + + def test_get_detect_queue_health_aligns_runtime_job_identity_with_active_job(self) -> None: + lease_row = (None, None, 0, 0) + throughput_rows = [] + step_throughput_rows = [] + runtime_display_rows = [] + + conn = MagicMock() + cursor_cm = MagicMock() + cursor = MagicMock() + conn.cursor.return_value = cursor_cm + cursor_cm.__enter__.return_value = cursor + db_cm = MagicMock() + db_cm.__enter__.return_value = conn + cursor.fetchone.return_value = lease_row + cursor.fetchall.side_effect = [throughput_rows, step_throughput_rows, runtime_display_rows] + + active_job = { + "job_id": 112, + "job_code": "sync-overseas-599", + "runtime_job_code": "sync-overseas-599", + "runtime_snapshot_job_code": "sync-overseas-599", + "status": "running", + "items_total": 5274, + "items_pending": 4959, + "items_claimed": 0, + "items_running": 840, + "items_completed": 274, + "items_blacklisted": 0, + "items_failed": 21, + "progress_percent": 5.59, + "display_items_claimed": 17, + "display_items_running": 1377, + "display_active_threads": 1377, + "display_max_threads": 81200, + "node_stats": [], + "distributed_node_stats": [], + "step_stats": [], + } + runtime_activity = { + "processed_recent": 93, + "completed_recent": 24, + "failed_recent": 69, + "blacklisted_recent": 0, + "focus_job_code": "sync-overseas-198", + "job_codes": ["sync-overseas-198", "sync-overseas-97"], + "step_code": "detect_360_site", + "step_stats": {}, + "nodes": {}, + } + runtime_snapshot = { + "job": { + "job_id": 112, + "job_code": "sync-overseas-599", + "status": "running", + "progress_percent": 5.59, + }, + "queue_health": { + "queue": { + "items_total": 5274, + "pending": 4959, + "claimed": 0, + "running": 1377, + "display_claimed": 17, + "display_running": 1377, + "completed": 274, + "blacklisted": 0, + "failed": 21, + }, + "nodes": [], + "steps": [], + }, + } + + with patch("app.services.detect_job_service.get_active_detect_job_summary", return_value=active_job): + with patch("app.services.detect_job_service._load_runtime_activity_snapshot", return_value=runtime_activity): + with patch("app.services.detect_job_service._load_latest_runtime_active_job_snapshot", return_value=runtime_snapshot): + with patch("app.services.detect_job_service.get_db", return_value=db_cm): + with patch("app.services.detect_job_service._load_disabled_managed_node_codes", return_value=set()): + health = get_detect_queue_health(window_minutes=15) + + self.assertEqual("sync-overseas-599", health["job"]["job_code"]) + self.assertEqual("sync-overseas-599", health["job"]["runtime_job_code"]) + self.assertEqual("sync-overseas-599", health["job"]["runtime_job_codes"][0]) + self.assertEqual(1377, health["queue"]["running"]) + self.assertEqual(1377, health["queue"]["display_running"]) + + def test_get_detect_queue_health_ignores_mismatched_runtime_overlay_for_unrelated_job(self) -> None: + lease_row = (None, None, 0, 0) + throughput_rows = [] + step_throughput_rows = [] + runtime_display_rows = [] + + conn = MagicMock() + cursor_cm = MagicMock() + cursor = MagicMock() + conn.cursor.return_value = cursor_cm + cursor_cm.__enter__.return_value = cursor + db_cm = MagicMock() + db_cm.__enter__.return_value = conn + cursor.fetchone.return_value = lease_row + cursor.fetchall.side_effect = [throughput_rows, step_throughput_rows, runtime_display_rows] + + active_job = { + "job_id": 1, + "job_code": "detect-20260421232649-acfa3d", + "runtime_job_code": "sync-overseas-40", + "status": "running", + "items_total": 14961, + "items_pending": 9478, + "items_claimed": 0, + "items_running": 1, + "items_completed": 4996, + "items_blacklisted": 0, + "items_failed": 486, + "progress_percent": 36.64, + "display_items_claimed": 0, + "display_items_running": 0, + "display_active_threads": 0, + "display_max_threads": 0, + "node_stats": [], + "distributed_node_stats": [ + { + "node_code": "mainland-worker-01", + "items_total": 0, + "items_pending": 0, + "items_claimed": 0, + "items_running": 0, + "display_running": 1, + "items_completed": 0, + "items_blacklisted": 0, + "items_failed": 0, + "metrics_source": "agent-heartbeat", + "region": "mainland", + "role": "worker", + "status": "busy", + "current_load": 1, + "active_threads": 1, + "max_threads": 1, + "last_heartbeat_at": datetime.now().isoformat(sep=" ", timespec="seconds"), + }, + { + "node_code": "mainland-controller-01", + "items_total": 1, + "items_pending": 0, + "items_claimed": 0, + "items_running": 1, + "display_running": 0, + "items_completed": 0, + "items_blacklisted": 0, + "items_failed": 0, + "metrics_source": "", + "region": "", + "role": "", + "status": "", + "current_load": 1, + "active_threads": 0, + "max_threads": 0, + "last_heartbeat_at": "", + }, + { + "node_code": "unassigned", + "items_total": 14960, + "items_pending": 14960, + "items_claimed": 0, + "items_running": 0, + "display_running": 0, + "items_completed": 0, + "items_blacklisted": 0, + "items_failed": 0, + "metrics_source": "central_queue", + }, + ], + "step_stats": [], + } + runtime_activity = { + "processed_recent": 52, + "completed_recent": 0, + "failed_recent": 0, + "blacklisted_recent": 0, + "focus_job_code": "sync-overseas-40", + "job_codes": ["sync-overseas-40", "sync-overseas-14"], + "step_code": "detect_register", + "nodes": { + "mainland-controller-01-k": { + "node_code": "mainland-controller-01-k", + "processed_recent": 20, + "completed_recent": 20, + "failed_recent": 0, + "blacklisted_recent": 0, + }, + "mainland-controller-01-a": { + "node_code": "mainland-controller-01-a", + "processed_recent": 16, + "completed_recent": 16, + "failed_recent": 0, + "blacklisted_recent": 0, + }, + "mainland-controller-01-s": { + "node_code": "mainland-controller-01-s", + "processed_recent": 16, + "completed_recent": 16, + "failed_recent": 0, + "blacklisted_recent": 0, + }, + }, + } + runtime_snapshot = { + "_created_at": datetime.now().isoformat(sep=" ", timespec="seconds"), + "job": { + "job_id": 551, + "job_code": "sync-overseas-17149", + "status": "running", + "progress_percent": 41.2, + }, + "queue_health": { + "queue": { + "items_total": 4000, + "pending": 0, + "claimed": 2352, + "running": 51, + "display_claimed": 2352, + "display_running": 51, + "completed": 0, + "blacklisted": 0, + "failed": 0, + }, + "nodes": [ + { + "node_code": "mainland-controller-01-k", + "items_total": 1200, + "items_pending": 0, + "items_claimed": 1179, + "items_running": 20, + "display_running": 20, + "current_load": 20, + "active_threads": 20, + "max_threads": 900, + "status": "busy", + "last_heartbeat_at": datetime.now().isoformat(sep=" ", timespec="seconds"), + }, + { + "node_code": "mainland-controller-01-a", + "items_total": 1400, + "items_pending": 0, + "items_claimed": 1383, + "items_running": 17, + "display_running": 17, + "current_load": 17, + "active_threads": 17, + "max_threads": 900, + "status": "busy", + "last_heartbeat_at": datetime.now().isoformat(sep=" ", timespec="seconds"), + }, + { + "node_code": "mainland-controller-01-s", + "items_total": 1400, + "items_pending": 0, + "items_claimed": 1395, + "items_running": 14, + "display_running": 14, + "current_load": 14, + "active_threads": 14, + "max_threads": 900, + "status": "busy", + "last_heartbeat_at": datetime.now().isoformat(sep=" ", timespec="seconds"), + }, + ], + "steps": [], + }, + } + + with patch("app.services.detect_job_service.settings.node_region", "overseas"): + with patch("app.services.detect_job_service.settings.node_role", "control"): + with patch("app.services.detect_job_service.settings.node_code", "overseas-control-01"): + with patch("app.services.detect_job_service.get_active_detect_job_summary", return_value=active_job): + with patch("app.services.detect_job_service._load_runtime_activity_snapshot", return_value=runtime_activity): + with patch("app.services.detect_job_service._load_latest_runtime_active_job_snapshot", return_value=runtime_snapshot): + with patch("app.services.detect_job_service._load_runtime_display_rows", return_value=runtime_display_rows): + with patch("app.services.detect_job_service.get_db", return_value=db_cm): + health = get_detect_queue_health(window_minutes=15) + + node_codes = [item["node_code"] for item in health["nodes"]] + self.assertEqual("detect-20260421232649-acfa3d", health["job"]["job_code"]) + self.assertEqual(2, health["queue"]["running"]) + self.assertEqual(2, health["queue"]["display_running"]) + self.assertCountEqual(["mainland-worker-01", "mainland-controller-01", "unassigned"], node_codes) + self.assertNotIn("mainland-controller-01-k", node_codes) + + def test_load_runtime_activity_snapshot_prefers_recent_runtime_snapshot_job_code(self) -> None: + event_rows = [ + ( + "mainland-worker-01-a", + "domain_completed", + "", + {"job_code": "sync-overseas-198", "detect_key": "detect_register"}, + datetime(2026, 4, 23, 22, 8, 0), + ), + ( + "mainland-controller-01-a", + "domain_started", + "", + {"job_code": "sync-overseas-599", "detect_key": "detect_wayback"}, + datetime(2026, 4, 23, 22, 12, 0), + ), + ( + "mainland-controller-01-a", + "domain_failed", + "", + {"job_code": "sync-overseas-599", "detect_key": "detect_wayback"}, + datetime(2026, 4, 23, 22, 12, 5), + ), + ] + fake_conn = _SequenceConn(fetchall_results=[event_rows]) + + with patch.object(detect_job_service_module.settings, "node_region", "overseas"): + with patch.object(detect_job_service_module.settings, "node_role", "control"): + with patch("app.services.detect_job_service.get_db", return_value=fake_conn): + with patch( + "app.services.detect_job_service._load_recent_runtime_snapshot_job_codes", + return_value=["sync-overseas-599"], + ): + snapshot = detect_job_service_module._load_runtime_activity_snapshot(15) + + self.assertEqual("sync-overseas-599", snapshot["focus_job_code"]) + self.assertEqual(["sync-overseas-599", "sync-overseas-198"], snapshot["job_codes"]) + self.assertEqual(1, snapshot["processed_recent"]) + self.assertEqual(1, snapshot["failed_recent"]) + self.assertEqual(0, snapshot["completed_recent"]) + self.assertEqual("detect_wayback", snapshot["step_code"]) + self.assertIn("mainland-controller-01-a", snapshot["nodes"]) + self.assertNotIn("mainland-worker-01-a", snapshot["nodes"]) + def test_normalize_detect_step_code_accepts_supported_single_step(self) -> None: self.assertEqual("detect_baidu_site", normalize_detect_step_code("detect_baidu_site")) self.assertEqual("detect_wayback", normalize_detect_step_code("detect_wayback")) diff --git a/domain-api/tests/test_detect_run_service.py b/domain-api/tests/test_detect_run_service.py new file mode 100644 index 0000000..e981fe2 --- /dev/null +++ b/domain-api/tests/test_detect_run_service.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +import unittest +from unittest.mock import patch + +import app.services.detect_run_service as detect_run_service + + +class _TraceLock: + def __init__(self, order: list[str]) -> None: + self.order = order + + def __enter__(self): + self.order.append("enter") + return self + + def __exit__(self, exc_type, exc, tb): + self.order.append("exit") + return False + + +class DetectRunServiceTests(unittest.TestCase): + def test_create_detect_run_snapshot_acquires_lock_before_loading_records(self) -> None: + order: list[str] = [] + + with patch.object(detect_run_service, "_DETECT_RUNS_LOCK", _TraceLock(order)): + with patch.object(detect_run_service, "_load", side_effect=lambda: order.append("load") or []): + with patch.object(detect_run_service, "_capture_worker_logs", return_value=[]): + with patch.object(detect_run_service, "_save", side_effect=lambda records: order.append("save")): + record = detect_run_service.create_detect_run_snapshot( + "start", + {"latest_start_time": "2026-04-23 15:00:00", "running": True}, + {"running": 1}, + {"thread_count": 1000}, + ) + + self.assertEqual("enter", order[0]) + self.assertIn("load", order) + self.assertIn("save", order) + self.assertEqual("exit", order[-1]) + self.assertEqual("starting", record["status"]) + + def test_sync_detect_runs_acquires_lock_before_mutating_records(self) -> None: + order: list[str] = [] + + with patch.object(detect_run_service, "_DETECT_RUNS_LOCK", _TraceLock(order)): + with patch.object(detect_run_service, "_load", side_effect=lambda: order.append("load") or []): + with patch.object(detect_run_service, "_capture_worker_logs", return_value=[]): + with patch.object(detect_run_service, "_save", side_effect=lambda records: order.append("save")): + records = detect_run_service.sync_detect_runs( + {"running": True, "detecting": True, "latest_start_time": "2026-04-23 15:00:00"}, + {"running": 1, "pending": 0}, + {"thread_count": 1000}, + active_job={"status": "running", "items_running": 1}, + ) + + self.assertEqual("enter", order[0]) + self.assertIn("load", order) + self.assertIn("save", order) + self.assertEqual("exit", order[-1]) + self.assertEqual("running", records[0]["status"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/domain-api/tests/test_detect_service_remote_logs.py b/domain-api/tests/test_detect_service_remote_logs.py new file mode 100644 index 0000000..fce338d --- /dev/null +++ b/domain-api/tests/test_detect_service_remote_logs.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +import unittest +from unittest.mock import patch + +from app.services import detect_service + + +class DetectServiceRemoteLogsTests(unittest.TestCase): + @patch("app.services.detect_service.list_debug_events") + def test_build_remote_log_snapshot_from_debug_events_filters_to_active_job_identity(self, mock_list_debug_events) -> None: + mock_list_debug_events.return_value = { + "records": [ + { + "id": 3, + "node_code": "mainland-controller-01-a", + "event_type": "worker_log", + "message": "旧任务日志", + "created_at": "2026-04-23 16:00:00", + "payload": { + "job_id": 10, + "job_code": "sync-overseas-10", + "cycle_token": "cycle-old", + "log_mode": "key", + }, + }, + { + "id": 4, + "node_code": "mainland-controller-01-a", + "event_type": "worker_log", + "message": "当前任务日志", + "created_at": "2026-04-23 16:01:00", + "payload": { + "job_id": 12, + "job_code": "sync-overseas-12", + "cycle_token": "cycle-12", + "log_mode": "key", + }, + }, + ] + } + + snapshot = detect_service._build_remote_log_snapshot_from_debug_events( + { + "job_id": 12, + "job_code": "sync-overseas-12", + "runtime_job_code": "sync-overseas-12", + "current_cycle_token": "cycle-12", + "node_stats": [{"node_code": "mainland-controller-01-a"}], + }, + enabled=True, + mode="key", + limit=20, + ) + + self.assertEqual(1, snapshot["line_count"]) + self.assertIn("当前任务日志", snapshot["lines"][0]) + self.assertNotIn("旧任务日志", "\n".join(snapshot["lines"])) + + +if __name__ == "__main__": + unittest.main() diff --git a/domain-api/tests/test_detect_service_status_fallback.py b/domain-api/tests/test_detect_service_status_fallback.py index cde0391..0ca3116 100644 --- a/domain-api/tests/test_detect_service_status_fallback.py +++ b/domain-api/tests/test_detect_service_status_fallback.py @@ -1,10 +1,47 @@ import unittest +from contextlib import ExitStack +from datetime import datetime, timedelta from unittest.mock import patch from app.services import detect_service class DetectServiceStatusFallbackTests(unittest.TestCase): + def test_filter_live_aggregate_runtime_nodes_excludes_disabled_managed_nodes(self) -> None: + now_text = datetime.now().isoformat(sep=" ", timespec="seconds") + rows = [ + { + "node_code": "mainland-controller-01", + "status": "busy", + "last_heartbeat_at": now_text, + "display_running": 1, + }, + { + "node_code": "mainland-worker-01", + "status": "busy", + "last_heartbeat_at": now_text, + "display_running": 1, + }, + ] + + with patch("app.services.detect_service._load_disabled_managed_node_codes", return_value={"mainland-worker-01"}): + filtered = detect_service._filter_live_aggregate_runtime_nodes(rows) + + self.assertEqual(["mainland-controller-01"], [item["node_code"] for item in filtered]) + + def setUp(self) -> None: + detect_service._DETECT_STATUS_CACHE_VALUE = None + detect_service._DETECT_STATUS_CACHE_EXPIRES_AT = 0.0 + + def test_resolve_capacity_node_code_treats_child_instance_suffix_as_single_process(self) -> None: + capacity_node_code, is_child_instance = detect_service._resolve_capacity_node_code( + "mainland-controller-01-ae", + {"node_thread_counts": {"overseas-control-01": 1}, "node_process_counts": {}}, + ) + + self.assertEqual("mainland-controller-01", capacity_node_code) + self.assertTrue(is_child_instance) + def test_get_detect_status_keeps_runtime_snapshot_when_db_is_unreachable(self) -> None: runtime_state = { "service_running": True, @@ -17,24 +54,39 @@ class DetectServiceStatusFallbackTests(unittest.TestCase): "available_proxy_count": 18, } - with patch("app.services.detect_service.ensure_runtime_schema"), \ - patch("app.services.detect_service.get_db", side_effect=RuntimeError("db down")), \ - patch("app.services.detect_service.get_settings_payload", return_value={"proxy_config": {"proxy_enable": True, "allow_direct": False, "proxy_urls": ["a"]}}), \ - patch("app.services.detect_service.get_runtime_settings", return_value={"worker_log_sync_enabled": False, "worker_log_sync_mode": "full"}), \ - patch("app.services.detect_service._load_recent_worker_lines", return_value=(True, "", [])), \ - patch("app.services.detect_service.detect_worker_runtime", return_value={"mode": "linux-systemd", "running": True, "process_count": 1, "latest_start_time": "2026-04-20 23:58:00", "message": "active/running"}), \ - patch("app.services.detect_service._load_runtime_state", return_value=runtime_state), \ - patch("app.services.detect_service._load_runtime_state_from_cluster_node", return_value={}), \ - patch("app.services.detect_service._extract_available_proxy_count", return_value=0), \ - patch("app.services.detect_service._extract_active_thread_snapshot", return_value={"active": 0, "max": 0}), \ - patch("app.services.detect_service._normalize_recent_warning", return_value=""), \ - patch("app.services.detect_service._build_proxy_runtime_snapshot", return_value={"state": "healthy", "label": "代理正常", "detail": "healthy", "direct_fallback_active": False, "reason": "healthy", "last_refresh_status": "ok", "last_refresh_time": "", "source_count": 2, "raw_items": 18, "validated_count": 18, "available_count": 18, "source_stats": [], "supplier_empty": False}), \ - patch("app.services.detect_service.resolve_thread_count", return_value={"effective_thread_count": 120, "default_thread_count": 5, "source": "node_override", "override_thread_count": 120, "node_code": "mainland-worker-01"}), \ - patch("app.services.detect_service.get_active_detect_job_summary", side_effect=RuntimeError("db down")), \ - patch("app.services.detect_service.sync_detect_runs", return_value=[]), \ - patch("app.services.detect_service._resolve_remote_log_snapshot", return_value={}), \ - patch("app.services.detect_service._extract_dependency_alerts", return_value=[]), \ - patch("app.services.detect_service.append_detect_result_projection_if_changed"): + with ExitStack() as stack: + stack.enter_context(patch.object(detect_service.settings, "node_region", "mainland")) + stack.enter_context(patch.object(detect_service.settings, "node_role", "worker")) + stack.enter_context(patch.object(detect_service.settings, "node_code", "mainland-worker-01")) + stack.enter_context(patch("app.services.detect_service.ensure_runtime_schema")) + stack.enter_context(patch("app.services.detect_service.get_db", side_effect=RuntimeError("db down"))) + stack.enter_context( + patch("app.services.detect_service.get_settings_payload", return_value={"proxy_config": {"proxy_enable": True, "allow_direct": False, "proxy_urls": ["a"]}}) + ) + stack.enter_context(patch("app.services.detect_service.get_runtime_settings", return_value={"worker_log_sync_enabled": False, "worker_log_sync_mode": "full"})) + stack.enter_context(patch("app.services.detect_service._load_recent_worker_lines", return_value=(True, "", []))) + stack.enter_context( + patch("app.services.detect_service.detect_worker_runtime", return_value={"mode": "linux-systemd", "running": True, "process_count": 1, "latest_start_time": "2026-04-20 23:58:00", "message": "active/running"}) + ) + stack.enter_context(patch("app.services.detect_service._load_runtime_state", return_value=runtime_state)) + stack.enter_context(patch("app.services.detect_service._load_runtime_state_from_cluster_node", return_value={})) + stack.enter_context(patch("app.services.detect_service._extract_available_proxy_count", return_value=0)) + stack.enter_context(patch("app.services.detect_service._extract_active_thread_snapshot", return_value={"active": 0, "max": 0})) + stack.enter_context(patch("app.services.detect_service._normalize_recent_warning", return_value="")) + stack.enter_context( + patch( + "app.services.detect_service._build_proxy_runtime_snapshot", + return_value={"state": "healthy", "label": "代理正常", "detail": "healthy", "direct_fallback_active": False, "reason": "healthy", "last_refresh_status": "ok", "last_refresh_time": "", "source_count": 2, "raw_items": 18, "validated_count": 18, "available_count": 18, "source_stats": [], "supplier_empty": False}, + ) + ) + stack.enter_context( + patch("app.services.detect_service.resolve_thread_count", return_value={"effective_thread_count": 120, "default_thread_count": 5, "source": "node_override", "override_thread_count": 120, "node_code": "mainland-worker-01"}) + ) + stack.enter_context(patch("app.services.detect_service.get_active_detect_job_summary", side_effect=RuntimeError("db down"))) + stack.enter_context(patch("app.services.detect_service.sync_detect_runs", return_value=[])) + stack.enter_context(patch("app.services.detect_service._resolve_remote_log_snapshot", return_value={})) + stack.enter_context(patch("app.services.detect_service._extract_dependency_alerts", return_value=[])) + stack.enter_context(patch("app.services.detect_service.append_detect_result_projection_if_changed")) payload = detect_service.get_detect_status() self.assertTrue(payload["worker_online"]) @@ -44,6 +96,56 @@ class DetectServiceStatusFallbackTests(unittest.TestCase): self.assertEqual(0, payload["progress"]["pending"]) self.assertEqual(0, payload["progress"]["completed"]) + def test_get_detect_status_skips_local_result_projection_on_overseas_control(self) -> None: + runtime_state = { + "service_running": False, + "detecting": False, + "active_threads": 0, + "max_threads": 1, + "phase": "idle", + "detail": "当前节点不承载本地检测执行", + "updated_at": "2026-04-24 02:10:00", + "available_proxy_count": 0, + } + + with ExitStack() as stack: + stack.enter_context(patch.object(detect_service.settings, "node_region", "overseas")) + stack.enter_context(patch.object(detect_service.settings, "node_role", "control")) + stack.enter_context(patch.object(detect_service.settings, "node_code", "overseas-control-01")) + stack.enter_context(patch("app.services.detect_service.ensure_runtime_schema")) + stack.enter_context(patch("app.services.detect_service.get_db", side_effect=RuntimeError("db down"))) + stack.enter_context( + patch("app.services.detect_service.get_settings_payload", return_value={"proxy_config": {"proxy_enable": False, "allow_direct": True, "proxy_urls": []}}) + ) + stack.enter_context(patch("app.services.detect_service.get_runtime_settings", return_value={"worker_log_sync_enabled": False, "worker_log_sync_mode": "key"})) + stack.enter_context(patch("app.services.detect_service._load_recent_worker_lines", return_value=(False, "", []))) + stack.enter_context( + patch("app.services.detect_service.detect_worker_runtime", return_value={"mode": "linux-systemd", "running": False, "process_count": 0, "latest_start_time": "", "message": "inactive"}) + ) + stack.enter_context(patch("app.services.detect_service._load_runtime_state", return_value=runtime_state)) + stack.enter_context(patch("app.services.detect_service._load_runtime_state_from_cluster_node", return_value={})) + stack.enter_context(patch("app.services.detect_service._extract_available_proxy_count", return_value=0)) + stack.enter_context(patch("app.services.detect_service._extract_active_thread_snapshot", return_value={"active": 0, "max": 0})) + stack.enter_context(patch("app.services.detect_service._normalize_recent_warning", return_value="")) + stack.enter_context( + patch( + "app.services.detect_service._build_proxy_runtime_snapshot", + return_value={"state": "disabled", "label": "不适用", "detail": "-", "direct_fallback_active": False, "reason": "not_applicable", "last_refresh_status": "", "last_refresh_time": "", "source_count": 0, "raw_items": 0, "validated_count": 0, "available_count": 0, "source_stats": [], "supplier_empty": False}, + ) + ) + stack.enter_context( + patch("app.services.detect_service.resolve_thread_count", return_value={"effective_thread_count": 1, "default_thread_count": 1, "source": "default", "override_thread_count": None, "node_code": "overseas-control-01"}) + ) + stack.enter_context(patch("app.services.detect_service.get_active_detect_job_summary", return_value=None)) + stack.enter_context(patch("app.services.detect_service.sync_detect_runs", return_value=[])) + stack.enter_context(patch("app.services.detect_service._resolve_remote_log_snapshot", return_value={})) + stack.enter_context(patch("app.services.detect_service._extract_dependency_alerts", return_value=[])) + mock_append = stack.enter_context(patch("app.services.detect_service.append_detect_result_projection_if_changed")) + + detect_service.get_detect_status() + + mock_append.assert_not_called() + def test_filter_lines_since_supports_journalctl_syslog_timestamps(self) -> None: lines = [ "Apr 21 20:12:42 mainland-controller python[1]: 当前实际线程数量: 323/4", @@ -64,6 +166,784 @@ class DetectServiceStatusFallbackTests(unittest.TestCase): self.assertEqual(lines, filtered) + def test_get_detect_status_uses_aggregate_capacity_for_overseas_control(self) -> None: + active_job = { + "job_id": 11, + "job_code": "sync-overseas-51", + "status": "running", + "items_pending": 5001, + "items_completed": 1788, + "items_failed": 0, + "items_blacklisted": 0, + "progress_percent": 40.49, + "display_items_running": 1432, + "display_active_threads": 138, + "distributed_node_stats": [ + { + "node_code": "mainland-controller-01", + "items_claimed": 73, + "items_running": 1432, + "display_running": 1432, + "active_threads": 138, + "max_threads": 2000, + }, + { + "node_code": "mainland-worker-01", + "items_completed": 1614, + "items_running": 0, + "active_threads": 0, + "max_threads": 0, + }, + { + "node_code": "unassigned", + "items_pending": 3495, + }, + ], + } + aggregate_queue_health = { + "has_active_job": True, + "job": { + "job_id": 11, + "job_code": "sync-overseas-51", + "status": "running", + "progress_percent": 40.49, + }, + "queue": { + "items_total": 8402, + "pending": 5001, + "claimed": 0, + "running": 5909, + "display_running": 5909, + "completed": 0, + "blacklisted": 0, + "failed": 0, + }, + "nodes": [ + { + "node_code": "mainland-controller-01-a", + "items_running": 1000, + "display_running": 1000, + "active_threads": 1000, + "max_threads": 1000, + }, + { + "node_code": "mainland-controller-01-b", + "items_running": 972, + "display_running": 972, + "active_threads": 972, + "max_threads": 1000, + }, + { + "node_code": "mainland-controller-01-c", + "items_running": 3937, + "display_running": 3937, + "active_threads": 3937, + "max_threads": 36000, + }, + ], + } + + def _resolve_thread_count(*, node_code=None, settings_payload=None): + if node_code == "mainland-controller-01" or str(node_code or "").startswith("mainland-controller-01-"): + return { + "effective_thread_count": 1000, + "default_thread_count": 1000, + "source": "default", + "override_thread_count": None, + "node_code": str(node_code or "mainland-controller-01"), + } + return { + "effective_thread_count": 1, + "default_thread_count": 1000, + "source": "node_override", + "override_thread_count": 1, + "node_code": "overseas-control-01", + } + + def _resolve_process_count(*, node_code=None, settings_payload=None): + if node_code == "mainland-controller-01": + return { + "effective_process_count": 80, + "default_process_count": 80, + "source": "default", + "override_process_count": None, + "node_code": "mainland-controller-01", + } + if str(node_code or "").startswith("mainland-controller-01-"): + return { + "effective_process_count": 1, + "default_process_count": 80, + "source": "child_instance", + "override_process_count": None, + "node_code": str(node_code or ""), + } + return { + "effective_process_count": 1, + "default_process_count": 80, + "source": "node_override", + "override_process_count": 1, + "node_code": "overseas-control-01", + } + + with ExitStack() as stack: + stack.enter_context(patch.object(detect_service.settings, "node_region", "overseas")) + stack.enter_context(patch.object(detect_service.settings, "node_role", "control")) + stack.enter_context(patch.object(detect_service.settings, "node_code", "overseas-control-01")) + stack.enter_context(patch("app.services.detect_service.ensure_runtime_schema")) + stack.enter_context(patch("app.services.detect_service.get_db", side_effect=RuntimeError("db down"))) + stack.enter_context( + patch( + "app.services.detect_service.get_settings_payload", + return_value={ + "proxy_config": {"proxy_enable": False, "allow_direct": True, "proxy_urls": []}, + "process_count": 80, + "node_process_counts": {}, + "thread_count": 1000, + "node_thread_counts": {"overseas-control-01": 1}, + }, + ) + ) + stack.enter_context( + patch("app.services.detect_service.get_runtime_settings", return_value={"worker_log_sync_enabled": False, "worker_log_sync_mode": "key"}) + ) + stack.enter_context( + patch("app.services.detect_service._load_recent_worker_lines", return_value=(False, "2026-04-23 00:00:00", ["stale line"])) + ) + stack.enter_context( + patch("app.services.detect_service.detect_worker_runtime", return_value={"mode": "linux-systemd", "running": False, "process_count": 0, "latest_start_time": "", "message": "inactive"}) + ) + stack.enter_context( + patch("app.services.detect_service._load_runtime_state", return_value={"service_running": True, "detecting": True, "active_threads": 2, "max_threads": 1, "detail": "当前实际线程数量: 2/1"}) + ) + stack.enter_context(patch("app.services.detect_service._load_runtime_state_from_cluster_node", return_value={})) + stack.enter_context(patch("app.services.detect_service._extract_available_proxy_count", return_value=0)) + stack.enter_context(patch("app.services.detect_service._extract_active_thread_snapshot", return_value={"active": 0, "max": 0})) + stack.enter_context(patch("app.services.detect_service._normalize_recent_warning", return_value="")) + stack.enter_context( + patch( + "app.services.detect_service._build_proxy_runtime_snapshot", + return_value={ + "state": "disabled", + "label": "未启用代理", + "detail": "-", + "direct_fallback_active": True, + "reason": "proxy_disabled", + "last_refresh_status": "", + "last_refresh_time": "", + "source_count": 0, + "raw_items": 0, + "validated_count": 0, + "available_count": 0, + "source_stats": [], + "supplier_empty": False, + }, + ) + ) + stack.enter_context(patch("app.services.detect_service.resolve_thread_count", side_effect=_resolve_thread_count)) + stack.enter_context(patch("app.services.detect_service.resolve_process_count", side_effect=_resolve_process_count)) + stack.enter_context(patch("app.services.detect_service.get_active_detect_job_summary", return_value=active_job)) + stack.enter_context(patch("app.services.detect_service.get_detect_queue_health", return_value=aggregate_queue_health)) + stack.enter_context( + patch( + "app.services.detect_service._load_runtime_states_from_cluster_nodes", + return_value={ + "mainland-controller-01": { + "node_code": "mainland-controller-01", + "available_proxy_count": 486, + "proxy_runtime_label": "代理正常", + "proxy_runtime_reason": "healthy", + "proxy_last_refresh_status": "复用共享代理快照 486 个", + "proxy_last_refresh_time": "2026-04-23 19:07:12", + "proxy_last_refresh_source_count": 6, + "proxy_last_refresh_total_items": 120, + "proxy_last_validated_count": 0, + } + }, + ) + ) + stack.enter_context(patch("app.services.detect_service.sync_detect_runs", return_value=[])) + stack.enter_context(patch("app.services.detect_service._resolve_remote_log_snapshot", return_value={})) + stack.enter_context(patch("app.services.detect_service._extract_dependency_alerts", return_value=[])) + stack.enter_context(patch("app.services.detect_service.append_detect_result_projection_if_changed")) + payload = detect_service.get_detect_status() + + self.assertFalse(payload["worker_online"]) + self.assertEqual({}, payload["runtime_state"]) + self.assertEqual(3, payload["worker_process_count"]) + self.assertEqual(3, payload["aggregate_process_count"]) + self.assertEqual(3, payload["aggregate_participating_node_count"]) + self.assertEqual( + ["mainland-controller-01-a", "mainland-controller-01-b", "mainland-controller-01-c"], + payload["aggregate_participating_node_codes"], + ) + self.assertEqual(3000, payload["aggregate_max_thread_count"]) + self.assertEqual(38000, payload["max_thread_count"]) + self.assertEqual(5909, payload["active_thread_count"]) + self.assertEqual(486, payload["available_proxy_count"]) + self.assertEqual("集群代理正常", payload["proxy_runtime_label"]) + self.assertIn("486", payload["proxy_last_refresh_status"]) + + def test_get_detect_status_uses_queue_running_when_aggregate_display_fields_are_zero(self) -> None: + active_job = { + "job_id": 60, + "job_code": "sync-overseas-255", + "status": "running", + "items_total": 8402, + "items_pending": 7865, + "items_running": 537, + "items_completed": 0, + "items_failed": 0, + "items_blacklisted": 0, + "progress_percent": 6.39, + "display_items_running": 0, + "display_active_threads": 0, + "display_max_threads": 0, + "distributed_node_stats": [], + } + aggregate_queue_health = { + "has_active_job": True, + "job": { + "job_id": 60, + "job_code": "sync-overseas-255", + "status": "running", + "progress_percent": 6.39, + }, + "queue": { + "items_total": 8402, + "pending": 7865, + "claimed": 0, + "running": 537, + "display_running": 0, + "completed": 0, + "blacklisted": 0, + "failed": 0, + }, + "nodes": [ + { + "node_code": "mainland-controller-01-a", + "items_running": 537, + "display_running": 537, + "active_threads": 537, + "max_threads": 1000, + } + ], + } + + with ExitStack() as stack: + stack.enter_context(patch.object(detect_service.settings, "node_region", "overseas")) + stack.enter_context(patch.object(detect_service.settings, "node_role", "control")) + stack.enter_context(patch.object(detect_service.settings, "node_code", "overseas-control-01")) + stack.enter_context(patch("app.services.detect_service.ensure_runtime_schema")) + stack.enter_context(patch("app.services.detect_service.get_db", side_effect=RuntimeError("db down"))) + stack.enter_context( + patch( + "app.services.detect_service.get_settings_payload", + return_value={ + "proxy_config": {"proxy_enable": False, "allow_direct": True, "proxy_urls": []}, + "process_count": 80, + "node_process_counts": {"mainland-controller-01": 24}, + "thread_count": 1000, + "node_thread_counts": {"overseas-control-01": 1, "mainland-controller-01-a": 1000}, + }, + ) + ) + stack.enter_context( + patch("app.services.detect_service.get_runtime_settings", return_value={"worker_log_sync_enabled": False, "worker_log_sync_mode": "key"}) + ) + stack.enter_context( + patch("app.services.detect_service._load_recent_worker_lines", return_value=(False, "2026-04-23 00:00:00", ["stale line"])) + ) + stack.enter_context( + patch("app.services.detect_service.detect_worker_runtime", return_value={"mode": "linux-systemd", "running": False, "process_count": 0, "latest_start_time": "", "message": "inactive"}) + ) + stack.enter_context( + patch("app.services.detect_service._load_runtime_state", return_value={"service_running": False, "detecting": False, "active_threads": 0, "max_threads": 1, "detail": ""}) + ) + stack.enter_context(patch("app.services.detect_service._load_runtime_state_from_cluster_node", return_value={})) + stack.enter_context(patch("app.services.detect_service._extract_available_proxy_count", return_value=0)) + stack.enter_context(patch("app.services.detect_service._extract_active_thread_snapshot", return_value={"active": 0, "max": 0})) + stack.enter_context(patch("app.services.detect_service._normalize_recent_warning", return_value="")) + stack.enter_context( + patch( + "app.services.detect_service._build_proxy_runtime_snapshot", + return_value={ + "state": "disabled", + "label": "未启用代理", + "detail": "-", + "direct_fallback_active": True, + "reason": "proxy_disabled", + "last_refresh_status": "", + "last_refresh_time": "", + "source_count": 0, + "raw_items": 0, + "validated_count": 0, + "available_count": 0, + "source_stats": [], + "supplier_empty": False, + }, + ) + ) + stack.enter_context( + patch( + "app.services.detect_service.resolve_thread_count", + return_value={ + "effective_thread_count": 1, + "default_thread_count": 1000, + "source": "node_override", + "override_thread_count": 1, + "node_code": "overseas-control-01", + }, + ) + ) + stack.enter_context( + patch( + "app.services.detect_service.resolve_process_count", + return_value={ + "effective_process_count": 1, + "default_process_count": 80, + "source": "node_override", + "override_process_count": 1, + "node_code": "overseas-control-01", + }, + ) + ) + stack.enter_context(patch("app.services.detect_service.get_active_detect_job_summary", return_value=active_job)) + stack.enter_context(patch("app.services.detect_service.get_detect_queue_health", return_value=aggregate_queue_health)) + stack.enter_context(patch("app.services.detect_service.sync_detect_runs", return_value=[])) + stack.enter_context(patch("app.services.detect_service._resolve_remote_log_snapshot", return_value={})) + stack.enter_context(patch("app.services.detect_service._extract_dependency_alerts", return_value=[])) + stack.enter_context(patch("app.services.detect_service.append_detect_result_projection_if_changed")) + payload = detect_service.get_detect_status() + + self.assertEqual(537, payload["progress"]["running"]) + self.assertEqual(537, payload["active_thread_count"]) + self.assertEqual(1000, payload["max_thread_count"]) + self.assertEqual(1, payload["aggregate_process_count"]) + + def test_get_detect_status_drops_stale_aggregate_nodes_from_running_and_process_counts(self) -> None: + live_heartbeat = datetime.now().isoformat(sep=" ", timespec="seconds") + stale_heartbeat = (datetime.now() - timedelta(minutes=8)).isoformat(sep=" ", timespec="seconds") + active_job = { + "job_id": 88, + "job_code": "sync-overseas-688", + "status": "running", + "items_total": 2600, + "items_pending": 500, + "items_running": 2100, + "items_completed": 0, + "items_failed": 0, + "items_blacklisted": 0, + "progress_percent": 80.77, + "display_items_running": 2100, + "display_active_threads": 2100, + "display_max_threads": 2000, + "distributed_node_stats": [ + { + "node_code": "mainland-controller-01-a", + "items_running": 300, + "display_running": 300, + "active_threads": 300, + "max_threads": 1000, + "status": "busy", + "last_heartbeat_at": live_heartbeat, + }, + { + "node_code": "mainland-controller-01-b", + "items_running": 1800, + "display_running": 1800, + "active_threads": 1800, + "max_threads": 1000, + "status": "stale", + "last_heartbeat_at": stale_heartbeat, + }, + ], + } + aggregate_queue_health = { + "has_active_job": True, + "job": { + "job_id": 88, + "job_code": "sync-overseas-688", + "status": "running", + "progress_percent": 80.77, + }, + "queue": { + "items_total": 2600, + "pending": 500, + "claimed": 0, + "running": 2100, + "display_running": 2100, + "completed": 0, + "blacklisted": 0, + "failed": 0, + }, + "nodes": [ + { + "node_code": "mainland-controller-01-a", + "items_running": 300, + "display_running": 300, + "active_threads": 300, + "max_threads": 1000, + "status": "busy", + "last_heartbeat_at": live_heartbeat, + }, + { + "node_code": "mainland-controller-01-b", + "items_running": 1800, + "display_running": 1800, + "active_threads": 1800, + "max_threads": 1000, + "status": "stale", + "last_heartbeat_at": stale_heartbeat, + }, + ], + } + + with ExitStack() as stack: + stack.enter_context(patch.object(detect_service.settings, "node_region", "overseas")) + stack.enter_context(patch.object(detect_service.settings, "node_role", "control")) + stack.enter_context(patch.object(detect_service.settings, "node_code", "overseas-control-01")) + stack.enter_context(patch("app.services.detect_service.ensure_runtime_schema")) + stack.enter_context(patch("app.services.detect_service.get_db", side_effect=RuntimeError("db down"))) + stack.enter_context( + patch( + "app.services.detect_service.get_settings_payload", + return_value={ + "proxy_config": {"proxy_enable": False, "allow_direct": True, "proxy_urls": []}, + "process_count": 80, + "node_process_counts": {"mainland-controller-01": 70}, + "thread_count": 1000, + "node_thread_counts": {"overseas-control-01": 1}, + }, + ) + ) + stack.enter_context( + patch("app.services.detect_service.get_runtime_settings", return_value={"worker_log_sync_enabled": False, "worker_log_sync_mode": "key"}) + ) + stack.enter_context( + patch("app.services.detect_service._load_recent_worker_lines", return_value=(False, "2026-04-23 00:00:00", ["stale line"])) + ) + stack.enter_context( + patch("app.services.detect_service.detect_worker_runtime", return_value={"mode": "linux-systemd", "running": False, "process_count": 0, "latest_start_time": "", "message": "inactive"}) + ) + stack.enter_context( + patch("app.services.detect_service._load_runtime_state", return_value={"service_running": False, "detecting": False, "active_threads": 0, "max_threads": 1, "detail": ""}) + ) + stack.enter_context(patch("app.services.detect_service._load_runtime_state_from_cluster_node", return_value={})) + stack.enter_context(patch("app.services.detect_service._extract_available_proxy_count", return_value=0)) + stack.enter_context(patch("app.services.detect_service._extract_active_thread_snapshot", return_value={"active": 0, "max": 0})) + stack.enter_context(patch("app.services.detect_service._normalize_recent_warning", return_value="")) + stack.enter_context( + patch( + "app.services.detect_service._build_proxy_runtime_snapshot", + return_value={ + "state": "disabled", + "label": "未启用代理", + "detail": "-", + "direct_fallback_active": True, + "reason": "proxy_disabled", + "last_refresh_status": "", + "last_refresh_time": "", + "source_count": 0, + "raw_items": 0, + "validated_count": 0, + "available_count": 0, + "source_stats": [], + "supplier_empty": False, + }, + ) + ) + stack.enter_context( + patch( + "app.services.detect_service.resolve_thread_count", + return_value={ + "effective_thread_count": 1, + "default_thread_count": 1000, + "source": "node_override", + "override_thread_count": 1, + "node_code": "overseas-control-01", + }, + ) + ) + stack.enter_context( + patch( + "app.services.detect_service.resolve_process_count", + side_effect=[ + { + "effective_process_count": 1, + "default_process_count": 80, + "source": "child_instance", + "override_process_count": None, + "node_code": "mainland-controller-01-a", + } + ], + ) + ) + stack.enter_context(patch("app.services.detect_service.get_active_detect_job_summary", return_value=active_job)) + stack.enter_context(patch("app.services.detect_service.get_detect_queue_health", return_value=aggregate_queue_health)) + stack.enter_context(patch("app.services.detect_service.sync_detect_runs", return_value=[])) + stack.enter_context(patch("app.services.detect_service._resolve_remote_log_snapshot", return_value={})) + stack.enter_context(patch("app.services.detect_service._extract_dependency_alerts", return_value=[])) + stack.enter_context(patch("app.services.detect_service.append_detect_result_projection_if_changed")) + payload = detect_service.get_detect_status() + + self.assertEqual(300, payload["progress"]["running"]) + self.assertEqual(300, payload["active_thread_count"]) + self.assertEqual(1000, payload["max_thread_count"]) + self.assertEqual(1, payload["aggregate_process_count"]) + self.assertEqual(1, payload["aggregate_participating_node_count"]) + self.assertEqual(["mainland-controller-01-a"], payload["aggregate_participating_node_codes"]) + + def test_get_detect_status_uses_event_proxy_counts_when_cluster_runtime_proxy_counts_are_missing(self) -> None: + active_job = { + "job_id": 77, + "job_code": "sync-overseas-482", + "status": "running", + "items_pending": 410, + "items_completed": 100, + "items_failed": 0, + "items_blacklisted": 0, + "progress_percent": 55.0, + "display_items_running": 807, + "display_active_threads": 807, + "current_cycle_token": "cycle-1", + "distributed_node_stats": [ + { + "node_code": "mainland-controller-01-u", + "items_running": 300, + "display_running": 300, + "active_threads": 300, + "max_threads": 1000, + }, + { + "node_code": "mainland-controller-01-v", + "items_running": 301, + "display_running": 301, + "active_threads": 301, + "max_threads": 1000, + }, + { + "node_code": "mainland-worker-01-a", + "items_running": 206, + "display_running": 206, + "active_threads": 206, + "max_threads": 1000, + }, + ], + "current_cycle_events": [ + { + "event_type": "worker_log", + "created_at": "2026-04-23 19:40:00", + "node_code": "mainland-controller-01-u", + "message": "当前可用代理数: 486", + "payload": {"cycle_token": "cycle-1", "log_mode": "key"}, + }, + { + "event_type": "worker_log", + "created_at": "2026-04-23 19:40:10", + "node_code": "mainland-controller-01-v", + "message": "代理池刷新完成，共 486 个可用代理，来源链接 6 个，原始 520 个", + "payload": {"cycle_token": "cycle-1", "log_mode": "key"}, + }, + { + "event_type": "worker_log", + "created_at": "2026-04-23 19:40:20", + "node_code": "mainland-worker-01-a", + "message": "共享刷新进行中，继续沿用缓存 321 个", + "payload": {"cycle_token": "cycle-1", "log_mode": "key"}, + }, + ], + } + aggregate_queue_health = { + "has_active_job": True, + "job": { + "job_id": 77, + "job_code": "sync-overseas-482", + "status": "running", + "progress_percent": 55.0, + }, + "queue": { + "items_total": 1317, + "pending": 410, + "claimed": 0, + "running": 807, + "display_running": 807, + "completed": 100, + "blacklisted": 0, + "failed": 0, + }, + "nodes": [ + { + "node_code": "mainland-controller-01-u", + "items_running": 300, + "display_running": 300, + "active_threads": 300, + "max_threads": 1000, + }, + { + "node_code": "mainland-controller-01-v", + "items_running": 301, + "display_running": 301, + "active_threads": 301, + "max_threads": 1000, + }, + { + "node_code": "mainland-worker-01-a", + "items_running": 206, + "display_running": 206, + "active_threads": 206, + "max_threads": 1000, + }, + ], + } + + def _resolve_thread_count(*, node_code=None, settings_payload=None): + if str(node_code or "").startswith("mainland-"): + return { + "effective_thread_count": 1000, + "default_thread_count": 1000, + "source": "default", + "override_thread_count": None, + "node_code": str(node_code or ""), + } + return { + "effective_thread_count": 1, + "default_thread_count": 1000, + "source": "node_override", + "override_thread_count": 1, + "node_code": "overseas-control-01", + } + + def _resolve_process_count(*, node_code=None, settings_payload=None): + normalized_node_code = str(node_code or "") + if normalized_node_code == "mainland-controller-01": + return { + "effective_process_count": 80, + "default_process_count": 80, + "source": "default", + "override_process_count": None, + "node_code": normalized_node_code, + } + if normalized_node_code == "mainland-worker-01": + return { + "effective_process_count": 60, + "default_process_count": 60, + "source": "default", + "override_process_count": None, + "node_code": normalized_node_code, + } + if normalized_node_code.startswith("mainland-"): + return { + "effective_process_count": 1, + "default_process_count": 80, + "source": "child_instance", + "override_process_count": None, + "node_code": normalized_node_code, + } + return { + "effective_process_count": 1, + "default_process_count": 80, + "source": "node_override", + "override_process_count": 1, + "node_code": "overseas-control-01", + } + + with ExitStack() as stack: + stack.enter_context(patch.object(detect_service.settings, "node_region", "overseas")) + stack.enter_context(patch.object(detect_service.settings, "node_role", "control")) + stack.enter_context(patch.object(detect_service.settings, "node_code", "overseas-control-01")) + stack.enter_context(patch("app.services.detect_service.ensure_runtime_schema")) + stack.enter_context(patch("app.services.detect_service.get_db", side_effect=RuntimeError("db down"))) + stack.enter_context( + patch( + "app.services.detect_service.get_settings_payload", + return_value={ + "proxy_config": {"proxy_enable": False, "allow_direct": True, "proxy_urls": []}, + "process_count": 80, + "node_process_counts": {"mainland-worker-01": 60}, + "thread_count": 1000, + "node_thread_counts": {"overseas-control-01": 1}, + }, + ) + ) + stack.enter_context( + patch("app.services.detect_service.get_runtime_settings", return_value={"worker_log_sync_enabled": False, "worker_log_sync_mode": "key"}) + ) + stack.enter_context( + patch("app.services.detect_service._load_recent_worker_lines", return_value=(False, "2026-04-23 00:00:00", ["stale line"])) + ) + stack.enter_context( + patch("app.services.detect_service.detect_worker_runtime", return_value={"mode": "linux-systemd", "running": False, "process_count": 0, "latest_start_time": "", "message": "inactive"}) + ) + stack.enter_context( + patch("app.services.detect_service._load_runtime_state", return_value={"service_running": False, "detecting": False, "active_threads": 0, "max_threads": 1, "detail": ""}) + ) + stack.enter_context(patch("app.services.detect_service._load_runtime_state_from_cluster_node", return_value={})) + stack.enter_context(patch("app.services.detect_service._extract_available_proxy_count", return_value=0)) + stack.enter_context(patch("app.services.detect_service._extract_active_thread_snapshot", return_value={"active": 0, "max": 0})) + stack.enter_context(patch("app.services.detect_service._normalize_recent_warning", return_value="")) + stack.enter_context( + patch( + "app.services.detect_service._build_proxy_runtime_snapshot", + return_value={ + "state": "warming_up", + "label": "等待首刷", + "detail": "-", + "direct_fallback_active": False, + "reason": "waiting_for_first_refresh", + "last_refresh_status": "未刷新", + "last_refresh_time": "", + "source_count": 0, + "raw_items": 0, + "validated_count": 0, + "available_count": 0, + "source_stats": [], + "supplier_empty": False, + }, + ) + ) + stack.enter_context(patch("app.services.detect_service.resolve_thread_count", side_effect=_resolve_thread_count)) + stack.enter_context(patch("app.services.detect_service.resolve_process_count", side_effect=_resolve_process_count)) + stack.enter_context(patch("app.services.detect_service.get_active_detect_job_summary", return_value=active_job)) + stack.enter_context(patch("app.services.detect_service.get_detect_queue_health", return_value=aggregate_queue_health)) + stack.enter_context( + patch( + "app.services.detect_service._load_runtime_states_from_cluster_nodes", + return_value={ + "mainland-controller-01": { + "node_code": "mainland-controller-01", + "available_proxy_count": 0, + "proxy_runtime_label": "", + "proxy_runtime_reason": "", + "proxy_last_refresh_status": "", + "proxy_last_refresh_time": "", + "proxy_last_refresh_source_count": 0, + "proxy_last_refresh_total_items": 0, + "proxy_last_validated_count": 0, + }, + "mainland-worker-01": { + "node_code": "mainland-worker-01", + "available_proxy_count": 0, + "proxy_runtime_label": "", + "proxy_runtime_reason": "", + "proxy_last_refresh_status": "", + "proxy_last_refresh_time": "", + "proxy_last_refresh_source_count": 0, + "proxy_last_refresh_total_items": 0, + "proxy_last_validated_count": 0, + }, + }, + ) + ) + stack.enter_context(patch("app.services.detect_service.sync_detect_runs", return_value=[])) + stack.enter_context(patch("app.services.detect_service._resolve_remote_log_snapshot", return_value={})) + stack.enter_context(patch("app.services.detect_service._extract_dependency_alerts", return_value=[])) + stack.enter_context(patch("app.services.detect_service.append_detect_result_projection_if_changed")) + payload = detect_service.get_detect_status() + + self.assertEqual(807, payload["available_proxy_count"]) + self.assertEqual("集群代理正常", payload["proxy_runtime_label"]) + self.assertIn("参与服务器 2 台", payload["proxy_runtime_detail"]) + if __name__ == "__main__": unittest.main() diff --git a/domain-api/tests/test_domains_service.py b/domain-api/tests/test_domains_service.py new file mode 100644 index 0000000..0fffb69 --- /dev/null +++ b/domain-api/tests/test_domains_service.py @@ -0,0 +1,143 @@ +from __future__ import annotations + +import unittest +from unittest.mock import patch + +from app.services import domains_service + + +class _FakeDomainsCursor: + def __init__(self) -> None: + self._fetchone_result = None + self._fetchall_result = [] + self.executed: list[tuple[str, tuple]] = [] + self.updated_detection_params = None + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def execute(self, sql: str, params=None) -> None: + normalized_sql = " ".join(str(sql or "").split()).lower() + tuple_params = tuple(params or ()) + self.executed.append((normalized_sql, tuple_params)) + if normalized_sql.startswith("select count(*)"): + self._fetchone_result = (1,) + return + if normalized_sql.startswith("select d.id, d.domain"): + self._fetchall_result = [ + ( + 1, + "a.com", + 0, + 0, + 0, + 0, + 1, + "", + None, + "", + 0, + None, + 7, + False, + None, + "", + {"status": True, "state": "passed"}, + {"status": True, "state": "passed"}, + True, + {"status": True, "state": "passed"}, + {"status": True, "state": "passed"}, + {"status": True, "state": "passed"}, + {"status": True, "state": "passed"}, + {"status": True, "state": "passed"}, + {"status": False, "state": "failed", "message": "juziseo failed"}, + {"status": False, "state": "blacklisted", "message": "jucha blacklisted"}, + ) + ] + return + if normalized_sql.startswith("update domains set"): + return + if normalized_sql.startswith("select id, baidu_history"): + self._fetchone_result = ( + 7, + {"status": False, "state": "failed", "message": "timeout", "checked_at": "2026-04-20 12:00:00", "step": "baidu_site"}, + {"status": True, "state": "passed", "message": "ok", "checked_at": "2026-04-20 12:00:00", "step": "baidu_site"}, + False, + {"status": False, "state": "failed", "message": "old", "checked_at": "2026-04-20 12:00:00", "step": "qihu360_site"}, + {"status": False, "state": "failed", "message": "old", "checked_at": "2026-04-20 12:00:00", "step": "google_site"}, + False, + ) + return + if normalized_sql.startswith("update domain_detections set"): + self.updated_detection_params = tuple_params + return + raise AssertionError(f"unexpected sql: {sql}") + + def fetchone(self): + return self._fetchone_result + + def fetchall(self): + return list(self._fetchall_result) + + +class _FakeDomainsConnection: + def __init__(self) -> None: + self.cursor_instance = _FakeDomainsCursor() + self.committed = False + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def cursor(self): + return self.cursor_instance + + def commit(self) -> None: + self.committed = True + + +class DomainsServiceTests(unittest.TestCase): + def test_build_domain_query_parts_supports_false_backlink_filter(self) -> None: + _from_clause, where_clause, params = domains_service._build_domain_query_parts({"backlink_gt_10": False}) + + self.assertIn("coalesce(dd.backlink_count_gt_10, false) = %s", where_clause) + self.assertEqual([False], params) + + @patch("app.services.domains_service.get_db") + def test_fetch_domains_step_summary_counts_juziseo_and_jucha_results(self, mock_get_db) -> None: + fake_conn = _FakeDomainsConnection() + mock_get_db.return_value = fake_conn + + result = domains_service.fetch_domains(page=1, page_size=20) + + summary = result["list"][0]["step_summary"] + self.assertEqual(1, summary["failed_count"]) + self.assertEqual(1, summary["blacklisted_count"]) + self.assertTrue(summary["has_failed"]) + self.assertTrue(summary["has_blacklisted_step"]) + + @patch("app.services.domains_service.get_db") + def test_batch_update_domains_preserves_detection_metadata_shape(self, mock_get_db) -> None: + fake_conn = _FakeDomainsConnection() + mock_get_db.return_value = fake_conn + + with patch("app.services.domains_service._now_text", return_value="2026-04-23 15:30:00"): + result = domains_service.batch_update_domains([42], {"baidu_site": "否"}) + + self.assertEqual(1, result["updated_count"]) + self.assertIsNotNone(fake_conn.cursor_instance.updated_detection_params) + updated_payload = fake_conn.cursor_instance.updated_detection_params[0] + self.assertEqual(False, updated_payload["status"]) + self.assertEqual("failed", updated_payload["state"]) + self.assertEqual("人工批量更新", updated_payload["message"]) + self.assertEqual("2026-04-23 15:30:00", updated_payload["checked_at"]) + self.assertTrue(updated_payload["manual_override"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/domain-api/tests/test_import_task_service.py b/domain-api/tests/test_import_task_service.py new file mode 100644 index 0000000..84ebd30 --- /dev/null +++ b/domain-api/tests/test_import_task_service.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +import app.services.import_task_service as import_task_service + + +class _TraceLock: + def __init__(self, order: list[str]) -> None: + self.order = order + + def __enter__(self): + self.order.append("enter") + return self + + def __exit__(self, exc_type, exc, tb): + self.order.append("exit") + return False + + +class ImportTaskServiceTests(unittest.TestCase): + @patch("app.services.import_task_service.import_domains_from_path") + @patch("app.services.import_task_service._update_task_with_log") + def test_run_import_task_acquires_execution_lock_before_marking_running( + self, + mock_update_task_with_log, + mock_import_domains_from_path, + ) -> None: + order: list[str] = [] + mock_update_task_with_log.side_effect = lambda *args, **kwargs: order.append("update") + mock_import_domains_from_path.return_value = { + "source_label": "TXT 导入", + "stats": {"total": 1, "valid": 1, "added": 1, "exists": 0, "invalid": 0}, + } + + with tempfile.TemporaryDirectory() as tmpdir: + path = Path(tmpdir) / "domains.txt" + path.write_text("a.com\n", encoding="utf-8") + + with patch.object(import_task_service, "_IMPORT_EXECUTION_LOCK", _TraceLock(order)): + import_task_service._run_import_task("task-1", str(path), source_type=7) + + self.assertEqual("enter", order[0]) + self.assertIn("update", order[1:]) + self.assertEqual("exit", order[-1]) + + +if __name__ == "__main__": + unittest.main() diff --git a/domain-api/tests/test_import_worker_service.py b/domain-api/tests/test_import_worker_service.py new file mode 100644 index 0000000..faa951d --- /dev/null +++ b/domain-api/tests/test_import_worker_service.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from app.services.import_worker_service import import_domains_from_path + + +class _FakeCursor: + def __init__(self) -> None: + self._fetchall_result = [] + self._fetchone_result = None + self.inserted_domains: list[str] = [] + self.inserted_detect_tasks: list[int] = [] + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def execute(self, sql: str, params=None) -> None: + normalized_sql = " ".join(str(sql or "").split()).lower() + params = params or () + if normalized_sql.startswith("select domain from domains where domain = any"): + self._fetchall_result = [] + return + if normalized_sql.startswith("insert into domains"): + domain = params[0] + self.inserted_domains.append(domain) + self._fetchone_result = (len(self.inserted_domains),) + return + if normalized_sql.startswith("insert into detect_tasks"): + self.inserted_detect_tasks.append(int(params[0])) + return + raise AssertionError(f"unexpected sql: {sql}") + + def fetchall(self): + return list(self._fetchall_result) + + def fetchone(self): + return self._fetchone_result + + +class _FakeConnection: + def __init__(self) -> None: + self.cursor_instance = _FakeCursor() + self.committed = False + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def cursor(self): + return self.cursor_instance + + def commit(self) -> None: + self.committed = True + + +class ImportWorkerServiceTests(unittest.TestCase): + @patch("app.services.import_worker_service.get_db") + def test_import_domains_from_path_skips_duplicate_domains_in_same_batch(self, mock_get_db) -> None: + fake_conn = _FakeConnection() + mock_get_db.return_value = fake_conn + + with tempfile.TemporaryDirectory() as tmpdir: + path = Path(tmpdir) / "domains.txt" + path.write_text("a.com\na.com\nb.net\ninvalid-domain\n", encoding="utf-8") + + result = import_domains_from_path(path, source_type=7) + + self.assertEqual(["a.com", "b.net"], fake_conn.cursor_instance.inserted_domains) + self.assertEqual([1, 2], fake_conn.cursor_instance.inserted_detect_tasks) + self.assertTrue(fake_conn.committed) + self.assertEqual( + { + "total": 4, + "valid": 3, + "added": 2, + "exists": 1, + "invalid": 1, + "failed": 0, + }, + result["stats"], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/domain-api/tests/test_ops_api_routes.py b/domain-api/tests/test_ops_api_routes.py index 5255cbd..47eef8c 100644 --- a/domain-api/tests/test_ops_api_routes.py +++ b/domain-api/tests/test_ops_api_routes.py @@ -4,6 +4,9 @@ import unittest from unittest.mock import patch from app.api.routes.ops import ( + ops_migration_execute, + ops_migration_preview, + ops_migration_source_profile, ops_doctor_decision, ops_go_live_bundle, ops_go_live_review, @@ -13,6 +16,39 @@ from app.api.routes.ops import ( class OpsApiRoutesTestCase(unittest.TestCase): + @patch("app.api.routes.ops.get_ops_migration_source_profile") + def test_migration_source_profile_route_returns_payload(self, mock_source_profile) -> None: + mock_source_profile.return_value = {"source_node": {"node_code": "overseas-control-01"}} + + response = ops_migration_source_profile() + + self.assertEqual(0, response.code) + self.assertEqual("overseas-control-01", response.data["source_node"]["node_code"]) + mock_source_profile.assert_called_once_with() + + @patch("app.api.routes.ops.preview_ops_migration") + def test_migration_preview_route_passes_payload(self, mock_preview_ops_migration) -> None: + mock_preview_ops_migration.return_value = (True, "ok", {"plan_steps": [{"key": "sync_env_files"}]}) + + payload = {"target_node_code": "node-a"} + response = ops_migration_preview(payload) + + self.assertEqual(0, response.code) + self.assertEqual("sync_env_files", response.data["plan_steps"][0]["key"]) + mock_preview_ops_migration.assert_called_once_with(payload) + + @patch("app.api.routes.ops.execute_ops_migration") + def test_migration_execute_route_returns_error_payload(self, mock_execute_ops_migration) -> None: + mock_execute_ops_migration.return_value = (False, "failed", {"blocking_reasons": ["ssh missing"]}) + + payload = {"target_node_code": "node-a"} + response = ops_migration_execute(payload) + + self.assertEqual(1, response.code) + self.assertEqual("failed", response.message) + self.assertEqual(["ssh missing"], response.data["blocking_reasons"]) + mock_execute_ops_migration.assert_called_once_with(payload) + @patch("app.api.routes.ops.get_ops_go_live_signoff") def test_go_live_signoff_route_uses_service_payload(self, mock_get_ops_go_live_signoff) -> None: mock_get_ops_go_live_signoff.return_value = { diff --git a/domain-api/tests/test_ops_migration_service.py b/domain-api/tests/test_ops_migration_service.py new file mode 100644 index 0000000..d880fa8 --- /dev/null +++ b/domain-api/tests/test_ops_migration_service.py @@ -0,0 +1,290 @@ +from __future__ import annotations + +import unittest +from unittest.mock import patch + +from app.services.ops_migration_service import execute_ops_migration, preview_ops_migration + + +class OpsMigrationServiceTestCase(unittest.TestCase): + @patch("app.services.ops_migration_service._collect_remote_checks") + @patch("app.services.ops_migration_service._inspect_target_database") + @patch("app.services.ops_migration_service._find_managed_node") + def test_preview_returns_blocking_reason_when_remote_tools_missing( + self, + mock_find_managed_node, + mock_inspect_target_database, + mock_collect_remote_checks, + ) -> None: + mock_find_managed_node.return_value = { + "node_code": "target-a", + "ssh_host": "10.0.0.8", + "ssh_user": "root", + "ssh_port": 22, + "auth_mode": "key", + } + mock_collect_remote_checks.return_value = { + "tools": { + "python3": True, + "node": False, + "npm": False, + "systemctl": True, + "psql": True, + "pg_dump": True, + "curl": True, + }, + "paths": { + "repo_exists": True, + "repo_git": True, + "domain_root_exists": True, + "api_root_exists": True, + "web_root_exists": True, + }, + "remote_db_config": { + "DB_HOST": "127.0.0.1", + "DB_PORT": "5432", + "DB_DATABASE": "domain", + "DB_USER": "domainuser", + }, + "blocking_reasons": ["目标机缺少 node。", "目标机缺少 npm。"], + "warnings": [], + } + mock_inspect_target_database.return_value = {"available": False} + + ok, message, data = preview_ops_migration({"target_node_code": "target-a"}) + + self.assertFalse(ok) + self.assertEqual("迁移预检查未通过", message) + self.assertIn("目标机缺少 node。", data["blocking_reasons"]) + self.assertIn("目标机缺少 npm。", data["blocking_reasons"]) + + @patch("app.services.ops_migration_service._collect_remote_checks") + @patch("app.services.ops_migration_service._inspect_target_database") + @patch("app.services.ops_migration_service._find_managed_node") + def test_preview_uses_remote_env_as_target_db_default( + self, + mock_find_managed_node, + mock_inspect_target_database, + mock_collect_remote_checks, + ) -> None: + mock_find_managed_node.return_value = { + "node_code": "target-a", + "ssh_host": "10.0.0.8", + "ssh_user": "root", + "ssh_port": 22, + "auth_mode": "key", + } + mock_collect_remote_checks.return_value = { + "tools": { + "python3": True, + "node": True, + "npm": True, + "systemctl": True, + "psql": True, + "pg_dump": True, + "curl": True, + }, + "paths": { + "repo_exists": True, + "repo_git": True, + "domain_root_exists": True, + "api_root_exists": True, + "web_root_exists": True, + }, + "git_commit": "abc123", + "remote_db_config": { + "DB_HOST": "127.0.0.1", + "DB_PORT": "5433", + "DB_DATABASE": "domain_remote", + "DB_USER": "remote_user", + }, + "blocking_reasons": [], + "warnings": [], + } + mock_inspect_target_database.return_value = { + "available": True, + "has_business_data": False, + } + + ok, message, data = preview_ops_migration({"target_node_code": "target-a", "overwrite_database": True}) + + self.assertTrue(ok) + self.assertEqual("迁移预检查完成", message) + self.assertEqual("domain_remote", data["target_db_config"]["database"]) + self.assertEqual("remote_user", data["target_db_config"]["user"]) + self.assertEqual(5433, data["target_db_config"]["port"]) + self.assertTrue(bool(data["execution_guard"]["token"])) + + @patch("app.services.ops_migration_service._run_remote_health_check") + @patch("app.services.ops_migration_service._restart_remote_services") + @patch("app.services.ops_migration_service._build_remote_frontend") + @patch("app.services.ops_migration_service._inspect_target_database") + @patch("app.services.ops_migration_service._collect_remote_checks") + @patch("app.services.ops_migration_service._find_managed_node") + def test_execute_rejects_missing_confirmation_text_for_nonempty_target_db( + self, + mock_find_managed_node, + mock_collect_remote_checks, + mock_inspect_target_database, + mock_build_remote_frontend, + mock_restart_remote_services, + mock_run_remote_health_check, + ) -> None: + mock_find_managed_node.return_value = { + "node_code": "target-a", + "ssh_host": "10.0.0.8", + "ssh_user": "root", + "ssh_port": 22, + "auth_mode": "key", + } + mock_collect_remote_checks.return_value = { + "tools": { + "python3": True, + "node": True, + "npm": True, + "systemctl": True, + "psql": True, + "pg_dump": True, + "curl": True, + }, + "paths": { + "repo_exists": True, + "repo_git": True, + "domain_root_exists": True, + "api_root_exists": True, + "web_root_exists": True, + }, + "git_commit": "abc123", + "remote_db_config": { + "DB_HOST": "127.0.0.1", + "DB_PORT": "5432", + "DB_DATABASE": "domain_remote", + "DB_USER": "remote_user", + }, + "blocking_reasons": [], + "warnings": [], + } + mock_inspect_target_database.return_value = { + "available": True, + "database": "domain_remote", + "has_business_data": True, + "public_table_count": 10, + "business_table_count": 5, + } + mock_build_remote_frontend.return_value = (True, "ok", {}) + mock_restart_remote_services.return_value = (True, "ok", {}) + mock_run_remote_health_check.return_value = (True, "ok", {}) + + preview_ok, _preview_message, preview_data = preview_ops_migration( + {"target_node_code": "target-a", "overwrite_database": True} + ) + + self.assertTrue(preview_ok) + token = preview_data["execution_guard"]["token"] + required_confirmation_text = preview_data["execution_guard"]["required_confirmation_text"] + self.assertEqual("OVERWRITE domain_remote", required_confirmation_text) + + execute_ok, execute_message, execute_data = execute_ops_migration( + { + "target_node_code": "target-a", + "overwrite_database": True, + "execute_confirmation_token": token, + } + ) + + self.assertFalse(execute_ok) + self.assertEqual("缺少数据库覆盖确认文案，执行被拒绝。", execute_message) + self.assertIn("missing execute_confirmation_text", execute_data["blocking_reasons"]) + + @patch("app.services.ops_migration_service._start_migration_dispatch_thread") + @patch("app.services.ops_migration_service.create_ops_job") + @patch("app.services.ops_migration_service._collect_remote_checks") + @patch("app.services.ops_migration_service._inspect_target_database") + @patch("app.services.ops_migration_service._find_managed_node") + def test_execute_creates_async_job_for_long_running_migration( + self, + mock_find_managed_node, + mock_inspect_target_database, + mock_collect_remote_checks, + mock_create_ops_job, + mock_start_thread, + ) -> None: + mock_find_managed_node.return_value = { + "node_code": "target-a", + "ssh_host": "10.0.0.8", + "ssh_user": "root", + "ssh_port": 22, + "auth_mode": "key", + } + mock_collect_remote_checks.return_value = { + "tools": { + "python3": True, + "node": True, + "npm": True, + "systemctl": True, + "psql": True, + "pg_dump": True, + "curl": True, + }, + "paths": { + "repo_exists": True, + "repo_git": True, + "domain_root_exists": True, + "api_root_exists": True, + "web_root_exists": True, + }, + "git_commit": "abc123", + "remote_db_config": { + "DB_HOST": "127.0.0.1", + "DB_PORT": "5432", + "DB_DATABASE": "domain_remote", + "DB_USER": "remote_user", + }, + "blocking_reasons": [], + "warnings": [], + } + mock_inspect_target_database.return_value = { + "available": True, + "database": "domain_remote", + "has_business_data": False, + "public_table_count": 0, + "business_table_count": 0, + } + mock_create_ops_job.return_value = ( + True, + "ok", + { + "job": { + "id": 88, + "job_code": "ops-20260422160000-abc123", + "action": "migration.execute", + "status": "queued", + "target_node_code": "target-a", + } + }, + ) + + preview_ok, _preview_message, preview_data = preview_ops_migration({"target_node_code": "target-a"}) + + self.assertTrue(preview_ok) + token = preview_data["execution_guard"]["token"] + + execute_ok, execute_message, execute_data = execute_ops_migration( + { + "target_node_code": "target-a", + "execute_confirmation_token": token, + } + ) + + self.assertTrue(execute_ok) + self.assertEqual("迁移任务已创建，后台开始执行。", execute_message) + self.assertEqual(88, execute_data["job"]["id"]) + self.assertEqual("migration.execute", mock_create_ops_job.call_args.args[0]["action"]) + self.assertFalse(bool(mock_create_ops_job.call_args.args[0]["run_now"])) + self.assertEqual("control-plane", mock_create_ops_job.call_args.args[0]["execution_mode"]) + self.assertEqual("", mock_create_ops_job.call_args.args[0]["payload"]["target_db_password"]) + mock_start_thread.assert_called_once_with(88) + + +if __name__ == "__main__": + unittest.main() diff --git a/domain-api/tests/test_ops_release_executor_core.py b/domain-api/tests/test_ops_release_executor_core.py index bcdaf1e..d163d45 100644 --- a/domain-api/tests/test_ops_release_executor_core.py +++ b/domain-api/tests/test_ops_release_executor_core.py @@ -9,9 +9,11 @@ from pathlib import Path from unittest.mock import patch from app.services.ops_release_executor_core import ( + _systemd_dropin_content, _pick_release_owner_group, build_remote_release_action_script, execute_release_action, + normalize_release_health_check_urls, ) @@ -36,6 +38,7 @@ def _build_release_archive() -> bytes: "README.txt": b"hello-release", "domain-api/deploy/systemd/domain-node-agent.service": b"[Service]\nEnvironmentFile=-/etc/default/domaincheck-worker\n", "domain-api/deploy/systemd/domain-worker.service": b"[Service]\nEnvironmentFile=-/etc/default/domaincheck-worker\n", + "domain-api/deploy/systemd/domain-worker@.service": b"[Service]\nEnvironmentFile=-/etc/default/domaincheck-worker-%i\n", "domain-api/deploy/systemd/domain-api.service": b"[Service]\nEnvironmentFile=-/etc/default/domaincheck-api\n", "domain-api/deploy/systemd/domain-sync-agent.service": b"[Service]\nEnvironmentFile=-/etc/default/domaincheck-worker\n", } @@ -47,6 +50,42 @@ def _build_release_archive() -> bytes: class OpsReleaseExecutorCoreTests(unittest.TestCase): + def test_normalize_release_health_check_urls_rewrites_runtime_status_probe(self) -> None: + self.assertEqual( + [ + "http://127.0.0.1:8100/health", + "http://127.0.0.1:8100/health", + "https://example.com/custom-health", + ], + normalize_release_health_check_urls( + [ + "http://127.0.0.1:8100/api/v1/runtime/status", + "http://127.0.0.1:8100/runtime/status?full=1", + "https://example.com/custom-health", + ] + ), + ) + + def test_api_service_template_limits_graceful_shutdown(self) -> None: + service_text = Path("domain-api/deploy/systemd/domain-api.service").read_text(encoding="utf-8") + + self.assertIn("--timeout-graceful-shutdown 15", service_text) + self.assertIn("TimeoutStopSec=20", service_text) + + def test_worker_service_template_uses_current_symlink(self) -> None: + service_text = Path("domain-api/deploy/systemd/domain-worker.service").read_text(encoding="utf-8") + + self.assertIn("WorkingDirectory=/opt/domaincheck/current/domainCheck", service_text) + self.assertIn( + "ExecStart=/opt/domaincheck/domainCheck/.venv/bin/python /opt/domaincheck/current/domainCheck/detect_worker.py", + service_text, + ) + + def test_api_service_dropin_uses_graceful_shutdown_timeout(self) -> None: + dropin_text = _systemd_dropin_content("domaincheck-api", "/opt/domaincheck") + + self.assertIn("--timeout-graceful-shutdown 15", dropin_text) + def test_pick_release_owner_group_prefers_service_identity_over_path_owner(self) -> None: with tempfile.TemporaryDirectory() as tmpdir: with patch( @@ -65,16 +104,29 @@ class OpsReleaseExecutorCoreTests(unittest.TestCase): def run_command(command: list[str], *, timeout: int = 60): commands.append(tuple(command)) + if command[:5] == ["systemctl", "list-units", "--type=service", "--all", "domaincheck-worker@*"]: + return 0, ( + "domaincheck-worker@a.service loaded active running domainCheck Worker Instance a\n" + "domaincheck-worker@ah.service loaded inactive dead domainCheck Worker Instance ah\n" + ), "" if command[:3] == ["systemctl", "show", "domaincheck-worker"] and "-p" in command: current_user = pwd.getpwuid(os.getuid()).pw_name current_group = grp.getgrgid(os.getgid()).gr_name return 0, f"{current_user}\n{current_group}\n", "" + if command[:3] == ["systemctl", "show", "domaincheck-worker@a.service"] and "-p" in command: + current_user = pwd.getpwuid(os.getuid()).pw_name + current_group = grp.getgrgid(os.getgid()).gr_name + return 0, f"{current_user}\n{current_group}\n", "" + if command[:3] == ["systemctl", "show", "domaincheck-worker@ah.service"] and "-p" in command: + current_user = pwd.getpwuid(os.getuid()).pw_name + current_group = grp.getgrgid(os.getgid()).gr_name + return 0, f"{current_user}\n{current_group}\n", "" if command[:2] == ["chown", "-R"]: return 0, "", "" if command[:2] == ["systemctl", "restart"]: return 0, "", "" if command[:2] == ["systemctl", "is-active"]: - return 0, "active", "" + return 0, "\n".join("active" for _ in command[2:]), "" return 0, "", "" with tempfile.TemporaryDirectory() as tmpdir: @@ -108,8 +160,13 @@ class OpsReleaseExecutorCoreTests(unittest.TestCase): self.assertTrue(any(cmd[:2] == ("chown", "-R") for cmd in commands)) self.assertIn(("systemctl", "daemon-reload"), commands) self.assertIn(("systemctl", "restart", "domaincheck-worker"), commands) + self.assertIn(("systemctl", "restart", "domaincheck-worker@a.service"), commands) + self.assertIn(("systemctl", "restart", "domaincheck-worker@ah.service"), commands) self.assertIn(("systemctl", "is-active", "domaincheck-worker"), commands) + self.assertIn(("systemctl", "is-active", "domaincheck-worker@a.service"), commands) + self.assertIn(("systemctl", "is-active", "domaincheck-worker@ah.service"), commands) self.assertTrue((systemd_root / "domaincheck-node-agent.service").exists()) + self.assertTrue((systemd_root / "domaincheck-worker@.service").exists()) self.assertTrue((systemd_root / "domaincheck-node-agent.service.d" / "current-path.conf").exists()) def test_build_remote_release_action_script_is_valid_python(self) -> None: @@ -128,6 +185,9 @@ class OpsReleaseExecutorCoreTests(unittest.TestCase): self.assertIn("False", script) self.assertIn("def collect_service_identity(", script) self.assertIn("def apply_release_permissions(", script) + self.assertIn("import grp", script) + self.assertIn("import os", script) + self.assertIn("import pwd", script) if __name__ == "__main__": diff --git a/domain-api/tests/test_ops_release_service_execution_modes.py b/domain-api/tests/test_ops_release_service_execution_modes.py index ac78276..df60861 100644 --- a/domain-api/tests/test_ops_release_service_execution_modes.py +++ b/domain-api/tests/test_ops_release_service_execution_modes.py @@ -38,6 +38,97 @@ class _EmptyConnection: class OpsReleaseServiceExecutionModeTests(unittest.TestCase): + @patch("app.services.ops_release_service.get_db") + @patch("app.services.ops_agent_service.ensure_ops_agent_schema") + @patch("app.services.ops_agent_service.get_managed_node_onboarding") + @patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state") + @patch("app.services.ops_job_service.list_managed_nodes") + @patch("app.services.cluster_runtime_service.get_cluster_snapshot") + def test_build_rollout_target_operational_readiness_reuses_managed_node_snapshot_for_onboarding( + self, + mock_get_cluster_snapshot, + mock_list_managed_nodes, + mock_list_managed_nodes_with_agent_state, + mock_get_managed_node_onboarding, + mock_ensure_ops_agent_schema, + mock_get_db, + ) -> None: + captured_nodes_payloads = [] + + def _fake_onboarding(node_code, **kwargs): + captured_nodes_payloads.append(kwargs.get("nodes_payload")) + return { + "onboarding_stage": {"code": "ready", "label": "已接管"}, + "summary": f"{node_code} ready", + "recovery_decision": { + "action": "noop", + "label": "当前无需额外恢复动作", + "summary": "当前节点暂无需要执行的接管恢复动作。", + "command_hint": "", + "window": "none", + }, + } + + mock_ensure_ops_agent_schema.return_value = None + mock_get_db.return_value = _EmptyConnection() + mock_get_cluster_snapshot.return_value = { + "nodes": [ + { + "node_code": "mainland-worker-01", + "region": "mainland", + "role": "worker", + "status": "online", + "current_load": 0, + "is_effective_worker": True, + }, + { + "node_code": "mainland-worker-02", + "region": "mainland", + "role": "worker", + "status": "online", + "current_load": 0, + "is_effective_worker": True, + }, + ] + } + mock_list_managed_nodes.return_value = [ + { + "node_code": "mainland-worker-01", + "region": "mainland", + "role": "worker", + "is_enabled": True, + "ssh_host": "121.204.244.248", + "ssh_user": "root", + "metadata": {}, + "last_seen_at": "", + }, + { + "node_code": "mainland-worker-02", + "region": "mainland", + "role": "worker", + "is_enabled": True, + "ssh_host": "121.204.244.249", + "ssh_user": "root", + "metadata": {}, + "last_seen_at": "", + }, + ] + mock_list_managed_nodes_with_agent_state.return_value = {"nodes": [{"node_code": "mainland-worker-01"}, {"node_code": "mainland-worker-02"}]} + mock_get_managed_node_onboarding.side_effect = _fake_onboarding + + readiness = build_rollout_target_operational_readiness( + [ + {"node_code": "mainland-worker-01", "region": "mainland", "role": "worker"}, + {"node_code": "mainland-worker-02", "region": "mainland", "role": "worker"}, + ], + execution_mode="remote-agent", + ) + + self.assertEqual(2, len(readiness["rows"])) + mock_list_managed_nodes_with_agent_state.assert_called_once() + self.assertEqual(2, len(captured_nodes_payloads)) + self.assertTrue(all(payload == {"nodes": [{"node_code": "mainland-worker-01"}, {"node_code": "mainland-worker-02"}]} for payload in captured_nodes_payloads)) + @patch("app.services.ops_release_service.get_db") @patch("app.services.ops_agent_service.ensure_ops_agent_schema") @patch("app.services.ops_agent_service.get_managed_node_onboarding") diff --git a/domain-api/tests/test_ops_schema_init.py b/domain-api/tests/test_ops_schema_init.py index 996cbf2..c8f8a67 100644 --- a/domain-api/tests/test_ops_schema_init.py +++ b/domain-api/tests/test_ops_schema_init.py @@ -3,6 +3,8 @@ from unittest.mock import MagicMock, patch import app.services.ops_agent_service as ops_agent_service import app.services.ops_job_service as ops_job_service +import app.services.ops_release_service as ops_release_service +from psycopg2 import errors class OpsSchemaInitTests(unittest.TestCase): @@ -10,6 +12,7 @@ class OpsSchemaInitTests(unittest.TestCase): def test_ensure_ops_schema_uses_advisory_lock_and_skips_after_ready(self, mock_get_db) -> None: conn = MagicMock() cursor = MagicMock() + cursor.fetchone.return_value = None db_ctx = MagicMock() cursor_ctx = MagicMock() db_ctx.__enter__.return_value = conn @@ -35,6 +38,94 @@ class OpsSchemaInitTests(unittest.TestCase): cursor.execute.assert_any_call(ops_job_service._OPS_SCHEMA_SQL) conn.commit.assert_called_once() + @patch("app.services.ops_job_service.get_db") + def test_ensure_ops_schema_skips_ddl_when_required_schema_already_exists(self, mock_get_db) -> None: + conn = MagicMock() + cursor = MagicMock() + cursor.fetchone.side_effect = [(f"public.{name}",) for name in ops_job_service._OPS_REQUIRED_TABLES] + cursor.fetchall.side_effect = [ + [(column,) for column in ops_job_service._OPS_REQUIRED_COLUMNS["ops_jobs"]], + [(column,) for column in ops_job_service._OPS_REQUIRED_COLUMNS["ops_job_steps"]], + ] + db_ctx = MagicMock() + cursor_ctx = MagicMock() + db_ctx.__enter__.return_value = conn + db_ctx.__exit__.return_value = False + cursor_ctx.__enter__.return_value = cursor + cursor_ctx.__exit__.return_value = False + conn.cursor.return_value = cursor_ctx + mock_get_db.return_value = db_ctx + + previous_ready = ops_job_service._OPS_SCHEMA_READY + ops_job_service._OPS_SCHEMA_READY = False + try: + ops_job_service.ensure_ops_schema() + finally: + ops_job_service._OPS_SCHEMA_READY = previous_ready + + self.assertFalse(any(call.args[0] == ops_job_service._OPS_SCHEMA_SQL for call in cursor.execute.call_args_list)) + conn.commit.assert_not_called() + + @patch("app.services.ops_job_service.get_db") + def test_ensure_ops_schema_accepts_deadlock_when_required_schema_already_exists(self, mock_get_db) -> None: + class _Cursor: + def __init__(self, *, raise_on_schema=False, fetchone_values=None, fetchall_values=None) -> None: + self.raise_on_schema = raise_on_schema + self.fetchone_values = list(fetchone_values or []) + self.fetchall_values = list(fetchall_values or []) + + def execute(self, sql, params=None): + if self.raise_on_schema and sql == ops_job_service._OPS_SCHEMA_SQL: + raise errors.DeadlockDetected() + + def fetchone(self): + if self.fetchone_values: + return self.fetchone_values.pop(0) + return None + + def fetchall(self): + if self.fetchall_values: + return self.fetchall_values.pop(0) + return [] + + class _CursorContext: + def __init__(self, cursor) -> None: + self.cursor = cursor + + def __enter__(self): + return self.cursor + + def __exit__(self, exc_type, exc, tb): + return False + + conn = MagicMock() + conn.cursor.side_effect = [ + _CursorContext(_Cursor(fetchone_values=[None])), + _CursorContext(_Cursor(raise_on_schema=True)), + _CursorContext( + _Cursor( + fetchone_values=[(f"public.{name}",) for name in ops_job_service._OPS_REQUIRED_TABLES], + fetchall_values=[ + [(column,) for column in ops_job_service._OPS_REQUIRED_COLUMNS["ops_jobs"]], + [(column,) for column in ops_job_service._OPS_REQUIRED_COLUMNS["ops_job_steps"]], + ], + ) + ), + ] + db_ctx = MagicMock() + db_ctx.__enter__.return_value = conn + mock_get_db.return_value = db_ctx + + previous_ready = ops_job_service._OPS_SCHEMA_READY + ops_job_service._OPS_SCHEMA_READY = False + try: + ops_job_service.ensure_ops_schema() + finally: + ops_job_service._OPS_SCHEMA_READY = previous_ready + + conn.rollback.assert_called_once() + conn.commit.assert_not_called() + @patch("app.services.ops_agent_service.ensure_ops_schema") @patch("app.services.ops_agent_service.get_db") def test_ensure_ops_agent_schema_uses_advisory_lock_and_skips_after_ready( @@ -44,6 +135,7 @@ class OpsSchemaInitTests(unittest.TestCase): ) -> None: conn = MagicMock() cursor = MagicMock() + cursor.fetchone.return_value = None db_ctx = MagicMock() cursor_ctx = MagicMock() db_ctx.__enter__.return_value = conn @@ -70,6 +162,224 @@ class OpsSchemaInitTests(unittest.TestCase): cursor.execute.assert_any_call(ops_agent_service._AGENT_SCHEMA_SQL) conn.commit.assert_called_once() + @patch("app.services.ops_agent_service.ensure_ops_schema") + @patch("app.services.ops_agent_service.get_db") + def test_ensure_ops_agent_schema_skips_ddl_when_required_schema_already_exists( + self, + mock_get_db, + mock_ensure_ops_schema, + ) -> None: + conn = MagicMock() + cursor = MagicMock() + cursor.fetchone.side_effect = [(f"public.{name}",) for name in ops_agent_service._OPS_AGENT_REQUIRED_TABLES] + cursor.fetchall.side_effect = [ + [(column,) for column in ops_agent_service._OPS_AGENT_REQUIRED_COLUMNS["ops_node_tokens"]], + [(column,) for column in ops_agent_service._OPS_AGENT_REQUIRED_COLUMNS["ops_job_events"]], + [(column,) for column in ops_agent_service._OPS_AGENT_REQUIRED_COLUMNS["ops_jobs"]], + ] + db_ctx = MagicMock() + cursor_ctx = MagicMock() + db_ctx.__enter__.return_value = conn + db_ctx.__exit__.return_value = False + cursor_ctx.__enter__.return_value = cursor + cursor_ctx.__exit__.return_value = False + conn.cursor.return_value = cursor_ctx + mock_get_db.return_value = db_ctx + + previous_ready = ops_agent_service._OPS_AGENT_SCHEMA_READY + ops_agent_service._OPS_AGENT_SCHEMA_READY = False + try: + ops_agent_service.ensure_ops_agent_schema() + finally: + ops_agent_service._OPS_AGENT_SCHEMA_READY = previous_ready + + self.assertFalse(any(call.args[0] == ops_agent_service._AGENT_SCHEMA_SQL for call in cursor.execute.call_args_list)) + conn.commit.assert_not_called() + self.assertEqual(1, mock_ensure_ops_schema.call_count) + + @patch("app.services.ops_agent_service.ensure_ops_schema") + @patch("app.services.ops_agent_service.get_db") + def test_ensure_ops_agent_schema_accepts_deadlock_when_required_schema_already_exists( + self, + mock_get_db, + mock_ensure_ops_schema, + ) -> None: + class _Cursor: + def __init__(self, *, raise_on_schema=False, fetchone_values=None, fetchall_values=None) -> None: + self.raise_on_schema = raise_on_schema + self.fetchone_values = list(fetchone_values or []) + self.fetchall_values = list(fetchall_values or []) + + def execute(self, sql, params=None): + if self.raise_on_schema and sql == ops_agent_service._AGENT_SCHEMA_SQL: + raise errors.DeadlockDetected() + + def fetchone(self): + if self.fetchone_values: + return self.fetchone_values.pop(0) + return None + + def fetchall(self): + if self.fetchall_values: + return self.fetchall_values.pop(0) + return [] + + class _CursorContext: + def __init__(self, cursor) -> None: + self.cursor = cursor + + def __enter__(self): + return self.cursor + + def __exit__(self, exc_type, exc, tb): + return False + + conn = MagicMock() + conn.cursor.side_effect = [ + _CursorContext(_Cursor(fetchone_values=[None, None])), + _CursorContext(_Cursor(raise_on_schema=True)), + _CursorContext( + _Cursor( + fetchone_values=[(f"public.{name}",) for name in ops_agent_service._OPS_AGENT_REQUIRED_TABLES], + fetchall_values=[ + [(column,) for column in ops_agent_service._OPS_AGENT_REQUIRED_COLUMNS["ops_node_tokens"]], + [(column,) for column in ops_agent_service._OPS_AGENT_REQUIRED_COLUMNS["ops_job_events"]], + [(column,) for column in ops_agent_service._OPS_AGENT_REQUIRED_COLUMNS["ops_jobs"]], + ], + ) + ), + ] + db_ctx = MagicMock() + db_ctx.__enter__.return_value = conn + mock_get_db.return_value = db_ctx + + previous_ready = ops_agent_service._OPS_AGENT_SCHEMA_READY + ops_agent_service._OPS_AGENT_SCHEMA_READY = False + try: + ops_agent_service.ensure_ops_agent_schema() + finally: + ops_agent_service._OPS_AGENT_SCHEMA_READY = previous_ready + + conn.rollback.assert_called_once() + conn.commit.assert_not_called() + self.assertEqual(1, mock_ensure_ops_schema.call_count) + + @patch("app.services.ops_release_service.get_db") + def test_ensure_ops_release_schema_uses_advisory_lock_and_skips_after_ready(self, mock_get_db) -> None: + conn = MagicMock() + cursor = MagicMock() + cursor.fetchone.return_value = None + db_ctx = MagicMock() + cursor_ctx = MagicMock() + db_ctx.__enter__.return_value = conn + db_ctx.__exit__.return_value = False + cursor_ctx.__enter__.return_value = cursor + cursor_ctx.__exit__.return_value = False + conn.cursor.return_value = cursor_ctx + mock_get_db.return_value = db_ctx + + previous_ready = ops_release_service._RELEASE_SCHEMA_READY + ops_release_service._RELEASE_SCHEMA_READY = False + try: + ops_release_service.ensure_ops_release_schema() + ops_release_service.ensure_ops_release_schema() + finally: + ops_release_service._RELEASE_SCHEMA_READY = previous_ready + + self.assertEqual(1, mock_get_db.call_count) + cursor.execute.assert_any_call( + "SELECT pg_advisory_xact_lock(%s)", + (ops_release_service._RELEASE_SCHEMA_ADVISORY_LOCK_KEY,), + ) + cursor.execute.assert_any_call(ops_release_service._RELEASE_SCHEMA_SQL) + conn.commit.assert_called_once() + + @patch("app.services.ops_release_service.get_db") + def test_ensure_ops_release_schema_skips_ddl_when_required_schema_already_exists(self, mock_get_db) -> None: + conn = MagicMock() + cursor = MagicMock() + cursor.fetchone.side_effect = [(f"public.{name}",) for name in ops_release_service._RELEASE_REQUIRED_TABLES] + cursor.fetchall.side_effect = [ + [(column,) for column in ops_release_service._RELEASE_REQUIRED_COLUMNS["ops_release_rollouts"]], + ] + db_ctx = MagicMock() + cursor_ctx = MagicMock() + db_ctx.__enter__.return_value = conn + db_ctx.__exit__.return_value = False + cursor_ctx.__enter__.return_value = cursor + cursor_ctx.__exit__.return_value = False + conn.cursor.return_value = cursor_ctx + mock_get_db.return_value = db_ctx + + previous_ready = ops_release_service._RELEASE_SCHEMA_READY + ops_release_service._RELEASE_SCHEMA_READY = False + try: + ops_release_service.ensure_ops_release_schema() + finally: + ops_release_service._RELEASE_SCHEMA_READY = previous_ready + + self.assertFalse(any(call.args[0] == ops_release_service._RELEASE_SCHEMA_SQL for call in cursor.execute.call_args_list)) + conn.commit.assert_not_called() + + @patch("app.services.ops_release_service.get_db") + def test_ensure_ops_release_schema_accepts_deadlock_when_required_schema_already_exists(self, mock_get_db) -> None: + class _Cursor: + def __init__(self, *, raise_on_schema=False, fetchone_values=None, fetchall_values=None) -> None: + self.raise_on_schema = raise_on_schema + self.fetchone_values = list(fetchone_values or []) + self.fetchall_values = list(fetchall_values or []) + + def execute(self, sql, params=None): + if self.raise_on_schema and sql == ops_release_service._RELEASE_SCHEMA_SQL: + raise errors.DeadlockDetected() + + def fetchone(self): + if self.fetchone_values: + return self.fetchone_values.pop(0) + return None + + def fetchall(self): + if self.fetchall_values: + return self.fetchall_values.pop(0) + return [] + + class _CursorContext: + def __init__(self, cursor) -> None: + self.cursor = cursor + + def __enter__(self): + return self.cursor + + def __exit__(self, exc_type, exc, tb): + return False + + conn = MagicMock() + conn.cursor.side_effect = [ + _CursorContext(_Cursor(fetchone_values=[None])), + _CursorContext(_Cursor(raise_on_schema=True)), + _CursorContext( + _Cursor( + fetchone_values=[(f"public.{name}",) for name in ops_release_service._RELEASE_REQUIRED_TABLES], + fetchall_values=[ + [(column,) for column in ops_release_service._RELEASE_REQUIRED_COLUMNS["ops_release_rollouts"]], + ], + ) + ), + ] + db_ctx = MagicMock() + db_ctx.__enter__.return_value = conn + mock_get_db.return_value = db_ctx + + previous_ready = ops_release_service._RELEASE_SCHEMA_READY + ops_release_service._RELEASE_SCHEMA_READY = False + try: + ops_release_service.ensure_ops_release_schema() + finally: + ops_release_service._RELEASE_SCHEMA_READY = previous_ready + + conn.rollback.assert_called_once() + conn.commit.assert_not_called() + if __name__ == "__main__": unittest.main() diff --git a/domain-api/tests/test_ops_service_activity.py b/domain-api/tests/test_ops_service_activity.py index 1cc7a06..15c0f0a 100644 --- a/domain-api/tests/test_ops_service_activity.py +++ b/domain-api/tests/test_ops_service_activity.py @@ -10,6 +10,35 @@ class OpsServiceActivityTests(unittest.TestCase): def _runtime_status(detect: Optional[dict] = None) -> dict: return {"detect": dict(detect or {})} + @patch("app.services.ops_service.get_ops_runbook", return_value={"control_sequences": []}) + @patch("app.services.ops_service.list_release_rollouts", return_value=[]) + @patch("app.services.ops_service.list_ops_jobs", return_value=[]) + @patch("app.services.ops_service.get_recent_ops_playbook_runs", return_value={"runs": []}) + @patch("app.services.ops_service.list_managed_nodes_with_agent_state", side_effect=AssertionError("should reuse managed nodes")) + @patch("app.services.ops_service.get_runtime_status", side_effect=AssertionError("should reuse runtime status")) + def test_activity_stream_reuses_provided_runtime_and_managed_snapshots( + self, + _mock_get_runtime_status, + _mock_list_managed_nodes_with_agent_state, + _mock_get_recent_ops_playbook_runs, + _mock_list_ops_jobs, + _mock_list_release_rollouts, + mock_get_ops_runbook, + ) -> None: + runtime_snapshot = self._runtime_status() + managed_snapshot = {"nodes": []} + payload = get_ops_activity_stream( + limit=10, + scan_limit=20, + runtime_status=runtime_snapshot, + managed_nodes_payload=managed_snapshot, + ) + + self.assertEqual([], payload["items"]) + self.assertEqual(0, payload["summary"]["total"]) + self.assertEqual(runtime_snapshot, mock_get_ops_runbook.call_args.kwargs["runtime_status"]) + self.assertEqual(managed_snapshot, mock_get_ops_runbook.call_args.kwargs["managed_nodes_payload"]) + @patch("app.services.ops_service.get_ops_runbook") @patch("app.services.ops_service.list_release_rollouts") @patch("app.services.ops_service.list_ops_jobs") diff --git a/domain-api/tests/test_redis_client.py b/domain-api/tests/test_redis_client.py new file mode 100644 index 0000000..b63b385 --- /dev/null +++ b/domain-api/tests/test_redis_client.py @@ -0,0 +1,27 @@ +import unittest +from unittest.mock import patch + +from app.core.redis_client import get_redis, reset_redis_client_for_tests + + +class ApiRedisClientTests(unittest.TestCase): + def tearDown(self) -> None: + reset_redis_client_for_tests() + + @patch("app.core.redis_client.redis.Redis") + @patch("app.core.redis_client.redis.BlockingConnectionPool") + def test_get_redis_reuses_singleton_client(self, mock_pool, mock_redis) -> None: + singleton = object() + mock_redis.return_value = singleton + + client_a = get_redis() + client_b = get_redis() + + self.assertIs(client_a, singleton) + self.assertIs(client_b, singleton) + mock_pool.assert_called_once() + mock_redis.assert_called_once() + + +if __name__ == "__main__": + unittest.main() diff --git a/domain-api/tests/test_runtime_api_routes.py b/domain-api/tests/test_runtime_api_routes.py new file mode 100644 index 0000000..b71bd78 --- /dev/null +++ b/domain-api/tests/test_runtime_api_routes.py @@ -0,0 +1,93 @@ +from __future__ import annotations + +import unittest +from unittest.mock import patch + +from app.api.routes import runtime as runtime_route + + +class RuntimeApiRoutesTests(unittest.TestCase): + @patch("app.api.routes.runtime.list_debug_events") + def test_runtime_debug_events_forwards_node_code_filter(self, mock_list_debug_events) -> None: + mock_list_debug_events.return_value = {"records": [], "has_more": False} + + response = runtime_route.runtime_debug_events(node_code="mainland-controller-01-a", limit=20) + + self.assertEqual(0, response.code) + mock_list_debug_events.assert_called_once_with( + limit=20, + service=None, + event_type=None, + source_region=None, + node_code="mainland-controller-01-a", + level=None, + before_id=None, + after_id=None, + created_after=None, + ) + + @patch("app.api.routes.runtime.get_runtime_status") + @patch("app.api.routes.runtime.get_sync_summary") + @patch("app.api.routes.runtime.get_debug_handoff_report") + def test_runtime_health_handover_filters_recent_issues_and_issue_groups_by_node_code( + self, + mock_get_debug_handoff_report, + mock_get_sync_summary, + mock_get_runtime_status, + ) -> None: + mock_get_sync_summary.return_value = {"latest_record": {}} + mock_get_runtime_status.return_value = {"readiness": {"status": "ready"}} + mock_get_debug_handoff_report.return_value = { + "overview": { + "recent_issues": [ + {"node_code": "mainland-controller-01", "message": "keep"}, + {"node_code": "mainland-worker-01", "message": "drop"}, + ] + }, + "recent_issues": [ + {"node_code": "mainland-controller-01", "message": "keep"}, + {"node_code": "mainland-worker-01", "message": "drop"}, + ], + "issue_groups": [ + {"node_code": "mainland-controller-01", "message": "keep"}, + {"node_code": "mainland-worker-01", "message": "drop"}, + ], + "failure_handoff": { + "recent_issues": [ + {"node_code": "mainland-controller-01", "message": "keep"}, + {"node_code": "mainland-worker-01", "message": "drop"}, + ], + "issue_groups": [ + {"node_code": "mainland-controller-01", "message": "keep"}, + {"node_code": "mainland-worker-01", "message": "drop"}, + ], + }, + } + + response = runtime_route.runtime_health_handover(node_code="mainland-controller-01") + + self.assertEqual(0, response.code) + self.assertEqual( + [{"node_code": "mainland-controller-01", "message": "keep"}], + response.data["recent_issues"], + ) + self.assertEqual( + [{"node_code": "mainland-controller-01", "message": "keep"}], + response.data["issue_groups"], + ) + self.assertEqual( + [{"node_code": "mainland-controller-01", "message": "keep"}], + response.data["overview"]["recent_issues"], + ) + self.assertEqual( + [{"node_code": "mainland-controller-01", "message": "keep"}], + response.data["failure_handoff"]["recent_issues"], + ) + self.assertEqual( + [{"node_code": "mainland-controller-01", "message": "keep"}], + response.data["failure_handoff"]["issue_groups"], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/domain-api/tests/test_runtime_control_service.py b/domain-api/tests/test_runtime_control_service.py new file mode 100644 index 0000000..b7061c1 --- /dev/null +++ b/domain-api/tests/test_runtime_control_service.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +import unittest +from unittest.mock import patch + +from app.services.runtime_control_service import runtime_action + + +class RuntimeControlServiceTests(unittest.TestCase): + @patch("app.services.runtime_control_service._emit_runtime_action_event") + @patch("app.services.runtime_control_service.send_worker_command") + def test_runtime_action_stop_detection_forwards_target_payload( + self, + mock_send_worker_command, + _mock_emit_runtime_action_event, + ) -> None: + mock_send_worker_command.return_value = (True, "已发送 Worker 控制指令") + + ok, message, data = runtime_action( + "stop_detection", + payload={"target_node_codes": ["mainland-controller-01-a", "mainland-controller-01-b"]}, + ) + + self.assertTrue(ok) + self.assertEqual("已发送 Worker 控制指令", message) + mock_send_worker_command.assert_called_once_with( + "stop_detection", + payload={"target_node_codes": ["mainland-controller-01-a", "mainland-controller-01-b"]}, + ) + self.assertEqual( + ["mainland-controller-01-a", "mainland-controller-01-b"], + data["payload"]["target_node_codes"], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/domain-api/tests/test_runtime_settings_service.py b/domain-api/tests/test_runtime_settings_service.py new file mode 100644 index 0000000..78e9f1b --- /dev/null +++ b/domain-api/tests/test_runtime_settings_service.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +import json +import unittest +from unittest.mock import Mock, patch + +from app.services.runtime_settings_service import update_runtime_settings + + +class RuntimeSettingsServiceTests(unittest.TestCase): + @patch("app.services.runtime_settings_service.get_redis") + @patch("app.services.runtime_settings_service.write_runtime_json") + @patch("app.services.runtime_settings_service.get_runtime_settings") + def test_update_runtime_settings_publishes_runtime_settings_refresh( + self, + mock_get_runtime_settings, + mock_write_runtime_json, + mock_get_redis, + ) -> None: + redis_client = Mock() + mock_get_redis.return_value = redis_client + mock_get_runtime_settings.return_value = { + "worker_mode": "linux-systemd", + "worker_service_name": "domaincheck-worker", + "api_service_name": "domaincheck-api", + "sync_agent_service_name": "domaincheck-sync-agent", + "worker_log_sync_enabled": False, + "worker_log_sync_mode": "key", + "control_node_autoresume_enabled": False, + "claim_recent_jobs_first": False, + "claim_recent_jobs_limit": 8, + "claim_recent_jobs_window_hours": 24, + "claim_batch_floor": 0, + "claim_batch_ceil": 0, + "submit_backlog_floor": 0, + "submit_backlog_ceil": 0, + "dispatch_cap_multiplier": 1, + "pending_buffer_cap_multiplier": 1, + } + + updated = update_runtime_settings( + { + "worker_log_sync_enabled": True, + "worker_log_sync_mode": "full", + "control_node_autoresume_enabled": True, + "claim_recent_jobs_first": True, + "claim_recent_jobs_limit": 6, + "claim_recent_jobs_window_hours": 72, + "claim_batch_floor": 200, + "claim_batch_ceil": 800, + "submit_backlog_floor": 500, + "submit_backlog_ceil": 1500, + "dispatch_cap_multiplier": 2, + "pending_buffer_cap_multiplier": 2, + } + ) + + mock_write_runtime_json.assert_called_once_with("runtime_settings.json", updated) + redis_client.set.assert_called_once() + redis_key, serialized = redis_client.set.call_args.args + self.assertEqual("domain_tool:runtime_settings", redis_key) + self.assertEqual(updated, json.loads(serialized)) + self.assertTrue(updated["control_node_autoresume_enabled"]) + self.assertTrue(updated["claim_recent_jobs_first"]) + self.assertEqual(6, updated["claim_recent_jobs_limit"]) + self.assertEqual(72, updated["claim_recent_jobs_window_hours"]) + self.assertEqual(200, updated["claim_batch_floor"]) + self.assertEqual(800, updated["claim_batch_ceil"]) + self.assertEqual(500, updated["submit_backlog_floor"]) + self.assertEqual(1500, updated["submit_backlog_ceil"]) + self.assertEqual(2, updated["dispatch_cap_multiplier"]) + self.assertEqual(2, updated["pending_buffer_cap_multiplier"]) + redis_client.publish.assert_called_once_with("domain_tool:config_update", "runtime_settings") + + +if __name__ == "__main__": + unittest.main() diff --git a/domain-api/tests/test_runtime_status_service.py b/domain-api/tests/test_runtime_status_service.py index 172b22a..34f449c 100644 --- a/domain-api/tests/test_runtime_status_service.py +++ b/domain-api/tests/test_runtime_status_service.py @@ -1,10 +1,273 @@ import unittest +from datetime import datetime, timedelta +from unittest.mock import MagicMock, patch +import app.services.runtime_status_service as runtime_status_service_module from app.services.detect_service import _slice_remote_log_lines_fairly -from app.services.runtime_status_service import _align_queue_health_with_backlog, _build_detect_node_row +from app.services.runtime_status_service import ( + _align_queue_health_with_backlog, + _build_detect_node_row, + _build_detect_observation_summary, + refresh_runtime_projection_snapshot, + get_runtime_status, +) class RuntimeStatusServiceTests(unittest.TestCase): + def setUp(self) -> None: + runtime_status_service_module._DOMAIN_INVENTORY_CACHE_VALUE = None + runtime_status_service_module._DOMAIN_INVENTORY_CACHE_EXPIRES_AT = 0.0 + runtime_status_service_module._RUNTIME_STATUS_CACHE_VALUE = None + runtime_status_service_module._RUNTIME_STATUS_CACHE_EXPIRES_AT = 0.0 + runtime_status_service_module._RUNTIME_STATUS_CACHE_SIGNATURE = () + + def test_build_detect_observation_summary_marks_real_running_state(self) -> None: + summary = _build_detect_observation_summary( + detect_payload={ + "active_job": { + "job_id": 308, + "job_code": "sync-overseas-16432", + "status": "running", + "progress_percent": 24.6, + "items_pending": 3999, + "items_claimed": 52, + "items_running": 136, + "items_completed": 971, + "items_failed": 0, + "items_blacklisted": 0, + "display_items_running": 222, + "display_active_threads": 222, + "display_max_threads": 2000, + "processed_recent": 172, + "processed_per_minute": 11.47, + "completed_recent": 2, + "failed_recent": 169, + "blacklisted_recent": 1, + }, + "active_thread_count": 222, + "max_thread_count": 2000, + "queue_health": { + "queue": { + "pending": 3999, + "claimed": 52, + "running": 136, + "display_running": 222, + "completed": 971, + "failed": 0, + "blacklisted": 0, + }, + "throughput": { + "processed_recent": 172, + "processed_per_minute": 11.47, + "completed_recent": 2, + "failed_recent": 169, + "blacklisted_recent": 1, + }, + "steps": [ + { + "step_code": "detect_register", + "step_name": "注册检测", + "items_pending": 2100, + "items_running": 40, + "processed_recent": 88, + "failed_recent": 100, + "blacklisted_recent": 0, + } + ], + }, + "participating_nodes": [ + { + "node_code": "mainland-controller-01-a", + "participation_label": "执行中", + "is_dispatch_active": True, + "items_claimed": 12, + "items_running": 100, + "active_threads": 100, + "max_threads": 1000, + "processed_recent": 80, + "processed_per_minute": 5.33, + }, + { + "node_code": "mainland-controller-01-b", + "participation_label": "执行中", + "is_dispatch_active": True, + "items_claimed": 8, + "items_running": 122, + "active_threads": 122, + "max_threads": 1000, + "processed_recent": 92, + "processed_per_minute": 6.13, + }, + ], + "participation_summary": { + "dispatch_active_nodes": 2, + "participating_nodes": 2, + }, + "backlog": { + "pending_total": 3999, + "claimed_total": 52, + "running_total": 136, + "completed_total": 971, + }, + }, + cluster_snapshot={"summary": {"online_worker_nodes": 1}}, + inventory_summary={ + "scope_label": "海外主库总盘子", + "authoritative": True, + "domains_total": 5690937, + "pending_total": 5690918, + "completed_total": 0, + "running_total": 7, + "blacklist_total": 0, + "failed_total": 12, + "processed_total": 12, + "remaining_total": 5690925, + }, + ) + + self.assertEqual("running", summary["state"]) + self.assertEqual("真跑中", summary["state_label"]) + self.assertEqual(2, summary["execution"]["active_processes"]) + self.assertEqual(222, summary["execution"]["active_threads"]) + self.assertEqual(169, summary["throughput"]["failed_recent"]) + self.assertEqual("detect_register", summary["top_steps"][0]["step_code"]) + self.assertEqual(5690937, summary["source_inventory"]["domains_total"]) + self.assertEqual(5244, summary["active_batch"]["effective_items_total"]) + self.assertIn("不等于全盘累计", summary["scope_hint"]) + + def test_build_detect_observation_summary_marks_not_running_when_only_backlog_left(self) -> None: + summary = _build_detect_observation_summary( + detect_payload={ + "active_job": { + "job_id": 401, + "job_code": "sync-overseas-404", + "status": "running", + "items_pending": 5000, + "items_claimed": 0, + "items_running": 0, + "items_completed": 0, + "items_failed": 0, + "items_blacklisted": 0, + }, + "queue_health": { + "queue": { + "pending": 5000, + "claimed": 0, + "running": 0, + "completed": 0, + "failed": 0, + "blacklisted": 0, + }, + "throughput": { + "processed_recent": 0, + "processed_per_minute": 0, + "completed_recent": 0, + "failed_recent": 0, + "blacklisted_recent": 0, + }, + "steps": [ + { + "step_code": "detect_register", + "step_name": "注册检测", + "items_pending": 5000, + "items_running": 0, + } + ], + }, + "participating_nodes": [], + "participation_summary": { + "dispatch_active_nodes": 0, + "participating_nodes": 0, + }, + "backlog": { + "pending_total": 5000, + "claimed_total": 0, + "running_total": 0, + }, + }, + cluster_snapshot={"summary": {"online_worker_nodes": 1}}, + inventory_summary={ + "scope_label": "海外主库总盘子", + "authoritative": True, + "domains_total": 5690937, + "pending_total": 5690918, + "completed_total": 0, + "running_total": 7, + "blacklist_total": 0, + "failed_total": 12, + "processed_total": 12, + "remaining_total": 5690925, + }, + ) + + self.assertEqual("not_running", summary["state"]) + self.assertEqual("没跑起来", summary["state_label"]) + self.assertEqual(0, summary["execution"]["active_processes"]) + self.assertIn("队列", summary["focus_hint"]) + self.assertEqual(5000, summary["active_batch"]["effective_items_total"]) + self.assertEqual(5690937, summary["source_inventory"]["domains_total"]) + + @patch("app.services.runtime_status_service.get_active_detect_job_summary") + @patch("app.services.runtime_status_service.get_db") + def test_load_detect_backlog_snapshot_prefers_active_job_snapshot( + self, + mock_get_db, + mock_get_active_detect_job_summary, + ) -> None: + mock_get_active_detect_job_summary.return_value = { + "items_pending": 321, + "items_claimed": 2, + "items_running": 9, + "items_completed": 50, + "items_failed": 3, + "display_active_threads": 11, + "step_stats": [ + {"step_code": "detect_register", "items_pending": 210}, + {"step_code": "detect_baidu", "items_pending": 111}, + ], + } + + backlog = runtime_status_service_module._load_detect_backlog_snapshot() + + self.assertEqual(321, backlog["pending_total"]) + self.assertEqual(2, backlog["claimed_total"]) + self.assertEqual(11, backlog["running_total"]) + self.assertEqual(50, backlog["completed_total"]) + self.assertEqual(3, backlog["failed_total"]) + self.assertEqual(210, backlog["register_pending"]) + self.assertEqual(111, backlog["downstream_pending"]) + mock_get_db.assert_not_called() + + @patch("app.services.runtime_status_service.get_active_detect_job_summary", return_value={}) + @patch("app.services.runtime_status_service.get_db") + def test_load_detect_backlog_snapshot_ignores_stale_pending_jobs( + self, + mock_get_db, + _mock_get_active_detect_job_summary, + ) -> None: + conn = MagicMock() + cursor_cm = MagicMock() + cursor = MagicMock() + cursor.fetchall.return_value = [ + (11, "running", datetime.now() - timedelta(hours=2)), + (10, "pending", datetime.now() - timedelta(minutes=30)), + (9, "pending", datetime.now() - timedelta(hours=8)), + ] + cursor.fetchone.return_value = (321, 2, 9, 50, 1, 3, 210, 111) + conn.cursor.return_value = cursor_cm + cursor_cm.__enter__.return_value = cursor + db_cm = MagicMock() + db_cm.__enter__.return_value = conn + mock_get_db.return_value = db_cm + + backlog = runtime_status_service_module._load_detect_backlog_snapshot() + + self.assertEqual(321, backlog["pending_total"]) + self.assertEqual(210, backlog["register_pending"]) + executed_sql, executed_params = cursor.execute.call_args_list[1][0] + self.assertIn("WHERE item.job_id = ANY(%s)", executed_sql) + self.assertEqual([11, 10], list(executed_params[0])) + def test_build_detect_node_row_merges_queue_running_metrics(self) -> None: row = _build_detect_node_row( node_code="mainland-worker-01", @@ -92,6 +355,42 @@ class RuntimeStatusServiceTests(unittest.TestCase): self.assertTrue(any("[mainland-controller-01]" in line for line in sliced)) self.assertTrue(any("[mainland-worker-01]" in line for line in sliced)) + def test_build_multi_region_readiness_skips_local_batch_warning_when_batches_are_not_applicable(self) -> None: + with patch("app.services.runtime_status_service.settings.node_region", "overseas"), \ + patch("app.services.runtime_status_service.settings.node_role", "control"): + readiness = runtime_status_service_module._build_multi_region_readiness( + cluster_snapshot={ + "nodes": [ + {"node_code": "overseas-control-01", "region": "overseas", "role": "control", "status": "online"}, + {"node_code": "mainland-controller-01", "region": "mainland", "role": "control", "status": "online"}, + {"node_code": "mainland-worker-01", "region": "mainland", "role": "worker", "status": "busy", "is_effective_worker": True}, + ], + "summary": { + "online_control_nodes": 2, + "online_worker_nodes": 1, + }, + }, + sync_summary={ + "enabled": False, + "source_region": "overseas", + "target_region": "mainland", + "detect_result_batches": { + "applicable": False, + "reason": "当前节点不承载本地检测执行，结果批次推送概览不适用。", + "state_counts": {"projected": 3, "failed": 1, "pushing": 1, "synced": 0}, + }, + }, + worker_runtime={"running": False}, + sync_agent_runtime={"running": False}, + ) + + warning_text = " ".join(readiness["warnings"]) + info_text = " ".join(readiness["info"]) + self.assertNotIn("结果批次仍待推送", warning_text) + self.assertNotIn("结果批次同步失败", warning_text) + self.assertIn("不适用", info_text) + self.assertFalse(readiness["sync"]["applicable"]) + def test_align_queue_health_with_backlog_prefers_larger_runtime_snapshot(self) -> None: aligned = _align_queue_health_with_backlog( { @@ -126,6 +425,227 @@ class RuntimeStatusServiceTests(unittest.TestCase): self.assertEqual(8, aligned["queue"]["failed"]) self.assertEqual(1790, aligned["queue"]["items_total"]) + def test_load_latest_remote_runtime_projection_backlog_skips_future_dated_rows(self) -> None: + now = datetime.now() + conn = MagicMock() + cursor_cm = MagicMock() + cursor = MagicMock() + cursor.fetchall.return_value = [ + ({"projection": {"backlog": {"pending_total": 9999}}}, now + timedelta(hours=4)), + ({"projection": {"backlog": {"pending_total": 321, "running_total": 8}}}, now - timedelta(minutes=2)), + ] + conn.cursor.return_value = cursor_cm + cursor_cm.__enter__.return_value = cursor + db_cm = MagicMock() + db_cm.__enter__.return_value = conn + + with patch("app.services.runtime_status_service.settings.node_region", "overseas"), \ + patch("app.services.runtime_status_service.settings.node_role", "control"), \ + patch("app.services.runtime_status_service.get_db", return_value=db_cm): + backlog = runtime_status_service_module._load_latest_remote_runtime_projection_backlog() + + self.assertEqual(321, backlog["pending_total"]) + self.assertEqual(8, backlog["running_total"]) + + def test_get_runtime_status_keeps_remote_aggregate_detect_metrics_on_overseas_control(self) -> None: + with patch("app.services.runtime_status_service.settings.node_region", "overseas"), \ + patch("app.services.runtime_status_service.settings.node_role", "control"), \ + patch("app.services.runtime_status_service.detect_worker_runtime", return_value={"running": False, "mode": "linux-systemd", "process_count": 0, "message": "inactive", "latest_start_time": ""}), \ + patch("app.services.runtime_status_service.detect_sync_agent_runtime", return_value={"running": False, "mode": "linux-systemd", "process_count": 0, "message": "inactive", "latest_start_time": ""}), \ + patch("app.services.runtime_status_service.get_runtime_settings", return_value={"worker_mode": "linux-systemd"}), \ + patch("app.services.runtime_status_service.get_detect_status", return_value={ + "phase_label": "", + "phase_detail": "", + "recent_event": "远端执行中", + "recent_warning": "", + "progress_percent": 40.49, + "progress": {"pending": 5001, "running": 138, "completed": 0, "failed": 0, "blacklisted": 0}, + "active_thread_count": 138, + "max_thread_count": 80000, + "aggregate_process_count": 80, + "aggregate_participating_node_count": 1, + "aggregate_participating_node_codes": ["mainland-controller-01"], + "aggregate_max_thread_count": 80000, + "aggregate_thread_count_per_process": 1000, + "available_proxy_count": 0, + "proxy_pool_count": 0, + "proxy_runtime_label": "", + "proxy_runtime_detail": "", + "proxy_runtime_reason": "", + "proxy_supplier_empty": False, + "proxy_last_refresh_status": "", + "proxy_last_refresh_time": "", + "proxy_last_refresh_source_count": 0, + "proxy_last_refresh_total_items": 0, + "proxy_last_validated_count": 0, + "proxy_last_available_count": 0, + "proxy_source_stats": [], + "dependency_alerts": [], + "active_job": {"job_code": "sync-overseas-51", "items_pending": 5001, "items_running": 0}, + "runs": [], + "remote_log_line_count": 0, + "remote_log_node_count": 0, + "remote_log_nodes": [], + "remote_log_node_summaries": [], + "remote_log_last_at": "", + "remote_log_last_line": "", + "remote_log_lines": [], + "aggregate_detect_view": True, + }), \ + patch("app.services.runtime_status_service.get_cluster_snapshot", return_value={ + "nodes": [{"node_code": "mainland-controller-01", "role": "control", "region": "mainland", "status": "busy", "is_effective_worker": True, "current_load": 1432, "metadata": {"active_threads": 138, "max_threads": 80000}}], + "summary": {"online_worker_nodes": 1, "dedicated_online_worker_nodes": 0, "online_control_nodes": 2}, + }), \ + patch("app.services.runtime_status_service.get_detect_queue_health", return_value={ + "has_active_job": True, + "job": {"job_id": 11, "job_code": "sync-overseas-51", "status": "running", "progress_percent": 40.49}, + "queue": {"items_total": 8402, "pending": 5001, "claimed": 0, "running": 1432, "display_running": 1432}, + "nodes": [{"node_code": "mainland-controller-01-a", "items_running": 1432, "display_running": 1432, "active_threads": 1432, "max_threads": 1000}], + "steps": [], + "runtime_activity": {}, + "window_minutes": 15, + }), \ + patch("app.services.runtime_status_service._load_detect_backlog_snapshot", return_value={"pending_total": 5001, "claimed_total": 0, "running_total": 0, "completed_total": 0, "blacklisted_total": 0, "failed_total": 0, "register_pending": 3495, "downstream_pending": 1506}), \ + patch("app.services.runtime_status_service._load_latest_remote_runtime_projection_backlog", return_value={}), \ + patch("app.services.runtime_status_service._load_latest_runtime_active_job_snapshot", return_value={}), \ + patch("app.services.runtime_status_service._load_domain_inventory_summary", return_value={"scope_label": "海外主库总盘子", "authoritative": True, "domains_total": 5690937, "pending_total": 5690918, "completed_total": 0, "running_total": 7, "blacklist_total": 0, "failed_total": 12, "processed_total": 12, "remaining_total": 5690925}), \ + patch("app.services.runtime_status_service.get_detect_capacity_plan", return_value={}), \ + patch("app.services.runtime_status_service.append_runtime_projection_if_changed"), \ + patch("app.services.runtime_status_service.get_sync_summary", return_value={}), \ + patch("app.services.runtime_status_service._build_multi_region_readiness", return_value={"status": "ready", "ready": True}), \ + patch("app.services.runtime_status_service.get_runtime_build_info", return_value={}): + payload = get_runtime_status() + + self.assertEqual(1, payload["detect"]["aggregate_process_count"]) + self.assertEqual(1432, payload["detect"]["active_thread_count"]) + self.assertEqual(1000, payload["detect"]["max_thread_count"]) + self.assertEqual("集群执行中", payload["detect"]["phase_label"]) + self.assertFalse(payload["detect"]["worker_online"]) + self.assertEqual("sync-overseas-51", payload["detect"]["active_job"]["job_code"]) + self.assertIn("observation_summary", payload["detect"]) + self.assertEqual("在跑但偏慢", payload["detect"]["observation_summary"]["state_label"]) + self.assertEqual(5690937, payload["detect"]["observation_summary"]["source_inventory"]["domains_total"]) + + def test_get_runtime_status_reuses_short_ttl_cache(self) -> None: + context_payload = { + "runtime_settings": { + "worker_mode": "linux-systemd", + "api_service_name": "domaincheck-api", + "worker_service_name": "domaincheck-worker", + "sync_agent_service_name": "domaincheck-sync-agent", + }, + "worker_runtime": { + "running": True, + "mode": "linux-systemd", + "process_count": 60, + "latest_start_time": "2026-04-24 17:10:28", + "message": "running", + "runtime_state": {"phase": "detecting"}, + }, + "worker_expected_on_this_node": True, + "detect_payload": { + "_detect_snapshot": {"thread_count": 1000}, + "active_thread_count": 321, + "max_thread_count": 60000, + "progress": {"pending": 10, "running": 3}, + "backlog": {"pending_total": 10, "running_total": 3}, + "progress_percent": 25.5, + "available_proxy_count": 12, + "proxy_pool_count": 20, + "proxy_runtime_label": "代理正常", + "proxy_runtime_detail": "ok", + "proxy_runtime_reason": "healthy", + "proxy_last_refresh_time": "2026-04-24 17:12:55", + "recent_event": "running", + "recent_warning": "", + }, + "cluster_snapshot": {"summary": {"online_worker_nodes": 1}}, + } + sync_summary = {"enabled": True} + readiness = {"status": "ready", "ready": True} + + with patch("app.services.runtime_status_service._build_runtime_detect_context", return_value=context_payload) as mock_context, \ + patch("app.services.runtime_status_service.detect_sync_agent_runtime", return_value={"running": True, "mode": "linux-systemd", "process_count": 1, "latest_start_time": "", "message": "ok"}), \ + patch("app.services.runtime_status_service.append_runtime_projection_if_changed") as mock_append, \ + patch("app.services.runtime_status_service.get_sync_summary", return_value=sync_summary) as mock_sync_summary, \ + patch("app.services.runtime_status_service._build_multi_region_readiness", return_value=readiness), \ + patch("app.services.runtime_status_service.get_runtime_build_info", return_value={"package_name": "pkg"}), \ + patch("app.services.runtime_status_service.time.monotonic", side_effect=[100.0, 100.1, 100.2]): + first = get_runtime_status() + second = get_runtime_status() + + self.assertEqual(first, second) + self.assertEqual(1, mock_context.call_count) + self.assertEqual(1, mock_append.call_count) + self.assertEqual(1, mock_sync_summary.call_count) + + @patch("app.services.runtime_status_service.append_runtime_projection_if_changed", return_value=321) + @patch( + "app.services.runtime_status_service.get_cluster_snapshot", + return_value={"nodes": [], "summary": {"online_worker_nodes": 3}}, + ) + @patch( + "app.services.runtime_status_service.get_active_detect_job_summary", + return_value={ + "job_id": 867, + "job_code": "sync-overseas-19835", + "status": "running", + "progress_percent": 99.1, + "items_total": 10000, + "items_pending": 4, + "items_claimed": 0, + "items_running": 5, + "items_completed": 55402, + "items_blacklisted": 0, + "items_failed": 0, + "display_items_running": 5, + "display_active_threads": 5, + "display_items_claimed": 0, + "display_max_threads": 1000, + "node_stats": [{"node_code": "mainland-controller-01", "items_running": 5}], + "raw_step_stats": [{"step_code": "detect_wayback", "items_pending": 4}], + }, + ) + @patch( + "app.services.runtime_status_service.detect_worker_runtime", + return_value={"running": True, "mode": "linux-systemd", "process_count": 60, "thread_count": 1000, "max_threads": 60000, "message": "ok"}, + ) + @patch("app.services.runtime_status_service.get_runtime_settings", return_value={"thread_count": 1000, "worker_mode": "linux-systemd"}) + @patch( + "app.services.runtime_status_service._load_detect_backlog_snapshot", + return_value={ + "pending_total": 4, + "claimed_total": 0, + "running_total": 5, + "completed_total": 55402, + "blacklisted_total": 0, + "failed_total": 0, + "register_pending": 0, + "downstream_pending": 4, + }, + ) + @patch("app.services.runtime_status_service._build_runtime_detect_context", side_effect=AssertionError("heavy path should not run")) + def test_refresh_runtime_projection_snapshot_uses_lightweight_context( + self, + mock_heavy_context, + mock_load_detect_backlog_snapshot, + mock_get_runtime_settings, + mock_detect_worker_runtime, + mock_get_active_detect_job_summary, + mock_get_cluster_snapshot, + mock_append_runtime_projection_if_changed, + ) -> None: + result = refresh_runtime_projection_snapshot() + + self.assertEqual(321, result["record_id"]) + mock_append_runtime_projection_if_changed.assert_called_once() + detect = mock_append_runtime_projection_if_changed.call_args.kwargs["detect"] + self.assertEqual("sync-overseas-19835", detect["active_job"]["job_code"]) + self.assertEqual(5, detect["active_thread_count"]) + self.assertEqual(4, detect["backlog"]["pending_total"]) + self.assertEqual(5, detect["queue_health"]["queue"]["running"]) + mock_heavy_context.assert_not_called() + if __name__ == "__main__": unittest.main() diff --git a/domain-api/tests/test_settings_service.py b/domain-api/tests/test_settings_service.py index 7cb1a67..ce7b375 100644 --- a/domain-api/tests/test_settings_service.py +++ b/domain-api/tests/test_settings_service.py @@ -1,6 +1,11 @@ import unittest -from app.services.settings_service import _normalize_thread_count, resolve_thread_count +from app.services.settings_service import ( + _normalize_process_count, + _normalize_thread_count, + resolve_process_count, + resolve_thread_count, +) class SettingsServiceTests(unittest.TestCase): @@ -26,6 +31,25 @@ class SettingsServiceTests(unittest.TestCase): self.assertEqual(512, resolved["effective_thread_count"]) self.assertEqual("node_override", resolved["source"]) + def test_normalize_process_count_rejects_non_positive_values(self) -> None: + with self.assertRaises(ValueError): + _normalize_process_count(0) + + def test_resolve_process_count_uses_large_node_override(self) -> None: + payload = { + "process_count": 80, + "node_process_counts": { + "mainland-controller-01": 96, + }, + } + + resolved = resolve_process_count(node_code="mainland-controller-01", settings_payload=payload) + + self.assertEqual(80, resolved["default_process_count"]) + self.assertEqual(96, resolved["override_process_count"]) + self.assertEqual(96, resolved["effective_process_count"]) + self.assertEqual("node_override", resolved["source"]) + if __name__ == "__main__": unittest.main() diff --git a/domain-api/tests/test_sync_agent.py b/domain-api/tests/test_sync_agent.py index 6ec1b24..fb7e09e 100644 --- a/domain-api/tests/test_sync_agent.py +++ b/domain-api/tests/test_sync_agent.py @@ -4,6 +4,9 @@ from unittest.mock import patch from app.sync_agent import ( _append_detect_result_projection_snapshot, _build_aligned_queue_health_snapshot, + _emit_runtime_debug_snapshots, + _maybe_trigger_overlap_start, + _run_sync_tick_once, _emit_structured_tick, _emit_sync_result_breakdown, _filter_runtime_events_for_job, @@ -187,6 +190,140 @@ class SyncAgentTests(unittest.TestCase): mock_process_detect_pipeline_now.assert_called_once_with(limit=5000) self.assertEqual("pipeline_tick_success", mock_push_debug_event.call_args.kwargs["event_type"]) + @patch("app.sync_agent._load_local_detect_backlog_snapshot", return_value={"pending_total": 12}) + @patch("app.sync_agent.list_recent_detect_run_events", return_value=[]) + @patch("app.sync_agent.get_detect_queue_health", return_value={"queue": {"overdue_leases": 0}}) + @patch("app.sync_agent.push_debug_event") + @patch("app.sync_agent._append_detect_result_projection_snapshot") + @patch("app.sync_agent._select_projection_job_snapshots", return_value=[{"job_id": 1, "job_code": "finished-job"}]) + @patch( + "app.sync_agent.get_active_detect_job_summary", + return_value={ + "job_id": 2, + "job_code": "running-job", + "status": "running", + "items_total": 100, + "items_pending": 10, + "items_claimed": 5, + "items_running": 7, + "items_completed": 70, + "items_failed": 8, + "progress_percent": 70.0, + "node_stats": [{"node_code": "mainland-controller-01", "items_total": 90}], + }, + ) + def test_emit_runtime_debug_snapshots_emits_projection_and_active_job( + self, + mock_get_active_detect_job_summary, + mock_select_projection_job_snapshots, + mock_append_detect_result_projection_snapshot, + mock_push_debug_event, + mock_get_detect_queue_health, + mock_list_recent_detect_run_events, + mock_load_local_detect_backlog_snapshot, + ) -> None: + _emit_runtime_debug_snapshots() + + mock_get_active_detect_job_summary.assert_called_once_with(event_limit=10) + mock_select_projection_job_snapshots.assert_called_once_with() + mock_append_detect_result_projection_snapshot.assert_called_once() + self.assertTrue(mock_push_debug_event.called) + self.assertEqual("active_job_snapshot", mock_push_debug_event.call_args_list[0].kwargs["event_type"]) + + @patch("app.sync_agent._emit_runtime_debug_snapshots") + @patch("app.sync_agent._run_pipeline_stage_processor", return_value=(True, "pipeline ok", {"stage": "pipeline"})) + @patch("app.sync_agent.pull_detect_task_batch_now", return_value=(True, "pull ok", {"stage": "pull"})) + @patch("app.sync_agent.push_runtime_projection_now", return_value=(True, "sync ok", {"stage": "sync"})) + def test_run_sync_tick_once_prioritizes_sync_before_pipeline( + self, + mock_push_runtime_projection_now, + mock_pull_detect_task_batch_now, + mock_run_pipeline_stage_processor, + mock_emit_runtime_debug_snapshots, + ) -> None: + with patch("app.sync_agent._maybe_trigger_overlap_start", return_value=(False, "no overlap", {})) as mock_overlap: + tick = _run_sync_tick_once() + + self.assertTrue(tick["sync"]["ok"]) + self.assertEqual("sync ok", tick["sync"]["message"]) + mock_push_runtime_projection_now.assert_called_once_with() + mock_pull_detect_task_batch_now.assert_called_once_with() + mock_run_pipeline_stage_processor.assert_called_once_with() + mock_overlap.assert_called_once_with() + mock_emit_runtime_debug_snapshots.assert_called_once_with() + self.assertEqual("no overlap", tick["overlap"]["message"]) + + @patch("app.sync_agent.send_worker_command") + @patch("app.sync_agent._select_overlap_target_node_codes") + @patch("app.sync_agent._select_overlap_start_candidate") + @patch("app.sync_agent.push_debug_event") + def test_maybe_trigger_overlap_start_dispatches_pending_job( + self, + mock_push_debug_event, + mock_select_overlap_start_candidate, + mock_select_overlap_target_node_codes, + mock_send_worker_command, + ) -> None: + mock_select_overlap_start_candidate.return_value = { + "job_id": 885, + "job_code": "sync-overseas-20592", + "task_mode": "domain_pipeline", + "items_pending": 64000, + "items_claimed": 0, + "items_running": 0, + "selection_reason": "overlap_tail_handoff", + } + mock_select_overlap_target_node_codes.return_value = [ + "mainland-controller-01-a", + "mainland-controller-01-da", + ] + mock_send_worker_command.return_value = (True, "已发送 Worker 控制指令: start_detection -> mainland-controller-01-a,mainland-controller-01-da") + + with patch("app.sync_agent._LAST_OVERLAP_JOB_ID", 0), patch("app.sync_agent._LAST_OVERLAP_TRIGGERED_AT", 0.0): + ok, message, data = _maybe_trigger_overlap_start() + + self.assertTrue(ok) + self.assertIn("mainland-controller-01-a", message) + self.assertEqual(885, data["job_id"]) + mock_send_worker_command.assert_called_once_with( + "start_detection", + payload={ + "job_id": 885, + "job_code": "sync-overseas-20592", + "target_job_id": 885, + "target_job_code": "sync-overseas-20592", + "task_mode": "domain_pipeline", + "source": "overlap-handoff", + "selection_reason": "overlap_tail_handoff", + "tail_handoff_candidate": True, + "target_node_codes": [ + "mainland-controller-01-a", + "mainland-controller-01-da", + ], + }, + ) + self.assertEqual("overlap_handoff_started", mock_push_debug_event.call_args.kwargs["event_type"]) + + @patch("app.sync_agent._select_overlap_start_candidate") + def test_maybe_trigger_overlap_start_honors_cooldown_for_same_job( + self, + mock_select_overlap_start_candidate, + ) -> None: + mock_select_overlap_start_candidate.return_value = { + "job_id": 885, + "job_code": "sync-overseas-20592", + "task_mode": "domain_pipeline", + "items_pending": 64000, + "selection_reason": "overlap_tail_handoff", + } + + with patch("app.sync_agent._LAST_OVERLAP_JOB_ID", 885), patch("app.sync_agent._LAST_OVERLAP_TRIGGERED_AT", __import__('time').time()): + ok, message, data = _maybe_trigger_overlap_start() + + self.assertFalse(ok) + self.assertIn("冷却中", message) + self.assertEqual(885, data["job_id"]) + if __name__ == "__main__": unittest.main() diff --git a/domain-api/tests/test_sync_push_service.py b/domain-api/tests/test_sync_push_service.py index 0565fd2..747e2b1 100644 --- a/domain-api/tests/test_sync_push_service.py +++ b/domain-api/tests/test_sync_push_service.py @@ -1,10 +1,20 @@ import unittest +from datetime import datetime, timedelta from unittest.mock import patch from app.services.sync_push_service import ( _build_task_pull_backlog_limits, _extract_detect_result_projection_events, + _load_latest_projection, + _load_pushable_projections, + _refresh_remote_runtime_node, + _load_local_detect_backlog_snapshot, + _push_projection_now, + _resolve_task_pull_request_limit, + _task_projection_limit, + pull_detect_task_batch_now, _resolve_detect_result_target_job_id, + _select_relevant_backlog_job_ids_from_rows, _should_throttle_task_pull, ingest_runtime_projection, ) @@ -23,6 +33,12 @@ class _FakeCursor: return self._rows.pop(0) return None + def fetchall(self): + if self._rows: + value = self._rows.pop(0) + return list(value or []) + return [] + def __enter__(self): return self @@ -48,23 +64,311 @@ class _FakeConnection: return False +class _FakeUrlopenResponse: + def __init__(self, payload): + self._payload = payload + + def read(self): + import json + + return json.dumps(self._payload, ensure_ascii=False).encode("utf-8") + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + class SyncPushServiceTests(unittest.TestCase): - def test_build_task_pull_backlog_limits_scales_with_thread_configuration(self) -> None: + @patch("app.services.sync_push_service.cleanup_imported_runtime_nodes_many") + @patch("app.services.sync_push_service.cleanup_imported_runtime_nodes") + @patch("app.services.sync_push_service.register_node_heartbeat") + def test_refresh_remote_runtime_node_imports_cluster_worker_rows_without_active_job( + self, + mock_register_node_heartbeat, + _mock_cleanup_imported_runtime_nodes, + mock_cleanup_imported_runtime_nodes_many, + ) -> None: + _refresh_remote_runtime_node( + source_region="mainland", + projection={ + "node": { + "node_code": "mainland-controller-01", + "region": "mainland", + "role": "control", + "hostname": "localhost", + "ip": "127.0.0.1", + }, + "worker_mode": "linux-systemd", + "phase_label": "等待中", + "phase_detail": "等待任务", + "proxy_runtime_label": "正常", + "proxy_runtime_reason": "healthy", + "active_thread_count": 0, + "max_thread_count": 60000, + "worker_online": True, + "detect_participating": False, + "active_job": { + "job_code": "", + "status": "", + "node_stats": [], + }, + "cluster_nodes": [ + { + "node_code": "mainland-controller-01", + "role": "control", + "status": "online", + "current_load": 0, + "active_threads": 0, + "max_threads": 60000, + "detect_participating": False, + }, + { + "node_code": "mainland-controller-01-a", + "role": "worker", + "status": "online", + "current_load": 0, + "active_threads": 0, + "max_threads": 1000, + "detect_participating": False, + }, + { + "node_code": "mainland-controller-01-b", + "role": "worker", + "status": "busy", + "current_load": 12, + "active_threads": 12, + "max_threads": 1000, + "detect_participating": True, + }, + ], + }, + ) + + self.assertEqual(3, mock_register_node_heartbeat.call_count) + worker_calls = [call.kwargs for call in mock_register_node_heartbeat.call_args_list[1:]] + self.assertEqual( + ["mainland-controller-01-a", "mainland-controller-01-b"], + [item["node_code"] for item in worker_calls], + ) + self.assertEqual(12, worker_calls[1]["current_load"]) + self.assertEqual( + ["mainland-controller-01-a", "mainland-controller-01-b"], + mock_cleanup_imported_runtime_nodes_many.call_args.kwargs["keep_node_codes"], + ) + + @patch("app.services.sync_push_service.settings") + def test_task_pull_request_limit_scales_with_estimated_threads(self, mock_settings) -> None: + mock_settings.node_code = "mainland-controller-01" + mock_settings.sync_batch_size = 5000 + + safe_limit = _resolve_task_pull_request_limit( + None, + settings_payload={ + "thread_count": 1000, + "process_count": 1, + "node_thread_counts": {"mainland-controller-01": 1000}, + "node_process_counts": {"mainland-controller-01": 60}, + }, + ) + + self.assertEqual(120000, safe_limit) + + @patch("app.services.sync_push_service.settings") + def test_task_projection_limit_allows_large_explicit_pull_request(self, mock_settings) -> None: + mock_settings.sync_batch_size = 5000 + + self.assertEqual(120000, _task_projection_limit(120000)) + + @patch("app.services.sync_push_service.settings") + @patch("app.services.sync_push_service.get_db") + def test_load_latest_projection_prefers_non_future_row_over_clock_skewed_history( + self, + mock_get_db, + mock_settings, + ) -> None: + now = datetime.now() + future_row = ( + 10081, + "mainland", + "overseas", + "projected", + {"projection": {"active_thread_count": 138}}, + now + timedelta(hours=4), + now + timedelta(hours=4), + ) + valid_row = ( + 11633, + "mainland", + "overseas", + "projected", + {"projection": {"active_thread_count": 6431}}, + now - timedelta(minutes=2), + now - timedelta(minutes=2), + ) + fake_conn = _FakeConnection(rows=[[future_row, valid_row]]) + mock_get_db.return_value = fake_conn + mock_settings.sync_source_region = "mainland" + mock_settings.node_region = "mainland" + mock_settings.sync_target_region = "overseas" + + with patch("app.services.sync_push_service.datetime") as mock_datetime: + mock_datetime.now.return_value = now + record = _load_latest_projection("runtime_projection") + + self.assertIsNotNone(record) + self.assertEqual(11633, record["id"]) + executed_sql, executed_params = fake_conn.cursor_obj.executed[0] + self.assertIn("CASE WHEN created_at <= %s THEN 0 ELSE 1 END", executed_sql) + self.assertEqual("runtime_projection", executed_params[0]) + self.assertEqual("mainland", executed_params[1]) + self.assertEqual("overseas", executed_params[2]) + self.assertEqual(now + timedelta(minutes=5), executed_params[3]) + + @patch("app.services.sync_push_service._latest_push_attempt") + @patch("app.services.sync_push_service.settings") + @patch("app.services.sync_push_service.get_db") + def test_load_pushable_projections_prefers_latest_unsent_records( + self, + mock_get_db, + mock_settings, + mock_latest_push_attempt, + ) -> None: + now = datetime.now() + old_success_rows = [ + ( + 10000 + index, + "mainland", + "overseas", + "projected", + {"projection_hash": f"old-{index}", "projection": {"job": {"job_id": index}}}, + now - timedelta(minutes=120 - index), + now - timedelta(minutes=120 - index), + ) + for index in range(100) + ] + newest_unsent = ( + 23360, + "mainland", + "overseas", + "projected", + {"projection_hash": "new-hash", "projection": {"job": {"job_id": 909, "job_code": "sync-overseas-28618"}}}, + now, + now, + ) + fake_conn = _FakeConnection(rows=[old_success_rows + [newest_unsent]]) + mock_get_db.return_value = fake_conn + mock_settings.sync_source_region = "mainland" + mock_settings.node_region = "mainland" + mock_settings.sync_target_region = "overseas" + mock_settings.sync_batch_size = 20 + + def _attempt_side_effect(source_record_id, target_region, sync_type): + if source_record_id == 23360: + return None + return {"status": "success", "created_at": now} + + mock_latest_push_attempt.side_effect = _attempt_side_effect + + records = _load_pushable_projections("detect_result_projection", limit=20) + + self.assertEqual([23360], [item["id"] for item in records]) + executed_sql, _ = fake_conn.cursor_obj.executed[0] + self.assertIn("ORDER BY created_at DESC, id DESC", executed_sql) + + @patch("app.services.sync_push_service.settings.node_code", "mainland-controller-01") + def test_build_task_pull_backlog_limits_scales_with_local_server_capacity(self) -> None: limits = _build_task_pull_backlog_limits( 5000, settings_payload={ "thread_count": 100, + "process_count": 80, "node_thread_counts": { - "mainland-controller-01": 2000, + "mainland-controller-01": 1000, + "mainland-controller-01-a": 1000, "mainland-worker-01": 1200, }, + "node_process_counts": { + "mainland-controller-01": 80, + "mainland-worker-01": 60, + }, }, ) - self.assertEqual(3200, limits["estimated_total_threads"]) - self.assertEqual(6400, limits["max_pending_total"]) - self.assertEqual(3200, limits["max_register_pending"]) - self.assertEqual(800, limits["max_downstream_pending"]) + self.assertEqual(80000, limits["estimated_total_threads"]) + self.assertEqual(160000, limits["max_pending_total"]) + self.assertEqual(80000, limits["max_register_pending"]) + self.assertEqual(20000, limits["max_downstream_pending"]) + + def test_select_relevant_backlog_job_ids_from_rows_skips_stale_pending_jobs(self) -> None: + now = datetime.now() + + job_ids = _select_relevant_backlog_job_ids_from_rows( + [ + (11, "running", now - timedelta(hours=10)), + (10, "pending", now - timedelta(minutes=30)), + (9, "pending", now - timedelta(hours=7)), + ], + freshness_hours=6, + limit=4, + ) + + self.assertEqual([11, 10], job_ids) + + @patch("app.services.sync_push_service.get_active_detect_job_summary") + @patch("app.services.sync_push_service.get_db") + def test_load_local_detect_backlog_snapshot_prefers_active_job_snapshot( + self, + mock_get_db, + mock_get_active_detect_job_summary, + ) -> None: + mock_get_active_detect_job_summary.return_value = { + "items_pending": 123, + "items_claimed": 4, + "items_running": 7, + "display_items_running": 9, + "step_stats": [ + {"step_code": "detect_register", "items_pending": 90}, + {"step_code": "detect_baidu", "items_pending": 33}, + ], + } + + backlog = _load_local_detect_backlog_snapshot() + + self.assertEqual(123, backlog["pending_total"]) + self.assertEqual(4, backlog["claimed_total"]) + self.assertEqual(9, backlog["running_total"]) + self.assertEqual(90, backlog["register_pending"]) + self.assertEqual(33, backlog["downstream_pending"]) + mock_get_db.assert_not_called() + + @patch("app.services.sync_push_service.get_active_detect_job_summary", return_value={}) + @patch("app.services.sync_push_service.get_db") + def test_load_local_detect_backlog_snapshot_only_counts_relevant_jobs( + self, + mock_get_db, + _mock_get_active_detect_job_summary, + ) -> None: + fake_conn = _FakeConnection( + rows=[ + [ + (11, "running", datetime.now() - timedelta(hours=2)), + (10, "pending", datetime.now() - timedelta(minutes=30)), + (9, "pending", datetime.now() - timedelta(hours=7)), + ], + (123, 4, 7, 90, 33), + ] + ) + mock_get_db.return_value = fake_conn + + backlog = _load_local_detect_backlog_snapshot() + + self.assertEqual(123, backlog["pending_total"]) + self.assertEqual(90, backlog["register_pending"]) + executed_sql, executed_params = fake_conn.cursor_obj.executed[1] + self.assertIn("WHERE item.job_id = ANY(%s)", executed_sql) + self.assertEqual([11, 10], list(executed_params[0])) def test_should_throttle_task_pull_when_register_backlog_overwhelms_downstream(self) -> None: should_throttle, reason = _should_throttle_task_pull( @@ -104,6 +408,224 @@ class SyncPushServiceTests(unittest.TestCase): self.assertFalse(should_throttle) self.assertEqual("", reason) + @patch("app.services.sync_push_service._should_throttle_task_pull", return_value=(False, "")) + @patch("app.services.sync_push_service._build_task_pull_backlog_limits", return_value={}) + @patch("app.services.sync_push_service._load_local_detect_backlog_snapshot", return_value={}) + @patch("app.services.sync_push_service._acquire_sync_pull_worker_wake_guard", return_value=True) + @patch("app.services.sync_push_service.ingest_detect_task_projection") + @patch("app.services.worker_control_service.send_worker_command") + @patch("app.services.sync_push_service.urllib.request.urlopen") + @patch("app.services.sync_push_service.settings") + @patch("app.services.sync_push_service.get_settings_payload") + def test_pull_detect_task_batch_now_starts_worker_with_projection_active_job_identity( + self, + mock_get_settings_payload, + mock_settings, + mock_urlopen, + mock_send_worker_command, + mock_ingest_detect_task_projection, + _mock_acquire_sync_pull_worker_wake_guard, + _mock_load_local_detect_backlog_snapshot, + _mock_build_task_pull_backlog_limits, + _mock_should_throttle_task_pull, + ) -> None: + mock_settings.node_region = "mainland" + mock_settings.node_role = "control" + mock_settings.node_code = "mainland-controller-01" + mock_settings.sync_target_api_base_url = "http://example.com" + mock_settings.sync_target_region = "overseas" + mock_settings.sync_batch_size = 200 + mock_settings.sync_shared_token = "" + mock_get_settings_payload.return_value = { + "thread_count": 1000, + "process_count": 1, + "node_thread_counts": {"mainland-controller-01": 1000}, + "node_process_counts": {"mainland-controller-01": 60}, + } + + projection = { + "batch_code": "task-20260423210000-aa11bb", + "active_job": { + "job_id": 376, + "job_code": "sync-overseas-376", + "current_cycle_token": "cycle-376", + }, + } + export_response = { + "code": 0, + "data": { + "source_record_id": 15164, + "projection_hash": "hash-15164", + "projection": projection, + }, + } + ack_response = {"code": 0, "data": {"acknowledged": True}} + mock_urlopen.side_effect = [ + _FakeUrlopenResponse(export_response), + _FakeUrlopenResponse(ack_response), + ] + mock_ingest_detect_task_projection.return_value = ( + True, + "任务批次接收成功", + { + "target_job_id": 1902, + "target_job_code": "sync-overseas-15164", + "queued_count": 1000, + }, + ) + mock_send_worker_command.return_value = (True, "started") + + ok, message, payload = pull_detect_task_batch_now(limit=1000) + + self.assertTrue(ok) + self.assertEqual("待检测任务批次拉取并入库成功", message) + self.assertTrue(payload["worker_start_ok"]) + export_request = mock_urlopen.call_args_list[0][0][0] + self.assertIn("limit=1000", export_request.full_url) + mock_send_worker_command.assert_called_once() + args, kwargs = mock_send_worker_command.call_args + self.assertEqual("start_detection", args[0]) + sent_payload = kwargs["payload"] + self.assertEqual(15164, sent_payload["source_record_id"]) + self.assertEqual(1902, sent_payload["target_job_id"]) + self.assertEqual("sync-overseas-15164", sent_payload["target_job_code"]) + self.assertEqual(376, sent_payload["job_id"]) + self.assertEqual("sync-overseas-376", sent_payload["job_code"]) + self.assertEqual("cycle-376", sent_payload["cycle_token"]) + + @patch("app.services.sync_push_service._should_throttle_task_pull", return_value=(False, "")) + @patch("app.services.sync_push_service._build_task_pull_backlog_limits", return_value={}) + @patch("app.services.sync_push_service._load_local_detect_backlog_snapshot", return_value={}) + @patch("app.services.sync_push_service._acquire_sync_pull_worker_wake_guard", return_value=False) + @patch("app.services.sync_push_service.ingest_detect_task_projection") + @patch("app.services.worker_control_service.send_worker_command") + @patch("app.services.sync_push_service.urllib.request.urlopen") + @patch("app.services.sync_push_service.settings") + @patch("app.services.sync_push_service.get_settings_payload") + def test_pull_detect_task_batch_now_skips_duplicate_worker_wake_within_short_window( + self, + mock_get_settings_payload, + mock_settings, + mock_urlopen, + mock_send_worker_command, + mock_ingest_detect_task_projection, + _mock_acquire_sync_pull_worker_wake_guard, + _mock_load_local_detect_backlog_snapshot, + _mock_build_task_pull_backlog_limits, + _mock_should_throttle_task_pull, + ) -> None: + mock_settings.node_region = "mainland" + mock_settings.node_role = "control" + mock_settings.node_code = "mainland-controller-01" + mock_settings.sync_target_api_base_url = "http://example.com" + mock_settings.sync_target_region = "overseas" + mock_settings.sync_batch_size = 200 + mock_settings.sync_shared_token = "" + mock_get_settings_payload.return_value = { + "thread_count": 1000, + "process_count": 1, + "node_thread_counts": {"mainland-controller-01": 1000}, + "node_process_counts": {"mainland-controller-01": 60}, + } + + projection = { + "batch_code": "task-20260423210000-aa11bb", + "active_job": { + "job_id": 376, + "job_code": "sync-overseas-376", + "current_cycle_token": "cycle-376", + }, + } + export_response = { + "code": 0, + "data": { + "source_record_id": 15164, + "projection_hash": "hash-15164", + "projection": projection, + }, + } + ack_response = {"code": 0, "data": {"acknowledged": True}} + mock_urlopen.side_effect = [ + _FakeUrlopenResponse(export_response), + _FakeUrlopenResponse(ack_response), + ] + mock_ingest_detect_task_projection.return_value = ( + True, + "任务批次接收成功", + { + "target_job_id": 1902, + "target_job_code": "sync-overseas-15164", + "queued_count": 1000, + }, + ) + + ok, message, payload = pull_detect_task_batch_now(limit=1000) + + self.assertTrue(ok) + self.assertEqual("待检测任务批次拉取并入库成功", message) + self.assertTrue(payload["worker_start_ok"]) + self.assertTrue(payload["worker_start_skipped"]) + self.assertIn("重复 Worker 唤起", payload["worker_start_message"]) + mock_send_worker_command.assert_not_called() + + @patch("app.services.sync_push_service._should_throttle_task_pull", return_value=(False, "")) + @patch("app.services.sync_push_service._build_task_pull_backlog_limits", return_value={}) + @patch("app.services.sync_push_service._load_local_detect_backlog_snapshot", return_value={}) + @patch("app.services.sync_push_service._acquire_sync_pull_worker_wake_guard", return_value=False) + @patch("app.services.sync_push_service.ingest_detect_task_projection") + @patch("app.services.sync_push_service.urllib.request.urlopen") + @patch("app.services.sync_push_service.settings") + @patch("app.services.sync_push_service.get_settings_payload") + def test_pull_detect_task_batch_now_uses_adaptive_limit_when_unspecified( + self, + mock_get_settings_payload, + mock_settings, + mock_urlopen, + mock_ingest_detect_task_projection, + _mock_acquire_sync_pull_worker_wake_guard, + _mock_load_local_detect_backlog_snapshot, + _mock_build_task_pull_backlog_limits, + _mock_should_throttle_task_pull, + ) -> None: + mock_settings.node_region = "mainland" + mock_settings.node_role = "control" + mock_settings.node_code = "mainland-controller-01" + mock_settings.sync_target_api_base_url = "http://example.com" + mock_settings.sync_target_region = "overseas" + mock_settings.sync_batch_size = 5000 + mock_settings.sync_shared_token = "" + mock_get_settings_payload.return_value = { + "thread_count": 1000, + "process_count": 1, + "node_thread_counts": {"mainland-controller-01": 1000}, + "node_process_counts": {"mainland-controller-01": 60}, + } + + export_response = { + "code": 0, + "data": { + "source_record_id": 15164, + "projection_hash": "hash-15164", + "projection": {"batch_code": "task-20260423210000-aa11bb", "active_job": {}}, + }, + } + ack_response = {"code": 0, "data": {"acknowledged": True}} + mock_urlopen.side_effect = [ + _FakeUrlopenResponse(export_response), + _FakeUrlopenResponse(ack_response), + ] + mock_ingest_detect_task_projection.return_value = ( + True, + "任务批次接收成功", + {"target_job_id": 1902, "target_job_code": "sync-overseas-15164", "queued_count": 1000}, + ) + + ok, _message, _payload = pull_detect_task_batch_now(limit=None) + + self.assertTrue(ok) + export_request = mock_urlopen.call_args_list[0][0][0] + self.assertIn("limit=120000", export_request.full_url) + def test_extract_detect_result_projection_events_adds_import_metadata(self) -> None: projection = { "job": { @@ -137,6 +659,79 @@ class SyncPushServiceTests(unittest.TestCase): self.assertTrue(event["payload"]["imported_from_projection"]) self.assertTrue(event["payload"]["import_fingerprint"]) + @patch("app.services.sync_push_service._push_projection_record") + @patch("app.services.sync_push_service._load_latest_projection") + @patch("app.services.runtime_status_service.refresh_runtime_projection_snapshot") + def test_push_projection_now_refreshes_runtime_projection_with_lightweight_snapshot( + self, + mock_refresh_runtime_projection_snapshot, + mock_load_latest_projection, + mock_push_projection_record, + ) -> None: + mock_refresh_runtime_projection_snapshot.return_value = { + "record_id": 10082, + "active_thread_count": 1972, + "max_thread_count": 80000, + "queue_display_running": 1972, + } + mock_load_latest_projection.return_value = { + "id": 10082, + "source_region": "mainland", + "target_region": "overseas", + "created_at": None, + "payload": {"projection_hash": "hash-10082", "projection": {}}, + } + mock_push_projection_record.return_value = ( + True, + "投影推送成功", + {"action": "push_sync", "sync_type": "runtime_projection", "source_record_id": 10082}, + ) + + ok, message, data = _push_projection_now("runtime_projection", "https://example.com/api/v1/runtime/sync-ingest") + + self.assertTrue(ok) + self.assertEqual("投影推送成功", message) + self.assertEqual(10082, data["source_record_id"]) + mock_refresh_runtime_projection_snapshot.assert_called_once_with(window_minutes=15) + mock_load_latest_projection.assert_called_once_with("runtime_projection") + mock_push_projection_record.assert_called_once() + + @patch("app.services.sync_push_service._push_projection_record") + @patch("app.services.sync_push_service._load_latest_projection") + @patch("app.services.sync_push_service._append_fast_runtime_projection_snapshot") + @patch("app.services.runtime_status_service.refresh_runtime_projection_snapshot") + def test_push_projection_now_can_use_fast_runtime_projection_path( + self, + mock_refresh_runtime_projection_snapshot, + mock_append_fast_runtime_projection_snapshot, + mock_load_latest_projection, + mock_push_projection_record, + ) -> None: + mock_append_fast_runtime_projection_snapshot.return_value = 10091 + mock_load_latest_projection.return_value = { + "id": 10091, + "source_region": "mainland", + "target_region": "overseas", + "created_at": None, + "payload": {"projection_hash": "hash-10091", "projection": {}}, + } + mock_push_projection_record.return_value = ( + True, + "投影推送成功", + {"action": "push_sync", "sync_type": "runtime_projection", "source_record_id": 10091}, + ) + + with patch.dict("os.environ", {"DOMAINCHECK_SYNC_RUNTIME_FAST_PROJECTION": "1"}, clear=False): + ok, message, data = _push_projection_now("runtime_projection", "https://example.com/api/v1/runtime/sync-ingest") + + self.assertTrue(ok) + self.assertEqual("投影推送成功", message) + self.assertEqual(10091, data["source_record_id"]) + mock_append_fast_runtime_projection_snapshot.assert_called_once() + mock_refresh_runtime_projection_snapshot.assert_not_called() + mock_load_latest_projection.assert_called_once_with("runtime_projection") + mock_push_projection_record.assert_called_once() + @patch("app.services.sync_push_service.get_db") def test_resolve_detect_result_target_job_id_prefers_matching_job_code(self, mock_get_db) -> None: fake_conn = _FakeConnection(rows=[(456,)]) diff --git a/domain-api/tests/test_sync_record_service.py b/domain-api/tests/test_sync_record_service.py index fb4e4c3..c9189b4 100644 --- a/domain-api/tests/test_sync_record_service.py +++ b/domain-api/tests/test_sync_record_service.py @@ -1,12 +1,61 @@ +import json import unittest +from datetime import datetime, timedelta from unittest.mock import patch from app.services.sync_record_service import ( _build_runtime_projection_payload, _collect_recent_domain_events, + _pick_latest_projection_row, + append_runtime_projection_if_changed, + get_detect_result_sync_batches, ) +class _FakeCursor: + def __init__(self, fetchone_values): + self.fetchone_values = list(fetchone_values or []) + self.executed = [] + + def execute(self, sql, params=None): + self.executed.append((sql, params)) + + def fetchone(self): + if self.fetchone_values: + return self.fetchone_values.pop(0) + return None + + def fetchall(self): + if self.fetchone_values: + value = self.fetchone_values.pop(0) + return list(value or []) + return [] + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + +class _FakeConnection: + def __init__(self, fetchone_values): + self.cursor_obj = _FakeCursor(fetchone_values) + self.committed = False + + def cursor(self): + return self.cursor_obj + + def commit(self): + self.committed = True + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + class SyncRecordServiceTests(unittest.TestCase): def test_collect_recent_domain_events_filters_and_keeps_latest_slice(self) -> None: active_job = { @@ -130,6 +179,682 @@ class SyncRecordServiceTests(unittest.TestCase): self.assertEqual("", projection["active_job"]["job_code"]) self.assertEqual([], projection["active_job"]["node_stats"]) + @patch("app.services.sync_record_service._resolve_local_ip", return_value="121.204.244.188") + @patch("app.services.sync_record_service.socket.gethostname", return_value="mainland-controller-01") + @patch("app.services.sync_record_service.settings") + def test_build_runtime_projection_payload_prefers_runtime_queue_nodes_for_multi_process_controller( + self, + mock_settings, + _mock_hostname, + _mock_resolve_ip, + ) -> None: + mock_settings.node_code = "mainland-controller-01" + mock_settings.node_region = "mainland" + mock_settings.node_role = "control" + mock_settings.sync_source_region = "mainland" + mock_settings.sync_target_region = "overseas" + + payload = _build_runtime_projection_payload( + detect={ + "worker_online": True, + "worker_mode": "linux-systemd", + "active_thread_count": 1, + "max_thread_count": 1000, + "aggregate_max_thread_count": 80000, + "phase_label": "运行中", + "phase_detail": "80 实例运行", + "proxy_runtime_label": "正常", + "proxy_runtime_reason": "healthy", + "progress": { + "pending": 5001, + "running": 0, + "completed": 0, + "blacklisted": 0, + "failed": 0, + }, + "backlog": {"pending_total": 1157325}, + "queue_health": { + "queue": { + "items_total": 5292, + "pending": 4578, + "claimed": 0, + "display_claimed": 120, + "running": 1972, + "display_running": 1972, + "completed": 690, + "blacklisted": 0, + "failed": 24, + "terminal": 714, + }, + "nodes": [ + { + "node_code": "mainland-controller-01-a", + "items_running": 1000, + "items_claimed": 0, + "display_running": 1000, + "active_threads": 1000, + "max_threads": 1000, + "region": "mainland", + "role": "control", + "status": "busy", + }, + { + "node_code": "mainland-controller-01-b", + "items_running": 972, + "items_claimed": 120, + "display_running": 972, + "active_threads": 972, + "max_threads": 1000, + "region": "mainland", + "role": "control", + "status": "busy", + }, + ], + }, + "active_job": { + "job_id": 55, + "job_code": "sync-overseas-55", + "status": "running", + "progress_percent": 13.49, + "items_total": 5292, + "items_terminal": 714, + "items_pending": 4578, + "items_running": 0, + "items_failed": 24, + "node_stats": [ + { + "node_code": "mainland-controller-01", + "items_running": 1, + "items_claimed": 0, + "items_total": 5292, + } + ], + }, + "dependency_alerts": [], + }, + cluster={ + "nodes_total": 80, + "nodes": [ + { + "node_code": "mainland-controller-01", + "current_load": 1, + "detect_participating": True, + } + ], + "summary": { + "online_worker_nodes": 65, + "dedicated_online_worker_nodes": 1, + "online_control_nodes": 65, + "busy_nodes": ["mainland-controller-01-a", "mainland-controller-01-b"], + "stale_nodes": [], + "offline_nodes": [], + }, + }, + source_region="mainland", + target_region="overseas", + ) + + projection = payload["projection"] + self.assertTrue(projection["detect_participating"]) + self.assertEqual(1972, projection["active_thread_count"]) + self.assertEqual(80000, projection["max_thread_count"]) + self.assertEqual(1972, projection["progress"]["running"]) + self.assertEqual(2, len(projection["active_job"]["node_stats"])) + self.assertEqual(2, len(projection["active_job"]["distributed_node_stats"])) + self.assertEqual(1, len(projection["cluster_nodes"])) + self.assertEqual("mainland-controller-01", projection["cluster_nodes"][0]["node_code"]) + self.assertEqual(1972, projection["active_job"]["items_running"]) + self.assertEqual(1972, projection["active_job"]["display_items_running"]) + self.assertEqual(80000, projection["active_job"]["display_max_threads"]) + + @patch("app.services.sync_record_service._resolve_local_ip", return_value="121.204.244.188") + @patch("app.services.sync_record_service.socket.gethostname", return_value="mainland-controller-01") + @patch("app.services.sync_record_service.settings") + @patch("app.services.sync_record_service.get_db") + def test_append_runtime_projection_if_changed_writes_heartbeat_for_unchanged_projection_after_interval( + self, + mock_get_db, + mock_settings, + _mock_hostname, + _mock_resolve_ip, + ) -> None: + mock_settings.node_code = "mainland-controller-01" + mock_settings.node_region = "mainland" + mock_settings.node_role = "control" + mock_settings.sync_source_region = "mainland" + mock_settings.sync_target_region = "overseas" + + detect = { + "worker_online": True, + "worker_mode": "linux-systemd", + "active_thread_count": 3200, + "max_thread_count": 80000, + "phase_label": "运行中", + "phase_detail": "80 实例运行", + "proxy_runtime_label": "正常", + "proxy_runtime_reason": "healthy", + "progress": { + "pending": 1200000, + "running": 6400, + "completed": 50000, + "blacklisted": 1200, + "failed": 88, + }, + "backlog": {"pending_total": 1200000}, + "active_job": { + "job_id": 55, + "job_code": "sync-overseas-55", + "status": "running", + "progress_percent": 12.5, + "items_total": 1300000, + "items_terminal": 51288, + "items_pending": 1200000, + "items_running": 6400, + "items_failed": 88, + "node_stats": [ + { + "node_code": "mainland-controller-01", + "items_running": 6400, + "items_claimed": 7000, + "items_total": 1300000, + } + ], + }, + "dependency_alerts": [], + } + cluster = { + "nodes_total": 1, + "nodes": [ + { + "node_code": "mainland-controller-01", + "current_load": 6400, + "detect_participating": True, + } + ], + "summary": { + "online_worker_nodes": 1, + "dedicated_online_worker_nodes": 0, + "online_control_nodes": 1, + "busy_nodes": ["mainland-controller-01"], + "stale_nodes": [], + "offline_nodes": [], + }, + } + + previous_payload = _build_runtime_projection_payload( + detect=detect, + cluster=cluster, + source_region="mainland", + target_region="overseas", + ) + fake_conn = _FakeConnection( + [ + [(previous_payload, datetime.now() - timedelta(seconds=90))], + (321,), + ] + ) + mock_get_db.return_value = fake_conn + + record_id = append_runtime_projection_if_changed( + detect=detect, + cluster=cluster, + source_region="mainland", + target_region="overseas", + ) + + self.assertEqual(321, record_id) + self.assertTrue(fake_conn.committed) + self.assertTrue( + any("INSERT INTO detect_sync_records" in sql for sql, _params in fake_conn.cursor_obj.executed) + ) + + @patch("app.services.sync_record_service._resolve_local_ip", return_value="121.204.244.188") + @patch("app.services.sync_record_service.socket.gethostname", return_value="mainland-controller-01") + @patch("app.services.sync_record_service.settings") + @patch("app.services.sync_record_service.get_db") + def test_append_runtime_projection_if_changed_writes_when_cluster_nodes_change_within_window( + self, + mock_get_db, + mock_settings, + _mock_hostname, + _mock_resolve_ip, + ) -> None: + mock_settings.node_code = "mainland-controller-01" + mock_settings.node_region = "mainland" + mock_settings.node_role = "control" + mock_settings.sync_source_region = "mainland" + mock_settings.sync_target_region = "overseas" + + previous_payload = { + "projection": { + "worker_online": True, + "worker_mode": "linux-systemd", + "phase_label": "运行中", + "phase_detail": "等待中", + "proxy_runtime_label": "正常", + "proxy_runtime_reason": "healthy", + "active_thread_count": 0, + "max_thread_count": 60000, + "progress": {"pending": 10000, "running": 0, "completed": 0, "blacklisted": 0, "failed": 0}, + "active_job": { + "job_id": None, + "job_code": "", + "status": "", + "items_total": 0, + "items_running": 0, + "items_claimed": 0, + "display_items_running": 0, + "display_items_claimed": 0, + "display_max_threads": 0, + "node_stats": [], + "distributed_node_stats": [], + }, + "cluster_summary": { + "nodes_total": 43, + "online_worker_nodes": 43, + "dedicated_online_worker_nodes": 42, + "online_control_nodes": 1, + "busy_nodes": [], + "stale_nodes": [], + "offline_nodes": [], + }, + "cluster_nodes": [ + { + "node_code": "mainland-controller-01", + "role": "control", + "status": "online", + "current_load": 0, + "active_threads": 0, + "max_threads": 60000, + "detect_participating": False, + } + ], + "dependency_alerts": [], + } + } + fake_conn = _FakeConnection( + [ + [(previous_payload, datetime.now() - timedelta(seconds=10))], + (911,), + ] + ) + mock_get_db.return_value = fake_conn + + record_id = append_runtime_projection_if_changed( + detect={ + "worker_online": True, + "worker_mode": "linux-systemd", + "phase_label": "运行中", + "phase_detail": "等待中", + "proxy_runtime_label": "正常", + "proxy_runtime_reason": "healthy", + "progress": {"pending": 10000, "running": 0, "completed": 0, "blacklisted": 0, "failed": 0}, + "backlog": {}, + "active_job": {}, + "dependency_alerts": [], + }, + cluster={ + "nodes_total": 61, + "nodes": [ + { + "node_code": "mainland-controller-01", + "role": "control", + "status": "online", + "current_load": 0, + "metadata": {"active_threads": 0, "max_threads": 60000}, + }, + { + "node_code": "mainland-controller-01-a", + "role": "worker", + "status": "online", + "current_load": 0, + "metadata": {"active_threads": 0, "max_threads": 1000}, + }, + ], + "summary": { + "online_worker_nodes": 61, + "dedicated_online_worker_nodes": 60, + "online_control_nodes": 1, + "busy_nodes": [], + "stale_nodes": [], + "offline_nodes": [], + }, + }, + source_region="mainland", + target_region="overseas", + ) + + self.assertEqual(911, record_id) + self.assertTrue(fake_conn.committed) + self.assertTrue( + any("INSERT INTO detect_sync_records" in sql for sql, _params in fake_conn.cursor_obj.executed) + ) + + @patch("app.services.sync_record_service._resolve_local_ip", return_value="121.204.244.188") + @patch("app.services.sync_record_service.socket.gethostname", return_value="mainland-controller-01") + @patch("app.services.sync_record_service.settings") + @patch("app.services.sync_record_service.get_db") + def test_append_runtime_projection_if_changed_skips_unchanged_projection_within_heartbeat_window( + self, + mock_get_db, + mock_settings, + _mock_hostname, + _mock_resolve_ip, + ) -> None: + mock_settings.node_code = "mainland-controller-01" + mock_settings.node_region = "mainland" + mock_settings.node_role = "control" + mock_settings.sync_source_region = "mainland" + mock_settings.sync_target_region = "overseas" + + detect = { + "worker_online": True, + "worker_mode": "linux-systemd", + "active_thread_count": 3200, + "max_thread_count": 80000, + "phase_label": "运行中", + "phase_detail": "80 实例运行", + "proxy_runtime_label": "正常", + "proxy_runtime_reason": "healthy", + "progress": { + "pending": 1200000, + "running": 6400, + "completed": 50000, + "blacklisted": 1200, + "failed": 88, + }, + "backlog": {"pending_total": 1200000}, + "active_job": { + "job_id": 55, + "job_code": "sync-overseas-55", + "status": "running", + "progress_percent": 12.5, + "items_total": 1300000, + "items_terminal": 51288, + "items_pending": 1200000, + "items_running": 6400, + "items_failed": 88, + "node_stats": [ + { + "node_code": "mainland-controller-01", + "items_running": 6400, + "items_claimed": 7000, + "items_total": 1300000, + } + ], + }, + "dependency_alerts": [], + } + cluster = { + "nodes_total": 1, + "nodes": [ + { + "node_code": "mainland-controller-01", + "current_load": 6400, + "detect_participating": True, + } + ], + "summary": { + "online_worker_nodes": 1, + "dedicated_online_worker_nodes": 0, + "online_control_nodes": 1, + "busy_nodes": ["mainland-controller-01"], + "stale_nodes": [], + "offline_nodes": [], + }, + } + + previous_payload = _build_runtime_projection_payload( + detect=detect, + cluster=cluster, + source_region="mainland", + target_region="overseas", + ) + fake_conn = _FakeConnection( + [ + [(previous_payload, datetime.now() - timedelta(seconds=10))], + ] + ) + mock_get_db.return_value = fake_conn + + record_id = append_runtime_projection_if_changed( + detect=detect, + cluster=cluster, + source_region="mainland", + target_region="overseas", + ) + + self.assertIsNone(record_id) + self.assertFalse(fake_conn.committed) + self.assertFalse( + any("INSERT INTO detect_sync_records" in sql for sql, _params in fake_conn.cursor_obj.executed) + ) + + @patch("app.services.sync_record_service._resolve_local_ip", return_value="121.204.244.188") + @patch("app.services.sync_record_service.socket.gethostname", return_value="mainland-controller-01") + @patch("app.services.sync_record_service.settings") + @patch("app.services.sync_record_service.get_db") + def test_append_runtime_projection_if_changed_writes_when_activity_signature_changes_within_window( + self, + mock_get_db, + mock_settings, + _mock_hostname, + _mock_resolve_ip, + ) -> None: + mock_settings.node_code = "mainland-controller-01" + mock_settings.node_region = "mainland" + mock_settings.node_role = "control" + mock_settings.sync_source_region = "mainland" + mock_settings.sync_target_region = "overseas" + + previous_detect = { + "worker_online": True, + "worker_mode": "linux-systemd", + "active_thread_count": 300, + "max_thread_count": 60000, + "phase_label": "运行中", + "phase_detail": "60 实例运行", + "proxy_runtime_label": "正常", + "proxy_runtime_reason": "healthy", + "progress": { + "pending": 5000, + "running": 600, + "completed": 1000, + "blacklisted": 0, + "failed": 10, + }, + "queue_health": { + "queue": { + "items_total": 6610, + "pending": 5000, + "claimed": 10, + "display_claimed": 10, + "running": 600, + "display_running": 600, + "completed": 1000, + "blacklisted": 0, + "failed": 10, + "terminal": 1010, + "display_max_threads": 60000, + }, + "nodes": [ + { + "node_code": "mainland-controller-01-a", + "items_running": 300, + "items_claimed": 10, + "display_running": 300, + "active_threads": 300, + "max_threads": 1000, + "region": "mainland", + "role": "control", + "status": "busy", + } + ], + }, + "active_job": { + "job_id": 867, + "job_code": "sync-overseas-19835", + "status": "running", + "progress_percent": 15.0, + "items_total": 6610, + "items_terminal": 1010, + "items_pending": 5000, + "items_claimed": 10, + "items_running": 600, + "items_failed": 10, + "node_stats": [ + { + "node_code": "mainland-controller-01-a", + "items_running": 300, + "items_claimed": 10, + "items_total": 6610, + } + ], + }, + "dependency_alerts": [], + } + current_detect = { + **previous_detect, + "active_thread_count": 900, + "queue_health": { + "queue": { + "items_total": 6610, + "pending": 4300, + "claimed": 30, + "display_claimed": 30, + "running": 900, + "display_running": 900, + "completed": 1370, + "blacklisted": 0, + "failed": 10, + "terminal": 1380, + "display_max_threads": 60000, + }, + "nodes": [ + { + "node_code": "mainland-controller-01-a", + "items_running": 420, + "items_claimed": 20, + "display_running": 420, + "active_threads": 420, + "max_threads": 1000, + "region": "mainland", + "role": "control", + "status": "busy", + }, + { + "node_code": "mainland-controller-01-b", + "items_running": 480, + "items_claimed": 10, + "display_running": 480, + "active_threads": 480, + "max_threads": 1000, + "region": "mainland", + "role": "control", + "status": "busy", + }, + ], + }, + "active_job": { + **previous_detect["active_job"], + "items_pending": 4300, + "items_claimed": 30, + "items_running": 900, + "node_stats": [ + { + "node_code": "mainland-controller-01-a", + "items_running": 420, + "items_claimed": 20, + "items_total": 6610, + }, + { + "node_code": "mainland-controller-01-b", + "items_running": 480, + "items_claimed": 10, + "items_total": 6610, + }, + ], + }, + } + cluster = { + "nodes_total": 2, + "nodes": [ + { + "node_code": "mainland-controller-01", + "current_load": 900, + "detect_participating": True, + } + ], + "summary": { + "online_worker_nodes": 60, + "dedicated_online_worker_nodes": 0, + "online_control_nodes": 60, + "busy_nodes": ["mainland-controller-01-a", "mainland-controller-01-b"], + "stale_nodes": [], + "offline_nodes": [], + }, + } + + previous_payload = _build_runtime_projection_payload( + detect=previous_detect, + cluster=cluster, + source_region="mainland", + target_region="overseas", + ) + fake_conn = _FakeConnection( + [ + [(previous_payload, datetime.now() - timedelta(seconds=10))], + (654,), + ] + ) + mock_get_db.return_value = fake_conn + + record_id = append_runtime_projection_if_changed( + detect=current_detect, + cluster=cluster, + source_region="mainland", + target_region="overseas", + ) + + self.assertEqual(654, record_id) + self.assertTrue(fake_conn.committed) + self.assertTrue( + any("INSERT INTO detect_sync_records" in sql for sql, _params in fake_conn.cursor_obj.executed) + ) + + def test_pick_latest_projection_row_skips_future_dated_rows(self) -> None: + rows = [ + ("future", datetime.now() + timedelta(hours=6)), + ("recent", datetime.now() - timedelta(seconds=10)), + ("older", datetime.now() - timedelta(minutes=2)), + ] + + selected = _pick_latest_projection_row(rows, created_at_index=1) + + self.assertEqual("recent", selected[0]) + + @patch("app.services.sync_record_service.get_db") + @patch("app.services.sync_record_service.settings") + def test_get_detect_result_sync_batches_marks_overseas_control_as_not_applicable( + self, + mock_settings, + mock_get_db, + ) -> None: + mock_settings.node_code = "overseas-control-01" + mock_settings.node_region = "overseas" + mock_settings.node_role = "control" + mock_settings.sync_source_region = "overseas" + mock_settings.sync_target_region = "mainland" + + payload = get_detect_result_sync_batches(limit=5) + + self.assertFalse(payload["applicable"]) + self.assertFalse(payload["local_worker_expected"]) + self.assertEqual(0, payload["jobs_total"]) + self.assertEqual([], payload["batches"]) + self.assertIn("不适用", payload["reason"]) + mock_get_db.assert_not_called() + if __name__ == "__main__": unittest.main() diff --git a/domain-api/tests/test_worker_control_service.py b/domain-api/tests/test_worker_control_service.py index d2a33cd..cc88d6a 100644 --- a/domain-api/tests/test_worker_control_service.py +++ b/domain-api/tests/test_worker_control_service.py @@ -1,8 +1,17 @@ import json +import subprocess import unittest from unittest.mock import Mock, patch -from app.services.worker_control_service import WORKER_CONTROL_CHANNEL, WORKER_PENDING_COMMAND_KEY, send_worker_command +from app.core.config import settings +from app.services.worker_control_service import ( + WORKER_CONTROL_CHANNEL, + WORKER_PENDING_COMMAND_KEY, + detect_worker_runtime, + send_worker_command, + start_worker, + stop_worker, +) class WorkerControlServiceTests(unittest.TestCase): @@ -11,10 +20,11 @@ class WorkerControlServiceTests(unittest.TestCase): redis_client = Mock() mock_get_redis.return_value = redis_client - ok, message = send_worker_command( - "start_detection", - payload={"job_id": 1, "job_code": "detect-20260419030000-abc123"}, - ) + with patch("app.services.worker_control_service._runtime_config", return_value={"worker_mode": "windows-local", "worker_service_name": "domaincheck-worker"}): + ok, message = send_worker_command( + "start_detection", + payload={"job_id": 1, "job_code": "detect-20260419030000-abc123"}, + ) self.assertTrue(ok) self.assertIn("已发送 Worker 控制指令", message) @@ -33,6 +43,309 @@ class WorkerControlServiceTests(unittest.TestCase): redis_client.publish.assert_called_once_with(WORKER_CONTROL_CHANNEL, serialized) + @patch("app.services.worker_control_service.get_redis") + def test_send_worker_command_scopes_pending_command_to_target_worker_instances(self, mock_get_redis) -> None: + redis_client = Mock() + mock_get_redis.return_value = redis_client + + with patch("app.services.worker_control_service._runtime_config", return_value={"worker_mode": "linux-systemd", "worker_service_name": "domaincheck-worker"}): + ok, message = send_worker_command( + "start_detection", + payload={ + "job_id": 2, + "target_node_codes": ["mainland-controller-01-a", "mainland-controller-01-b"], + }, + ) + + self.assertTrue(ok) + self.assertIn("mainland-controller-01-a,mainland-controller-01-b", message) + self.assertEqual(2, redis_client.set.call_count) + + set_keys = [call.args[0] for call in redis_client.set.call_args_list] + self.assertEqual( + [ + f"{WORKER_PENDING_COMMAND_KEY}:mainland-controller-01-a", + f"{WORKER_PENDING_COMMAND_KEY}:mainland-controller-01-b", + ], + set_keys, + ) + + serialized = redis_client.set.call_args_list[0].args[1] + payload = json.loads(serialized) + self.assertEqual(["mainland-controller-01-a", "mainland-controller-01-b"], payload["target_node_codes"]) + redis_client.publish.assert_called_once_with(WORKER_CONTROL_CHANNEL, serialized) + + @patch("app.services.worker_control_service._expand_linux_worker_control_units") + @patch("app.services.worker_control_service.get_redis") + def test_send_worker_command_expands_local_linux_worker_instances_when_targets_unspecified( + self, + mock_get_redis, + mock_expand_linux_worker_control_units, + ) -> None: + redis_client = Mock() + mock_get_redis.return_value = redis_client + mock_expand_linux_worker_control_units.return_value = [ + "domaincheck-worker", + "domaincheck-worker@a.service", + "domaincheck-worker@b.service", + ] + + with patch("app.services.worker_control_service._runtime_config", return_value={"worker_mode": "linux-systemd", "worker_service_name": "domaincheck-worker"}), \ + patch.object(settings, "node_code", "mainland-controller-01"): + ok, message = send_worker_command("start_detection", payload={"job_id": 3}) + + self.assertTrue(ok) + self.assertIn("mainland-controller-01,mainland-controller-01-a,mainland-controller-01-b", message) + self.assertEqual(3, redis_client.set.call_count) + set_keys = [call.args[0] for call in redis_client.set.call_args_list] + self.assertEqual( + [ + f"{WORKER_PENDING_COMMAND_KEY}:mainland-controller-01", + f"{WORKER_PENDING_COMMAND_KEY}:mainland-controller-01-a", + f"{WORKER_PENDING_COMMAND_KEY}:mainland-controller-01-b", + ], + set_keys, + ) + serialized = redis_client.set.call_args_list[0].args[1] + payload = json.loads(serialized) + self.assertEqual( + ["mainland-controller-01", "mainland-controller-01-a", "mainland-controller-01-b"], + payload["target_node_codes"], + ) + redis_client.publish.assert_called_once_with(WORKER_CONTROL_CHANNEL, serialized) + + @patch("app.services.worker_control_service._build_direct_redis_client") + @patch("app.services.worker_control_service.get_redis") + def test_send_worker_command_falls_back_to_direct_redis_client( + self, + mock_get_redis, + mock_build_direct_redis_client, + ) -> None: + mock_get_redis.side_effect = RecursionError("maximum recursion depth exceeded") + direct_client = Mock() + mock_build_direct_redis_client.return_value = direct_client + + with patch("app.services.worker_control_service._runtime_config", return_value={"worker_mode": "windows-local", "worker_service_name": "domaincheck-worker"}): + ok, message = send_worker_command("start_detection", payload={"job_id": 9}) + + self.assertTrue(ok) + self.assertIn("已发送 Worker 控制指令", message) + direct_client.set.assert_called_once() + serialized = direct_client.set.call_args.args[1] + payload = json.loads(serialized) + self.assertEqual("start_detection", payload["action"]) + self.assertEqual(9, payload["job_id"]) + direct_client.publish.assert_called_once_with(WORKER_CONTROL_CHANNEL, serialized) + direct_client.close.assert_called_once() + + @patch("app.services.worker_control_service._probe_linux_worker_instance_count", return_value=0) + @patch("app.services.worker_control_service._run_shell") + @patch("app.services.worker_control_service.probe_systemd_service") + @patch("app.services.worker_control_service._runtime_config") + def test_detect_worker_runtime_prefers_fast_pgrep_probe( + self, + mock_runtime_config, + mock_probe_systemd_service, + mock_run_shell, + _mock_instance_count, + ) -> None: + mock_runtime_config.return_value = { + "worker_mode": "linux-systemd", + "worker_service_name": "domaincheck-worker", + } + mock_probe_systemd_service.return_value = { + "mode": "linux-systemd", + "service_name": "domaincheck-worker", + "running": True, + "process_count": 1, + "latest_start_time": "2026-04-22 23:00:00", + "message": "active/running", + } + mock_run_shell.return_value = subprocess.CompletedProcess( + args=["bash", "-lc", "pgrep -fc '[d]etect_worker.py' || true"], + returncode=0, + stdout="80\n", + stderr="", + ) + + runtime = detect_worker_runtime() + + self.assertTrue(runtime["running"]) + self.assertEqual(80, runtime["process_count"]) + self.assertEqual(1, mock_run_shell.call_count) + + @patch("app.services.worker_control_service._probe_linux_worker_instance_count", return_value=0) + @patch("app.services.worker_control_service._run_shell") + @patch("app.services.worker_control_service.probe_systemd_service") + @patch("app.services.worker_control_service._runtime_config") + def test_detect_worker_runtime_falls_back_when_pgrep_probe_is_unavailable( + self, + mock_runtime_config, + mock_probe_systemd_service, + mock_run_shell, + _mock_instance_count, + ) -> None: + mock_runtime_config.return_value = { + "worker_mode": "linux-systemd", + "worker_service_name": "domaincheck-worker", + } + mock_probe_systemd_service.return_value = { + "mode": "linux-systemd", + "service_name": "domaincheck-worker", + "running": True, + "process_count": 1, + "latest_start_time": "2026-04-22 23:00:00", + "message": "active/running", + } + mock_run_shell.side_effect = [ + subprocess.CompletedProcess( + args=["bash", "-lc", "pgrep -fc '[d]etect_worker.py' || true"], + returncode=0, + stdout="", + stderr="pgrep: command not found\n", + ), + subprocess.CompletedProcess( + args=["bash", "-lc", "ps -eo args= | grep '[d]etect_worker.py' | wc -l"], + returncode=0, + stdout="12\n", + stderr="", + ), + ] + + runtime = detect_worker_runtime() + + self.assertTrue(runtime["running"]) + self.assertEqual(12, runtime["process_count"]) + self.assertEqual(2, mock_run_shell.call_count) + + @patch("app.services.worker_control_service._probe_linux_worker_process_count", return_value=7) + @patch("app.services.worker_control_service._probe_linux_worker_instance_count", return_value=0) + @patch("app.services.worker_control_service.probe_systemd_service") + @patch("app.services.worker_control_service._runtime_config") + def test_detect_worker_runtime_keeps_service_offline_when_only_unmanaged_processes_exist( + self, + mock_runtime_config, + mock_probe_systemd_service, + _mock_instance_count, + _mock_process_count, + ) -> None: + mock_runtime_config.return_value = { + "worker_mode": "linux-systemd", + "worker_service_name": "domaincheck-worker", + } + mock_probe_systemd_service.return_value = { + "mode": "linux-systemd", + "service_name": "domaincheck-worker", + "running": False, + "process_count": 0, + "latest_start_time": "", + "message": "inactive/dead", + } + + runtime = detect_worker_runtime() + + self.assertFalse(runtime["running"]) + self.assertEqual(7, runtime["process_count"]) + self.assertIn("unmanaged worker processes", runtime["message"]) + + @patch("app.services.worker_control_service._probe_linux_worker_process_count", return_value=30) + @patch("app.services.worker_control_service._probe_linux_worker_instance_count", return_value=3) + @patch("app.services.worker_control_service.probe_systemd_service") + @patch("app.services.worker_control_service._runtime_config") + def test_detect_worker_runtime_accepts_active_template_instances_when_base_service_is_inactive( + self, + mock_runtime_config, + mock_probe_systemd_service, + _mock_instance_count, + _mock_process_count, + ) -> None: + mock_runtime_config.return_value = { + "worker_mode": "linux-systemd", + "worker_service_name": "domaincheck-worker", + } + mock_probe_systemd_service.return_value = { + "mode": "linux-systemd", + "service_name": "domaincheck-worker", + "running": False, + "process_count": 0, + "latest_start_time": "", + "message": "inactive/dead", + } + + runtime = detect_worker_runtime() + + self.assertTrue(runtime["running"]) + self.assertEqual(30, runtime["process_count"]) + self.assertEqual("template instances active (3)", runtime["message"]) + + @patch("app.services.worker_control_service._expand_linux_worker_control_units") + @patch("app.services.worker_control_service._run_systemctl") + @patch("app.services.worker_control_service._runtime_config") + def test_start_worker_includes_template_instances( + self, + mock_runtime_config, + mock_run_systemctl, + mock_expand_units, + ) -> None: + mock_runtime_config.return_value = { + "worker_mode": "linux-systemd", + "worker_service_name": "domaincheck-worker", + } + mock_expand_units.return_value = [ + "domaincheck-worker", + "domaincheck-worker@a", + "domaincheck-worker@b", + ] + mock_run_systemctl.return_value = subprocess.CompletedProcess( + args=["systemctl", "start", "domaincheck-worker", "domaincheck-worker@a", "domaincheck-worker@b"], + returncode=0, + stdout="", + stderr="", + ) + + ok, message = start_worker() + + self.assertTrue(ok) + self.assertIn("附带 2 个实例", message) + mock_run_systemctl.assert_called_once_with( + ["start", "domaincheck-worker", "domaincheck-worker@a", "domaincheck-worker@b"], + timeout=45, + ) + + @patch("app.services.worker_control_service._expand_linux_worker_control_units") + @patch("app.services.worker_control_service._run_systemctl") + @patch("app.services.worker_control_service._runtime_config") + def test_stop_worker_includes_template_instances( + self, + mock_runtime_config, + mock_run_systemctl, + mock_expand_units, + ) -> None: + mock_runtime_config.return_value = { + "worker_mode": "linux-systemd", + "worker_service_name": "domaincheck-worker", + } + mock_expand_units.return_value = [ + "domaincheck-worker", + "domaincheck-worker@a", + "domaincheck-worker@b", + ] + mock_run_systemctl.return_value = subprocess.CompletedProcess( + args=["systemctl", "stop", "domaincheck-worker", "domaincheck-worker@a", "domaincheck-worker@b"], + returncode=0, + stdout="", + stderr="", + ) + + ok, message = stop_worker() + + self.assertTrue(ok) + self.assertIn("附带 2 个实例", message) + mock_run_systemctl.assert_called_once_with( + ["stop", "domaincheck-worker", "domaincheck-worker@a", "domaincheck-worker@b"], + timeout=45, + ) + if __name__ == "__main__": unittest.main() diff --git a/domain-web/src/api/modules.ts b/domain-web/src/api/modules.ts index 880bb38..4466317 100644 --- a/domain-web/src/api/modules.ts +++ b/domain-web/src/api/modules.ts @@ -103,6 +103,9 @@ export const opsApi = { dispatchJob: (jobId: number) => http.post(`/ops/jobs/${jobId}/dispatch`), issueAgentToken: (payload: Record) => http.post("/ops/agent/tokens", payload), buildAgentBootstrapPlan: (payload: Record) => http.post("/ops/agent/bootstrap-plan", payload), + migrationSourceProfile: () => http.get("/ops/migration/source-profile"), + previewMigration: (payload: Record) => http.post("/ops/migration/preview", payload), + executeMigration: (payload: Record) => http.post("/ops/migration/execute", payload), releases: (params?: Record) => http.get("/ops/releases", { params }), latestRelease: (channel = "stable") => http.get("/ops/releases/latest", { params: { channel } }), latestReleasePackageMetadata: () => http.get("/ops/releases/package-metadata/latest"), diff --git a/domain-web/src/layouts/MainLayout.vue b/domain-web/src/layouts/MainLayout.vue index 59efe8d..27e5389 100644 --- a/domain-web/src/layouts/MainLayout.vue +++ b/domain-web/src/layouts/MainLayout.vue @@ -68,7 +68,8 @@ const menuItems = [ { path: "/runtime", label: "运行中心" }, { path: "/runtime-debug", label: "运行调试" }, { path: "/ops-center", label: "运维中枢" }, - { path: "/managed-nodes", label: "托管节点" }, + { path: "/managed-nodes", label: "服务器管理" }, + { path: "/migration", label: "迁移向导" }, { path: "/settings", label: "系统设置" }, { path: "/imports", label: "域名导入" }, { path: "/sensitive-words", label: "敏感词配置" }, diff --git a/domain-web/src/router/index.ts b/domain-web/src/router/index.ts index 83f9da6..7974755 100644 --- a/domain-web/src/router/index.ts +++ b/domain-web/src/router/index.ts @@ -17,7 +17,8 @@ const routes: RouteRecordRaw[] = [ { path: "runtime", name: "runtime", component: () => import("@/views/runtime/RuntimeView.vue"), meta: { title: "运行中心", description: "看检测任务有没有真正跑起来，重点盯 running、ppm、步骤队列和节点负载。" } }, { path: "runtime-debug", name: "runtime-debug", component: () => import("@/views/runtime/RuntimeDebugView.vue"), meta: { title: "运行调试", description: "排查步骤卡点、事件流、调试日志和运行时异常。" } }, { path: "ops-center", name: "ops-center", component: () => import("@/views/ops/OpsCenterView.vue"), meta: { title: "运维中枢", description: "这里是总驾驶舱，负责发布、运维任务、巡检、接管进度和整体运维视角。" } }, - { path: "managed-nodes", name: "managed-nodes", component: () => import("@/views/ops/ManagedNodesView.vue"), meta: { title: "托管节点", description: "这里专门管理机器本身：新增节点、填 SSH、启用停用、维护接入信息。" } }, + { path: "managed-nodes", name: "managed-nodes", component: () => import("@/views/ops/ManagedNodesView.vue"), meta: { title: "服务器管理", description: "这里专门管理服务器本身：新增机器、填 SSH、启用停用、维护接入信息。" } }, + { path: "migration", name: "migration", component: () => import("@/views/ops/MigrationView.vue"), meta: { title: "迁移向导", description: "面向已 clone 代码的新机器，先预检，再预览计划，最后按选项同步配置、数据库和服务。" } }, { path: "settings", name: "settings", component: () => import("@/views/settings/SettingsView.vue"), meta: { title: "系统设置", description: "维护系统基础配置、运行参数和默认行为。" } }, { path: "imports", name: "imports", component: () => import("@/views/imports/ImportsView.vue"), meta: { title: "域名导入", description: "导入待检测域名数据，建立任务源。" } }, { path: "sensitive-words", name: "sensitive-words", component: () => import("@/views/sensitive-words/SensitiveWordsView.vue"), meta: { title: "敏感词配置", description: "维护过滤词、黑名单词和筛选规则。" } }, diff --git a/domain-web/src/views/dashboard/DashboardView.vue b/domain-web/src/views/dashboard/DashboardView.vue index 6441b06..cddf481 100644 --- a/domain-web/src/views/dashboard/DashboardView.vue +++ b/domain-web/src/views/dashboard/DashboardView.vue @@ -43,7 +43,7 @@
当前速度
{{ processedPerMinuteText }}
-
近窗完成 {{ opsSummary.completed_recent || 0 }} / 失败 {{ opsSummary.failed_recent || 0 }} / 黑名单 {{ opsSummary.blacklisted_recent || 0 }}
+
近窗完成 {{ opsSummary.completed_recent || 0 }} / 失败 {{ opsSummary.failed_recent || 0 }} / 近窗黑名单 {{ opsSummary.blacklisted_recent || 0 }}
预计剩余
@@ -61,18 +61,25 @@
注册待处理 {{ backlogRegisterPendingText }} / 后续步骤 {{ backlogDownstreamPendingText }}
-
有效执行节点
-
{{ opsSummary.active_execution_nodes || 0 }}
-
在线 Worker {{ opsSummary.online_worker_nodes || 0 }} / 独立 Worker {{ opsSummary.dedicated_online_worker_nodes || 0 }} / 当前线程负载 {{ queueDisplayRunningText }}
+
有效执行服务器
+
{{ activeExecutionServerCount }}
+
在线 Worker 服务器 {{ opsSummary.online_worker_nodes || 0 }} / 当前参与进程 {{ participatingProcessCountText }} / 当前参与线程 {{ queueDisplayRunningText }}
+
+
+
集群代理
+
{{ clusterProxyAvailableText }}
+
{{ clusterProxySummaryText }}
- + + + @@ -85,29 +92,36 @@ /> - + + + + - - + + + - - - + + + + - - + + @@ -117,7 +131,7 @@ - + @@ -160,6 +174,21 @@ const opsSummary = ref>({}); const bottleneckStep = ref>({}); const stepQueue = ref[]>([]); const nodeThroughput = ref[]>([]); +const inventoryBlacklistedTotal = ref(0); + +const resolveServerCode = (rawNodeCode?: string) => { + const normalized = String(rawNodeCode || "").trim(); + if (!normalized) return ""; + const parts = normalized.split("-"); + if (parts.length >= 2) { + const suffix = parts[parts.length - 1]; + const parent = parts.slice(0, -1).join("-"); + if (/^[a-z]{1,3}$/i.test(suffix) && /\d$/.test(parent)) { + return parent; + } + } + return normalized; +}; const clearRefreshTimer = () => { if (timer) { @@ -190,6 +219,58 @@ const processedPerMinuteText = computed(() => { const activeJobDisplayRunning = computed(() => Number(activeJob.value.items_display_running || activeJob.value.items_running || 0)); const queueDisplayRunningText = computed(() => String(Number(opsSummary.value.queue_display_running_total || 0))); +const participatingProcessCountText = computed(() => String(Number(opsSummary.value.aggregate_process_count || 0))); +const clusterProxyAvailableText = computed(() => String(Number(opsSummary.value.cluster_proxy_available_count || 0))); +const clusterProxySummaryText = computed(() => { + const label = String(opsSummary.value.cluster_proxy_runtime_label || "未知"); + const detail = String(opsSummary.value.cluster_proxy_runtime_detail || opsSummary.value.cluster_proxy_last_refresh_status || "").trim(); + return detail ? `${label} / ${detail}` : label; +}); +const serverThroughput = computed(() => { + const grouped = new Map>(); + for (const rawRow of nodeThroughput.value || []) { + const serverCode = resolveServerCode(String(rawRow?.node_code || "")); + if (!serverCode) continue; + const bucket = grouped.get(serverCode) || { + server_code: serverCode, + process_count: 0, + items_running: 0, + display_running: 0, + items_claimed: 0, + current_load: 0, + active_threads: 0, + max_threads: 0, + processed_recent: 0, + processed_per_minute: 0, + completed_recent: 0, + failed_recent: 0, + blacklisted_recent: 0 + }; + bucket.process_count += 1; + bucket.items_running += Number(rawRow?.items_running || 0); + bucket.display_running += Number(rawRow?.display_running || 0); + bucket.items_claimed += Number(rawRow?.items_claimed || 0); + bucket.current_load += Number(rawRow?.current_load || 0); + bucket.active_threads += Number(rawRow?.active_threads || 0); + bucket.max_threads += Number(rawRow?.max_threads || 0); + bucket.processed_recent += Number(rawRow?.processed_recent || 0); + bucket.processed_per_minute = Number((bucket.processed_per_minute + Number(rawRow?.processed_per_minute || 0)).toFixed(2)); + bucket.completed_recent += Number(rawRow?.completed_recent || 0); + bucket.failed_recent += Number(rawRow?.failed_recent || 0); + bucket.blacklisted_recent += Number(rawRow?.blacklisted_recent || 0); + grouped.set(serverCode, bucket); + } + return [...grouped.values()].sort((a, b) => ( + Number(b.processed_recent || 0) - Number(a.processed_recent || 0) + || Number(b.display_running || 0) - Number(a.display_running || 0) + || String(a.server_code || "").localeCompare(String(b.server_code || "")) + )); +}); +const activeExecutionServerCount = computed(() => serverThroughput.value.filter((item) => + Number(item.items_running || 0) > 0 + || Number(item.items_claimed || 0) > 0 + || Number(item.processed_recent || 0) > 0 +).length); const backlogPendingText = computed(() => String(Number(opsSummary.value.backlog_pending_total || 0))); @@ -240,11 +321,18 @@ const loadOverview = async (showError = true) => { backlog_running_total: data.backlog_running_total || 0, backlog_register_pending_total: data.backlog_register_pending_total || 0, backlog_downstream_pending_total: data.backlog_downstream_pending_total || 0, - queue_display_running_total: data.queue_display_running_total || 0 + queue_display_running_total: data.queue_display_running_total || 0, + aggregate_process_count: data.aggregate_process_count || 0, + aggregate_active_thread_count: data.aggregate_active_thread_count || 0, + cluster_proxy_available_count: data.cluster_proxy_available_count || 0, + cluster_proxy_runtime_label: data.cluster_proxy_runtime_label || "", + cluster_proxy_runtime_detail: data.cluster_proxy_runtime_detail || "", + cluster_proxy_last_refresh_status: data.cluster_proxy_last_refresh_status || "" }; bottleneckStep.value = data.bottleneck_step || {}; stepQueue.value = Array.isArray(data.step_queue) ? data.step_queue : []; nodeThroughput.value = Array.isArray(data.node_throughput) ? data.node_throughput : []; + inventoryBlacklistedTotal.value = Number(data.blacklist_total || 0); stats.value = [ { label: "总域名", value: String(data.domains_total), note: "数据库域名总量" }, { label: "库存待检测", value: String(data.pending_total), note: "全库尚未跑完的域名" }, @@ -266,9 +354,14 @@ const loadOverview = async (showError = true) => { { label: "API", value: data.api_status, note: "当前 API 运行状态" }, { label: "本机 Worker", value: localWorkerValue, note: localWorkerNote }, { - label: "有效执行节点", + label: "有效执行服务器", value: `${data.cluster_worker_status || "offline"} / ${data.cluster_online_worker_nodes || 0}`, - note: `可执行检测的在线节点，独立 Worker ${data.cluster_dedicated_online_worker_nodes || 0}，控制面 ${data.cluster_online_control_nodes || 0}` + note: `可执行检测的在线服务器，当前参与进程 ${data.aggregate_process_count || 0}，当前参与线程 ${data.queue_display_running_total || 0}` + }, + { + label: "集群代理", + value: String(data.cluster_proxy_available_count || 0), + note: String(data.cluster_proxy_runtime_label || "未知") }, { label: "模式", value: data.worker_mode, note: "当前 Worker 运行模式" } ]; diff --git a/domain-web/src/views/detect/DetectView.vue b/domain-web/src/views/detect/DetectView.vue index c31fe6e..0724623 100644 --- a/domain-web/src/views/detect/DetectView.vue +++ b/domain-web/src/views/detect/DetectView.vue @@ -31,8 +31,10 @@ {{ status.worker_process_count }} {{ status.thread_count }} {{ threadCountSummary }} - {{ threadSnapshotActive }} / {{ threadSnapshotMax }} - {{ participatingNodeCount }} + {{ threadSnapshotActive }} / {{ threadSnapshotMax }} + {{ participatingServerCount }} + {{ aggregateProcessCount }} + {{ aggregateMaxThreadCount }} {{ status.proxy_enable ? "是" : "否" }} {{ status.allow_direct ? "是" : "否" }} {{ status.proxy_pool_count }} @@ -114,7 +116,8 @@ - + + @@ -136,9 +139,9 @@
{{ currentPhaseDetail }}
-
活跃线程
+
参与线程
{{ threadSnapshotActive }} / {{ threadSnapshotMax }}
-
当前参与节点实际活跃线程 / 配置线程上限
+
当前参与服务器汇总线程 / 配置线程上限
当前速度
@@ -161,9 +164,10 @@ - + + @@ -216,25 +220,39 @@ :title="queueAlertText" /> - - + + + - +
- - + + + - + @@ -287,10 +305,12 @@
回到底部 - {{ isAggregateDetectView ? "参与节点" : "本机 Worker" }}：{{ isAggregateDetectView ? participatingNodeCount : (status.worker_online ? "在线" : "离线") }} + {{ isAggregateDetectView ? "参与服务器" : "本机 Worker" }}：{{ isAggregateDetectView ? participatingServerCount : (status.worker_online ? "在线" : "离线") }} + 参与进程：{{ aggregateProcessCount }} + 参与线程：{{ threadSnapshotActive }} 阶段：{{ selectedRun.phase_label }} 运行中：{{ displayRunningStat }} - 线程：{{ runtimeThreadSummary }} + {{ isAggregateDetectView ? "线程上限" : "线程配置" }}：{{ runtimeThreadSummary }}
{{ selectedRun.phase_detail }}
@@ -322,7 +342,13 @@ - + @@ -338,6 +364,7 @@ import PageCard from "@/components/PageCard.vue"; import { detectApi, settingsApi } from "@/api/modules"; const DETECT_LAST_ACTION_KEY = "domaincheck:detect:last-action"; +const sixRowTableMaxHeight = 336; const router = useRouter(); const status = ref({ worker_online: false, @@ -354,6 +381,10 @@ const status = ref({ thread_count_node_code: "", active_thread_count: 0, max_thread_count: 0, + aggregate_process_count: 0, + aggregate_participating_node_count: 0, + aggregate_participating_node_codes: [] as string[], + aggregate_max_thread_count: 0, proxy_enable: false, allow_direct: false, proxy_pool_count: 0, @@ -392,12 +423,77 @@ const status = ref({ } }); +const resolveServerCode = (rawNodeCode?: string) => { + const normalized = String(rawNodeCode || "").trim(); + if (!normalized) return ""; + const parts = normalized.split("-"); + if (parts.length >= 2) { + const suffix = parts[parts.length - 1]; + const parent = parts.slice(0, -1).join("-"); + if (/^[a-z]{1,3}$/i.test(suffix) && /\d$/.test(parent)) { + return parent; + } + } + return normalized; +}; + +const aggregateServerRows = (rows: any[]) => { + const grouped = new Map>(); + for (const rawItem of rows || []) { + const serverCode = resolveServerCode(String(rawItem?.node_code || "")); + if (!serverCode || serverCode === "unassigned") continue; + const bucket = grouped.get(serverCode) || { + server_code: serverCode, + process_count: 0, + items_total: 0, + items_pending: 0, + items_claimed: 0, + items_running: 0, + items_completed: 0, + items_blacklisted: 0, + items_failed: 0, + processed_recent: 0, + processed_per_minute: 0, + completed_recent: 0, + blacklisted_recent: 0, + failed_recent: 0, + display_running: 0, + current_load: 0, + active_threads: 0, + max_threads: 0 + }; + bucket.process_count += 1; + bucket.items_total += Number(rawItem?.items_total || 0); + bucket.items_pending += Number(rawItem?.items_pending || 0); + bucket.items_claimed += Number(rawItem?.items_claimed || 0); + bucket.items_running += Number(rawItem?.items_running || 0); + bucket.items_completed += Number(rawItem?.items_completed || 0); + bucket.items_blacklisted += Number(rawItem?.items_blacklisted || 0); + bucket.items_failed += Number(rawItem?.items_failed || 0); + bucket.processed_recent += Number(rawItem?.processed_recent || 0); + bucket.processed_per_minute = Number((bucket.processed_per_minute + Number(rawItem?.processed_per_minute || 0)).toFixed(2)); + bucket.completed_recent += Number(rawItem?.completed_recent || 0); + bucket.blacklisted_recent += Number(rawItem?.blacklisted_recent || 0); + bucket.failed_recent += Number(rawItem?.failed_recent || 0); + bucket.display_running += Number(rawItem?.display_running || 0); + bucket.current_load += Number(rawItem?.current_load || 0); + bucket.active_threads += Number(rawItem?.active_threads || 0); + bucket.max_threads += Number(rawItem?.max_threads || 0); + grouped.set(serverCode, bucket); + } + return [...grouped.values()].sort((a, b) => ( + Number(b.processed_recent || 0) - Number(a.processed_recent || 0) + || Number(b.display_running || 0) - Number(a.display_running || 0) + || String(a.server_code || "").localeCompare(String(b.server_code || "")) + )); +}; + const threadCountSummary = computed(() => { const effective = Number(status.value.thread_count || 0); const defaultValue = Number(status.value.thread_count_default || 0); const nodeCode = String(status.value.thread_count_node_code || "").trim(); if (status.value.thread_count_source === "node_override" && status.value.thread_count_override) { - return `${effective}（节点 ${nodeCode || "-"} 单独覆盖，默认 ${defaultValue}）`; + return `${effective}（服务器 ${nodeCode || "-"} 单独覆盖，默认 ${defaultValue}）`; } return `${effective}（默认）`; }); @@ -415,7 +511,8 @@ const queueSummary = ref({ }, throughput: { processed_recent: 0, - processed_per_minute: 0 + processed_per_minute: 0, + blacklisted_recent: 0 }, nodes: [] as any[] }); @@ -466,7 +563,20 @@ const activeJobDisplayClaimed = computed(() => Number(activeJob.value?.display_i const activeJobDisplayRunning = computed(() => Math.max( Number(activeJob.value?.items_running ?? 0), Number(activeJob.value?.display_active_threads ?? 0), + Number(activeJob.value?.display_items_running ?? 0), )); +const aggregateProcessCount = computed(() => { + if (isAggregateDetectView.value) { + return currentParticipatingProcessRows.value.length || Number(status.value.aggregate_process_count || status.value.worker_process_count || 0); + } + return Number(status.value.worker_process_count || 0); +}); +const aggregateMaxThreadCount = computed(() => { + if (isAggregateDetectView.value) { + return Number(status.value.aggregate_max_thread_count || 0); + } + return Number(status.value.max_thread_count || status.value.thread_count || 0); +}); const threadSnapshotActive = computed(() => { const jobValue = Number(activeJob.value?.display_active_threads ?? 0); if (jobValue > 0) { @@ -476,6 +586,10 @@ const threadSnapshotActive = computed(() => { }); const threadSnapshotMax = computed(() => { const jobValue = Number(activeJob.value?.display_max_threads ?? 0); + const aggregateValue = Number(status.value.aggregate_max_thread_count || 0); + if (aggregateValue > 0) { + return Math.max(jobValue, aggregateValue); + } if (jobValue > 0) { return jobValue; } @@ -490,9 +604,23 @@ const activeJobDisplayNodeStats = computed(() => { } return Array.isArray(activeJob.value?.node_stats) ? activeJob.value.node_stats : []; }); -const participatingNodeCount = computed(() => { - const rows = activeJobDisplayNodeStats.value.filter((item) => String(item?.node_code || "").trim() && String(item?.node_code || "").trim() !== "unassigned"); - return rows.length || 0; +const currentParticipatingProcessRows = computed(() => + activeJobDisplayNodeStats.value.filter((item) => { + const nodeCode = String(item?.node_code || "").trim(); + if (!nodeCode || nodeCode === "unassigned") return false; + return [ + Number(item?.items_claimed || 0), + Number(item?.items_running || 0), + Number(item?.display_running || 0), + Number(item?.current_load || 0), + Number(item?.active_threads || 0) + ].some((value) => value > 0); + }) +); +const activeJobDisplayServerStats = computed(() => aggregateServerRows(activeJobDisplayNodeStats.value)); +const queueSummaryServerRows = computed(() => aggregateServerRows(Array.isArray(queueSummary.value.nodes) ? queueSummary.value.nodes : [])); +const participatingServerCount = computed(() => { + return activeJobDisplayServerStats.value.length || Number(status.value.aggregate_participating_node_count || 0) || 0; }); const queueProcessedPerMinute = computed(() => Number(queueSummary.value.throughput?.processed_per_minute || 0)); const queueProcessedPerHour = computed(() => Number((queueProcessedPerMinute.value * 60).toFixed(2))); @@ -521,7 +649,7 @@ const estimatedRemainingText = computed(() => { }); const runtimeThreadSummary = computed(() => { if (isAggregateDetectView.value) { - return `${threadSnapshotActive.value} / ${threadSnapshotMax.value}（参与节点汇总）`; + return `${threadSnapshotActive.value} / ${threadSnapshotMax.value}（参与服务器汇总）`; } return threadCountSummary.value; }); @@ -532,16 +660,32 @@ const activeJobSummaryText = computed(() => { const job = activeJob.value; const displayClaimed = Number(job.display_items_claimed ?? job.items_claimed ?? 0); const displayRunning = activeJobDisplayRunning.value; - const displayActiveThreads = Number(job.display_active_threads ?? 0); - const displayMaxThreads = Number(job.display_max_threads ?? 0); - return `状态：${jobStatusText(job.status)} / 总数 ${job.items_total || 0} / 待领 ${job.items_pending || 0} / 已领 ${displayClaimed} / 实时执行 ${displayRunning} / 活跃线程 ${displayActiveThreads}/${displayMaxThreads} / 速度 ${queueProcessedPerMinute.value} 项/分钟 / 预计剩余 ${estimatedRemainingText.value}`; + const displayActiveThreads = threadSnapshotActive.value; + const displayMaxThreads = threadSnapshotMax.value; + return `状态：${jobStatusText(job.status)} / 总数 ${job.items_total || 0} / 待领 ${job.items_pending || 0} / 已领 ${displayClaimed} / 实时执行 ${displayRunning} / 参与线程 ${displayActiveThreads}/${displayMaxThreads} / 速度 ${queueProcessedPerMinute.value} 项/分钟 / 预计剩余 ${estimatedRemainingText.value}`; +}); +const selectedRun = computed(() => { + if (isAggregateDetectView.value) { + return null; + } + return runs.value.find((item) => item.run_id === selectedRunId.value) || runs.value[0] || null; +}); +const currentPhaseLabel = computed(() => { + if (selectedRun.value?.phase_label) { + return selectedRun.value.phase_label; + } + if (isAggregateDetectView.value && activeJob.value?.job_code) { + return "集群执行中"; + } + return status.value.worker_online ? "运行中" : "未启动"; }); -const selectedRun = computed(() => runs.value.find((item) => item.run_id === selectedRunId.value) || runs.value[0] || null); -const currentPhaseLabel = computed(() => selectedRun.value?.phase_label || (status.value.worker_online ? "运行中" : "未启动")); const currentPhaseDetail = computed(() => { if (selectedRun.value?.phase_detail) { return selectedRun.value.phase_detail; } + if (isAggregateDetectView.value && activeJob.value?.job_code) { + return status.value.recent_event || `当前汇总执行由 ${participatingServerCount.value} 台服务器、${aggregateProcessCount.value} 个进程承担。`; + } if (status.value.recent_event) { return status.value.recent_event; } @@ -629,7 +773,7 @@ const remoteLogMirrorText = computed(() => { }); const detectScopeSummaryText = computed(() => { if (isAggregateDetectView.value) { - return `当前检测状态：${currentPhaseLabel}；当前页展示的是多节点汇总运行态，参与节点 ${participatingNodeCount.value} 台，实时活跃线程 ${threadSnapshotActive.value}/${threadSnapshotMax.value}。`; + return `当前检测状态：${currentPhaseLabel}；当前页展示的是多服务器汇总运行态，参与服务器 ${participatingServerCount.value} 台，参与进程 ${aggregateProcessCount.value} 个，参与线程 ${threadSnapshotActive.value}/${threadSnapshotMax.value}。`; } return `当前检测状态：${currentPhaseLabel}；本机 Worker ${status.value.worker_online ? "在线" : "离线"}。`; }); diff --git a/domain-web/src/views/ops/ManagedNodesView.vue b/domain-web/src/views/ops/ManagedNodesView.vue index e541b98..a1c4ef6 100644 --- a/domain-web/src/views/ops/ManagedNodesView.vue +++ b/domain-web/src/views/ops/ManagedNodesView.vue @@ -1,5 +1,5 @@ - + - + @@ -509,7 +663,7 @@
-

动作留痕

+

最近操作记录

刷新页面后仍会保留最近 8 条动作记录
@@ -528,12 +682,12 @@
-

集群节点

- 统一查看控制面 / 执行面节点最近心跳与负载 +

所有接入服务器

+ 统一看每台服务器最近的心跳、负载和并发设置
- 在线控制面 {{ runtime.cluster?.summary?.online_control_nodes || 0 }} - 有效执行节点 {{ runtime.cluster?.summary?.online_worker_nodes || 0 }} + 在线控制服务器 {{ runtime.cluster?.summary?.online_control_nodes || 0 }} + 有效执行服务器 {{ runtime.cluster?.summary?.online_worker_nodes || 0 }} 独立 Worker {{ runtime.cluster?.summary?.dedicated_online_worker_nodes || 0 }} 忙碌 {{ runtime.cluster?.summary?.status_counts?.busy || 0 }} 失活 {{ runtime.cluster?.summary?.status_counts?.stale || 0 }} @@ -547,8 +701,14 @@ style="margin-bottom: 12px" :title="clusterAlertText" /> - - + + @@ -566,7 +726,7 @@ - + - + - + @@ -602,19 +762,19 @@
-

同步摘要

- 为后续跨地域结果同步预留的配置与记录观察入口 +

跨区回传情况

+ 看运行信息和检测结果有没有正常从另一边带回来
推送 {{ runtime.sync?.enabled ? "已启用" : "未启用" }} 源地域 {{ runtime.sync?.source_region || "-" }} 目标地域 {{ runtime.sync?.target_region || "-" }} 总记录 {{ runtime.sync?.records_total || 0 }} - 投影 {{ runtime.sync?.type_counts?.runtime_projection || 0 }} - 结果投影 {{ runtime.sync?.type_counts?.detect_result_projection || 0 }} - 推送记录 {{ runtime.sync?.type_counts?.runtime_push || 0 }} - 接收记录 {{ runtime.sync?.type_counts?.runtime_ingest || 0 }} - 结果接收 {{ runtime.sync?.type_counts?.detect_result_ingest || 0 }} + 本地整理 {{ runtime.sync?.type_counts?.runtime_projection || 0 }} + 结果整理 {{ runtime.sync?.type_counts?.detect_result_projection || 0 }} + 已发出 {{ runtime.sync?.type_counts?.runtime_push || 0 }} + 已收到 {{ runtime.sync?.type_counts?.runtime_ingest || 0 }} + 结果已收到 {{ runtime.sync?.type_counts?.detect_result_ingest || 0 }}
- - - - - + + + + + - +
-

结果批次同步

- 按检测任务查看结果投影是否已推送、已接收或仍有积压 +

每批任务结果回传情况

+ 看每批任务的结果有没有发出来、收到了没有、还卡在哪
已接收 {{ runtime.sync?.detect_result_batches?.state_counts?.synced || 0 }} @@ -654,16 +814,16 @@ :data="runtime.sync?.detect_result_batches?.batches || []" border size="small" - empty-text="当前还没有可观察的结果同步批次" + empty-text="当前还没有可观察的结果回传记录" > - + - + - + - + - + - + @@ -711,13 +871,13 @@
-

活跃任务事件

- 当前任务的最近分发、领取与执行事件 +

这批任务最近发生了什么

+ 这里看这批任务最近的派发、接手和执行变化
- - + +
@@ -779,6 +939,7 @@ import { runtimeApi, settingsApi } from "@/api/modules"; const RUNTIME_LAST_ACTION_KEY = "domaincheck:runtime:last-action"; const RUNTIME_ACTION_HISTORY_KEY = "domaincheck:runtime:action-history"; +const sixRowSmallTableMaxHeight = 280; const router = useRouter(); const runtime = reactive>({ node: {}, @@ -953,6 +1114,115 @@ const proxyDiagnosisText = () => { return runtime.detect?.proxy_runtime_detail || "当前暂无代理运行说明"; }; +const observationSummary = computed(() => runtime.detect?.observation_summary || null); +const observationSourceInventory = computed(() => observationSummary.value?.source_inventory || {}); +const observationActiveBatch = computed(() => observationSummary.value?.active_batch || {}); +const clusterWorkerProcessRows = computed(() => + (Array.isArray(runtime.cluster?.nodes) ? runtime.cluster.nodes : []).filter((item: Record) => { + const role = String(item?.role || "").trim().toLowerCase(); + return role === "worker"; + }) +); +const liveWorkerProcessCount = computed(() => { + const runtimeReported = Number(runtime.worker?.process_count || runtime.worker_process_count || 0); + if (runtimeReported > 0) { + return runtimeReported; + } + return clusterWorkerProcessRows.value.filter((item: Record) => { + const status = String(item?.status || "").trim().toLowerCase(); + return status && !["offline", "stale"].includes(status); + }).length; +}); +const busyWorkerProcessCount = computed(() => { + const observationReported = Math.max( + Number(observationSummary.value?.execution?.active_processes || 0), + Number(observationSummary.value?.execution?.dispatch_active_nodes || 0) + ); + if (observationReported > 0) { + return observationReported; + } + return clusterWorkerProcessRows.value.filter((item: Record) => { + const status = String(item?.status || "").trim().toLowerCase(); + const currentLoad = Number(item?.current_load || 0); + const activeThreads = Number(item?.metadata?.active_threads || 0); + return status === "busy" || currentLoad > 0 || activeThreads > 0; + }).length; +}); +const observationTermHintText = + "先记一个最容易混的点：这里看到的很多数字，算的是任务记录，不直接等于域名个数。一个域名拆成多个步骤后，会出现多条记录。"; +const observationBatchScopeText = computed(() => { + const effective = Number(observationActiveBatch.value?.effective_items_total || 0); + const raw = Number(observationActiveBatch.value?.raw_items_total || 0); + const progress = Number(observationActiveBatch.value?.progress_percent || 0); + if (!effective && !raw) { + return "这里看的不是全库总进度，而是当前这一批拆出来的任务记录。现在还没有足够样本。"; + } + return `这里看的不是全库域名总数，而是当前这一批拆出来的任务记录。页面现在能直接对上的有 ${effective} 条，系统历史流水里一共记了 ${raw} 条，目前大约跑到 ${progress}%。`; +}); +const observationBatchStartText = computed(() => { + const startedAt = String(observationActiveBatch.value?.started_at || "").trim(); + if (!startedAt) { + return "这批任务的开始时间目前还没有拿到。"; + } + return `这批任务从 ${startedAt} 开始跑。`; +}); +const observationBatchQueueText = computed(() => { + const pending = Number(observationActiveBatch.value?.pending || 0); + const claimed = Number(observationActiveBatch.value?.claimed || 0); + const running = Number(observationActiveBatch.value?.running || 0); + return `还没开始 ${pending} = 还没被真正处理；已接手待执行 ${claimed} = 已被 worker 接走但还没真正开跑；正在执行 ${running} = 当前正在执行。`; +}); +const observationBatchResultText = computed(() => { + const completed = Number(observationActiveBatch.value?.completed || 0); + const failed = Number(observationActiveBatch.value?.failed || 0); + const blacklisted = Number(observationActiveBatch.value?.blacklisted || 0); + return `按页面现在能直接确认到的结果看，已完成 ${completed} 条，失败 ${failed} 条，进黑名单 ${blacklisted} 条。`; +}); +const observationBatchRawText = computed(() => { + const rawCompleted = Number(observationActiveBatch.value?.raw_completed || 0); + const rawFailed = Number(observationActiveBatch.value?.raw_failed || 0); + const rawBlacklisted = Number(observationActiveBatch.value?.raw_blacklisted || 0); + return `如果把历史重试和拆分步骤都算进去，累计已完成 ${rawCompleted} 条，失败 ${rawFailed} 条，进黑名单 ${rawBlacklisted} 条，所以这组数字通常会比上面大。`; +}); + +const observationStateTagType = (state?: string) => { + if (state === "running") return "success"; + if (state === "slow") return "warning"; + if (state === "not_running" || state === "stale") return "danger"; + return "info"; +}; + +const observationFocusAlertType = (summary?: Record) => { + const failedRecent = Number(summary?.throughput?.failed_recent || 0); + const blacklistedRecent = Number(summary?.throughput?.blacklisted_recent || 0); + const state = String(summary?.state || "").trim(); + if (state === "not_running" || state === "stale") return "error"; + if (failedRecent > Math.max(blacklistedRecent, 0)) return "warning"; + return "info"; +}; + +const observationNodeText = (item: Record) => { + const nodeCode = String(item?.node_code || "-"); + const running = Number(item?.items_running || 0); + const claimed = Number(item?.items_claimed || 0); + const activeThreads = Number(item?.active_threads || 0); + const maxThreads = Number(item?.max_threads || 0); + const processedRecent = Number(item?.processed_recent || 0); + const participationLabel = String(item?.participation_label || "参与中"); + const threadText = maxThreads > 0 ? `${activeThreads}/${maxThreads}` : `${activeThreads}`; + return `${nodeCode} · ${participationLabel} · 正在执行 ${running} / 已接手待执行 ${claimed} / 线程 ${threadText} / 近窗 ${processedRecent}`; +}; + +const observationStepText = (item: Record) => { + const stepName = String(item?.step_name || item?.step_code || "-"); + const pending = Number(item?.pending || 0); + const running = Number(item?.running || 0); + const failedRecent = Number(item?.failed_recent || 0); + const blacklistedRecent = Number(item?.blacklisted_recent || 0); + const processedRecent = Number(item?.processed_recent || 0); + return `${stepName} · 还没开始 ${pending} / 正在执行 ${running} / 近窗 ${processedRecent} / 失败 ${failedRecent} / 黑名单 ${blacklistedRecent}`; +}; + const localWorkerExpected = computed(() => Boolean(runtime.worker?.expected_on_this_node ?? true)); const localWorkerSummaryText = computed(() => { @@ -1004,19 +1274,19 @@ const clusterAlertText = computed(() => { const staleNodes = Array.isArray(summary.stale_nodes) ? summary.stale_nodes : []; const offlineNodes = Array.isArray(summary.offline_nodes) ? summary.offline_nodes : []; const parts = [ - `当前有效执行节点 ${summary.online_worker_nodes || 0}，其中独立 Worker ${summary.dedicated_online_worker_nodes || 0}，控制面兼跑 ${participationSummary.controller_worker_nodes || 0}` + `当前能真正执行任务的服务器有 ${summary.online_worker_nodes || 0} 台，其中独立 Worker ${summary.dedicated_online_worker_nodes || 0} 台，控制面兼跑 ${participationSummary.controller_worker_nodes || 0} 台` ]; parts.push( - `真正执行/领任务 ${participationSummary.dispatch_active_nodes || 0} 台，近窗刚有吞吐 ${participationSummary.recent_only_nodes || 0} 台，在线未参与 ${participationSummary.non_participating_nodes || 0} 台` + `现在正在领任务或执行任务的有 ${participationSummary.dispatch_active_nodes || 0} 台，最近 15 分钟刚有产出的有 ${participationSummary.recent_only_nodes || 0} 台，挂在线但没干活的有 ${participationSummary.non_participating_nodes || 0} 台` ); if (busy) { - parts.push(`忙碌节点：${busy}`); + parts.push(`当前比较忙的服务器：${busy}`); } if (staleNodes.length) { - parts.push(`失活节点 ${staleNodes.length} 个${staleNodes.length <= 2 ? `：${staleNodes.join("、")}` : "，详见下方节点表"}`); + parts.push(`失活服务器 ${staleNodes.length} 台${staleNodes.length <= 2 ? `：${staleNodes.join("、")}` : "，详见下方服务器表"}`); } if (offlineNodes.length) { - parts.push(`离线节点 ${offlineNodes.length} 个${offlineNodes.length <= 2 ? `：${offlineNodes.join("、")}` : "，详见下方节点表"}`); + parts.push(`离线服务器 ${offlineNodes.length} 台${offlineNodes.length <= 2 ? `：${offlineNodes.join("、")}` : "，详见下方服务器表"}`); } return parts.join("；"); }); @@ -1024,12 +1294,14 @@ const clusterAlertText = computed(() => { const participatingNodesAlertText = computed(() => { const rows = Array.isArray(runtime.detect?.participating_nodes) ? runtime.detect.participating_nodes : []; if (!rows.length) { - return "当前没有节点正在领任务、执行任务或产生近窗吞吐。"; + return "当前没有服务器在领任务、执行任务，最近 15 分钟里也没有新的处理产出。"; } const names = rows.map((item: Record) => String(item.node_code || "-")).join("、"); const dispatchActiveCount = rows.filter((item: Record) => ["running", "claimed"].includes(String(item.participation_state || "").trim())).length; const recentOnlyCount = rows.filter((item: Record) => String(item.participation_state || "").trim() === "recent_throughput").length; - return `当前参与检测节点 ${rows.length} 台：正在执行/领任务 ${dispatchActiveCount} 台，近窗刚有吞吐 ${recentOnlyCount} 台。节点：${names}`; + const processCount = Number(runtime.detect?.aggregate_process_count || 0); + const processText = processCount > 0 ? `，汇总进程 ${processCount} 个` : ""; + return `当前一共有 ${rows.length} 台服务器在这批任务里出现过动作：其中 ${dispatchActiveCount} 台正在领任务或执行，${recentOnlyCount} 台最近 15 分钟刚有产出${processText}。服务器：${names}`; }); const syncAlertText = computed(() => { @@ -1037,16 +1309,15 @@ const syncAlertText = computed(() => { const statusCounts = sync.status_counts || {}; const typeCounts = sync.type_counts || {}; const parts = [ - sync.enabled ? "当前已启用同步推送骨架" : "当前未启用同步推送，适合作为单地域或开发态运行", - `源地域 ${sync.source_region || "-"}`, - `目标地域 ${sync.target_region || "-"}`, - `运行投影 ${typeCounts.runtime_projection || 0} / 结果投影 ${typeCounts.detect_result_projection || 0} / 推送 ${typeCounts.runtime_push || 0} / 运行接收 ${typeCounts.runtime_ingest || 0} / 结果接收 ${typeCounts.detect_result_ingest || 0}`, - `成功 ${statusCounts.success || 0} / 待处理 ${statusCounts.pending || 0} / 失败 ${statusCounts.failed || 0}`, + sync.enabled ? "当前已经开启跨区回传" : "当前没有开启跨区回传，更适合单地区或开发环境", + `数据从 ${sync.source_region || "-"} 发往 ${sync.target_region || "-"}`, + `本地整理 ${typeCounts.runtime_projection || 0} / 结果整理 ${typeCounts.detect_result_projection || 0} / 已发出 ${typeCounts.runtime_push || 0} / 已收到 ${typeCounts.runtime_ingest || 0} / 结果已收到 ${typeCounts.detect_result_ingest || 0}`, + `成功 ${statusCounts.success || 0} / 等待处理 ${statusCounts.pending || 0} / 失败 ${statusCounts.failed || 0}`, ]; const batchStates = sync.detect_result_batches?.state_counts || {}; if (sync.detect_result_batches?.jobs_total) { parts.push( - `结果批次 已接收 ${batchStates.synced || 0} / 已送达 ${batchStates.delivered || 0} / 推送中 ${batchStates.pushing || 0} / 待推送 ${batchStates.projected || 0} / 失败 ${batchStates.failed || 0}` + `每批任务结果回传：已收到 ${batchStates.synced || 0} / 已送达 ${batchStates.delivered || 0} / 发送中 ${batchStates.pushing || 0} / 还没发 ${batchStates.projected || 0} / 失败 ${batchStates.failed || 0}` ); } if (sync.target_api_base_url) { @@ -1069,23 +1340,23 @@ const workerLogSyncMode = computed(() => { const workerLogSyncDescription = computed(() => { if (!workerLogSyncEnabled.value) { - return "当前已关闭远端日志回传；跨机联调时不会把 Worker 过程日志镜像到当前后台。"; + return "当前已经关闭远端日志带回；跨机联调时，不会把 Worker 的过程日志同步到这个后台。"; } const lineCount = Number(runtime.detect?.log_sync?.line_count || 0); const sourceNodes = Array.isArray(runtime.detect?.log_sync?.source_nodes) ? runtime.detect.log_sync.source_nodes : []; const lastAt = String(runtime.detect?.log_sync?.last_at || "").trim(); const observationSuffix = lineCount > 0 - ? ` 当前已收到 ${lineCount} 条回传日志，来源节点 ${sourceNodes.length || 0} 台${lastAt ? `，最近一条 ${lastAt}` : ""}。` + ? ` 当前已经收到 ${lineCount} 条日志，来自 ${sourceNodes.length || 0} 台服务器${lastAt ? `，最近一条在 ${lastAt}` : ""}。` : " 当前还没有收到远端日志样本。"; if (workerLogSyncMode.value === "full") { - return `当前为全量回传；会尽量保留更多 Worker 执行过程，适合短时深度排障，但日志量会更大。${observationSuffix}`; + return `当前为全量带回；会尽量保留更多 Worker 执行过程，适合短时间深入排查，但日志量会更大。${observationSuffix}`; } - return `当前为关键回传；只同步阶段、异常、代理与关键执行事件，适合日常联调观察。${observationSuffix}`; + return `当前为关键带回；只同步阶段、异常、代理和关键执行事件，适合日常观察。${observationSuffix}`; }); const workerLogSyncNodesText = computed(() => { const nodes = Array.isArray(runtime.detect?.log_sync?.source_nodes) ? runtime.detect.log_sync.source_nodes : []; - return nodes.length ? nodes.join("、") : "当前还没有远端节点回传样本"; + return nodes.length ? nodes.join("、") : "当前还没有远端服务器带回样本"; }); const workerLogSyncLastAtText = computed(() => { @@ -1124,14 +1395,14 @@ const nonParticipatingNodes = computed(() => { const nonParticipatingNodesAlertText = computed(() => { if (!nonParticipatingNodes.value.length) { - return "当前所有有效执行节点要么正在参与检测，要么当前没有额外在线未参与节点。"; + return "当前所有有效执行服务器要么正在干活，要么现在没有额外挂在线却没干活的服务器。"; } const names = nonParticipatingNodes.value.map((item: Record) => String(item.node_code || "-")).join("、"); const loadSyncingCount = nonParticipatingNodes.value.filter( (item: Record) => String(item.participation_state || "").trim() === "load_syncing" ).length; const standbyCount = nonParticipatingNodes.value.length - loadSyncingCount; - return `当前在线但未参与节点 ${nonParticipatingNodes.value.length} 台：在线待命 ${standbyCount} 台，负载待确认 ${loadSyncingCount} 台。节点：${names}`; + return `当前有 ${nonParticipatingNodes.value.length} 台服务器在线但还没干活：纯待命 ${standbyCount} 台，负载还在确认 ${loadSyncingCount} 台。服务器：${names}`; }); const nonParticipationStateText = (row: Record) => { @@ -1171,9 +1442,9 @@ const standbyReasonText = (row: Record) => { const metadata = row?.metadata || {}; if (metadata?.detail) return String(metadata.detail); if (metadata?.phase_detail) return String(metadata.phase_detail); - if (metadata?.phase) return `当前阶段：${metadata.phase}`; - if (metadata?.phase_label) return `当前阶段：${metadata.phase_label}`; - return "节点在线，当前未领任务、未执行任务，也没有近窗吞吐。"; + if (metadata?.phase) return `当前状态：${metadata.phase}`; + if (metadata?.phase_label) return `当前状态：${metadata.phase_label}`; + return "这台服务器在线，但现在还没领任务、没在执行，最近 15 分钟里也没有处理产出。"; }; const hasNodeOverride = (nodeCode?: string) => { @@ -1470,6 +1741,119 @@ onBeforeUnmount(clearRefreshTimer); font-size: 13px; } +.observer-panel { + margin-bottom: 16px; + padding: 18px; + border: 1px solid #dbe4ee; + border-radius: 18px; + background: linear-gradient(135deg, #f8fbff 0%, #eef6ff 100%); +} + +.observer-panel__header { + display: flex; + align-items: flex-start; + justify-content: space-between; + gap: 16px; +} + +.observer-panel__eyebrow { + color: #2563eb; + font-size: 12px; + font-weight: 700; +} + +.observer-panel__title { + margin-top: 4px; + color: #0f172a; + font-size: 26px; + font-weight: 800; +} + +.observer-panel__reason { + margin-top: 6px; + color: #475569; + font-size: 13px; + line-height: 1.6; +} + +.observer-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(220px, 1fr)); + gap: 12px; + margin-top: 16px; +} + +.observer-card { + padding: 14px 16px; + border-radius: 14px; + border: 1px solid #dbe4ee; + background: rgba(255, 255, 255, 0.85); +} + +.observer-card__label { + color: #64748b; + font-size: 12px; +} + +.observer-card__value { + margin-top: 8px; + color: #0f172a; + font-size: 20px; + font-weight: 700; + line-height: 1.4; +} + +.observer-card__note { + margin-top: 8px; + color: #64748b; + font-size: 12px; + line-height: 1.5; +} + +.observer-lists { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(280px, 1fr)); + gap: 14px; + margin-top: 14px; +} + +.observer-list { + padding: 14px 16px; + border-radius: 14px; + border: 1px solid #dbe4ee; + background: rgba(255, 255, 255, 0.72); +} + +.observer-list__title { + color: #475569; + font-size: 13px; + font-weight: 700; +} + +.observer-token-list { + display: flex; + flex-wrap: wrap; + gap: 8px; + margin-top: 12px; +} + +.observer-token { + display: inline-flex; + align-items: center; + min-height: 30px; + padding: 0 12px; + border-radius: 999px; + background: #eff6ff; + color: #1e3a8a; + font-size: 12px; + line-height: 1.4; +} + +.observer-token--soft { + background: #f8fafc; + color: #334155; +} + .summary-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(220px, 1fr)); diff --git a/domain-web/src/views/settings/SettingsView.vue b/domain-web/src/views/settings/SettingsView.vue index 17c2527..6cdd653 100644 --- a/domain-web/src/views/settings/SettingsView.vue +++ b/domain-web/src/views/settings/SettingsView.vue @@ -9,29 +9,49 @@ + + + + + +
+
+ 当前没有服务器级单独覆盖配置，所有服务器都会走默认进程数。 +
+
+ + + 删除 +
+
+ 新增服务器覆盖 +
+
+ + - +
- 当前没有单独覆盖配置，所有节点都会走默认线程数。 + 当前没有服务器级单独覆盖配置，所有服务器都会走默认线程数。
- + 删除
- 新增节点覆盖 + 新增服务器覆盖
@@ -71,7 +91,7 @@ type="info" show-icon style="margin-top: 12px" - title="关闭时不追加远端日志镜像；关键模式回传阶段/代理/异常等关键过程；全量模式会追加更多执行过程，便于临时分析。" + title="默认会按服务器级别统一下发：80 进程、每进程 1000 线程；这里只保留服务器级覆盖，不建议人工管理进程实例级并发。关闭时不追加远端日志镜像；关键模式回传阶段/代理/异常等关键过程；全量模式会追加更多执行过程，便于临时分析。" /> @@ -223,6 +243,25 @@ type NodeThreadOverrideRow = { thread_count: number; }; +type NodeProcessOverrideRow = { + node_code: string; + process_count: number; +}; + +const resolveServerCode = (rawNodeCode: string) => { + const normalized = String(rawNodeCode || "").trim(); + if (!normalized) return ""; + const parts = normalized.split("-"); + if (parts.length >= 2) { + const suffix = parts[parts.length - 1]; + const parent = parts.slice(0, -1).join("-"); + if (/^[a-z]{1,3}$/i.test(suffix) && /\d$/.test(parent)) { + return parent; + } + } + return normalized; +}; + const DETECT_LABELS: Record = { detect_register: "检查注册", detect_baidu_site: "百度 site 查询", @@ -235,9 +274,11 @@ const DETECT_LABELS: Record = { }; const loading = ref(true); -const threadCount = ref(2); +const processCount = ref(80); +const threadCount = ref(1000); const currentNodeCode = ref(""); const nodeThreadOverrides = ref([]); +const nodeProcessOverrides = ref([]); const detectItems = ref([]); const backups = ref([]); const proxyConfig = ref>({ @@ -328,12 +369,32 @@ const moveItem = (index: number, offset: -1 | 1) => { }; const normalizeNodeThreadOverrides = (payload: Record) => { - nodeThreadOverrides.value = Object.entries(payload || {}) - .map(([node_code, thread_count]) => ({ - node_code: String(node_code || "").trim(), - thread_count: Number(thread_count || 0) - })) - .filter((item) => item.node_code && item.thread_count > 0) + const collapsed = Object.entries(payload || {}).reduce>((result, [rawNodeCode, threadCountValue]) => { + const node_code = resolveServerCode(String(rawNodeCode || "").trim()); + const thread_count = Number(threadCountValue || 0); + if (!node_code || thread_count <= 0) { + return result; + } + result[node_code] = Math.max(Number(result[node_code] || 0), thread_count); + return result; + }, {}); + nodeThreadOverrides.value = Object.entries(collapsed) + .map(([node_code, thread_count]) => ({ node_code, thread_count })) + .sort((a, b) => a.node_code.localeCompare(b.node_code)); +}; + +const normalizeNodeProcessOverrides = (payload: Record) => { + const collapsed = Object.entries(payload || {}).reduce>((result, [rawNodeCode, processCountValue]) => { + const node_code = resolveServerCode(String(rawNodeCode || "").trim()); + const process_count = Number(processCountValue || 0); + if (!node_code || process_count <= 0) { + return result; + } + result[node_code] = Math.max(Number(result[node_code] || 0), process_count); + return result; + }, {}); + nodeProcessOverrides.value = Object.entries(collapsed) + .map(([node_code, process_count]) => ({ node_code, process_count })) .sort((a, b) => a.node_code.localeCompare(b.node_code)); }; @@ -348,10 +409,21 @@ const addNodeThreadOverride = () => { }); }; +const addNodeProcessOverride = () => { + nodeProcessOverrides.value.push({ + node_code: "", + process_count: processCount.value || 80 + }); +}; + const removeNodeThreadOverride = (index: number) => { nodeThreadOverrides.value.splice(index, 1); }; +const removeNodeProcessOverride = (index: number) => { + nodeProcessOverrides.value.splice(index, 1); +}; + const loadBackups = async () => { try { const response = await settingsApi.getSettingsBackups(); @@ -379,8 +451,10 @@ const loadSettings = async () => { loading.value = true; try { const response = await settingsApi.getSettings(); + processCount.value = response.data.process_count || 80; threadCount.value = response.data.thread_count; currentNodeCode.value = response.data.current_node_code || ""; + normalizeNodeProcessOverrides(response.data.node_process_counts || {}); normalizeNodeThreadOverrides(response.data.node_thread_counts || {}); normalizeDetectItems(response.data.detect_options || {}); @@ -430,7 +504,19 @@ const buildSettingsPayload = () => { return result; }, {}); + const nodeProcessCounts = nodeProcessOverrides.value.reduce>((result, item) => { + const nodeCode = String(item.node_code || "").trim(); + const processCountValue = Number(item.process_count || 0); + if (!nodeCode || processCountValue <= 0) { + return result; + } + result[nodeCode] = processCountValue; + return result; + }, {}); + return { + process_count: processCount.value, + node_process_counts: nodeProcessCounts, thread_count: threadCount.value, node_thread_counts: nodeThreadCounts, detect_options: detectOptions, diff --git a/domainCheck/app/core/detect_engine.py b/domainCheck/app/core/detect_engine.py index 8ca9799..3160df3 100644 --- a/domainCheck/app/core/detect_engine.py +++ b/domainCheck/app/core/detect_engine.py @@ -11,6 +11,7 @@ import time from loguru import logger from app.utils.database import Database +from app.utils.detection_results import normalize_detector_result from app.detectors.rdap_detector import RDAPDetector from app.detectors.wayback_detector import WaybackDetector from app.detectors.baidu_detector import BaiduDetector @@ -32,6 +33,10 @@ class DetectEngine: """ 检测引擎 """ + + OUTCOME_SUCCESS = "success" + OUTCOME_BLACKLISTED = "blacklisted" + OUTCOME_FAILED = "failed" def __init__(self): """ @@ -49,6 +54,9 @@ class DetectEngine: self.jucha_detector = JuchaDetector() def detect_domain(self, domain_id): + return self._detect_domain_with_outcome(domain_id) == self.OUTCOME_SUCCESS + + def _detect_domain_with_outcome(self, domain_id): """ 检测域名 @@ -69,24 +77,30 @@ class DetectEngine: self.db.update_domain_detect_status(domain_id, DETECT_STATUS_RUNNING) # 1. 基础检测 - if not self._basic_detect(domain_id, domain): + basic_outcome = self._basic_detect(domain_id, domain) + if basic_outcome != self.OUTCOME_SUCCESS: logger.info(f"基础检测失败，停止后续检测: {domain}") - return False + if basic_outcome == self.OUTCOME_FAILED: + self.db.update_domain_detect_status(domain_id, DETECT_STATUS_FAILED) + return basic_outcome # 2. 深度检测 - if not self._deep_detect(domain_id, domain): + deep_outcome = self._deep_detect(domain_id, domain) + if deep_outcome != self.OUTCOME_SUCCESS: logger.info(f"深度检测失败: {domain}") - return False + if deep_outcome == self.OUTCOME_FAILED: + self.db.update_domain_detect_status(domain_id, DETECT_STATUS_FAILED) + return deep_outcome # 更新检测状态为正常 self.db.update_domain_detect_status(domain_id, DETECT_STATUS_COMPLETED) logger.info(f"域名检测完成: {domain}") - return True + return self.OUTCOME_SUCCESS except Exception as e: logger.error(f"检测域名出错: {e}") # 更新检测状态为检测失败 self.db.update_domain_detect_status(domain_id, DETECT_STATUS_FAILED) - return False + return self.OUTCOME_FAILED def _basic_detect(self, domain_id, domain): """ @@ -94,7 +108,7 @@ class DetectEngine: :param domain_id: 域名ID :param domain: 域名 - :return: bool - 是否检测通过 + :return: str - 检测结果 """ # 1. 检查是否为一口价域名 is_ykj = self.db.is_ykj_domain(domain_id) @@ -108,7 +122,7 @@ class DetectEngine: if self.db.is_blacklisted(domain): logger.info(f"域名在黑名单中: {domain}") self.db.update_domain_detect_status(domain_id, DETECT_STATUS_BLACKLISTED) - return False + return self.OUTCOME_BLACKLISTED # 4. 时光机快照年份采集 snapshot_years = self.wayback_detector.get_snapshot_years(domain) @@ -120,9 +134,9 @@ class DetectEngine: logger.info(f"域名包含敏感词: {domain}") self.db.update_domain_detect_status(domain_id, DETECT_STATUS_BLACKLISTED) self.db.add_to_blacklist(domain, "快照包含敏感词") - return False + return self.OUTCOME_BLACKLISTED - return True + return self.OUTCOME_SUCCESS def _deep_detect(self, domain_id, domain): """ @@ -130,38 +144,52 @@ class DetectEngine: :param domain_id: 域名ID :param domain: 域名 - :return: bool - 是否检测通过 + :return: str - 检测结果 """ - # 1. 百度历史/Site - baidu_history = self.baidu_detector.check_history(domain) - baidu_site = self.baidu_detector.check_site(domain) - - # 2. 360 Site - qihu360_site = self.qihu360_detector.check_site(domain) - - # 3. Google Site - google_site = self.google_detector.check_site(domain) - - # 4. 站长之家 - chinaz_info = self.chinaz_detector.check_domain(domain) - - # 5. 爱站网 - aizhan_info = self.aizhan_detector.check_domain(domain) - - # 6. 桔子SEO - juziseo_info = self.juziseo_detector.check_domain(domain) - - # 7. 聚查 - jucha_info = self.jucha_detector.check_domain(domain) - - # 检查是否有风险 - if self._check_risk(domain_id, domain, baidu_history, baidu_site, qihu360_site, google_site, chinaz_info, aizhan_info, juziseo_info, jucha_info): - return False - + detector_results = {} + detector_steps = [ + ("baidu_history", lambda: self.baidu_detector.check_history(domain)), + ("baidu_site", lambda: self.baidu_detector.check_site(domain)), + ("qihu360_site", lambda: self.qihu360_detector.check_site(domain)), + ("google_site", lambda: self.google_detector.check_site(domain)), + ("chinaz_info", lambda: self.chinaz_detector.check_domain(domain)), + ("aizhan_info", lambda: self.aizhan_detector.check_domain(domain)), + ("juziseo_info", lambda: self.juziseo_detector.check_domain(domain)), + ("jucha_info", lambda: self.jucha_detector.check_domain(domain)), + ] + + for detector_name, runner in detector_steps: + detector_results[detector_name] = normalize_detector_result(detector_name, runner()) + + detector_error = detector_results[detector_name].get("error") + if detector_error: + logger.error( + f"深度检测存在第三方检测错误: {domain}, detector={detector_name}, error={detector_error}" + ) + return self.OUTCOME_FAILED + + if self._check_risk( + domain_id, + domain, + detector_results.get("baidu_history"), + detector_results.get("baidu_site"), + detector_results.get("qihu360_site"), + detector_results.get("google_site"), + detector_results.get("chinaz_info"), + detector_results.get("aizhan_info"), + detector_results.get("juziseo_info"), + detector_results.get("jucha_info"), + ): + self._persist_detection_results(domain_id, detector_results) + return self.OUTCOME_BLACKLISTED + # 保存检测结果 - self.db.add_detection_result(domain_id, baidu_history, baidu_site, qihu360_site, google_site, chinaz_info, aizhan_info, juziseo_info, jucha_info) + persisted = self._persist_detection_results(domain_id, detector_results) + if not persisted: + logger.error(f"保存检测结果失败: {domain}") + return self.OUTCOME_FAILED - return True + return self.OUTCOME_SUCCESS def _check_risk(self, domain_id, domain, baidu_history, baidu_site, qihu360_site, google_site, chinaz_info, aizhan_info, juziseo_info, jucha_info): """ @@ -180,7 +208,9 @@ class DetectEngine: :return: bool - 是否有风险 """ # 检查百度历史过灰 - if baidu_history and '灰' in str(baidu_history): + if ( + isinstance(baidu_history, dict) and baidu_history.get('has_gray') + ) or (baidu_history and '灰' in str(baidu_history)): logger.info(f"百度历史过灰: {domain}") self.db.update_domain_detect_status(domain_id, DETECT_STATUS_BLACKLISTED) self.db.add_to_blacklist(domain, "百度历史过灰") @@ -215,7 +245,7 @@ class DetectEngine: # 检查WHOIS状态 if jucha_info and 'whois' in jucha_info: - if jucha_info['whois'].get('status') in ['clientHold', 'serverHold']: + if jucha_info['whois'].get('whois_status') in ['clientHold', 'serverHold']: logger.info(f"WHOIS状态异常: {domain}") self.db.update_domain_detect_status(domain_id, DETECT_STATUS_BLACKLISTED) self.db.add_to_blacklist(domain, "WHOIS状态异常") @@ -225,11 +255,24 @@ class DetectEngine: if jucha_info and 'intercept' in jucha_info: if not jucha_info['intercept'].get('normal', True): logger.info(f"拦截检测异常: {domain}") - self.db.update_domain_detect_status(domain_id, 4) # 4 表示黑名单 + self.db.update_domain_detect_status(domain_id, DETECT_STATUS_BLACKLISTED) self.db.add_to_blacklist(domain, "拦截检测异常") return True return False + + def _persist_detection_results(self, domain_id, detector_results): + return self.db.add_detection_result( + domain_id, + detector_results.get("baidu_history"), + detector_results.get("baidu_site"), + detector_results.get("qihu360_site"), + detector_results.get("google_site"), + detector_results.get("chinaz_info"), + detector_results.get("aizhan_info"), + detector_results.get("juziseo_info"), + detector_results.get("jucha_info"), + ) def _contains_sensitive_words(self, text): """ @@ -264,11 +307,12 @@ class DetectEngine: self.db.update_task_status(task_id, 1) # 1 表示执行中 # 执行检测 - success = self.detect_domain(domain_id) + outcome = self._detect_domain_with_outcome(domain_id) # 更新任务状态 - if success: + if outcome in (self.OUTCOME_SUCCESS, self.OUTCOME_BLACKLISTED): self.db.update_task_status(task_id, 2) # 2 表示完成 + return True else: # 增加重试次数 retry_count = task.get('retry_count', 0) + 1 @@ -277,8 +321,7 @@ class DetectEngine: self.db.update_task_status(task_id, 0) # 0 表示待执行 else: self.db.update_task_status(task_id, 3) # 3 表示失败 - - return success + return False except Exception as e: logger.error(f"处理任务出错: {e}") # 更新任务状态为失败 diff --git a/domainCheck/app/detectors/aizhan_detector.py b/domainCheck/app/detectors/aizhan_detector.py index 93d4a15..72c678e 100644 --- a/domainCheck/app/detectors/aizhan_detector.py +++ b/domainCheck/app/detectors/aizhan_detector.py @@ -64,7 +64,7 @@ class AizhanDetector(BaseDetector): } else: self._log_warning(f"爱站网查询失败: {response.status_code}") - return {'title': '', 'risk': '', 'has_sensitive': False} + return {'error': f'HTTP {response.status_code}'} except Exception as e: return self._handle_exception(e, domain) @@ -127,4 +127,4 @@ class AizhanDetector(BaseDetector): if word in title: return True - return False \ No newline at end of file + return False diff --git a/domainCheck/app/detectors/baidu_detector.py b/domainCheck/app/detectors/baidu_detector.py index df49b2d..13d2dad 100644 --- a/domainCheck/app/detectors/baidu_detector.py +++ b/domainCheck/app/detectors/baidu_detector.py @@ -83,10 +83,9 @@ class BaiduDetector(BaseDetector): } else: self._log_warning(f"百度site查询失败: {response.status_code}") - return {'has_收录': False, 'subdomains': []} + return {'error': f'HTTP {response.status_code}'} except Exception as e: - self._handle_exception(e, domain) - return {'has_收录': False, 'subdomains': []} + return self._handle_exception(e, domain) def check_history(self, domain): """ @@ -123,10 +122,9 @@ class BaiduDetector(BaseDetector): } else: self._log_warning(f"百度历史查询失败: {response.status_code}") - return {'has_history': False, 'has_gray': False} + return {'error': f'HTTP {response.status_code}'} except Exception as e: - self._handle_exception(e, domain) - return {'has_history': False, 'has_gray': False} + return self._handle_exception(e, domain) def _extract_subdomains(self, content, domain): """ @@ -148,4 +146,4 @@ class BaiduDetector(BaseDetector): return subdomains except Exception as e: self._handle_exception(e, domain) - return [] \ No newline at end of file + return [] diff --git a/domainCheck/app/detectors/chinaz_detector.py b/domainCheck/app/detectors/chinaz_detector.py index c8ac4df..0cc9d39 100644 --- a/domainCheck/app/detectors/chinaz_detector.py +++ b/domainCheck/app/detectors/chinaz_detector.py @@ -64,7 +64,7 @@ class ChinazDetector(BaseDetector): } else: self._log_warning(f"站长之家查询失败: {response.status_code}") - return {'title': '', 'category': '', 'has_sensitive': False} + return {'error': f'HTTP {response.status_code}'} except Exception as e: return self._handle_exception(e, domain) @@ -127,4 +127,4 @@ class ChinazDetector(BaseDetector): if word in title: return True - return False \ No newline at end of file + return False diff --git a/domainCheck/app/detectors/google_detector.py b/domainCheck/app/detectors/google_detector.py index 35a8642..6620d89 100644 --- a/domainCheck/app/detectors/google_detector.py +++ b/domainCheck/app/detectors/google_detector.py @@ -74,7 +74,6 @@ class GoogleDetector(BaseDetector): } else: self._log_warning(f"Google site查询失败: {response.status_code}") - return {'has_收录': False} + return {'error': f'HTTP {response.status_code}'} except Exception as e: - self._handle_exception(e, domain) - return {'has_收录': False} \ No newline at end of file + return self._handle_exception(e, domain) diff --git a/domainCheck/app/detectors/jucha_detector.py b/domainCheck/app/detectors/jucha_detector.py index 5109325..13f8551 100644 --- a/domainCheck/app/detectors/jucha_detector.py +++ b/domainCheck/app/detectors/jucha_detector.py @@ -81,10 +81,9 @@ class JuchaDetector(BaseDetector): return whois_info else: self._log_warning(f"聚查WHOIS查询失败: {response.status_code}") - return {'status': ''} + return {'error': f'HTTP {response.status_code}'} except Exception as e: - self._handle_exception(e, domain) - return {'status': ''} + return self._handle_exception(e, domain) def check_beian(self, domain): """ @@ -113,10 +112,9 @@ class JuchaDetector(BaseDetector): return beian_info else: self._log_warning(f"聚查备案查询失败: {response.status_code}") - return {'has_beian': False, 'beian_year': '', 'is_enterprise': False, 'beian_match': False} + return {'error': f'HTTP {response.status_code}'} except Exception as e: - self._handle_exception(e, domain) - return {'has_beian': False, 'beian_year': '', 'is_enterprise': False, 'beian_match': False} + return self._handle_exception(e, domain) def check_intercept(self, domain): """ @@ -147,10 +145,9 @@ class JuchaDetector(BaseDetector): } else: self._log_warning(f"聚查拦截查询失败: {response.status_code}") - return {'normal': False} + return {'error': f'HTTP {response.status_code}'} except Exception as e: - self._handle_exception(e, domain) - return {'normal': False} + return self._handle_exception(e, domain) def _extract_whois_info(self, content): """ @@ -225,4 +222,4 @@ class JuchaDetector(BaseDetector): return False except Exception as e: self._handle_exception(e, 'check_intercept_status') - return False \ No newline at end of file + return False diff --git a/domainCheck/app/detectors/juziseo_detector.py b/domainCheck/app/detectors/juziseo_detector.py index 54af1fa..54becb2 100644 --- a/domainCheck/app/detectors/juziseo_detector.py +++ b/domainCheck/app/detectors/juziseo_detector.py @@ -93,10 +93,9 @@ class JuziseoDetector(BaseDetector): } else: self._log_warning(f"桔子SEO历史查询失败: {response.status_code}") - return {'has_sensitive': False, 'has_baidu_history': False, 'has_subdomains': False, 'is_simplified': True} + return {'error': f'HTTP {response.status_code}'} except Exception as e: - self._handle_exception(e, domain) - return {'has_sensitive': False, 'has_baidu_history': False, 'has_subdomains': False, 'is_simplified': True} + return self._handle_exception(e, domain) def check_backlink(self, domain): """ @@ -131,10 +130,9 @@ class JuziseoDetector(BaseDetector): } else: self._log_warning(f"桔子SEO外链查询失败: {response.status_code}") - return {'has_sensitive': False, 'has_subdomains': False} + return {'error': f'HTTP {response.status_code}'} except Exception as e: - self._handle_exception(e, domain) - return {'has_sensitive': False, 'has_subdomains': False} + return self._handle_exception(e, domain) def _extract_history_info(self, content): """ @@ -211,4 +209,4 @@ class JuziseoDetector(BaseDetector): if word in content: return True - return False \ No newline at end of file + return False diff --git a/domainCheck/app/detectors/qihu360_detector.py b/domainCheck/app/detectors/qihu360_detector.py index 68264f5..ecf4fe4 100644 --- a/domainCheck/app/detectors/qihu360_detector.py +++ b/domainCheck/app/detectors/qihu360_detector.py @@ -79,10 +79,9 @@ class Qihu360Detector(BaseDetector): } else: self._log_warning(f"360 site查询失败: {response.status_code}") - return {'has_收录': False, 'subdomains': []} + return {'error': f'HTTP {response.status_code}'} except Exception as e: - self._handle_exception(e, domain) - return {'has_收录': False, 'subdomains': []} + return self._handle_exception(e, domain) def _extract_subdomains(self, content, domain): """ @@ -104,4 +103,4 @@ class Qihu360Detector(BaseDetector): return subdomains except Exception as e: self._handle_exception(e, domain) - return [] \ No newline at end of file + return [] diff --git a/domainCheck/app/detectors/rdap_detector.py b/domainCheck/app/detectors/rdap_detector.py index edf5d56..49ae713 100644 --- a/domainCheck/app/detectors/rdap_detector.py +++ b/domainCheck/app/detectors/rdap_detector.py @@ -73,7 +73,7 @@ class RDAPDetector(BaseDetector): return 2 # 可注册 # 检查域名状态 - statuses = result.get('status', []) + statuses = result.get('statuses', []) if 'clientHold' in statuses: return 7 # clientHold elif 'serverHold' in statuses: @@ -125,4 +125,4 @@ class RDAPDetector(BaseDetector): elif event_action == 'last update': result['last_update'] = event_date - return result \ No newline at end of file + return result diff --git a/domainCheck/app/detectors/wayback_detector.py b/domainCheck/app/detectors/wayback_detector.py index 561f042..08e4267 100644 --- a/domainCheck/app/detectors/wayback_detector.py +++ b/domainCheck/app/detectors/wayback_detector.py @@ -17,13 +17,13 @@ import zlib from base64 import b64decode, b64encode from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait -import redis import requests from requests.adapters import HTTPAdapter from urllib3.util.retry import Retry from app.config import config from app.detectors.base import BaseDetector +from app.utils.redis_client import get_redis_client class WaybackDetector(BaseDetector): @@ -51,6 +51,7 @@ class WaybackDetector(BaseDetector): def _build_session(self): session = requests.Session() + session.trust_env = False retry = Retry( total=max(0, config.WAYBACK_RETRY_COUNT), backoff_factor=0.5, @@ -106,15 +107,7 @@ class WaybackDetector(BaseDetector): def _build_redis_client(self): try: - client = redis.Redis( - host=config.REDIS_HOST, - port=config.REDIS_PORT, - password=config.REDIS_PASSWORD, - db=config.REDIS_DB, - decode_responses=True, - socket_connect_timeout=5, - socket_timeout=5, - ) + client = get_redis_client(role="standard") client.ping() return client except Exception: @@ -489,6 +482,14 @@ class WaybackDetector(BaseDetector): trimmed.append(item) return trimmed + def _resolve_scan_record_fetch_limit(self): + max_records = max(1, int(getattr(config, "WAYBACK_MAX_RECORDS", 8) or 8)) + # 扫描阶段最终只会保留最近的少量快照；如果每次都把整个 CDX 历史拉下来， + # 在快照特别多的域名上会白白浪费很多秒。这里改成“最近一小窗”，同时留出 + # 重复 digest/标题的缓冲空间，避免把最新几条里重复记录全部裁没了。 + recent_window = max(12, max_records * 6) + return -recent_window + def scan_snapshots(self, domain, sensitive_words=None, stop_on_first_hit=True, recent_years=None): sensitive_words = sensitive_words or config.load_sensitive_words() request_errors = [] @@ -512,12 +513,17 @@ class WaybackDetector(BaseDetector): 'request_errors': [f"wayback_backoff_active:{round(transient_backoff_remaining, 2)}s"], 'elapsed_seconds': 0.0, } + transient_request_failures = 0 + transient_request_failure_threshold = 2 + latest_fetch = self._fetch_cdx_records_with_meta(domain, limit=-1, fast_latest=True) latest_record = (latest_fetch.get('records') or [None])[0] + latest_fetch_transient_failure = False if latest_fetch.get('error'): request_errors.append(f"latest_cdx: {latest_fetch.get('error')}") if self._is_transient_request_error(latest_fetch.get('error')): - self._trip_transient_backoff() + latest_fetch_transient_failure = True + transient_request_failures += 1 latest_timestamp = (latest_record or {}).get('timestamp') latest_digest = (latest_record or {}).get('digest', '') cutoff_year = self._resolve_recent_year_cutoff(recent_years) @@ -547,26 +553,6 @@ class WaybackDetector(BaseDetector): transient_snapshot_failures = 0 transient_snapshot_failure_threshold = max(2, domain_concurrency) - if (not latest_timestamp) and latest_fetch.get('error') and self._is_transient_request_error(latest_fetch.get('error')): - return { - 'snapshot_years': [], - 'has_sensitive_content': False, - 'matched_word': None, - 'matched_timestamp': None, - 'matched_title': None, - 'backlink_count': 0, - 'backlink_count_gt_10': False, - 'checked_snapshot_count': 0, - 'fetched_snapshot_count': 0, - 'failed_snapshot_count': max(1, failed_snapshot_count), - 'unique_title_count': 0, - 'duplicate_title_skipped': 0, - 'digest_duplicate_skipped': 0, - 'request_error_count': len(request_errors), - 'request_errors': request_errors, - 'elapsed_seconds': round(time.time() - started_at, 2), - } - if latest_timestamp: latest_result = self._fetch_snapshot_title(domain, latest_timestamp) checked_snapshot_count = 1 @@ -605,40 +591,55 @@ class WaybackDetector(BaseDetector): if latest_error: request_errors.append(f"latest_snapshot: {latest_error}") if latest_error and self._is_transient_request_error(latest_error): - transient_snapshot_failures += 1 - self._trip_transient_backoff() - return { - 'snapshot_years': [], - 'has_sensitive_content': False, - 'matched_word': None, - 'matched_timestamp': None, - 'matched_title': None, - 'backlink_count': 0, - 'backlink_count_gt_10': False, - 'checked_snapshot_count': checked_snapshot_count, - 'fetched_snapshot_count': fetched_snapshot_count, - 'failed_snapshot_count': failed_snapshot_count, - 'unique_title_count': unique_title_count, - 'duplicate_title_skipped': duplicate_title_skipped, - 'digest_duplicate_skipped': digest_duplicate_skipped, - 'request_error_count': len(request_errors), - 'request_errors': request_errors, - 'elapsed_seconds': round(time.time() - started_at, 2), - } + transient_request_failures += 1 cached_records = self._load_cached_records(domain) if cached_records is not None: records = cached_records else: - records_fetch = self._fetch_cdx_records_with_meta(domain) - if records_fetch.get('error'): - request_errors.append(f"records_cdx: {records_fetch.get('error')}") - if self._is_transient_request_error(records_fetch.get('error')): - self._trip_transient_backoff() - records = records_fetch.get('records') or [] - if records: - self._save_cached_records(domain, records) - self._save_cached_timestamps(domain, [item['timestamp'] for item in records]) + if latest_fetch_transient_failure and not latest_timestamp: + records = [] + else: + records_fetch = self._fetch_cdx_records_with_meta( + domain, + limit=self._resolve_scan_record_fetch_limit(), + ) + if records_fetch.get('error'): + request_errors.append(f"records_cdx: {records_fetch.get('error')}") + if self._is_transient_request_error(records_fetch.get('error')): + transient_request_failures += 1 + records = records_fetch.get('records') or [] + if records: + self._save_cached_records(domain, records) + self._save_cached_timestamps(domain, [item['timestamp'] for item in records]) + + if ( + not latest_timestamp + and not records + and ( + transient_request_failures >= transient_request_failure_threshold + or latest_fetch_transient_failure + ) + ): + self._trip_transient_backoff() + return { + 'snapshot_years': [], + 'has_sensitive_content': False, + 'matched_word': None, + 'matched_timestamp': None, + 'matched_title': None, + 'backlink_count': 0, + 'backlink_count_gt_10': False, + 'checked_snapshot_count': checked_snapshot_count, + 'fetched_snapshot_count': fetched_snapshot_count, + 'failed_snapshot_count': max(1, failed_snapshot_count), + 'unique_title_count': 0, + 'duplicate_title_skipped': duplicate_title_skipped, + 'digest_duplicate_skipped': digest_duplicate_skipped, + 'request_error_count': len(request_errors), + 'request_errors': request_errors, + 'elapsed_seconds': round(time.time() - started_at, 2), + } records = self._filter_records_recent_years(records, recent_years=recent_years) records = sorted(records, key=lambda item: item.get('timestamp', ''), reverse=True) @@ -660,12 +661,12 @@ class WaybackDetector(BaseDetector): digest_seen.add(digest) pending_records.append(item) - with ThreadPoolExecutor(max_workers=domain_concurrency) as executor: - pending = {} - index = 0 - finished_count = 1 if latest_timestamp else 0 - stop_requested = False - + executor = ThreadPoolExecutor(max_workers=domain_concurrency) + pending = {} + index = 0 + finished_count = 1 if latest_timestamp else 0 + stop_requested = False + try: while (index < len(pending_records) or pending) and not stop_requested: while index < len(pending_records) and len(pending) < domain_concurrency and not stop_requested: timestamp = pending_records[index]['timestamp'] @@ -694,8 +695,8 @@ class WaybackDetector(BaseDetector): request_errors.append(f"snapshot:{timestamp}: {error_message}") if error_message and self._is_transient_request_error(error_message): transient_snapshot_failures += 1 - self._trip_transient_backoff() if transient_snapshot_failures >= transient_snapshot_failure_threshold: + self._trip_transient_backoff() stop_requested = True continue @@ -722,10 +723,13 @@ class WaybackDetector(BaseDetector): ) if config.WAYBACK_REQUEST_DELAY > 0: time.sleep(config.WAYBACK_REQUEST_DELAY) - + finally: if stop_requested: - for future in pending: + for future in list(pending.keys()): future.cancel() + executor.shutdown(wait=False, cancel_futures=True) + else: + executor.shutdown(wait=True) return { 'snapshot_years': years, diff --git a/domainCheck/app/ui/domain_filter.py b/domainCheck/app/ui/domain_filter.py index 05648cb..4ccd22f 100644 --- a/domainCheck/app/ui/domain_filter.py +++ b/domainCheck/app/ui/domain_filter.py @@ -8,6 +8,8 @@ @explain : 域名筛选界面 ''' +import json + from PySide6.QtWidgets import QWidget, QVBoxLayout, QHBoxLayout, QGridLayout, QPushButton, QLabel, QComboBox, QDateEdit, QCheckBox, QTableWidget, QTableWidgetItem, QHeaderView, QFileDialog, QLineEdit, QSpinBox, QInputDialog from PySide6.QtGui import QIntValidator from PySide6.QtCore import Qt, QDate, QThread, Signal @@ -15,6 +17,11 @@ from loguru import logger from app.core.export_manager import ExportManager from app.utils.database import Database +from app.utils.detection_results import ( + build_manual_detection_result, + load_detection_result, + resolve_detection_status, +) from app.utils.status_codes import ( DETECT_STATUS_BLACKLISTED, DETECT_STATUS_COMPLETED, @@ -205,9 +212,9 @@ class UpdateThread(QThread): baidu_history_value = self.update_values['baidu_history'] if baidu_history_value != '不更新': status_value = status_mappings['百度历史收录状态'][baidu_history_value] - # 转换为JSON格式 - import json - json_value = json.dumps({"status": status_value}) + json_value = json.dumps( + build_manual_detection_result(status_value, legacy_key='has_history') + ) if detection_id: cur.execute("UPDATE domain_detections SET baidu_history = %s WHERE domain_id = %s", (json_value, domain_info['id'])) else: @@ -219,9 +226,9 @@ class UpdateThread(QThread): baidu_site_value = self.update_values['baidu_site'] if baidu_site_value != '不更新': status_value = status_mappings['百度site收录状态'][baidu_site_value] - # 转换为JSON格式 - import json - json_value = json.dumps({"status": status_value}) + json_value = json.dumps( + build_manual_detection_result(status_value, legacy_key='has_收录') + ) if detection_id: cur.execute("UPDATE domain_detections SET baidu_site = %s WHERE domain_id = %s", (json_value, domain_info['id'])) else: @@ -244,9 +251,9 @@ class UpdateThread(QThread): qihu360_site_value = self.update_values['qihu360_site'] if qihu360_site_value != '不更新': status_value = status_mappings['360 site收录状态'][qihu360_site_value] - # 转换为JSON格式 - import json - json_value = json.dumps({"status": status_value}) + json_value = json.dumps( + build_manual_detection_result(status_value, legacy_key='has_收录') + ) if detection_id: cur.execute("UPDATE domain_detections SET qihu360_site = %s WHERE domain_id = %s", (json_value, domain_info['id'])) else: @@ -258,9 +265,9 @@ class UpdateThread(QThread): google_site_value = self.update_values['google_site'] if google_site_value != '不更新': status_value = status_mappings['Google site收录状态'][google_site_value] - # 转换为JSON格式 - import json - json_value = json.dumps({"status": status_value}) + json_value = json.dumps( + build_manual_detection_result(status_value, legacy_key='has_收录') + ) if detection_id: cur.execute("UPDATE domain_detections SET google_site = %s WHERE domain_id = %s", (json_value, domain_info['id'])) else: @@ -1082,28 +1089,14 @@ class DomainFilterWidget(QWidget): # 百度历史收录状态 baidu_history = domain.get('baidu_history') - if baidu_history is None: - baidu_history = {} - elif isinstance(baidu_history, str): - import json - try: - baidu_history = json.loads(baidu_history) - except: - baidu_history = {} - baidu_history_status = '是' if baidu_history.get('status') else '否' + baidu_history = load_detection_result(baidu_history) + baidu_history_status = '是' if resolve_detection_status(baidu_history, 'has_history') else '否' self.table_widget.setItem(row, 12, QTableWidgetItem(baidu_history_status)) # 百度site收录状态 baidu_site = domain.get('baidu_site') - if baidu_site is None: - baidu_site = {} - elif isinstance(baidu_site, str): - import json - try: - baidu_site = json.loads(baidu_site) - except: - baidu_site = {} - baidu_site_status = '是' if baidu_site.get('status') else '否' + baidu_site = load_detection_result(baidu_site) + baidu_site_status = '是' if resolve_detection_status(baidu_site, 'has_收录') else '否' self.table_widget.setItem(row, 13, QTableWidgetItem(baidu_site_status)) # title是否有中文 @@ -1113,28 +1106,14 @@ class DomainFilterWidget(QWidget): # 360site收录 qihu360_site = domain.get('qihu360_site') - if qihu360_site is None: - qihu360_site = {} - elif isinstance(qihu360_site, str): - import json - try: - qihu360_site = json.loads(qihu360_site) - except: - qihu360_site = {} - qihu360_site_status = '是' if qihu360_site.get('status') else '否' + qihu360_site = load_detection_result(qihu360_site) + qihu360_site_status = '是' if resolve_detection_status(qihu360_site, 'has_收录') else '否' self.table_widget.setItem(row, 15, QTableWidgetItem(qihu360_site_status)) # Google site收录状态 google_site = domain.get('google_site') - if google_site is None: - google_site = {} - elif isinstance(google_site, str): - import json - try: - google_site = json.loads(google_site) - except: - google_site = {} - google_site_status = '是' if google_site.get('status') else '否' + google_site = load_detection_result(google_site) + google_site_status = '是' if resolve_detection_status(google_site, 'has_收录') else '否' self.table_widget.setItem(row, 16, QTableWidgetItem(google_site_status)) # 友情链接数量 diff --git a/domainCheck/app/utils/database.py b/domainCheck/app/utils/database.py index 804a270..c6cc949 100644 --- a/domainCheck/app/utils/database.py +++ b/domainCheck/app/utils/database.py @@ -8,11 +8,15 @@ @explain : 数据库操作类 ''' +import hashlib import json import os import socket import threading import time +import uuid +from datetime import datetime, timedelta +from pathlib import Path import psycopg2 import redis @@ -20,6 +24,7 @@ from loguru import logger from psycopg2 import extensions from psycopg2.extras import Json from app.config import config +from app.utils.redis_client import get_redis_client from app.utils.status_codes import ( DETECT_STATUS_BLACKLISTED, DETECT_STATUS_COMPLETED, @@ -34,6 +39,93 @@ from app.utils.status_codes import ( _POOL_VERBOSE_LOGS = bool(int(os.getenv("DOMAINCHECK_DB_POOL_VERBOSE_LOGS", "0") or 0)) +_DETECT_JOB_ITEM_RECYCLE_BATCH_SIZE = 2000 +_DETECT_JOB_ITEM_RECYCLE_MAX_BATCHES = 6 +_DETECT_JOB_ITEM_RELEASE_BATCH_SIZE = 2000 +_DETECT_JOB_ITEM_RELEASE_MAX_BATCHES = 6 +_RUNTIME_INDEX_BUILD_LOCK_SCOPE = "detect-job-items-runtime-indexes" +_RUNTIME_REQUIRED_INDEX_DDL = { + "idx_detect_job_items_job_domain_step": """ + CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS idx_detect_job_items_job_domain_step + ON detect_job_items(job_id, domain_id, step_code) + """, + "idx_detect_job_items_claim_step_ready": """ + CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_detect_job_items_claim_step_ready + ON detect_job_items(status, step_code, lease_expires_at, create_time, id) + WHERE step_code <> '' AND status IN ('pending', 'failed') + """, + "idx_detect_job_items_claim_job_step_ready": """ + CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_detect_job_items_claim_job_step_ready + ON detect_job_items(job_id, status, step_code, lease_expires_at, create_time, id) + WHERE step_code <> '' AND status IN ('pending', 'failed') + """, + "idx_detect_job_items_stalled_job_activity": """ + CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_detect_job_items_stalled_job_activity + ON detect_job_items( + job_id, + status, + (COALESCE(updated_at, started_at, create_time)), + id + ) + WHERE status IN ('claimed', 'running') + """, + "idx_detect_job_items_release_node_job": """ + CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_detect_job_items_release_node_job + ON detect_job_items(claimed_by, job_id, status, id) + WHERE claimed_by <> '' + AND status IN ('claimed', 'running') + """, + "idx_detect_sync_records_scope_created": """ + CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_detect_sync_records_scope_created + ON detect_sync_records(sync_type, source_region, target_region, created_at DESC, id DESC) + """, + "idx_detect_sync_records_source_record_created": """ + CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_detect_sync_records_source_record_created + ON detect_sync_records( + sync_type, + source_region, + target_region, + ((payload_json->>'source_record_id')), + created_at DESC, + id DESC + ) + """, + "idx_detect_sync_records_source_record_hash_created": """ + CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_detect_sync_records_source_record_hash_created + ON detect_sync_records( + sync_type, + source_region, + target_region, + ((payload_json->>'source_record_id')), + ((payload_json->>'projection_hash')), + created_at DESC, + id DESC + ) + """, + "idx_detect_sync_records_runtime_push_lookup": """ + CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_detect_sync_records_runtime_push_lookup + ON detect_sync_records( + source_region, + target_region, + ((payload_json->>'sync_type')), + ((payload_json->>'source_record_id')), + created_at DESC, + id DESC + ) + WHERE sync_type = 'runtime_push' + """, +} +_RUNTIME_REQUIRED_INDEX_TABLES = { + "idx_detect_job_items_job_domain_step": "detect_job_items", + "idx_detect_job_items_claim_step_ready": "detect_job_items", + "idx_detect_job_items_claim_job_step_ready": "detect_job_items", + "idx_detect_job_items_stalled_job_activity": "detect_job_items", + "idx_detect_job_items_release_node_job": "detect_job_items", + "idx_detect_sync_records_scope_created": "detect_sync_records", + "idx_detect_sync_records_source_record_created": "detect_sync_records", + "idx_detect_sync_records_source_record_hash_created": "detect_sync_records", + "idx_detect_sync_records_runtime_push_lookup": "detect_sync_records", +} _STEP_CLAIM_PRIORITY = { @@ -53,6 +145,243 @@ _STEP_CLAIM_PRIORITY = { } +def _select_preferred_claim_job_ids(job_rows, *, limit=8, recent_hours=24, now=None): + safe_limit = max(1, int(limit or 1)) + safe_recent_hours = max(1, int(recent_hours or 1)) + normalized_rows = [] + for row in list(job_rows or []): + if not isinstance(row, (list, tuple)) or len(row) < 3: + continue + try: + job_id = int(row[0] or 0) + except Exception: + continue + if job_id <= 0: + continue + status = str(row[1] or "").strip().lower() + activity_at = row[2] + if activity_at is None: + continue + reference_now = now + if reference_now is None: + reference_now = ( + datetime.now(activity_at.tzinfo) + if getattr(activity_at, "tzinfo", None) is not None + else datetime.now() + ) + if status != "running" and activity_at < reference_now - timedelta(hours=safe_recent_hours): + continue + normalized_rows.append((job_id, status, activity_at)) + + normalized_rows.sort( + key=lambda item: ( + 0 if item[1] == "running" else 1, + -item[2].timestamp(), + -item[0], + ), + ) + preferred_job_ids = [] + seen_job_ids = set() + for job_id, _status, _activity_at in normalized_rows: + if job_id in seen_job_ids: + continue + preferred_job_ids.append(job_id) + seen_job_ids.add(job_id) + if len(preferred_job_ids) >= safe_limit: + break + return preferred_job_ids + + +def _build_pg_advisory_lock_key(scope: str) -> int: + normalized_scope = str(scope or "").strip() or "domaincheck-default" + digest = hashlib.sha1(normalized_scope.encode("utf-8")).digest() + raw_value = int.from_bytes(digest[:8], "big", signed=False) + return raw_value - (1 << 64) if raw_value >= (1 << 63) else raw_value + + +def _build_claim_token(node_code, thread_id=None): + normalized_node_code = str(node_code or "").strip() or "unknown" + normalized_thread_id = int( + thread_id if thread_id is not None else (threading.current_thread().ident or 0) + ) + node_fragment = normalized_node_code.replace(" ", "_")[:24] + thread_fragment = format(normalized_thread_id & 0xFFFFFFFF, "x") + entropy = ( + f"{normalized_node_code}|{normalized_thread_id}|" + f"{time.time_ns()}|{uuid.uuid4().hex}" + ) + digest = hashlib.sha1(entropy.encode("utf-8")).hexdigest()[:24] + return f"{node_fragment}-{thread_fragment}-{digest}"[:64] + + +def _read_local_json_config(filename, default): + for root in _local_config_search_roots(): + candidate = root / str(filename or "").strip() + try: + if candidate.exists(): + with candidate.open("r", encoding="utf-8") as handle: + return json.load(handle) + except Exception: + continue + return default + + +def _local_config_search_roots(): + seen = set() + roots = [] + + def append_root(raw_path): + text = str(raw_path or "").strip() + if not text: + return + path = Path(text).expanduser().resolve() + key = str(path) + if key in seen: + return + seen.add(key) + roots.append(path) + + explicit_root = str(os.getenv("DOMAINCHECK_CONFIG_ROOT", "") or "").strip() + append_root(explicit_root) + append_root(Path.cwd()) + + module_root = Path(__file__).resolve().parents[2] + append_root(module_root) + + for base_root in list(roots): + normalized = str(base_root) + if f"{os.sep}releases{os.sep}" in normalized: + install_root = normalized.split(f"{os.sep}releases{os.sep}", 1)[0] + append_root(Path(install_root) / "current" / "domainCheck") + append_root(Path(install_root) / "domainCheck") + elif f"{os.sep}current{os.sep}" in normalized: + install_root = normalized.split(f"{os.sep}current{os.sep}", 1)[0] + append_root(Path(install_root) / "current" / "domainCheck") + append_root(Path(install_root) / "domainCheck") + elif base_root.name == "domainCheck": + append_root(base_root.parent / "current" / "domainCheck") + append_root(base_root.parent / "domainCheck") + + return roots + + +def _normalize_scaling_int(value, default): + try: + normalized = int(value) + except Exception: + return int(default) + return max(1, normalized) + + +def _resolve_scaling_override(overrides, *candidate_codes): + if not isinstance(overrides, dict): + return None + for candidate in candidate_codes: + normalized_candidate = str(candidate or "").strip() + if not normalized_candidate: + continue + if normalized_candidate not in overrides: + continue + try: + return max(1, int(overrides.get(normalized_candidate) or 0)) + except Exception: + continue + return None + + +def _load_local_worker_scaling_hints(): + node_code = str(getattr(config, "NODE_CODE", "") or "").strip() + parent_node_code = str(os.getenv("WORKER_PARENT_NODE_CODE", "") or "").strip() + + thread_count_payload = _read_local_json_config("thread_count.json", {"thread_count": "1000"}) + node_thread_counts = _read_local_json_config("node_thread_counts.json", {}) + process_count_payload = _read_local_json_config("process_count.json", {"process_count": "1"}) + node_process_counts = _read_local_json_config("node_process_counts.json", {}) + + default_thread_count = _normalize_scaling_int(thread_count_payload.get("thread_count", 1000), 1000) + default_process_count = _normalize_scaling_int(process_count_payload.get("process_count", 1), 1) + + effective_thread_count = _resolve_scaling_override(node_thread_counts, node_code, parent_node_code) + if effective_thread_count is None: + effective_thread_count = default_thread_count + + effective_process_count = _resolve_scaling_override(node_process_counts, node_code, parent_node_code) + if effective_process_count is None: + effective_process_count = default_process_count + + return { + "node_code": node_code, + "parent_node_code": parent_node_code, + "thread_count": effective_thread_count, + "process_count": effective_process_count, + } + + +def _is_controller_scaling_hints(scaling_hints): + normalized_hints = dict(scaling_hints or {}) + node_role = str(getattr(config, "NODE_ROLE", "") or "").strip().lower() + if node_role == "control": + return True + node_code = str(normalized_hints.get("node_code") or "").strip().lower() + parent_node_code = str(normalized_hints.get("parent_node_code") or "").strip().lower() + return "controller" in node_code or "controller" in parent_node_code + + +def _default_pool_total_budget(process_count, scaling_hints=None): + normalized = max(1, int(process_count or 1)) + if _is_controller_scaling_hints(scaling_hints): + if normalized >= 96: + return 1000 + if normalized >= 48: + return 720 + if normalized >= 24: + return 480 + if normalized >= 48: + return 240 + if normalized >= 24: + return 320 + return 480 + + +def _resolve_db_pool_limits(): + scaling_hints = _load_local_worker_scaling_hints() + raw_pool_size = str(os.getenv("DB_POOL_SIZE", "") or "").strip() + raw_pool_warm_size = str(os.getenv("DB_POOL_WARM_SIZE", "") or "").strip() + raw_pool_idle_keep_max = str(os.getenv("DB_POOL_IDLE_KEEP_MAX", "") or "").strip() + + effective_process_count = max(1, int(scaling_hints.get("process_count", 1) or 1)) + effective_thread_count = max(1, int(scaling_hints.get("thread_count", 1000) or 1000)) + + if raw_pool_size: + pool_size = max(1, int(raw_pool_size)) + else: + default_budget = _default_pool_total_budget( + effective_process_count, + scaling_hints=scaling_hints, + ) + total_budget = max(32, int(os.getenv("DOMAINCHECK_DB_POOL_TOTAL_BUDGET", str(default_budget)) or default_budget)) + per_process_budget = max(4, total_budget // effective_process_count) + thread_window = max(8, min(24, max(1, effective_thread_count // 96))) + pool_size = max(4, min(24, min(per_process_budget, thread_window))) + + if raw_pool_warm_size: + pool_warm_size = max(1, min(pool_size, int(raw_pool_warm_size))) + else: + pool_warm_size = max(1, min(pool_size, max(1, min(4, pool_size // 4)))) + + if raw_pool_idle_keep_max: + pool_idle_keep_max = max(pool_warm_size, min(pool_size, int(raw_pool_idle_keep_max))) + else: + pool_idle_keep_max = max(pool_warm_size, min(pool_size, max(2, pool_size // 2))) + + return { + "pool_size": pool_size, + "pool_warm_size": pool_warm_size, + "pool_idle_keep_max": pool_idle_keep_max, + "scaling_hints": scaling_hints, + } + + def _detect_job_item_step_priority(step_code: str) -> int: normalized = str(step_code or "").strip() if not normalized: @@ -75,6 +404,11 @@ def _resolve_step_claim_quota(limit: int) -> int: if configured_quota > 0: return max(1, configured_quota) normalized_limit = max(1, int(limit or 1)) + # Large controller pools should be allowed to claim a full-step window; + # otherwise a 2k-thread worker can get pinned near ~500 inflight items when + # a single hot step dominates the queue. + if normalized_limit >= 1024: + return normalized_limit return max(64, min(normalized_limit, max(1, normalized_limit // 4))) @@ -98,14 +432,16 @@ class Database: self.database = database or config.DB_DATABASE self.user = user or config.DB_USER self.password = password or config.DB_PASSWORD - + + pool_limits = _resolve_db_pool_limits() + # 数据库连接池 self.connection_pool = [] - self.pool_size = max(1, int(config.DB_POOL_SIZE)) # 连接池大小 - self.pool_warm_size = max(1, min(self.pool_size, int(getattr(config, 'DB_POOL_WARM_SIZE', 16) or 16))) + self.pool_size = max(1, int(pool_limits["pool_size"])) # 连接池大小 + self.pool_warm_size = max(1, min(self.pool_size, int(pool_limits["pool_warm_size"]))) self.pool_idle_keep_max = max( self.pool_warm_size, - min(self.pool_size, int(getattr(config, 'DB_POOL_IDLE_KEEP_MAX', self.pool_size) or self.pool_size)), + min(self.pool_size, int(pool_limits["pool_idle_keep_max"])), ) self.pool_healthcheck_interval = max( 0.0, @@ -117,19 +453,23 @@ class Database: self._pool_initialized = False self.total_connections = 0 self._connection_last_healthcheck = {} - + + scaling_hints = dict(pool_limits.get("scaling_hints") or {}) + logger.info( + "数据库连接池限额: " + f"node={scaling_hints.get('node_code') or '-'} " + f"parent={scaling_hints.get('parent_node_code') or '-'} " + f"process_count={int(scaling_hints.get('process_count', 1) or 1)} " + f"thread_count={int(scaling_hints.get('thread_count', 0) or 0)} " + f"pool_size={self.pool_size} warm={self.pool_warm_size} idle_keep={self.pool_idle_keep_max}" + ) + # 初始化连接池 self._init_connection_pool() # 初始化 Redis 客户端 try: - self.redis_client = redis.Redis( - host=config.REDIS_HOST, - port=config.REDIS_PORT, - password=config.REDIS_PASSWORD, - db=config.REDIS_DB, - decode_responses=True - ) + self.redis_client = get_redis_client(role="standard") # 测试连接 self.redis_client.ping() logger.info(f"Redis 连接成功: {config.REDIS_HOST}:{config.REDIS_PORT}") @@ -142,6 +482,61 @@ class Database: self.redis_client = None self.use_redis = False self.use_bloom_filter = False + self._active_detect_job_cache_lock = threading.Lock() + self._active_detect_job_cache_payload = None + self._active_detect_job_cache_fresh_until = 0.0 + self._active_detect_job_cache_stale_until = 0.0 + + def _clone_active_detect_job_payload(self, payload): + if isinstance(payload, dict): + return dict(payload) + return payload + + def _ensure_active_detect_job_cache_state(self): + if not hasattr(self, "_active_detect_job_cache_lock") or self._active_detect_job_cache_lock is None: + self._active_detect_job_cache_lock = threading.Lock() + if not hasattr(self, "_active_detect_job_cache_payload"): + self._active_detect_job_cache_payload = None + if not hasattr(self, "_active_detect_job_cache_fresh_until"): + self._active_detect_job_cache_fresh_until = 0.0 + if not hasattr(self, "_active_detect_job_cache_stale_until"): + self._active_detect_job_cache_stale_until = 0.0 + + def _get_local_active_detect_job_cache(self, now_ts: float, *, allow_stale: bool = False): + self._ensure_active_detect_job_cache_state() + with self._active_detect_job_cache_lock: + payload = self._clone_active_detect_job_payload(self._active_detect_job_cache_payload) + fresh_until = float(getattr(self, "_active_detect_job_cache_fresh_until", 0.0) or 0.0) + stale_until = float(getattr(self, "_active_detect_job_cache_stale_until", 0.0) or 0.0) + if payload is None: + return None + if now_ts <= fresh_until: + return payload + if allow_stale and now_ts <= stale_until: + return payload + return None + + def _set_local_active_detect_job_cache(self, payload, *, now_ts: float, fresh_ttl_seconds: int, stale_ttl_seconds: int): + self._ensure_active_detect_job_cache_state() + fresh_until = now_ts + max(1, int(fresh_ttl_seconds or 1)) + stale_until = fresh_until + max(0, int(stale_ttl_seconds or 0)) + with self._active_detect_job_cache_lock: + self._active_detect_job_cache_payload = self._clone_active_detect_job_payload(payload) + self._active_detect_job_cache_fresh_until = fresh_until + self._active_detect_job_cache_stale_until = stale_until + + def _read_cached_active_detect_job_from_redis(self, cache_key: str): + if not self.redis_client: + return None + try: + cached_payload = self.redis_client.get(cache_key) + if cached_payload: + cached = json.loads(cached_payload) + if isinstance(cached, dict): + return cached + except Exception as e: + logger.debug(f"读取活动任务缓存失败: {e}") + return None def _init_connection_pool(self): """ @@ -181,6 +576,41 @@ class Database: connect_timeout=5, application_name="domaincheck-worker", ) + + def _prepare_pooled_connection(self, conn): + if not conn or conn.closed: + return False + try: + if conn.get_transaction_status() != extensions.TRANSACTION_STATUS_IDLE: + conn.rollback() + last_healthcheck = float(self._connection_last_healthcheck.get(id(conn), 0.0) or 0.0) + now = time.monotonic() + if self.pool_healthcheck_interval > 0 and now - last_healthcheck >= self.pool_healthcheck_interval: + cur = conn.cursor() + try: + cur.execute("SELECT 1") + cur.fetchone() + finally: + try: + cur.close() + except Exception: + pass + self._connection_last_healthcheck[id(conn)] = now + return True + except Exception: + return False + + def _discard_connection(self, conn): + conn_id = id(conn) if conn is not None else 0 + try: + if conn and not conn.closed: + conn.close() + except Exception: + pass + with self.pool_condition: + self._connection_last_healthcheck.pop(conn_id, None) + self.total_connections = max(0, self.total_connections - 1) + self.pool_condition.notify() def _init_bloom_filter(self): """ @@ -226,37 +656,17 @@ class Database: while True: create_new = False + pooled_conn = None with self.pool_condition: while self.connection_pool: - conn = self.connection_pool.pop() - if conn and not conn.closed: - try: - if conn.get_transaction_status() != extensions.TRANSACTION_STATUS_IDLE: - conn.rollback() - last_healthcheck = float(self._connection_last_healthcheck.get(id(conn), 0.0) or 0.0) - now = time.monotonic() - if self.pool_healthcheck_interval > 0 and now - last_healthcheck >= self.pool_healthcheck_interval: - cur = conn.cursor() - cur.execute("SELECT 1") - cur.fetchone() - cur.close() - self._connection_last_healthcheck[id(conn)] = now - if _POOL_VERBOSE_LOGS: - logger.debug(f"线程 {thread_id} 从连接池获取连接成功") - return conn, conn.cursor() - except Exception: - self._connection_last_healthcheck.pop(id(conn), None) - try: - conn.close() - except Exception: - pass - self.total_connections = max(0, self.total_connections - 1) - continue + pooled_conn = self.connection_pool.pop() + if pooled_conn: + break - if self.total_connections < self.pool_size: + if pooled_conn is None and self.total_connections < self.pool_size: self.total_connections += 1 create_new = True - else: + elif pooled_conn is None: remaining = deadline - time.monotonic() if remaining <= 0: logger.warning( @@ -271,6 +681,14 @@ class Database: ) self.pool_condition.wait(timeout=min(0.5, remaining)) + if pooled_conn is not None: + if self._prepare_pooled_connection(pooled_conn): + if _POOL_VERBOSE_LOGS: + logger.debug(f"线程 {thread_id} 从连接池获取连接成功") + return pooled_conn, pooled_conn.cursor() + self._discard_connection(pooled_conn) + continue + if create_new: try: conn = self._create_connection() @@ -312,19 +730,16 @@ class Database: pass if conn and not conn.closed: + try: + if conn.get_transaction_status() != extensions.TRANSACTION_STATUS_IDLE: + conn.rollback() + except Exception: + self._discard_connection(conn) + logger.warning("连接归还前回滚失败，已关闭连接") + return + close_conn = False + conn_id = id(conn) with self.pool_condition: - try: - if conn.get_transaction_status() != extensions.TRANSACTION_STATUS_IDLE: - conn.rollback() - except Exception: - try: - self._connection_last_healthcheck.pop(id(conn), None) - conn.close() - finally: - self.total_connections = max(0, self.total_connections - 1) - self.pool_condition.notify() - logger.warning("连接归还前回滚失败，已关闭连接") - return if len(self.connection_pool) < self.pool_idle_keep_max: self.connection_pool.append(conn) self.pool_condition.notify() @@ -333,12 +748,17 @@ class Database: else: # 启动阶段只预热少量连接，高并发跑起来后允许保留更多空闲连接， # 避免本地数据库在“建连/关连”之间来回抖动。 - self._connection_last_healthcheck.pop(id(conn), None) - conn.close() + self._connection_last_healthcheck.pop(conn_id, None) self.total_connections = max(0, self.total_connections - 1) self.pool_condition.notify() + close_conn = True if _POOL_VERBOSE_LOGS: logger.debug(f"空闲连接超过保留阈值({self.pool_idle_keep_max})，已关闭多余连接") + if close_conn: + try: + conn.close() + except Exception: + pass except Exception as e: logger.error(f"关闭数据库连接失败: {e}") try: @@ -504,6 +924,14 @@ class Database: self.close(conn, cur) def ensure_cluster_runtime_tables(self): + if self._cluster_runtime_schema_ready(): + return True + if self._cluster_runtime_schema_basics_present(): + if not self._runtime_index_repair_enabled(): + logger.warning("多机运行库索引存在缺口/无效，默认跳过自动DDL修复") + return False + self._ensure_cluster_runtime_indexes() + return self._cluster_runtime_schema_ready() sql = """ CREATE TABLE IF NOT EXISTS detect_worker_nodes ( node_code VARCHAR(64) PRIMARY KEY, @@ -558,6 +986,22 @@ class Database: CREATE INDEX IF NOT EXISTS idx_detect_job_items_status_lease ON detect_job_items(status, lease_expires_at); + CREATE INDEX IF NOT EXISTS idx_detect_job_items_claim_ready + ON detect_job_items(status, create_time, id) + WHERE step_code <> ''; + + CREATE INDEX IF NOT EXISTS idx_detect_job_items_claim_job_ready + ON detect_job_items(job_id, status, create_time, id) + WHERE step_code <> ''; + + CREATE INDEX IF NOT EXISTS idx_detect_job_items_claim_step_ready + ON detect_job_items(status, step_code, lease_expires_at, create_time, id) + WHERE step_code <> '' AND status IN ('pending', 'failed'); + + CREATE INDEX IF NOT EXISTS idx_detect_job_items_claim_job_step_ready + ON detect_job_items(job_id, status, step_code, lease_expires_at, create_time, id) + WHERE step_code <> '' AND status IN ('pending', 'failed'); + ALTER TABLE detect_jobs ADD COLUMN IF NOT EXISTS task_mode VARCHAR(32) NOT NULL DEFAULT 'domain_pipeline', ADD COLUMN IF NOT EXISTS step_code VARCHAR(64) NOT NULL DEFAULT ''; @@ -599,6 +1043,203 @@ class Database: """ return self.execute(sql) + def _cluster_runtime_schema_basics_present(self): + sql = """ + SELECT + EXISTS ( + SELECT 1 + FROM information_schema.tables + WHERE table_schema = current_schema() + AND table_name = 'detect_worker_nodes' + ) AS has_detect_worker_nodes, + EXISTS ( + SELECT 1 + FROM information_schema.tables + WHERE table_schema = current_schema() + AND table_name = 'detect_jobs' + ) AS has_detect_jobs, + EXISTS ( + SELECT 1 + FROM information_schema.tables + WHERE table_schema = current_schema() + AND table_name = 'detect_job_items' + ) AS has_detect_job_items, + EXISTS ( + SELECT 1 + FROM information_schema.tables + WHERE table_schema = current_schema() + AND table_name = 'detect_run_events' + ) AS has_detect_run_events, + EXISTS ( + SELECT 1 + FROM information_schema.tables + WHERE table_schema = current_schema() + AND table_name = 'detect_sync_records' + ) AS has_detect_sync_records, + EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_schema = current_schema() + AND table_name = 'detect_jobs' + AND column_name = 'task_mode' + ) AS has_detect_jobs_task_mode, + EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_schema = current_schema() + AND table_name = 'detect_jobs' + AND column_name = 'step_code' + ) AS has_detect_jobs_step_code, + EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_schema = current_schema() + AND table_name = 'detect_job_items' + AND column_name = 'step_code' + ) AS has_detect_job_items_step_code, + EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_schema = current_schema() + AND table_name = 'detect_job_items' + AND column_name = 'step_payload_json' + ) AS has_detect_job_items_step_payload, + EXISTS ( + SELECT 1 + FROM information_schema.columns + WHERE table_schema = current_schema() + AND table_name = 'detect_job_items' + AND column_name = 'result_payload_json' + ) AS has_detect_job_items_result_payload + """ + try: + row = self.fetch_one(sql) + except Exception as e: + logger.debug(f"检测多机运行库 schema 状态失败: {e}") + return False + if isinstance(row, dict): + return all(bool(value) for value in row.values()) + if isinstance(row, (list, tuple)): + return all(bool(value) for value in row) + return False + + def _cluster_runtime_missing_indexes(self): + table_names = sorted(set(_RUNTIME_REQUIRED_INDEX_TABLES.values())) + sql = """ + SELECT + idx.relname AS index_name, + pg_index.indisvalid AS is_valid + FROM pg_class AS idx + JOIN pg_index ON pg_index.indexrelid = idx.oid + JOIN pg_class AS tbl ON tbl.oid = pg_index.indrelid + JOIN pg_namespace AS ns ON ns.oid = tbl.relnamespace + WHERE ns.nspname = current_schema() + AND tbl.relname = ANY(%s) + AND idx.relname = ANY(%s) + """ + try: + rows = self.fetch_all(sql, (table_names, list(_RUNTIME_REQUIRED_INDEX_DDL.keys()),)) + except Exception as e: + logger.debug(f"检测多机运行库索引状态失败: {e}") + return list(_RUNTIME_REQUIRED_INDEX_DDL.keys()) + states = { + index_name: False + for index_name in _RUNTIME_REQUIRED_INDEX_DDL.keys() + } + for row in list(rows or []): + index_name = str((row or {}).get("index_name") or "").strip() + if index_name in states: + states[index_name] = bool((row or {}).get("is_valid")) + return [index_name for index_name, is_valid in states.items() if not is_valid] + + def _cluster_runtime_invalid_indexes(self): + table_names = sorted(set(_RUNTIME_REQUIRED_INDEX_TABLES.values())) + sql = """ + SELECT + idx.relname AS index_name + FROM pg_class AS idx + JOIN pg_index ON pg_index.indexrelid = idx.oid + JOIN pg_class AS tbl ON tbl.oid = pg_index.indrelid + JOIN pg_namespace AS ns ON ns.oid = tbl.relnamespace + WHERE ns.nspname = current_schema() + AND tbl.relname = ANY(%s) + AND idx.relname = ANY(%s) + AND NOT pg_index.indisvalid + """ + try: + rows = self.fetch_all(sql, (table_names, list(_RUNTIME_REQUIRED_INDEX_DDL.keys()),)) + except Exception as e: + logger.debug(f"检测多机运行库无效索引状态失败: {e}") + return [] + return [ + str((row or {}).get("index_name") or "").strip() + for row in list(rows or []) + if str((row or {}).get("index_name") or "").strip() in _RUNTIME_REQUIRED_INDEX_DDL + ] + + def _runtime_index_repair_enabled(self): + raw = str(os.getenv("DOMAINCHECK_RUNTIME_INDEX_REPAIR_ENABLED", "") or "").strip().lower() + return raw in {"1", "true", "yes", "on"} + + def _ensure_cluster_runtime_indexes(self): + missing_indexes = self._cluster_runtime_missing_indexes() + if not missing_indexes: + return True + + conn = None + cur = None + lock_acquired = False + try: + conn = self._create_connection() + conn.autocommit = True + cur = conn.cursor() + cur.execute("SELECT pg_try_advisory_lock(%s)", (_build_pg_advisory_lock_key(_RUNTIME_INDEX_BUILD_LOCK_SCOPE),)) + lock_row = cur.fetchone() + lock_acquired = bool((lock_row or [False])[0]) + if not lock_acquired: + logger.info("多机运行库索引补齐进行中，当前进程跳过重复建索引") + return False + + current_missing = set(self._cluster_runtime_missing_indexes()) + current_invalid = set(self._cluster_runtime_invalid_indexes()) + for index_name in missing_indexes: + if index_name not in current_missing: + continue + ddl = str(_RUNTIME_REQUIRED_INDEX_DDL.get(index_name) or "").strip() + if not ddl: + continue + if index_name in current_invalid: + if not self._runtime_index_repair_enabled(): + logger.warning(f"检测到无效多机运行库索引，跳过自动重建: {index_name}") + continue + logger.warning(f"检测到无效多机运行库索引，准备重建: {index_name}") + cur.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {index_name}") + logger.info(f"补齐多机运行库索引: {index_name}") + cur.execute(ddl) + return True + except Exception as e: + logger.warning(f"补齐多机运行库索引失败: {e}") + return False + finally: + if cur and lock_acquired: + try: + cur.execute("SELECT pg_advisory_unlock(%s)", (_build_pg_advisory_lock_key(_RUNTIME_INDEX_BUILD_LOCK_SCOPE),)) + except Exception: + pass + try: + if cur: + cur.close() + except Exception: + pass + try: + if conn: + conn.close() + except Exception: + pass + + def _cluster_runtime_schema_ready(self): + return self._cluster_runtime_schema_basics_present() and not self._cluster_runtime_missing_indexes() + def register_cluster_node(self, node_code, region, role, status='online', current_load=0, metadata=None): try: ip_addr = '' @@ -640,13 +1281,23 @@ class Database: logger.warning(f"注册检测节点失败: {e}") return False - def claim_detect_job_items(self, node_code, limit=1000, lease_seconds=3600, job_id=None): + def claim_detect_job_items( + self, + node_code, + limit=1000, + lease_seconds=3600, + job_id=None, + *, + prefer_recent_jobs=False, + preferred_recent_job_limit=8, + preferred_recent_job_window_hours=24, + ): """ 领取一批待执行的任务项。 """ conn = None cur = None - claim_token = f"{node_code}-{int(threading.current_thread().ident or 0)}-{int(__import__('time').time())}" + claim_token = _build_claim_token(node_code) try: conn, cur = self.connect() if not conn or not cur: @@ -657,20 +1308,48 @@ class Database: normalized_lease_seconds = max(60, int(lease_seconds or 3600)) step_claim_quota = _resolve_step_claim_quota(normalized_limit) rows = [] + preferred_job_ids = [] - def _claim_rows(batch_limit: int, step_condition_sql: str = "", step_condition_params: tuple = ()) -> list: + if normalized_job_id is None and bool(prefer_recent_jobs): + cur.execute( + """ + SELECT id, status, COALESCE(started_at, created_at) AS activity_at + FROM detect_jobs + WHERE status IN ('pending', 'running') + ORDER BY + COALESCE(started_at, created_at) DESC, + CASE WHEN status = 'running' THEN 0 ELSE 1 END ASC, + id DESC + LIMIT %s + """, + (max(8, int(preferred_recent_job_limit or 8) * 4),), + ) + preferred_job_ids = _select_preferred_claim_job_ids( + cur.fetchall() or [], + limit=preferred_recent_job_limit, + recent_hours=preferred_recent_job_window_hours, + ) + + def _claim_rows( + batch_limit: int, + step_condition_sql: str = "", + step_condition_params: tuple = (), + job_scope_sql: str = "", + job_scope_params: tuple = (), + ) -> list: safe_batch_limit = max(1, int(batch_limit or 0)) if safe_batch_limit <= 0: return [] extra_condition = f"\n AND {step_condition_sql}" if step_condition_sql else "" + job_scope_clause = f"\n AND {job_scope_sql}" if job_scope_sql else "" cur.execute( f""" WITH picked AS ( SELECT id FROM detect_job_items WHERE status IN ('pending', 'failed') - AND COALESCE(step_code, '') <> '' - AND (%s IS NULL OR job_id = %s) + AND step_code <> '' + {job_scope_clause} AND (lease_expires_at IS NULL OR lease_expires_at < CURRENT_TIMESTAMP){extra_condition} ORDER BY create_time ASC, id ASC FOR UPDATE SKIP LOCKED @@ -713,8 +1392,7 @@ class Database: ORDER BY updated.id ASC """, ( - normalized_job_id, - normalized_job_id, + *job_scope_params, *step_condition_params, safe_batch_limit, node_code, @@ -724,33 +1402,40 @@ class Database: ) return cur.fetchall() or [] - remaining = normalized_limit - for step_code in _ordered_step_claim_codes(): - if remaining <= 0: - break - claimed_rows = _claim_rows( - min(remaining, step_claim_quota), - "COALESCE(step_code, '') = %s", - (str(step_code or "").strip(),), - ) - if claimed_rows: - rows.extend(claimed_rows) - remaining -= len(claimed_rows) + def _claim_with_job_scope(job_scope_sql: str = "", job_scope_params: tuple = ()) -> list: + scoped_rows = [] + remaining = normalized_limit + for step_code in _ordered_step_claim_codes(): + if remaining <= 0: + break + claimed_rows = _claim_rows( + min(remaining, step_claim_quota), + "COALESCE(step_code, '') = %s", + (str(step_code or "").strip(),), + job_scope_sql, + job_scope_params, + ) + if claimed_rows: + scoped_rows.extend(claimed_rows) + remaining -= len(claimed_rows) + if remaining > 0: + scoped_rows.extend(_claim_rows(remaining, job_scope_sql=job_scope_sql, job_scope_params=job_scope_params)) + return scoped_rows - if remaining > 0: - rows.extend(_claim_rows(remaining)) + if normalized_job_id is not None: + rows.extend(_claim_with_job_scope("job_id = %s", (normalized_job_id,))) + else: + if preferred_job_ids: + rows.extend(_claim_with_job_scope("job_id = ANY(%s)", (preferred_job_ids,))) + remaining = max(0, normalized_limit - len(rows)) + if remaining > 0: + original_limit = normalized_limit + try: + normalized_limit = remaining + rows.extend(_claim_with_job_scope()) + finally: + normalized_limit = original_limit - touched_job_ids = sorted({row[1] for row in rows}) - for job_id in touched_job_ids: - cur.execute( - """ - UPDATE detect_jobs - SET status = CASE WHEN status = 'pending' THEN 'running' ELSE status END, - started_at = COALESCE(started_at, CURRENT_TIMESTAMP) - WHERE id = %s - """, - (job_id,), - ) conn.commit() return [ { @@ -785,6 +1470,135 @@ class Database: finally: self.close(conn, cur) + def claim_restart_released_detect_job_items( + self, + node_code, + job_id, + *, + limit=1000, + lease_seconds=3600, + ): + """ + 优先回收节点重启后放回 pending 的当前 job 尾批任务项。 + + 这批条目如果长期不被重新 claim，会让旧 job 维持 running， + 从而拖住下一批的自然接棒。 + """ + conn = None + cur = None + claim_token = _build_claim_token(node_code) + try: + conn, cur = self.connect() + if not conn or not cur: + logger.error("优先领取重启回收尾批任务失败: 无法获取数据库连接") + return [] + normalized_job_id = int(job_id) if job_id not in (None, "", 0, "0") else None + if not normalized_job_id: + return [] + normalized_limit = max(1, int(limit or 1)) + normalized_lease_seconds = max(60, int(lease_seconds or 3600)) + cur.execute( + """ + WITH picked AS ( + SELECT id + FROM detect_job_items + WHERE job_id = %s + AND status = 'pending' + AND step_code <> '' + AND (lease_expires_at IS NULL OR lease_expires_at < CURRENT_TIMESTAMP) + AND last_error IN ( + 'released after worker restart', + 'released before execution after worker restart' + ) + ORDER BY + CASE + WHEN last_error = 'released after worker restart' THEN 0 + ELSE 1 + END ASC, + create_time ASC, + id ASC + FOR UPDATE SKIP LOCKED + LIMIT %s + ), + updated AS ( + UPDATE detect_job_items AS item + SET status = 'claimed', + claimed_by = %s, + claim_token = %s, + lease_expires_at = CURRENT_TIMESTAMP + (%s || ' seconds')::interval, + attempt_count = item.attempt_count + 1, + started_at = COALESCE(item.started_at, CURRENT_TIMESTAMP), + updated_at = CURRENT_TIMESTAMP + FROM picked + WHERE item.id = picked.id + RETURNING item.id, item.job_id, item.domain_id, item.claim_token, item.step_code, item.step_payload_json + ) + SELECT + updated.id, + updated.job_id, + updated.domain_id, + updated.claim_token, + updated.step_code, + detect_jobs.task_mode, + detect_jobs.job_code, + detect_jobs.step_code, + updated.step_payload_json, + domains.domain, + domains.source_type, + domains.register_status, + domains.detect_status, + domains.use_status, + domains.expire_date, + domains.jucha_status, + domains.juziseo_status + FROM updated + JOIN detect_jobs ON detect_jobs.id = updated.job_id + JOIN domains ON domains.id = updated.domain_id + ORDER BY updated.id ASC + """, + ( + normalized_job_id, + normalized_limit, + node_code, + claim_token, + normalized_lease_seconds, + ), + ) + rows = cur.fetchall() or [] + conn.commit() + return [ + { + "job_item_id": row[0], + "job_id": row[1], + "id": row[2], + "claim_token": row[3], + "item_step_code": row[4], + "task_mode": row[5], + "job_code": row[6], + "step_code": row[4] or row[7], + "step_payload": row[8], + "domain": row[9], + "source_type": row[10], + "register_status": row[11], + "detect_status": row[12], + "use_status": row[13], + "expire_date": row[14], + "jucha_status": row[15], + "juziseo_status": row[16], + } + for row in rows + ] + except Exception as e: + try: + if conn: + conn.rollback() + except Exception: + pass + logger.error(f"优先领取重启回收尾批任务失败: {e}") + return [] + finally: + self.close(conn, cur) + def has_dispatchable_detect_job_items(self): """ 是否仍存在可继续派发的标准步骤任务项。 @@ -816,37 +1630,62 @@ class Database: """ conn = None cur = None + advisory_lock_key = _build_pg_advisory_lock_key("detect-job-items-recycle-expired") try: conn, cur = self.connect() if not conn or not cur: return 0 - cur.execute( - """ - WITH recycled AS ( - UPDATE detect_job_items - SET status = 'pending', - claimed_by = '', - claim_token = '', - lease_expires_at = NULL, - updated_at = CURRENT_TIMESTAMP, - last_error = CASE - WHEN status = 'running' THEN 'lease expired while running' - WHEN status = 'claimed' THEN 'lease expired before running' - ELSE last_error - END - WHERE status IN ('claimed', 'running') - AND lease_expires_at IS NOT NULL - AND lease_expires_at < CURRENT_TIMESTAMP - RETURNING job_id + cur.execute("SELECT pg_try_advisory_lock(%s)", (advisory_lock_key,)) + lock_row = cur.fetchone() + if not bool((lock_row or [False])[0]): + try: + conn.rollback() + except Exception: + pass + return 0 + recycled_count = 0 + touched_job_ids = set() + for _ in range(_DETECT_JOB_ITEM_RECYCLE_MAX_BATCHES): + cur.execute( + """ + WITH expired_candidates AS ( + SELECT id, job_id, status + FROM detect_job_items + WHERE status IN ('claimed', 'running') + AND lease_expires_at IS NOT NULL + AND lease_expires_at < CURRENT_TIMESTAMP + ORDER BY lease_expires_at ASC, id ASC + FOR UPDATE SKIP LOCKED + LIMIT %s + ), + recycled AS ( + UPDATE detect_job_items AS item + SET status = 'pending', + claimed_by = '', + claim_token = '', + lease_expires_at = NULL, + updated_at = CURRENT_TIMESTAMP, + last_error = CASE + WHEN expired_candidates.status = 'running' THEN 'lease expired while running' + WHEN expired_candidates.status = 'claimed' THEN 'lease expired before running' + ELSE item.last_error + END + FROM expired_candidates + WHERE item.id = expired_candidates.id + RETURNING expired_candidates.job_id + ) + SELECT count(*), array_remove(array_agg(DISTINCT job_id), NULL) + FROM recycled + """, + (_DETECT_JOB_ITEM_RECYCLE_BATCH_SIZE,), ) - SELECT count(*), array_remove(array_agg(DISTINCT job_id), NULL) - FROM recycled - """ - ) - row = cur.fetchone() - recycled_count = int((row or [0])[0] or 0) - touched_job_ids = (row or [0, []])[1] or [] - for job_id in touched_job_ids: + row = cur.fetchone() + batch_count = int((row or [0])[0] or 0) + recycled_count += batch_count + touched_job_ids.update(item for item in ((row or [0, []])[1] or []) if item is not None) + if batch_count < _DETECT_JOB_ITEM_RECYCLE_BATCH_SIZE: + break + for job_id in sorted(touched_job_ids): self._refresh_detect_job_status_with_cursor(cur, job_id) conn.commit() return recycled_count @@ -859,45 +1698,296 @@ class Database: logger.error(f"回收过期检测任务失败: {e}") return 0 finally: + if cur: + try: + cur.execute("SELECT pg_advisory_unlock(%s)", (advisory_lock_key,)) + if conn: + conn.commit() + except Exception: + try: + if conn: + conn.rollback() + except Exception: + pass self.close(conn, cur) - def release_detect_job_items_for_node(self, node_code): + def recycle_stalled_detect_job_items(self, job_id, *, stall_seconds=1800, batch_size=None): """ - 释放指定节点遗留的 claimed/running 任务项，供节点重启后快速恢复。 + 定向回收长时间无活动的 claimed/running 任务项。 """ conn = None cur = None + try: + normalized_job_id = int(job_id or 0) + except Exception: + normalized_job_id = 0 + if normalized_job_id <= 0: + return 0 + safe_stall_seconds = max(300, int(stall_seconds or 1800)) + safe_batch_size = max( + 1, + min( + int(batch_size or _DETECT_JOB_ITEM_RECYCLE_BATCH_SIZE), + _DETECT_JOB_ITEM_RECYCLE_BATCH_SIZE, + ), + ) + advisory_lock_key = _build_pg_advisory_lock_key( + f"detect-job-items-recycle-stalled:{normalized_job_id}" + ) try: conn, cur = self.connect() if not conn or not cur: return 0 + cur.execute("SELECT pg_try_advisory_lock(%s)", (advisory_lock_key,)) + lock_row = cur.fetchone() + if not bool((lock_row or [False])[0]): + try: + conn.rollback() + except Exception: + pass + return 0 cur.execute( """ - WITH released AS ( - UPDATE detect_job_items + WITH stalled_candidates AS ( + SELECT id, job_id, status + FROM detect_job_items + WHERE job_id = %s + AND status IN ('claimed', 'running') + AND COALESCE(updated_at, started_at, create_time) + < CURRENT_TIMESTAMP - (%s || ' seconds')::interval + ORDER BY COALESCE(updated_at, started_at, create_time) ASC, id ASC + FOR UPDATE SKIP LOCKED + LIMIT %s + ), + recycled AS ( + UPDATE detect_job_items AS item SET status = 'pending', claimed_by = '', claim_token = '', lease_expires_at = NULL, updated_at = CURRENT_TIMESTAMP, last_error = CASE - WHEN status = 'running' THEN 'released after worker restart' - WHEN status = 'claimed' THEN 'released before execution after worker restart' - ELSE last_error + WHEN stalled_candidates.status = 'running' THEN 'stalled while running' + WHEN stalled_candidates.status = 'claimed' THEN 'stalled before running' + ELSE item.last_error END - WHERE claimed_by = %s - AND status IN ('claimed', 'running') - RETURNING job_id + FROM stalled_candidates + WHERE item.id = stalled_candidates.id + RETURNING stalled_candidates.job_id ) SELECT count(*), array_remove(array_agg(DISTINCT job_id), NULL) - FROM released + FROM recycled """, - (node_code,), + ( + normalized_job_id, + safe_stall_seconds, + safe_batch_size, + ), ) row = cur.fetchone() - released_count = int((row or [0])[0] or 0) - touched_job_ids = (row or [0, []])[1] or [] - for job_id in touched_job_ids: + recycled_count = int((row or [0])[0] or 0) + touched_job_ids = [item for item in ((row or [0, []])[1] or []) if item is not None] + for touched_job_id in sorted(set(touched_job_ids)): + self._refresh_detect_job_status_with_cursor(cur, touched_job_id) + conn.commit() + return recycled_count + except Exception as e: + try: + if conn: + conn.rollback() + except Exception: + pass + logger.error(f"回收长挂检测任务失败: job_id={normalized_job_id}, error={e}") + return 0 + finally: + if cur: + try: + cur.execute("SELECT pg_advisory_unlock(%s)", (advisory_lock_key,)) + if conn: + conn.commit() + except Exception: + try: + if conn: + conn.rollback() + except Exception: + pass + self.close(conn, cur) + + def release_detect_job_item(self, job_item_id, claim_token, reason=''): + """ + 释放单个 claimed/running 任务项，供会话切换时快速回到 pending。 + """ + conn = None + cur = None + try: + conn, cur = self.connect() + if not conn or not cur: + return False + cur.execute( + """ + UPDATE detect_job_items + SET status = 'pending', + claimed_by = '', + claim_token = '', + lease_expires_at = NULL, + updated_at = CURRENT_TIMESTAMP, + last_error = CASE + WHEN %s <> '' THEN %s + ELSE last_error + END + WHERE id = %s + AND claim_token = %s + AND status IN ('claimed', 'running') + RETURNING job_id + """, + ( + str(reason or '')[:1000], + str(reason or '')[:1000], + int(job_item_id or 0), + str(claim_token or '').strip(), + ), + ) + row = cur.fetchone() + conn.commit() + return bool(row) + except Exception as e: + try: + if conn: + conn.rollback() + except Exception: + pass + logger.error(f"释放单个检测任务项失败: {e}") + return False + finally: + self.close(conn, cur) + + def release_detect_job_items_batch(self, items): + """ + 批量释放 claimed/running 任务项，降低会话切换时的逐条写锁竞争。 + """ + normalized_items = [] + for item in list(items or []): + if not isinstance(item, (list, tuple)) or len(item) < 2: + continue + try: + job_item_id = int(item[0] or 0) + except Exception: + continue + claim_token = str(item[1] or "").strip() + reason = "" + if len(item) >= 3: + reason = str(item[2] or "").strip()[:1000] + if job_item_id <= 0 or not claim_token: + continue + normalized_items.append((job_item_id, claim_token, reason)) + if not normalized_items: + return 0 + + conn = None + cur = None + try: + conn, cur = self.connect() + if not conn or not cur: + return -1 + values_sql = ",".join( + cur.mogrify("(%s, %s, %s)", item).decode("utf-8") + for item in normalized_items + ) + cur.execute( + f""" + UPDATE detect_job_items AS item + SET status = 'pending', + claimed_by = '', + claim_token = '', + lease_expires_at = NULL, + updated_at = CURRENT_TIMESTAMP, + last_error = CASE + WHEN batch.reason <> '' THEN batch.reason + ELSE item.last_error + END + FROM (VALUES {values_sql}) AS batch(id, claim_token, reason) + WHERE item.id = batch.id + AND item.claim_token = batch.claim_token + AND item.status IN ('claimed', 'running') + RETURNING item.id + """ + ) + updated_rows = cur.fetchall() or [] + conn.commit() + return len(updated_rows) + except Exception as e: + try: + if conn: + conn.rollback() + except Exception: + pass + logger.error(f"批量释放检测任务项失败: {e}") + return -1 + finally: + self.close(conn, cur) + + def release_detect_job_items_for_node(self, node_code): + """ + 释放指定节点遗留的 claimed/running 任务项，供节点重启后快速恢复。 + """ + normalized_flag = str(os.getenv("DOMAINCHECK_ENABLE_NODE_ITEM_RELEASE", "1") or "1").strip().lower() + if normalized_flag in { + "0", + "false", + "no", + "off", + "disable", + "disabled", + }: + return 0 + conn = None + cur = None + try: + conn, cur = self.connect() + if not conn or not cur: + return 0 + released_count = 0 + touched_job_ids = set() + for _ in range(_DETECT_JOB_ITEM_RELEASE_MAX_BATCHES): + cur.execute( + """ + WITH release_candidates AS ( + SELECT id, job_id, status + FROM detect_job_items + WHERE claimed_by = %s + AND status IN ('claimed', 'running') + ORDER BY id ASC + FOR UPDATE SKIP LOCKED + LIMIT %s + ), + released AS ( + UPDATE detect_job_items AS item + SET status = 'pending', + claimed_by = '', + claim_token = '', + lease_expires_at = NULL, + updated_at = CURRENT_TIMESTAMP, + last_error = CASE + WHEN release_candidates.status = 'running' THEN 'released after worker restart' + WHEN release_candidates.status = 'claimed' THEN 'released before execution after worker restart' + ELSE item.last_error + END + FROM release_candidates + WHERE item.id = release_candidates.id + RETURNING release_candidates.job_id + ) + SELECT count(*), array_remove(array_agg(DISTINCT job_id), NULL) + FROM released + """, + (node_code, _DETECT_JOB_ITEM_RELEASE_BATCH_SIZE), + ) + row = cur.fetchone() + batch_count = int((row or [0])[0] or 0) + released_count += batch_count + touched_job_ids.update(item for item in ((row or [0, []])[1] or []) if item is not None) + if batch_count < _DETECT_JOB_ITEM_RELEASE_BATCH_SIZE: + break + for job_id in sorted(touched_job_ids): self._refresh_detect_job_status_with_cursor(cur, job_id) conn.commit() return released_count @@ -912,41 +2002,451 @@ class Database: finally: self.close(conn, cur) + def release_detect_job_items_for_node_job(self, node_code, job_id): + """ + 只释放指定节点在某个 job 上遗留的 claimed/running 任务项，供尾批定向回收。 + """ + normalized_flag = str(os.getenv("DOMAINCHECK_ENABLE_NODE_ITEM_RELEASE", "1") or "1").strip().lower() + if normalized_flag in { + "0", + "false", + "no", + "off", + "disable", + "disabled", + }: + return 0 + normalized_node_code = str(node_code or "").strip() + try: + normalized_job_id = int(job_id or 0) + except Exception: + normalized_job_id = 0 + if not normalized_node_code or normalized_job_id <= 0: + return 0 + conn = None + cur = None + try: + conn, cur = self.connect() + if not conn or not cur: + return 0 + released_count = 0 + touched_job_ids = set() + for _ in range(_DETECT_JOB_ITEM_RELEASE_MAX_BATCHES): + cur.execute( + """ + WITH release_candidates AS ( + SELECT id, job_id, status + FROM detect_job_items + WHERE claimed_by = %s + AND job_id = %s + AND status IN ('claimed', 'running') + ORDER BY id ASC + FOR UPDATE SKIP LOCKED + LIMIT %s + ), + released AS ( + UPDATE detect_job_items AS item + SET status = 'pending', + claimed_by = '', + claim_token = '', + lease_expires_at = NULL, + updated_at = CURRENT_TIMESTAMP, + last_error = CASE + WHEN release_candidates.status = 'running' THEN 'released after worker restart' + WHEN release_candidates.status = 'claimed' THEN 'released before execution after worker restart' + ELSE item.last_error + END + FROM release_candidates + WHERE item.id = release_candidates.id + RETURNING release_candidates.job_id + ) + SELECT count(*), array_remove(array_agg(DISTINCT job_id), NULL) + FROM released + """, + ( + normalized_node_code, + normalized_job_id, + _DETECT_JOB_ITEM_RELEASE_BATCH_SIZE, + ), + ) + row = cur.fetchone() + batch_count = int((row or [0])[0] or 0) + released_count += batch_count + touched_job_ids.update(item for item in ((row or [0, []])[1] or []) if item is not None) + if batch_count < _DETECT_JOB_ITEM_RELEASE_BATCH_SIZE: + break + for touched_job_id in sorted(touched_job_ids): + self._refresh_detect_job_status_with_cursor(cur, touched_job_id) + conn.commit() + return released_count + except Exception as e: + try: + if conn: + conn.rollback() + except Exception: + pass + logger.error( + f"释放节点定向任务失败: node={normalized_node_code}, job_id={normalized_job_id}, error={e}" + ) + return 0 + finally: + self.close(conn, cur) + def get_active_detect_job(self): """ 获取当前活动中的检测任务摘要，供 Worker 重启后自动回挂。 """ - return self.fetch_one( + cache_key = "domaincheck:active_detect_job_summary:v1" + cache_lock_key = f"{cache_key}:refresh_lock" + now_ts = time.time() + try: + cache_ttl_seconds = max( + 1, + int(os.getenv("DOMAINCHECK_ACTIVE_JOB_CACHE_TTL_SECONDS", "3") or 3), + ) + except (TypeError, ValueError): + cache_ttl_seconds = 3 + try: + local_cache_ttl_seconds = max( + 1, + int( + os.getenv( + "DOMAINCHECK_ACTIVE_JOB_LOCAL_CACHE_TTL_SECONDS", + str(cache_ttl_seconds), + ) + or cache_ttl_seconds + ), + ) + except (TypeError, ValueError): + local_cache_ttl_seconds = cache_ttl_seconds + try: + stale_cache_ttl_seconds = max( + 0, + int(os.getenv("DOMAINCHECK_ACTIVE_JOB_CACHE_STALE_SECONDS", "10") or 10), + ) + except (TypeError, ValueError): + stale_cache_ttl_seconds = 10 + try: + cache_refresh_lock_seconds = max( + 1, + int(os.getenv("DOMAINCHECK_ACTIVE_JOB_CACHE_LOCK_SECONDS", "2") or 2), + ) + except (TypeError, ValueError): + cache_refresh_lock_seconds = 2 + + local_cached = self._get_local_active_detect_job_cache(now_ts) + if local_cached is not None: + return local_cached + + cached = self._read_cached_active_detect_job_from_redis(cache_key) + if isinstance(cached, dict): + self._set_local_active_detect_job_cache( + cached, + now_ts=now_ts, + fresh_ttl_seconds=local_cache_ttl_seconds, + stale_ttl_seconds=stale_cache_ttl_seconds, + ) + return self._clone_active_detect_job_payload(cached) + + refresh_lock_token = "" + if self.redis_client: + try: + refresh_lock_token = str(uuid.uuid4()) + acquired = self.redis_client.set( + cache_lock_key, + refresh_lock_token, + nx=True, + ex=cache_refresh_lock_seconds, + ) + if not acquired: + stale_cached = self._get_local_active_detect_job_cache(now_ts, allow_stale=True) + if stale_cached is not None: + return stale_cached + time.sleep(0.05) + cached = self._read_cached_active_detect_job_from_redis(cache_key) + if isinstance(cached, dict): + self._set_local_active_detect_job_cache( + cached, + now_ts=time.time(), + fresh_ttl_seconds=local_cache_ttl_seconds, + stale_ttl_seconds=stale_cache_ttl_seconds, + ) + return self._clone_active_detect_job_payload(cached) + refresh_lock_token = "" + except Exception as e: + logger.debug(f"获取活动任务缓存刷新锁失败: {e}") + refresh_lock_token = "" + tail_handoff_enabled = str( + os.getenv("DOMAINCHECK_TAIL_HANDOFF_ENABLED", "1") or "1" + ).strip().lower() not in {"0", "false", "off", "no"} + try: + tail_handoff_active_max = max( + 0, + int( + os.getenv( + "DOMAINCHECK_TAIL_HANDOFF_MAX_ACTIVE_ITEMS", + "128", + ) + or 128 + ), + ) + except (TypeError, ValueError): + tail_handoff_active_max = 128 + try: + tail_handoff_pending_max = max( + 0, + int( + os.getenv( + "DOMAINCHECK_TAIL_HANDOFF_MAX_PENDING_ITEMS", + "256", + ) + or 256 + ), + ) + except (TypeError, ValueError): + tail_handoff_pending_max = 256 + try: + tail_handoff_min_pending = max( + 1, + int( + os.getenv( + "DOMAINCHECK_TAIL_HANDOFF_MIN_PENDING_ITEMS", + "1", + ) + or 1 + ), + ) + except (TypeError, ValueError): + tail_handoff_min_pending = 1 + try: + running_stall_seconds = max( + 60, + int( + os.getenv( + "DOMAINCHECK_RUNNING_JOB_STALL_SECONDS", + "900", + ) + or 900 + ), + ) + except (TypeError, ValueError): + running_stall_seconds = 900 + row = self.fetch_one( """ + WITH tail_config AS ( + SELECT + %s::boolean AS handoff_enabled, + %s::integer AS handoff_active_max, + %s::integer AS handoff_pending_max, + %s::integer AS handoff_min_pending, + %s::integer AS running_stall_seconds + ), + candidate_jobs AS ( + SELECT + job.id, + job.job_code, + job.task_mode, + job.status, + job.started_at, + job.created_at + FROM detect_jobs AS job + WHERE job.status IN ('pending', 'running') + ORDER BY + CASE WHEN job.status = 'running' THEN 0 ELSE 1 END ASC, + COALESCE(job.started_at, job.created_at) DESC, + job.id DESC + LIMIT 20 + ), + job_counts AS ( + SELECT + job.id, + job.job_code, + job.task_mode, + job.status, + job.started_at, + job.created_at, + COALESCE(item_counts.items_pending, 0) AS items_pending, + COALESCE(item_counts.items_claimed, 0) AS items_claimed, + COALESCE(item_counts.items_running, 0) AS items_running, + COALESCE(item_counts.items_claimed, 0) + COALESCE(item_counts.items_running, 0) AS items_active, + item_counts.latest_unfinished_activity_at, + item_counts.latest_active_activity_at + FROM candidate_jobs AS job + LEFT JOIN LATERAL ( + SELECT + count(*) FILTER (WHERE item.status = 'pending') AS items_pending, + count(*) FILTER (WHERE item.status = 'claimed') AS items_claimed, + count(*) FILTER (WHERE item.status = 'running') AS items_running, + max(COALESCE(item.updated_at, item.started_at, item.create_time)) AS latest_unfinished_activity_at, + max(COALESCE(item.updated_at, item.started_at, item.create_time)) + FILTER (WHERE item.status IN ('claimed', 'running')) AS latest_active_activity_at + FROM detect_job_items AS item + WHERE item.job_id = job.id + AND item.status IN ('pending', 'claimed', 'running') + ) AS item_counts ON TRUE + WHERE + COALESCE(item_counts.items_pending, 0) > 0 + OR COALESCE(item_counts.items_claimed, 0) > 0 + OR COALESCE(item_counts.items_running, 0) > 0 + ), + job_health AS ( + SELECT + job.*, + CASE + WHEN job.status = 'running' + AND job.items_active > 0 + AND cfg.running_stall_seconds > 0 + AND COALESCE( + job.latest_active_activity_at, + job.latest_unfinished_activity_at, + job.started_at, + job.created_at + ) <= (NOW() - (cfg.running_stall_seconds * INTERVAL '1 second')) + THEN TRUE + ELSE FALSE + END AS running_job_stalled + FROM job_counts AS job + CROSS JOIN tail_config AS cfg + ), + tail_gate AS ( + SELECT + COALESCE( + bool_or( + job.status = 'pending' + AND job.items_pending >= cfg.handoff_min_pending + ), + FALSE + ) AS has_handoff_pending, + COALESCE( + bool_or( + job.status = 'running' + AND NOT job.running_job_stalled + AND ( + job.items_active > cfg.handoff_active_max + OR job.items_pending > cfg.handoff_pending_max + ) + ), + FALSE + ) AS has_non_tail_running + FROM job_health AS job + CROSS JOIN tail_config AS cfg + ) SELECT job.id, job.job_code, job.task_mode, job.status, - count(item.*) FILTER (WHERE item.status = 'pending') AS items_pending, - count(item.*) FILTER (WHERE item.status = 'claimed') AS items_claimed, - count(item.*) FILTER (WHERE item.status = 'running') AS items_running, - count(item.*) FILTER (WHERE item.status IN ('completed', 'blacklisted')) AS items_completed, - count(item.*) FILTER (WHERE item.status = 'failed') AS items_failed, - max(item.updated_at) AS latest_item_updated_at, - max(item.create_time) AS latest_item_created_at - FROM detect_jobs AS job - LEFT JOIN detect_job_items AS item - ON item.job_id = job.id - WHERE job.status IN ('pending', 'running') - GROUP BY job.id, job.job_code, job.task_mode, job.status + job.items_pending, + job.items_claimed, + job.items_running, + 0 AS items_completed, + 0 AS items_failed, + job.latest_unfinished_activity_at AS latest_item_updated_at, + NULL::timestamp AS latest_item_created_at, + job.running_job_stalled, + CASE + WHEN cfg.handoff_enabled + AND gate.has_handoff_pending + AND NOT gate.has_non_tail_running + AND job.status = 'pending' + AND job.items_pending >= cfg.handoff_min_pending + THEN TRUE + ELSE FALSE + END AS tail_handoff_candidate, + CASE + WHEN cfg.handoff_enabled + AND gate.has_handoff_pending + AND NOT gate.has_non_tail_running + AND job.status = 'pending' + AND job.items_pending >= cfg.handoff_min_pending + THEN 'tail_handoff_pending' + WHEN job.running_job_stalled + THEN 'running_job_stalled' + WHEN job.items_claimed > 0 OR job.items_running > 0 + THEN 'running_job_active' + WHEN job.items_pending > 0 + THEN 'pending_job' + ELSE 'inactive' + END AS selection_reason + FROM job_health AS job + CROSS JOIN tail_gate AS gate + CROSS JOIN tail_config AS cfg ORDER BY CASE - WHEN count(item.*) FILTER (WHERE item.status IN ('claimed', 'running')) > 0 THEN 0 - WHEN count(item.*) FILTER (WHERE item.status = 'pending') > 0 THEN 1 - ELSE 2 + WHEN cfg.handoff_enabled + AND gate.has_handoff_pending + AND NOT gate.has_non_tail_running + AND job.status = 'pending' + AND job.items_pending >= cfg.handoff_min_pending + THEN 0 + WHEN (job.items_claimed > 0 OR job.items_running > 0) + AND NOT job.running_job_stalled + THEN 1 + WHEN job.items_pending > 0 THEN 2 + WHEN job.running_job_stalled THEN 3 + ELSE 4 END ASC, + CASE + WHEN cfg.handoff_enabled + AND gate.has_handoff_pending + AND NOT gate.has_non_tail_running + AND job.status = 'pending' + AND job.items_pending >= cfg.handoff_min_pending + THEN job.items_pending + ELSE NULL + END DESC NULLS LAST, CASE WHEN job.status = 'running' THEN 0 ELSE 1 END ASC, - COALESCE(max(item.updated_at), max(item.create_time), job.started_at, job.created_at) DESC, + COALESCE(job.started_at, job.created_at) DESC, job.id DESC LIMIT 1 - """ + """, + ( + tail_handoff_enabled, + tail_handoff_active_max, + tail_handoff_pending_max, + tail_handoff_min_pending, + running_stall_seconds, + ), ) + try: + if isinstance(row, dict): + cache_row = {} + for key, value in row.items(): + if isinstance(value, datetime): + cache_row[key] = value.isoformat() + else: + cache_row[key] = value + self._set_local_active_detect_job_cache( + cache_row, + now_ts=time.time(), + fresh_ttl_seconds=local_cache_ttl_seconds, + stale_ttl_seconds=stale_cache_ttl_seconds, + ) + if self.redis_client: + try: + self.redis_client.setex( + cache_key, + cache_ttl_seconds, + json.dumps(cache_row, ensure_ascii=True, default=str), + ) + except Exception as e: + logger.debug(f"写入活动任务缓存失败: {e}") + return row + self._set_local_active_detect_job_cache( + row, + now_ts=time.time(), + fresh_ttl_seconds=local_cache_ttl_seconds, + stale_ttl_seconds=stale_cache_ttl_seconds, + ) + return row + finally: + if self.redis_client and refresh_lock_token: + try: + current_owner = self.redis_client.get(cache_lock_key) + if current_owner and str(current_owner) == refresh_lock_token: + self.redis_client.delete(cache_lock_key) + except Exception: + pass def mark_detect_job_item_running(self, job_item_id, claim_token): return self.execute( @@ -994,9 +2494,11 @@ class Database: WHERE item.id = batch.id AND item.claim_token = batch.claim_token AND item.status = 'claimed' + RETURNING item.job_id """ ) - updated_count = int(cur.rowcount or 0) + updated_rows = cur.fetchall() or [] + updated_count = len(updated_rows) conn.commit() return updated_count except Exception as e: @@ -1218,10 +2720,11 @@ class Database: WHERE item.id = batch.id AND item.claim_token = batch.claim_token AND item.status IN ('claimed', 'running') - RETURNING item.id + RETURNING item.id, item.job_id """ ) - updated_item_ids = {int(row[0]) for row in (cur.fetchall() or [])} + updated_rows = cur.fetchall() or [] + updated_item_ids = {int(row[0]) for row in updated_rows} updated_count = len(updated_item_ids) filtered_event_rows = [row for row in event_rows if int(row[0]) in updated_item_ids] if filtered_event_rows: @@ -1259,38 +2762,78 @@ class Database: def _refresh_detect_job_status_with_cursor(self, cur, job_id): cur.execute( """ - SELECT - job.task_mode, - count(*) FILTER (WHERE detect_job_items.status = 'pending') AS pending_count, - count(*) FILTER (WHERE detect_job_items.status IN ('claimed', 'running')) AS dispatch_active_count, - count(*) FILTER (WHERE detect_job_items.status = 'failed') AS failed_count, - count(*) FILTER (WHERE detect_job_items.status IN ('completed', 'blacklisted')) AS done_count, - count(*) FILTER ( - WHERE detect_job_items.status IN ('completed', 'blacklisted', 'failed') - AND COALESCE(detect_job_items.step_code, '') <> '' - AND COALESCE(result_payload_json->>'controller_processed', 'false') <> 'true' - ) AS unprocessed_terminal_count - FROM detect_job_items - JOIN detect_jobs AS job ON job.id = detect_job_items.job_id - WHERE job_id = %s - GROUP BY job.task_mode + SELECT COALESCE(task_mode, '') + FROM detect_jobs + WHERE id = %s """, (job_id,), ) - row = cur.fetchone() - task_mode = str((row or [""])[0] or "").strip() - pending_count = int((row or ["", 0])[1] or 0) - dispatch_active_count = int((row or ["", 0, 0])[2] or 0) - failed_count = int((row or ["", 0, 0, 0])[3] or 0) - done_count = int((row or ["", 0, 0, 0, 0])[4] or 0) - unprocessed_terminal_count = int((row or ["", 0, 0, 0, 0, 0])[5] or 0) - if dispatch_active_count > 0 or (task_mode == 'domain_pipeline' and unprocessed_terminal_count > 0): + task_mode_row = cur.fetchone() + task_mode = str((task_mode_row or [""])[0] or "").strip() + + def _exists(query, *params): + cur.execute(query, params) + row = cur.fetchone() + return bool((row or [False])[0]) + + dispatch_active_exists = _exists( + """ + SELECT EXISTS ( + SELECT 1 + FROM detect_job_items AS item + WHERE item.job_id = %s + AND item.status IN ('claimed', 'running') + LIMIT 1 + ) + """, + job_id, + ) + unprocessed_terminal_exists = False + if task_mode == 'domain_pipeline': + unprocessed_terminal_exists = _exists( + """ + SELECT EXISTS ( + SELECT 1 + FROM detect_job_items AS item + WHERE item.job_id = %s + AND item.status IN ('completed', 'blacklisted', 'failed') + AND COALESCE(item.step_code, '') <> '' + AND COALESCE(item.result_payload_json->>'controller_processed', 'false') <> 'true' + LIMIT 1 + ) + """, + job_id, + ) + if dispatch_active_exists or unprocessed_terminal_exists: cur.execute( - "UPDATE detect_jobs SET status = 'running', started_at = COALESCE(started_at, CURRENT_TIMESTAMP) WHERE id = %s", + """ + UPDATE detect_jobs + SET status = 'running', + started_at = COALESCE(started_at, CURRENT_TIMESTAMP), + finished_at = NULL + WHERE id = %s + AND ( + status <> 'running' + OR started_at IS NULL + OR finished_at IS NOT NULL + ) + """, (job_id,), ) return - if pending_count > 0: + pending_exists = _exists( + """ + SELECT EXISTS ( + SELECT 1 + FROM detect_job_items AS item + WHERE item.job_id = %s + AND item.status = 'pending' + LIMIT 1 + ) + """, + job_id, + ) + if pending_exists: cur.execute( """ UPDATE detect_jobs @@ -1301,10 +2844,34 @@ class Database: (job_id,), ) return + failed_exists = _exists( + """ + SELECT EXISTS ( + SELECT 1 + FROM detect_job_items AS item + WHERE item.job_id = %s + AND item.status = 'failed' + LIMIT 1 + ) + """, + job_id, + ) + done_exists = _exists( + """ + SELECT EXISTS ( + SELECT 1 + FROM detect_job_items AS item + WHERE item.job_id = %s + AND item.status IN ('completed', 'blacklisted') + LIMIT 1 + ) + """, + job_id, + ) final_status = 'completed' - if failed_count > 0 and done_count > 0: + if failed_exists and done_exists: final_status = 'partial_failed' - elif failed_count > 0: + elif failed_exists: final_status = 'failed' cur.execute( """ diff --git a/domainCheck/app/utils/detection_results.py b/domainCheck/app/utils/detection_results.py new file mode 100644 index 0000000..22c9133 --- /dev/null +++ b/domainCheck/app/utils/detection_results.py @@ -0,0 +1,194 @@ +import json + + +def build_detection_result( + *, + status=None, + state=None, + message="", + error=None, + **payload, +): + result = dict(payload) + result["status"] = bool(status) if status is not None else None + result["message"] = message or "" + + if error: + result["state"] = "error" + result["error"] = str(error) + if not result["message"]: + result["message"] = str(error) + else: + result["state"] = state or _default_state_for_status(status) + result.pop("error", None) + + return result + + +def _default_state_for_status(status): + if status is True: + return "positive" + if status is False: + return "negative" + return "ok" + + +def load_detection_result(value): + if value is None: + return {} + if isinstance(value, dict): + return value + if isinstance(value, str): + try: + decoded = json.loads(value) + except Exception: + return {} + return decoded if isinstance(decoded, dict) else {} + return {} + + +def resolve_detection_status(value, *legacy_keys): + data = load_detection_result(value) + if not data: + return False + + status = data.get("status") + if status is not None: + return bool(status) + + for key in legacy_keys: + if data.get(key) is not None: + return bool(data.get(key)) + return False + + +def build_manual_detection_result(status, *, legacy_key=None, message="人工更新"): + payload = {} + if legacy_key: + payload[legacy_key] = bool(status) + return build_detection_result( + status=bool(status), + state="manual", + message=message, + **payload, + ) + + +def normalize_detector_result(name, result): + result = load_detection_result(result) + if result.get("error"): + return build_detection_result(error=result.get("error"), **_without_meta(result)) + + if name == "baidu_history": + has_history = bool(result.get("has_history")) + has_gray = bool(result.get("has_gray")) + return build_detection_result( + status=has_history, + state="risk" if has_gray else None, + has_history=has_history, + has_gray=has_gray, + ) + + if name in {"baidu_site", "qihu360_site", "google_site"}: + has_index = bool(result.get("has_收录")) + normalized = build_detection_result( + status=has_index, + has_收录=has_index, + subdomains=list(result.get("subdomains", []) or []), + ) + return normalized + + if name == "chinaz_info": + return build_detection_result( + status=None, + title=result.get("title", ""), + category=result.get("category", ""), + has_sensitive=bool(result.get("has_sensitive")), + ) + + if name == "aizhan_info": + return build_detection_result( + status=None, + title=result.get("title", ""), + risk=result.get("risk", ""), + has_sensitive=bool(result.get("has_sensitive")), + ) + + if name == "juziseo_info": + history = normalize_detector_result("juziseo_history", result.get("history")) + backlink = normalize_detector_result("juziseo_backlink", result.get("backlink")) + nested_error = history.get("error") or backlink.get("error") + return build_detection_result( + status=None, + error=nested_error, + history=history, + backlink=backlink, + ) + + if name == "juziseo_history": + return build_detection_result( + status=None, + state="risk" if result.get("has_sensitive") or result.get("has_subdomains") else None, + has_sensitive=bool(result.get("has_sensitive")), + has_baidu_history=bool(result.get("has_baidu_history")), + has_subdomains=bool(result.get("has_subdomains")), + is_simplified=bool(result.get("is_simplified", True)), + ) + + if name == "juziseo_backlink": + return build_detection_result( + status=None, + state="risk" if result.get("has_sensitive") or result.get("has_subdomains") else None, + has_sensitive=bool(result.get("has_sensitive")), + has_subdomains=bool(result.get("has_subdomains")), + ) + + if name == "jucha_info": + whois = normalize_detector_result("jucha_whois", result.get("whois")) + beian = normalize_detector_result("jucha_beian", result.get("beian")) + intercept = normalize_detector_result("jucha_intercept", result.get("intercept")) + nested_error = whois.get("error") or beian.get("error") or intercept.get("error") + return build_detection_result( + status=None, + error=nested_error, + whois=whois, + beian=beian, + intercept=intercept, + ) + + if name == "jucha_whois": + whois_status = result.get("status", "") + hold = whois_status in {"clientHold", "serverHold"} + return build_detection_result( + status=None, + state="risk" if hold else None, + whois_status=whois_status, + ) + + if name == "jucha_beian": + has_beian = bool(result.get("has_beian")) + return build_detection_result( + status=has_beian, + has_beian=has_beian, + beian_year=result.get("beian_year", ""), + is_enterprise=bool(result.get("is_enterprise")), + beian_match=bool(result.get("beian_match")), + ) + + if name == "jucha_intercept": + normal = bool(result.get("normal")) + return build_detection_result( + status=normal, + state="risk" if not normal else None, + normal=normal, + ) + + return build_detection_result(status=None, **result) + + +def _without_meta(result): + return { + key: value + for key, value in result.items() + if key not in {"status", "state", "message", "error"} + } diff --git a/domainCheck/app/utils/redis_client.py b/domainCheck/app/utils/redis_client.py new file mode 100644 index 0000000..9f19bae --- /dev/null +++ b/domainCheck/app/utils/redis_client.py @@ -0,0 +1,129 @@ +from __future__ import annotations + +import os +import threading + +import redis + +from app.config import config + + +_CLIENTS: dict[tuple[str, bool], redis.Redis] = {} +_LOCK = threading.Lock() + + +def _safe_int(raw_value: object, default: int, minimum: int) -> int: + try: + parsed = int(raw_value) + except Exception: + parsed = default + return max(minimum, parsed) + + +def _safe_float(raw_value: object, default: float, minimum: float) -> float: + try: + parsed = float(raw_value) + except Exception: + parsed = default + return max(minimum, parsed) + + +def _pool_options(role: str, *, decode_responses: bool) -> dict: + normalized_role = str(role or "standard").strip().lower() or "standard" + node_code = str(getattr(config, "NODE_CODE", "") or "").strip() or "unknown" + + if normalized_role == "pubsub": + return { + "host": config.REDIS_HOST, + "port": config.REDIS_PORT, + "password": config.REDIS_PASSWORD or None, + "db": config.REDIS_DB, + "decode_responses": decode_responses, + "socket_connect_timeout": _safe_float( + os.getenv("DOMAINCHECK_REDIS_PUBSUB_CONNECT_TIMEOUT", "30"), + 30.0, + 1.0, + ), + "socket_timeout": _safe_float( + os.getenv("DOMAINCHECK_REDIS_PUBSUB_SOCKET_TIMEOUT", "60"), + 60.0, + 1.0, + ), + "health_check_interval": _safe_int( + os.getenv("DOMAINCHECK_REDIS_HEALTH_CHECK_INTERVAL", "30"), + 30, + 0, + ), + "retry_on_timeout": True, + "max_connections": _safe_int( + os.getenv("DOMAINCHECK_REDIS_PUBSUB_MAX_CONNECTIONS", "2"), + 2, + 1, + ), + "timeout": _safe_float( + os.getenv("DOMAINCHECK_REDIS_PUBSUB_POOL_TIMEOUT", "5"), + 5.0, + 0.1, + ), + "client_name": f"domaincheck:pubsub:{node_code}:{os.getpid()}", + } + + return { + "host": config.REDIS_HOST, + "port": config.REDIS_PORT, + "password": config.REDIS_PASSWORD or None, + "db": config.REDIS_DB, + "decode_responses": decode_responses, + "socket_connect_timeout": _safe_float( + os.getenv("DOMAINCHECK_REDIS_CONNECT_TIMEOUT", "5"), + 5.0, + 0.5, + ), + "socket_timeout": _safe_float( + os.getenv("DOMAINCHECK_REDIS_SOCKET_TIMEOUT", "10"), + 10.0, + 0.5, + ), + "health_check_interval": _safe_int( + os.getenv("DOMAINCHECK_REDIS_HEALTH_CHECK_INTERVAL", "30"), + 30, + 0, + ), + "retry_on_timeout": True, + "max_connections": _safe_int( + os.getenv("DOMAINCHECK_REDIS_MAX_CONNECTIONS", "12"), + 12, + 1, + ), + "timeout": _safe_float( + os.getenv("DOMAINCHECK_REDIS_POOL_TIMEOUT", "1.5"), + 1.5, + 0.1, + ), + "client_name": f"domaincheck:standard:{node_code}:{os.getpid()}", + } + + +def get_redis_client(*, role: str = "standard", decode_responses: bool = True) -> redis.Redis: + normalized_role = str(role or "standard").strip().lower() or "standard" + cache_key = (normalized_role, bool(decode_responses)) + with _LOCK: + cached = _CLIENTS.get(cache_key) + if cached is not None: + return cached + + pool = redis.BlockingConnectionPool(**_pool_options(normalized_role, decode_responses=decode_responses)) + client = redis.Redis(connection_pool=pool) + _CLIENTS[cache_key] = client + return client + + +def reset_redis_clients_for_tests() -> None: + with _LOCK: + clients = list(_CLIENTS.values()) + _CLIENTS.clear() + for client in clients: + try: + client.close() + except Exception: + pass diff --git a/domainCheck/detect/c360.py b/domainCheck/detect/c360.py index aa0ac95..3ad4b70 100644 --- a/domainCheck/detect/c360.py +++ b/domainCheck/detect/c360.py @@ -12,6 +12,7 @@ import re # 正则表达式模块 import os import threading import time +from collections import OrderedDict import requests # HTTP请求库 from loguru import logger # 日志记录 from requests.adapters import HTTPAdapter @@ -23,7 +24,12 @@ SO_SEARCH_URL = 'https://www.so.com/s' # 360搜索URL SO_REFERER_TEMPLATE = 'https://www.so.com/s?ie=utf-8&q=site%3A{domain}' # Referer模板 SEARCH_PATTERN = r'target="_blank">([^<]+)' # 搜索结果匹配模式 BLOCKED_CODE = 3 # 拦截状态码 -_SESSION_LOCAL = threading.local() +_DIRECT_HTTP = requests.Session() +_DIRECT_HTTP.trust_env = False +_DIRECT_HTTP.mount("http://", HTTPAdapter(pool_connections=128, pool_maxsize=256, max_retries=0)) +_DIRECT_HTTP.mount("https://", HTTPAdapter(pool_connections=128, pool_maxsize=256, max_retries=0)) +_PROXY_SESSIONS = OrderedDict() +_PROXY_SESSION_LOCK = threading.Lock() SO_TIMEOUT_PROXY = max( 1.0, float(os.getenv("DOMAINCHECK_360_TIMEOUT_PROXY", "1.4") or 1.4), @@ -34,6 +40,20 @@ SO_TIMEOUT_DIRECT = max( ) +def _proxy_session_cache_limit() -> int: + try: + return max(1, int(os.getenv("DOMAINCHECK_360_PROXY_SESSION_CACHE_SIZE", "128") or 128)) + except Exception: + return 128 + + +def _close_session_quietly(session) -> None: + try: + session.close() + except Exception: + pass + + def _resolve_360_timeout(proxies: Optional[Dict] = None, budget_seconds: Optional[float] = None) -> float: timeout = float(SO_TIMEOUT_PROXY if proxies else SO_TIMEOUT_DIRECT) if budget_seconds not in (None, "", 0, "0"): @@ -41,15 +61,27 @@ def _resolve_360_timeout(proxies: Optional[Dict] = None, budget_seconds: Optiona return max(0.6, timeout) -def _get_session(): - session = getattr(_SESSION_LOCAL, "session", None) - if session is not None: +def _get_session(proxies: Optional[Dict] = None): + proxy_url = "" + if proxies: + proxy_url = str(proxies.get("https") or proxies.get("http") or "").strip() + if not proxy_url: + return _DIRECT_HTTP + + with _PROXY_SESSION_LOCK: + session = _PROXY_SESSIONS.get(proxy_url) + if session is not None: + _PROXY_SESSIONS.move_to_end(proxy_url) + return session + session = requests.Session() + session.trust_env = False + session.mount("http://", HTTPAdapter(pool_connections=64, pool_maxsize=128, max_retries=0)) + session.mount("https://", HTTPAdapter(pool_connections=64, pool_maxsize=128, max_retries=0)) + _PROXY_SESSIONS[proxy_url] = session + while len(_PROXY_SESSIONS) > _proxy_session_cache_limit(): + _, stale_session = _PROXY_SESSIONS.popitem(last=False) + _close_session_quietly(stale_session) return session - session = requests.Session() - session.mount("http://", HTTPAdapter(pool_connections=64, pool_maxsize=128, max_retries=0)) - session.mount("https://", HTTPAdapter(pool_connections=64, pool_maxsize=128, max_retries=0)) - _SESSION_LOCAL.session = session - return session def check_domain( @@ -95,7 +127,7 @@ def check_domain( try: session_started_at = time.perf_counter() logger.info(f'360阶段: domain={domain} | stage=session_init_start') - session = _get_session() + session = _get_session(proxies) logger.info( f'360阶段: domain={domain} | stage=session_init_done ' f'| elapsed_ms={int((time.perf_counter() - session_started_at) * 1000)}' @@ -103,20 +135,28 @@ def check_domain( # 发送GET请求获取搜索结果 request_started_at = time.perf_counter() logger.info(f'360阶段: domain={domain} | stage=request_start | proxy={"yes" if proxies else "no"}') - response = session.get( - SO_SEARCH_URL, - params=params, - headers=headers, - proxies=proxies, - timeout=_resolve_360_timeout(proxies, budget_seconds=budget_seconds), - ) - logger.info( - f'360阶段: domain={domain} | stage=request_done | status={response.status_code} ' - f'| elapsed_ms={int((time.perf_counter() - request_started_at) * 1000)}' - ) + response = None + try: + response = session.get( + SO_SEARCH_URL, + params=params, + headers=headers, + proxies=proxies, + timeout=_resolve_360_timeout(proxies, budget_seconds=budget_seconds), + ) + logger.info( + f'360阶段: domain={domain} | stage=request_done | status={response.status_code} ' + f'| elapsed_ms={int((time.perf_counter() - request_started_at) * 1000)}' + ) - # 解析响应内容 - response_html = response.content.decode('utf-8', errors='ignore') + # 解析响应内容 + response_html = response.content.decode('utf-8', errors='ignore') + finally: + try: + if response is not None: + response.close() + except Exception: + pass # 使用正则表达式提取搜索结果 search_results = re.findall(SEARCH_PATTERN, response_html) diff --git a/domainCheck/detect/geetest2.py b/domainCheck/detect/geetest2.py index 8dd3889..52bf184 100644 --- a/domainCheck/detect/geetest2.py +++ b/domainCheck/detect/geetest2.py @@ -15,10 +15,23 @@ import subprocess import execjs +def _resolve_geetest_asset_timeout(): + connect_timeout = max( + 0.3, + float(os.getenv("DOMAINCHECK_GEETEST_ASSET_CONNECT_TIMEOUT", "2.0") or 2.0), + ) + read_timeout = max( + 0.5, + float(os.getenv("DOMAINCHECK_GEETEST_ASSET_READ_TIMEOUT", "4.0") or 4.0), + ) + return (connect_timeout, read_timeout) + + class slide(): def __init__(self): self.headers = randomUA() self.t = round(time.time() * 1000) + self.asset_timeout = _resolve_geetest_asset_timeout() def __ease_out_expo(self, sep): if sep == 1: @@ -83,7 +96,7 @@ class slide(): for idv, p_url in vdict.items(): p_url = 'http://static.geetest.com/' + p_url # print(p_url) - vcode = requests.get(p_url, headers=self.headers) + vcode = requests.get(p_url, headers=self.headers, timeout=self.asset_timeout) text = vcode.content if idv == 'bg': text = self.tp_huanyuan(text) @@ -131,7 +144,7 @@ class Geetest2(): "callback": f"geetest_{self.t}" } try: - respe = http.get(url=url, params=params, proxies=proxies).text + respe = http.get(url=url, params=params, proxies=proxies, timeout=self.timeout).text except Exception as e: if proxy: if pparmas: diff --git a/domainCheck/detect/jucha.py b/domainCheck/detect/jucha.py index b46ffe5..f2809f2 100644 --- a/domainCheck/detect/jucha.py +++ b/domainCheck/detect/jucha.py @@ -13,7 +13,6 @@ import base64 # Base64编码解码 import hashlib # 哈希加密 import json # JSON数据处理 import os # 操作系统接口 -import pickle # 序列化反序列化 import random # 随机数生成 import shutil import subprocess # 子进程管理 @@ -24,6 +23,7 @@ import requests # HTTP请求库 from loguru import logger # 日志记录 from requests.cookies import RequestsCookieJar # Cookie管理 +from detect.locked_pickle import load_pickle_locked, save_pickle_atomic # # 移除自定义子进程类的替换，避免影响其他模块 @@ -109,12 +109,19 @@ class JC(object): # 聚查网API封装类 def __init__(self, proxies: dict = None): # 初始化JC类 self.session = requests.Session() # 创建会话对象 - self.session.timeout = 10 # 设置超时时间 self.session.proxies = proxies # 设置代理 + self.request_timeout = max( + 0.5, + float(os.getenv("DOMAINCHECK_JUCHA_TIMEOUT", "10") or 10), + ) self.base_url = "https://www.jucha.com" # 设置基础URL self.cookie = requests.cookies.RequestsCookieJar() # 初始化CookieJar self.juming_cookie = requests.cookies.RequestsCookieJar() # 初始化聚名网CookieJar + def _request(self, method: str, url: str, **kwargs): + kwargs.setdefault("timeout", self.request_timeout) + return getattr(self.session, method)(url, **kwargs) + def _get_headers(self, referer: str = None) -> dict: # 获取请求头（可自定义referer） headers = self.headers.copy() # 复制默认请求头 if referer: # 如果指定了referer @@ -139,7 +146,7 @@ class JC(object): # 聚查网API封装类 return False, f'滑块验证码失败，{retry_times}次内未成功' # 返回失败 def _check_request(self, url: str, data: dict, headers: dict, cookies=None) -> dict: # 检查请求并处理验证码 - res = self.session.post(url=url, data=data, headers=headers, cookies=cookies) # 发送POST请求 + res = self._request("post", url=url, data=data, headers=headers, cookies=cookies) # 发送POST请求 try: response = res.json() # 尝试解析JSON except Exception as e: # JSON解析失败 @@ -179,7 +186,7 @@ class JC(object): # 聚查网API封装类 "seed": calculate_seed(286) # 种子值 } url = f"{self.base_url}/captcha/init" # 初始化URL - response = self.session.post(url, headers=self.headers, data=data, cookies=self.cookie).json() # 发送请求 + response = self._request("post", url, headers=self.headers, data=data, cookies=self.cookie).json() # 发送请求 if response['code'] == 1: # 初始化成功 self.captchaId = response['data']['captchaId'] # 保存验证码ID self.encryptionPublicKey = response['data']['encryptionPublicKey'] # 保存加密公钥 @@ -234,8 +241,13 @@ class JC(object): # 聚查网API封装类 url = f"{self.base_url}/captcha/verify" # 验证URL verify_headers = self.headers.copy() # 复制请求头 verify_headers['Content-Type'] = 'application/json' # 设置内容类型（仅用于验证请求） - verify_response = self.session.post(url, headers=verify_headers, data=verify_data, - cookies=self.cookie) # 发送验证请求 + verify_response = self._request( + "post", + url, + headers=verify_headers, + data=verify_data, + cookies=self.cookie, + ) # 发送验证请求 if verify_response.cookies: # 如果响应中有新的Cookie self.cookie.update(verify_response.cookies) # 合并Cookie（而不是替换） response = verify_response.json() # 解析JSON响应 @@ -254,20 +266,12 @@ class JC(object): # 聚查网API封装类 return False, "Node.js脚本未输出任何内容" # 返回失败 def save_cookies(self, filepath="jucha_cookies.pkl"): # 保存Cookie到文件 - with open(filepath, "wb") as f: # 以二进制写入模式打开文件 - pickle.dump(self.cookie, f) # 序列化保存Cookie + save_pickle_atomic(filepath, self.cookie) logger.info(f"已保存Cookie到文件: {filepath}") # 保存到Redis try: - import redis - from app.config import config - redis_client = redis.Redis( - host=config.REDIS_HOST, - port=config.REDIS_PORT, - password=config.REDIS_PASSWORD, - db=config.REDIS_DB, - decode_responses=True - ) + from app.utils.redis_client import get_redis_client + redis_client = get_redis_client(role="standard") # 将cookie转换为字典 cookie_dict = {} # 检查self.cookie的类型 @@ -292,25 +296,29 @@ class JC(object): # 聚查网API封装类 def load_cookies(self, filepath="jucha_cookies.pkl"): # 从文件加载Cookie try: # 尝试加载 - with open(filepath, "rb") as f: # 以二进制读取模式打开文件 - loaded_cookie = pickle.load(f) # 反序列化加载Cookie - # 检查加载的cookie类型 - if isinstance(loaded_cookie, dict): - # 如果是字典，转换为RequestsCookieJar - cookie_jar = requests.cookies.RequestsCookieJar() - for name, value in loaded_cookie.items(): - cookie_jar.set(name, value) - self.cookie = cookie_jar - else: - self.cookie = loaded_cookie + loaded_cookie = load_pickle_locked( + filepath, + default_factory=requests.cookies.RequestsCookieJar, + ) + # 检查加载的cookie类型 + if isinstance(loaded_cookie, dict): + # 如果是字典，转换为RequestsCookieJar + cookie_jar = requests.cookies.RequestsCookieJar() + for name, value in loaded_cookie.items(): + cookie_jar.set(name, value) + self.cookie = cookie_jar + else: + self.cookie = loaded_cookie except Exception as e: # 加载失败 logger.error(f"加载Cookie失败: {e}") self.cookie = requests.cookies.RequestsCookieJar() # 创建空的CookieJar def load_juming_cookies(self, filepath="juming_cookies.pkl"): # 从文件加载聚名网Cookie try: # 尝试加载 - with open(filepath, "rb") as f: # 以二进制读取模式打开文件 - self.juming_cookie = pickle.load(f) # 反序列化加载Cookie + self.juming_cookie = load_pickle_locked( + filepath, + default_factory=requests.cookies.RequestsCookieJar, + ) except: # 加载失败 self.juming_cookie = requests.cookies.RequestsCookieJar() # 创建空的CookieJar @@ -318,7 +326,8 @@ class JC(object): # 聚查网API封装类 params = { # 请求参数 'platform': 'juming', # 平台标识 } - res = self.session.get( # 发送GET请求 + res = self._request( # 发送GET请求 + "get", url=f'{self.base_url}/home/login/get_auth_url', # 获取授权URL params=params, # 请求参数 headers=self.headers, # 请求头 @@ -333,7 +342,8 @@ class JC(object): # 聚查网API封装类 combined_cookies = requests.cookies.RequestsCookieJar() combined_cookies.update(self.juming_cookie) combined_cookies.update(self.cookie) - response = self.session.get( # 发送GET请求 + response = self._request( # 发送GET请求 + "get", url=url, # 跳转URL headers=self.headers, # 请求头 cookies=combined_cookies, # 合并后的Cookie @@ -345,7 +355,8 @@ class JC(object): # 聚查网API封装类 self.cookie[name] = value # 直接赋值，覆盖同名Cookie if response.status_code == 302: # 重定向状态码 url = response.headers['location'] # 获取重定向URL - response = self.session.get( # 发送GET请求 + response = self._request( # 发送GET请求 + "get", url=url, # 重定向URL headers=self.headers, # 请求头 allow_redirects=False # 不自动重定向 diff --git a/domainCheck/detect/juming.py b/domainCheck/detect/juming.py index 6fc4aeb..147f436 100644 --- a/domainCheck/detect/juming.py +++ b/domainCheck/detect/juming.py @@ -12,7 +12,6 @@ import base64 # Base64编码模块 import hashlib # 哈希算法模块 import json # JSON处理模块 import os # 操作系统接口模块 -import pickle # 序列化模块 import random # 随机数生成模块 import shutil import subprocess # 子进程管理模块 @@ -23,6 +22,7 @@ import requests # HTTP请求库 from loguru import logger # 日志记录库 from requests.cookies import RequestsCookieJar # Cookie处理 +from detect.locked_pickle import load_pickle_locked, save_pickle_atomic # 常量定义 BASE_URL = "https://www.juming.com" # 聚名网基础URL @@ -170,11 +170,18 @@ class JM(object): # 聚名网API封装类 def __init__(self, proxies: Optional[Dict] = None): # 初始化方法 self.session = requests.Session() # 创建会话对象 - self.session.timeout = REQUEST_TIMEOUT # 设置超时时间 self.session.proxies = proxies # 设置代理 + self.request_timeout = max( + 0.5, + float(os.getenv("DOMAINCHECK_JUMING_TIMEOUT", str(REQUEST_TIMEOUT)) or REQUEST_TIMEOUT), + ) self.base_url = BASE_URL # 设置基础URL logger.debug(proxies) + def _request(self, method: str, url: str, **kwargs): + kwargs.setdefault("timeout", self.request_timeout) + return getattr(self.session, method)(url, **kwargs) + def captcha_init(self) -> Tuple[bool, str]: # 初始化滑块验证码 """ 初始化滑块验证码 @@ -194,7 +201,7 @@ class JM(object): # 聚名网API封装类 # self.cookie.clear() # self.cookie.update({'acw_sc__v2': '69d948ed49117d89433d4316a79ada6f00d68f26'}) # logger.info(self.cookie) - response = self.session.post(url, headers=self.headers, data=data, cookies=self.cookie).json() # 发送请求 + response = self._request("post", url, headers=self.headers, data=data, cookies=self.cookie).json() # 发送请求 logger.info(response) if response['code'] == 1: # 初始化成功 @@ -271,7 +278,7 @@ class JM(object): # 聚名网API封装类 url = f"{self.base_url}/captcha/verify" # 验证URL self.headers['Content-Type'] = 'application/json' # 设置内容类型 - response = self.session.post(url, headers=self.headers, data=verify_data, cookies=self.cookie).json() # 发送验证请求 + response = self._request("post", url, headers=self.headers, data=verify_data, cookies=self.cookie).json() # 发送验证请求 if response['code'] == 1: # 验证成功 self.token = response['data']['token'] # 保存token self.session_id = response['data']['session_id'] # 保存会话ID @@ -294,7 +301,7 @@ class JM(object): # 聚名网API封装类 """ url = f"{self.base_url}/user_zh/wxdl_ewm" # 二维码URL try: # 异常处理 - response = self.session.post(url, headers=self.headers).json() # 发送请求 + response = self._request("post", url, headers=self.headers).json() # 发送请求 self.token = response['data']["token"] # 保存token return response['code'] == 1, response['msg'], response['data']['url'] if response['code'] == 1 else None # 返回结果 except Exception as e: # 异常处理 @@ -338,7 +345,8 @@ class JM(object): # 聚名网API封装类 } login_url = f"{self.base_url}/user_zh/p_login" # 登录URL - response = self.session.post( # 发送登录请求 + response = self._request( # 发送登录请求 + "post", url=login_url, headers=self.headers, json=data, @@ -384,7 +392,7 @@ class JM(object): # 聚名网API封装类 url = f"{self.base_url}/ykj/get_list" # 列表URL try: # 异常处理 self.headers['Content-Type'] = 'application/x-www-form-urlencoded' # 设置内容类型 - res = self.session.post(url, headers=self.headers, data=data, cookies=self.cookie) # 发送请求 + res = self._request("post", url, headers=self.headers, data=data, cookies=self.cookie) # 发送请求 response = res.json() # 解析响应 if response['code'] == -401: # 需要验证码 @@ -427,7 +435,7 @@ class JM(object): # 聚名网API封装类 """ url = f"{self.base_url}/newcha/del_down?scsj={date}" # 下载URL try: # 异常处理 - response = self.session.get(url, headers=self.headers, cookies=self.cookie, allow_redirects=False) # 发送请求 + response = self._request("get", url, headers=self.headers, cookies=self.cookie, allow_redirects=False) # 发送请求 url = response.headers['Location'] # 获取重定向URL response = self.session.get(url, timeout=DOWNLOAD_TIMEOUT) # 下载文件 response_text = response.content.decode('utf-8') # 解码内容 @@ -443,20 +451,12 @@ class JM(object): # 聚名网API封装类 Args: filepath: 文件路径 """ - with open(filepath, "wb") as f: # 以二进制写入模式打开文件 - pickle.dump(self.cookie, f) # 序列化保存 + save_pickle_atomic(filepath, self.cookie) # 保存到Redis try: - import redis - from app.config import config - redis_client = redis.Redis( - host=config.REDIS_HOST, - port=config.REDIS_PORT, - password=config.REDIS_PASSWORD, - db=config.REDIS_DB, - decode_responses=True - ) + from app.utils.redis_client import get_redis_client + redis_client = get_redis_client(role="standard") # 将cookie转换为字典 cookie_dict = {} for cookie in self.cookie: @@ -473,8 +473,10 @@ class JM(object): # 聚名网API封装类 filepath: 文件路径 """ try: # 异常处理 - with open(filepath, "rb") as f: # 以二进制读取模式打开文件 - self.cookie = pickle.load(f) # 反序列化加载 + self.cookie = load_pickle_locked( + filepath, + default_factory=requests.cookies.RequestsCookieJar, + ) except: # 异常处理 self.cookie = requests.cookies.RequestsCookieJar() # 创建空Cookie diff --git a/domainCheck/detect/juziseo.py b/domainCheck/detect/juziseo.py index ece397a..3f33efa 100644 --- a/domainCheck/detect/juziseo.py +++ b/domainCheck/detect/juziseo.py @@ -10,9 +10,7 @@ # 导入标准库 import os # 操作系统接口 -import pickle # 序列化反序列化 import re -import time # 时间处理 from Crypto.Cipher import AES from Crypto.Util.Padding import pad import base64 @@ -22,6 +20,7 @@ from loguru import logger # 日志记录 from requests.cookies import RequestsCookieJar # Cookie管理 from detect.geetest2 import Geetest2 +from detect.locked_pickle import load_pickle_locked, save_pickle_atomic @@ -69,30 +68,29 @@ class Juziseo: self.gt = None self.session = requests.Session() # 创建会话对象 self.session.proxies = proxies # 设置代理 - self.session.timeout = 10 # 设置超时时间10秒 + self.request_timeout = max( + 0.5, + float(os.getenv("DOMAINCHECK_JUZISEO_TIMEOUT", "10") or 10), + ) self.base_url = "https://seo.juziseo.com" # 设置基础URL + def _request(self, method: str, url: str, **kwargs): + kwargs.setdefault("timeout", self.request_timeout) + return getattr(self.session, method)(url, **kwargs) + def save_cookies(self, filepath="juziseo_cookies.pkl"): """ 保存Cookie到文件和Redis :param filepath: 文件路径 """ - with open(filepath, "wb") as f: # 以二进制写入模式打开文件 - pickle.dump(self.cookie, f) # 序列化保存Cookie + save_pickle_atomic(filepath, self.cookie) logger.info(f"已保存桔子SEO Cookie到 {filepath}") # 保存到Redis try: - import redis - from app.config import config - redis_client = redis.Redis( - host=config.REDIS_HOST, - port=config.REDIS_PORT, - password=config.REDIS_PASSWORD, - db=config.REDIS_DB, - decode_responses=True - ) + from app.utils.redis_client import get_redis_client + redis_client = get_redis_client(role="standard") # 将cookie转换为字典 cookie_dict = {} for cookie in self.cookie: @@ -108,8 +106,10 @@ class Juziseo: :param filepath: 文件路径 """ try: # 尝试加载 - with open(filepath, "rb") as f: # 以二进制读取模式打开文件 - self.cookie = pickle.load(f) # 反序列化加载Cookie + self.cookie = load_pickle_locked( + filepath, + default_factory=requests.cookies.RequestsCookieJar, + ) # logger.info(f"已从 {filepath} 加载桔子SEO Cookie") except: # 加载失败 self.cookie = requests.cookies.RequestsCookieJar() # 创建空的CookieJar @@ -117,7 +117,7 @@ class Juziseo: def start_msg_captcha_servlet(self): url = f"{self.base_url}/class/gtcode/msg/StartMsgCaptchaServlet.php" - response = self.session.get(url, headers=self.headers, cookies=self.cookie).json() + response = self._request("get", url, headers=self.headers, cookies=self.cookie).json() logger.info(response) self.gt = response['gt'] self.challenge = response['challenge'] @@ -161,7 +161,7 @@ class Juziseo: logger.debug(data) # 发送登录请求 url = f"{self.base_url}/account/ajax/login_process/" - response = self.session.post(url, headers=self.headers, data=data, cookies=self.cookie) + response = self._request("post", url, headers=self.headers, data=data, cookies=self.cookie) logger.info(response.text) # 解析响应 result = response.json() @@ -190,11 +190,11 @@ class Juziseo: '_post_type': 'ajax', } url = f"{self.base_url}/snapshot/save/" - response = self.session.post(url, headers=self.headers, data=data, cookies=self.cookie).json() + response = self._request("post", url, headers=self.headers, data=data, cookies=self.cookie).json() # logger.info(response) if response['errno'] == 1: url = response['rsm']['url'] - response_html = self.session.get(url, headers=self.headers, cookies=self.cookie).content.decode('utf-8') + response_html = self._request("get", url, headers=self.headers, cookies=self.cookie).content.decode('utf-8') title_sensitive_words_match = re.search(r'标题敏感词\D*(\d+)', response_html, re.S) title_suspected_sensitive_words_match = re.search(r'标题有疑似敏感词\D*(\d+)', response_html, re.S) content_sensitive_words_match = re.search(r'内容敏感词\D*(\d+)', response_html, re.S) @@ -221,11 +221,11 @@ class Juziseo: '_post_type': 'ajax', } url = f"{self.base_url}/domain_rank/save_domain/" - response = self.session.post(url, headers=self.headers, data=data, cookies=self.cookie).json() + response = self._request("post", url, headers=self.headers, data=data, cookies=self.cookie).json() logger.info(response) if response['errno'] == 1: url = response['rsm']['url'] - response_html = self.session.get(url, headers=self.headers, cookies=self.cookie).content.decode('utf-8') + response_html = self._request("get", url, headers=self.headers, cookies=self.cookie).content.decode('utf-8') subdomain_match = re.search(r'子域名：\D*(\d+)', response_html, re.S) if subdomain_match: return False, f"存在子域名: {subdomain_match.group(1)}" diff --git a/domainCheck/detect/locked_pickle.py b/domainCheck/detect/locked_pickle.py new file mode 100644 index 0000000..41ed0f8 --- /dev/null +++ b/domainCheck/detect/locked_pickle.py @@ -0,0 +1,91 @@ +# -*- coding: UTF-8 -*- +"""Helpers for cross-process safe pickle persistence.""" + +import os +import pickle +import tempfile +import time +from contextlib import contextmanager +from typing import Callable, TypeVar + +T = TypeVar("T") + +try: # pragma: no cover - platform specific + import fcntl +except ImportError: # pragma: no cover - platform specific + fcntl = None + +try: # pragma: no cover - platform specific + import msvcrt +except ImportError: # pragma: no cover - platform specific + msvcrt = None + + +def _acquire_platform_lock(lock_file, *, timeout_seconds: float) -> None: + deadline = time.monotonic() + max(0.1, float(timeout_seconds or 0.0)) + while True: + try: + if fcntl is not None: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + elif msvcrt is not None: + msvcrt.locking(lock_file.fileno(), msvcrt.LK_NBLCK, 1) + return + except (BlockingIOError, OSError): + if time.monotonic() >= deadline: + raise TimeoutError(f"lock acquire timed out for {lock_file.name}") + time.sleep(0.05) + + +def _release_platform_lock(lock_file) -> None: + try: + if fcntl is not None: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) + elif msvcrt is not None: + lock_file.seek(0) + msvcrt.locking(lock_file.fileno(), msvcrt.LK_UNLCK, 1) + except Exception: + pass + + +@contextmanager +def _locked_path(path: str, *, timeout_seconds: float = 10.0): + normalized_path = os.path.abspath(path) + directory = os.path.dirname(normalized_path) or "." + os.makedirs(directory, exist_ok=True) + lock_path = f"{normalized_path}.lock" + with open(lock_path, "a+b") as lock_file: + _acquire_platform_lock(lock_file, timeout_seconds=timeout_seconds) + try: + yield normalized_path + finally: + _release_platform_lock(lock_file) + + +def save_pickle_atomic(path: str, value, *, timeout_seconds: float = 10.0) -> None: + with _locked_path(path, timeout_seconds=timeout_seconds) as normalized_path: + directory = os.path.dirname(normalized_path) or "." + fd, temp_path = tempfile.mkstemp( + prefix=f".{os.path.basename(normalized_path)}.", + suffix=".tmp", + dir=directory, + ) + try: + with os.fdopen(fd, "wb") as temp_file: + pickle.dump(value, temp_file) + temp_file.flush() + os.fsync(temp_file.fileno()) + os.replace(temp_path, normalized_path) + finally: + if os.path.exists(temp_path): + try: + os.remove(temp_path) + except OSError: + pass + + +def load_pickle_locked(path: str, *, default_factory: Callable[[], T], timeout_seconds: float = 10.0) -> T: + with _locked_path(path, timeout_seconds=timeout_seconds) as normalized_path: + if not os.path.exists(normalized_path): + return default_factory() + with open(normalized_path, "rb") as source: + return pickle.load(source) diff --git a/domainCheck/detect/register.py b/domainCheck/detect/register.py index 68c7549..ef76758 100644 --- a/domainCheck/detect/register.py +++ b/domainCheck/detect/register.py @@ -11,6 +11,7 @@ import json import os import threading +from collections import OrderedDict from datetime import datetime, timezone, timedelta # 日期时间处理，用于时区转换 from typing import Optional @@ -22,10 +23,24 @@ _DIRECT_HTTP = requests.Session() _DIRECT_HTTP.trust_env = False _DIRECT_HTTP.mount("http://", requests.adapters.HTTPAdapter(pool_connections=256, pool_maxsize=512, max_retries=0)) _DIRECT_HTTP.mount("https://", requests.adapters.HTTPAdapter(pool_connections=256, pool_maxsize=512, max_retries=0)) -_PROXY_MANAGERS = {} +_PROXY_MANAGERS = OrderedDict() _PROXY_MANAGER_LOCK = threading.Lock() +def _proxy_manager_cache_limit() -> int: + try: + return max(1, int(os.getenv("DOMAINCHECK_REGISTER_PROXY_SESSION_CACHE_SIZE", "128") or 128)) + except Exception: + return 128 + + +def _close_session_quietly(session) -> None: + try: + session.close() + except Exception: + pass + + def _get_http_manager(proxies: Optional[dict] = None): proxy_url = "" if proxies: @@ -34,12 +49,17 @@ def _get_http_manager(proxies: Optional[dict] = None): return _DIRECT_HTTP with _PROXY_MANAGER_LOCK: manager = _PROXY_MANAGERS.get(proxy_url) - if manager is None: - manager = requests.Session() - manager.trust_env = False - manager.mount("http://", requests.adapters.HTTPAdapter(pool_connections=128, pool_maxsize=256, max_retries=0)) - manager.mount("https://", requests.adapters.HTTPAdapter(pool_connections=128, pool_maxsize=256, max_retries=0)) - _PROXY_MANAGERS[proxy_url] = manager + if manager is not None: + _PROXY_MANAGERS.move_to_end(proxy_url) + return manager + manager = requests.Session() + manager.trust_env = False + manager.mount("http://", requests.adapters.HTTPAdapter(pool_connections=128, pool_maxsize=256, max_retries=0)) + manager.mount("https://", requests.adapters.HTTPAdapter(pool_connections=128, pool_maxsize=256, max_retries=0)) + _PROXY_MANAGERS[proxy_url] = manager + while len(_PROXY_MANAGERS) > _proxy_manager_cache_limit(): + _, stale_manager = _PROXY_MANAGERS.popitem(last=False) + _close_session_quietly(stale_manager) return manager diff --git a/domainCheck/detect_worker.py b/domainCheck/detect_worker.py index c8eb3ed..42dd7e1 100644 --- a/domainCheck/detect_worker.py +++ b/domainCheck/detect_worker.py @@ -9,8 +9,11 @@ ''' import os +import signal import sys import json +import gc +import math import time import socket import threading @@ -20,6 +23,7 @@ import schedule import urllib.error import urllib.parse import urllib.request +from pathlib import Path from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import datetime from loguru import logger @@ -31,12 +35,15 @@ from PySide6.QtGui import QIcon from app.utils.database import Database from app.config import config from app.detectors.wayback_detector import WaybackDetector +from app.utils.redis_client import get_redis_client from app.utils.status_codes import ( DETECT_STATUS_BLACKLISTED, DETECT_STATUS_COMPLETED, DETECT_STATUS_FAILED, DETECT_STATUS_RUNNING, REGISTER_STATUS_AVAILABLE, + REGISTER_STATUS_CLIENT_HOLD, + REGISTER_STATUS_SERVER_HOLD, REVIEW_STATUS_PENDING, THIRD_PARTY_STATUS_DONE, ) @@ -48,7 +55,8 @@ CONTROL_CHANNEL = "domain_tool:worker_control" RUNTIME_STATE_KEY = "domain_tool:detect_runtime_state" PENDING_CONTROL_KEY = "domain_tool:worker_pending_command" RUNTIME_SETTINGS_KEY = "domain_tool:runtime_settings" -REMOTE_DEBUG_EVENT_TIMEOUT = 5 +SHARED_PROXY_POOL_KEY_PREFIX = "domain_tool:shared_proxy_pool" +SHARED_PROXY_REFRESH_LOCK_KEY_PREFIX = "domain_tool:shared_proxy_refresh_lock" def _raise_nofile_soft_limit(): @@ -62,6 +70,41 @@ def _raise_nofile_soft_limit(): logger.warning(f"提升 NOFILE 软限制失败: {exc}") +def _resolve_worker_thread_stack_size_bytes() -> int: + worker_mode = str(os.getenv("WORKER_MODE", "") or "").strip() + raw_override = str(os.getenv("DOMAINCHECK_THREAD_STACK_SIZE_KB", "") or "").strip() + if not raw_override and worker_mode != "linux-systemd": + return 0 + default_kb = "1024" if worker_mode == "linux-systemd" else "0" + try: + target_kb = int(raw_override or default_kb or 0) + except Exception: + target_kb = int(default_kb or 0) + if target_kb <= 0: + return 0 + return max(256 * 1024, target_kb * 1024) + + +def _configure_worker_thread_stack_size() -> int: + target_bytes = int(_resolve_worker_thread_stack_size_bytes() or 0) + try: + current_bytes = int(threading.stack_size() or 0) + except Exception: + current_bytes = 0 + if target_bytes <= 0: + return current_bytes + if current_bytes == target_bytes: + return current_bytes + try: + threading.stack_size(target_bytes) + applied_bytes = int(threading.stack_size() or 0) + logger.info(f"已设置检测线程默认栈大小: {current_bytes} -> {applied_bytes}") + return applied_bytes + except Exception as exc: + logger.warning(f"设置检测线程默认栈大小失败: target={target_bytes}, error={exc}") + return current_bytes + + def _resolve_worker_log_file() -> str: candidate_dirs = [ str(getattr(config, "LOG_DIR", "") or "").strip(), @@ -84,6 +127,111 @@ def _resolve_worker_log_file() -> str: continue return os.path.join("/tmp", "detect_worker.log") + +def _worker_local_config_roots() -> list[Path]: + seen = set() + roots: list[Path] = [] + + def append_root(raw_path) -> None: + text = str(raw_path or "").strip() + if not text: + return + path = Path(text).expanduser().resolve() + key = str(path) + if key in seen: + return + seen.add(key) + roots.append(path) + + append_root(str(os.getenv("DOMAINCHECK_CONFIG_ROOT", "") or "").strip()) + append_root(Path.cwd()) + + script_root = Path(__file__).resolve().parent + append_root(script_root) + + for base_root in list(roots): + normalized = str(base_root) + if f"{os.sep}releases{os.sep}" in normalized: + install_root = normalized.split(f"{os.sep}releases{os.sep}", 1)[0] + append_root(Path(install_root) / "current" / "domainCheck") + append_root(Path(install_root) / "domainCheck") + elif f"{os.sep}current{os.sep}" in normalized: + install_root = normalized.split(f"{os.sep}current{os.sep}", 1)[0] + append_root(Path(install_root) / "current" / "domainCheck") + append_root(Path(install_root) / "domainCheck") + elif base_root.name == "domainCheck": + append_root(base_root.parent / "current" / "domainCheck") + append_root(base_root.parent / "domainCheck") + + return roots + + +def _read_worker_local_json_config(filename: str): + normalized_name = str(filename or "").strip() + if not normalized_name: + return None + for root in _worker_local_config_roots(): + candidate = root / normalized_name + try: + if candidate.exists(): + with candidate.open("r", encoding="utf-8") as handle: + return json.load(handle) + except Exception: + continue + return None + + +def pending_control_key(node_code=None): + normalized_node_code = str(node_code or config.NODE_CODE or "").strip() + if not normalized_node_code: + return PENDING_CONTROL_KEY + return f"{PENDING_CONTROL_KEY}:{normalized_node_code}" + + +def control_target_node_codes(control_payload) -> list[str]: + if not isinstance(control_payload, dict): + return [] + + normalized_targets: list[str] = [] + + def append_target(raw_value): + normalized_value = str(raw_value or "").strip() + if normalized_value and normalized_value not in normalized_targets: + normalized_targets.append(normalized_value) + + list_keys = ("target_node_codes", "node_codes") + scalar_keys = ("target_node_code", "node_code") + + for key in list_keys: + raw_value = control_payload.get(key) + if isinstance(raw_value, (list, tuple, set)): + for item in raw_value: + append_target(item) + elif isinstance(raw_value, str) and raw_value.strip(): + for item in raw_value.split(","): + append_target(item) + if normalized_targets: + return normalized_targets + + for key in scalar_keys: + raw_value = control_payload.get(key) + if raw_value not in (None, ""): + append_target(raw_value) + if normalized_targets: + return normalized_targets + + return normalized_targets + + +def control_targets_current_worker(control_payload, *, node_code=None) -> bool: + normalized_node_code = str(node_code or config.NODE_CODE or "").strip() + if not normalized_node_code: + return True + target_node_codes = control_target_node_codes(control_payload) + if not target_node_codes: + return True + return normalized_node_code in target_node_codes + class DetectThread(QThread): """ 检测线程 @@ -868,6 +1016,30 @@ class DetectWorker: self._pending_restart_reason = "" self._detect_session_seq = 0 self._detect_session_owner = 0 + self._worker_started_at = time.time() + self._last_worker_activity_at = self._worker_started_at + self._last_explicit_start_signal_at = 0.0 + self._last_explicit_start_payload = {} + self._last_detection_start_at = 0.0 + self._last_detection_start_source = "" + self._last_detection_start_job_id = None + self._last_detection_start_job_code = "" + self._last_detection_start_task_mode = "" + self._cached_active_detect_job = None + self._last_active_detect_job_refresh_at = 0.0 + self._explicit_claim_scope_job_id = None + self._explicit_claim_scope_job_code = "" + self._explicit_claim_scope_until = 0.0 + self._explicit_claim_scope_source = "" + self._ignored_targeted_job_id = None + self._ignored_targeted_job_code = "" + self._ignored_targeted_job_until = 0.0 + self._ignored_targeted_job_targets = "" + self._last_explicit_scope_claim_log_at = 0.0 + self._last_explicit_scope_claim_log_key = "" + self._last_explicit_scope_probe_log_at = 0.0 + self._last_job_status_refresh_probe_log_at = 0.0 + self._idle_recycle_requested = False self._last_thread_count_refresh_at = 0.0 self.runtime_heartbeat_interval = max( 3.0, @@ -883,20 +1055,38 @@ class DetectWorker: self.current_job_id = None self.current_job_code = "" self.current_job_task_mode = "" + self._restart_release_handoff_job_id = None + self._restart_release_handoff_job_code = "" + self._restart_release_handoff_until = 0.0 + self._restart_release_handoff_reason = "" self.runtime_settings = { "worker_log_sync_enabled": False, "worker_log_sync_mode": "key", "worker_step_trace_enabled": True, "worker_step_trace_sync_full": True, + "claim_batch_floor": 0, + "claim_batch_ceil": 0, + "submit_backlog_floor": 0, + "submit_backlog_ceil": 0, + "dispatch_cap_multiplier": 1, + "pending_buffer_cap_multiplier": 1, } self._last_synced_worker_log = "" self._worker_log_sync_queue = Queue(maxsize=50000) self._worker_log_sync_stop = threading.Event() self._worker_log_sync_drop_count = 0 self._last_worker_log_sync_drop_notice_at = 0.0 + self._last_worker_log_sync_degrade_notice_at = 0.0 + self._remote_debug_event_failure_streak = 0 + self._remote_debug_event_cooldown_until = 0.0 + self._last_remote_debug_event_notice_at = 0.0 self._running_mark_lock = threading.Lock() self._pending_running_marks = collections.deque() self._last_running_mark_flush_at = 0.0 + self._job_release_lock = threading.Lock() + self._pending_job_releases = collections.deque() + self._pending_job_release_reasons = {} + self._last_job_release_flush_at = 0.0 self._job_finalize_lock = threading.Lock() self._pending_job_finalizations = collections.deque() self._last_job_finalize_flush_at = 0.0 @@ -911,7 +1101,8 @@ class DetectWorker: self._last_review_status_flush_at = 0.0 self._completed_future_lock = threading.Lock() self._completed_futures = collections.deque() - + self._last_job_item_release_at = 0.0 + # 初始化数据库连接 self.db = Database() try: @@ -930,7 +1121,7 @@ class DetectWorker: recycled = self.db.recycle_running_domains(DETECT_STATUS_FAILED) if recycled: logger.warning(f"Worker 启动时回收了 {recycled} 个遗留的检测中域名，已标记为失败待重试") - released = self.db.release_detect_job_items_for_node(config.NODE_CODE) + released = self._maybe_release_node_job_items(reason="worker_bootstrap", min_interval_seconds=0.0) if released: logger.warning(f"Worker 启动时释放了 {released} 个当前节点遗留任务项，已重新回到 pending") except Exception as e: @@ -938,17 +1129,7 @@ class DetectWorker: # 初始化Redis连接 try: - self.redis_client = redis.Redis( - host=config.REDIS_HOST, - port=config.REDIS_PORT, - password=config.REDIS_PASSWORD, - db=config.REDIS_DB, - decode_responses=True, - socket_connect_timeout=10, - socket_timeout=10, - retry_on_timeout=True, - health_check_interval=30 - ) + self.redis_client = get_redis_client(role="standard") # 测试连接 self.redis_client.ping() logger.info(f"Redis 连接成功: {config.REDIS_HOST}:{config.REDIS_PORT}") @@ -965,7 +1146,35 @@ class DetectWorker: self.detect_options = self.load_detect_options() self.proxy_config = self.load_proxy_config() self.thread_count = self.load_thread_count() # 从配置文件加载线程数 + self.process_count = self.load_process_count() # 从配置文件加载进程数 self.runtime_settings = self.load_runtime_settings() + self.register_single_machine_mode_enabled = self._env_flag_enabled( + "DOMAINCHECK_REGISTER_SINGLE_MACHINE_MODE" + ) + self.single_machine_site_direct_fallback_enabled = self._env_flag_enabled( + "DOMAINCHECK_SINGLE_MACHINE_SITE_DIRECT_FALLBACK", + default=self.register_single_machine_mode_enabled, + ) + self.single_machine_aizhan_direct_first_enabled = self._env_flag_enabled( + "DOMAINCHECK_SINGLE_MACHINE_AIZHAN_DIRECT_FIRST", + default=False, + ) + self.single_machine_baidu_direct_first_enabled = self._env_flag_enabled( + "DOMAINCHECK_SINGLE_MACHINE_BAIDU_DIRECT_FIRST", + default=False, + ) + self.aizhan_remote_disconnect_degrade_enabled = self._env_flag_enabled( + "DOMAINCHECK_AIZHAN_REMOTE_DISCONNECT_DEGRADE", + default=False, + ) + self.aizhan_external_fast_degrade_enabled = self._env_flag_enabled( + "DOMAINCHECK_AIZHAN_EXTERNAL_FAST_DEGRADE", + default=False, + ) + self.register_single_machine_direct_streak_attempts = max( + 0, + int(os.getenv("DOMAINCHECK_REGISTER_DIRECT_STREAK_ATTEMPTS", "2") or 2), + ) if self.register_single_machine_mode_enabled else 0 # 初始化代理池 self.proxy_pool = [] @@ -978,8 +1187,10 @@ class DetectWorker: self.proxy_last_validated_count = 0 self.proxy_last_available_count = 0 self.proxy_last_source_stats = [] + self._last_proxy_clock_skew_warning_at = 0.0 self.proxy_refresh_cooldown_seconds = 30 self.proxy_next_refresh_time = 0.0 + self._last_proxy_refresh_reason_at = {} self.proxy_max_reuse_count = max(8, self.thread_count * 4) self.proxy_step_wait_timeout_seconds = max( 0.0, @@ -989,6 +1200,58 @@ class DetectWorker: 0.0, float(os.getenv("DOMAINCHECK_PROXY_DIRECT_FALLBACK_GRACE", "0.35") or 0.35), ) + self.proxy_direct_fallback_grace_overrides = { + "注册状态检测": max( + 0.0, + float(os.getenv("DOMAINCHECK_PROXY_DIRECT_FALLBACK_GRACE_REGISTER", "0.08") or 0.08), + ), + "360检测": max( + 0.0, + float(os.getenv("DOMAINCHECK_PROXY_DIRECT_FALLBACK_GRACE_360", "0.12") or 0.12), + ), + "百度site检测": max( + 0.0, + float(os.getenv("DOMAINCHECK_PROXY_DIRECT_FALLBACK_GRACE_BAIDU", "0.12") or 0.12), + ), + "站长之家检测": max( + 0.0, + float(os.getenv("DOMAINCHECK_PROXY_DIRECT_FALLBACK_GRACE_CHINAZ", "0.12") or 0.12), + ), + "爱站网检测": max( + 0.0, + float(os.getenv("DOMAINCHECK_PROXY_DIRECT_FALLBACK_GRACE_AIZHAN", "0.12") or 0.12), + ), + "时光机检测": max( + 0.0, + float(os.getenv("DOMAINCHECK_PROXY_DIRECT_FALLBACK_GRACE_WAYBACK", "0.2") or 0.2), + ), + } + self.proxy_direct_retry_wait_seconds = max( + 0.0, + float(os.getenv("DOMAINCHECK_DIRECT_RETRY_PROXY_WAIT", "1.2") or 1.2), + ) + self.proxy_direct_retry_wait_overrides = { + "注册状态检测": max( + 0.0, + float(os.getenv("DOMAINCHECK_DIRECT_RETRY_PROXY_WAIT_REGISTER", "0.2") or 0.2), + ), + "360检测": max( + 0.0, + float(os.getenv("DOMAINCHECK_DIRECT_RETRY_PROXY_WAIT_360", "0.25") or 0.25), + ), + "百度site检测": max( + 0.0, + float(os.getenv("DOMAINCHECK_DIRECT_RETRY_PROXY_WAIT_BAIDU", "0.25") or 0.25), + ), + "站长之家检测": max( + 0.0, + float(os.getenv("DOMAINCHECK_DIRECT_RETRY_PROXY_WAIT_CHINAZ", "0.25") or 0.25), + ), + "爱站网检测": max( + 0.0, + float(os.getenv("DOMAINCHECK_DIRECT_RETRY_PROXY_WAIT_AIZHAN", "0.25") or 0.25), + ), + } self.proxy_step_retry_max_attempts = max( 1, int(os.getenv("DOMAINCHECK_PROXY_STEP_MAX_ATTEMPTS", "12") or 12), @@ -1066,6 +1329,7 @@ class DetectWorker: self.proxy_source_by_key = {} self.proxy_source_failure_counts = {} self.proxy_source_quarantine_until = {} + self.proxy_active_leases = {} self.redis_sub_thread = None self._last_autoresume_skip_log_at = 0.0 self._last_autoresume_success_log_at = 0.0 @@ -1097,6 +1361,7 @@ class DetectWorker: logger.info(f"检测选项: {self.detect_options}") logger.info(f"代理配置: {self.proxy_config}") logger.info(f"检测线程数: {self.thread_count}") + logger.info(f"检测进程数: {self.process_count}") # Redis订阅线程将在start方法中启动 self._update_runtime_state("idle", "Worker 已启动，等待检测指令") @@ -1134,6 +1399,10 @@ class DetectWorker: ) except Exception as e: logger.debug(f"运行态心跳续期失败: {e}") + try: + self._maybe_recycle_idle_worker(trigger_reason="heartbeat") + except Exception as e: + logger.debug(f"空闲 Worker 生命周期检查失败: {e}") def _start_worker_log_sync_loop(self): existing = getattr(self, "_worker_log_sync_thread", None) @@ -1272,6 +1541,66 @@ class DetectWorker: } ) + def _enqueue_job_release(self, job_item_id, claim_token, reason=""): + normalized_claim_token = str(claim_token or "").strip() + try: + normalized_job_item_id = int(job_item_id or 0) + except Exception: + normalized_job_item_id = 0 + if normalized_job_item_id <= 0 or not normalized_claim_token: + return False + release_key = (normalized_job_item_id, normalized_claim_token) + normalized_reason = str(reason or "").strip()[:1000] + with self._job_release_lock: + existing_reason = str( + getattr(self, "_pending_job_release_reasons", {}).get(release_key, "") or "" + ).strip() + if release_key not in getattr(self, "_pending_job_release_reasons", {}): + self._pending_job_releases.append(release_key) + # Keep the newest non-empty reason, but avoid growing the queue for duplicates. + self._pending_job_release_reasons[release_key] = normalized_reason or existing_reason + return True + + def _flush_pending_job_releases(self, *, force=False, batch_limit=128): + now_ts = time.time() + if not force and now_ts - float(getattr(self, "_last_job_release_flush_at", 0.0) or 0.0) < 0.20: + return 0 + batch = [] + with self._job_release_lock: + if not self._pending_job_releases: + if force: + self._last_job_release_flush_at = now_ts + return 0 + while self._pending_job_releases and len(batch) < max(1, int(batch_limit or 1)): + release_key = self._pending_job_releases.popleft() + reason = str(self._pending_job_release_reasons.pop(release_key, "") or "").strip() + batch.append((int(release_key[0]), str(release_key[1]), reason)) + updated_count = 0 + flush_started_at = time.perf_counter() + try: + updated_count = int(self.db.release_detect_job_items_batch(batch) or 0) + except Exception as e: + logger.debug(f"批量释放任务项回队列失败: {e}") + updated_count = -1 + if updated_count < 0: + with self._job_release_lock: + for item in reversed(batch): + release_key = (int(item[0]), str(item[1])) + self._pending_job_releases.appendleft(release_key) + self._pending_job_release_reasons[release_key] = str(item[2] or "").strip()[:1000] + logger.warning( + f"批量释放任务项回队列失败，已回退待重试队列: batch={len(batch)} | force={1 if force else 0}" + ) + updated_count = 0 + flush_elapsed_ms = int((time.perf_counter() - flush_started_at) * 1000) + if batch and (flush_elapsed_ms >= 20 or len(batch) >= 64 or force): + logger.info( + f"任务项回队列批量释放: batch={len(batch)} | updated={updated_count} " + f"| elapsed_ms={flush_elapsed_ms} | force={1 if force else 0}" + ) + self._last_job_release_flush_at = now_ts + return updated_count + def _flush_pending_job_finalizations(self, *, force=False, batch_limit=512): now_ts = time.time() if not force and now_ts - float(getattr(self, "_last_job_finalize_flush_at", 0.0) or 0.0) < 0.05: @@ -1452,30 +1781,259 @@ class DetectWorker: def _consume_pending_control_command(self): if not self.use_redis or self.redis_client is None: return + pending_keys = [pending_control_key()] + if pending_keys[0] != PENDING_CONTROL_KEY: + pending_keys.append(PENDING_CONTROL_KEY) + + payload = None + payload_key = "" try: - payload = self.redis_client.getdel(PENDING_CONTROL_KEY) + for key in pending_keys: + payload = self.redis_client.getdel(key) + if payload: + payload_key = key + break except Exception: try: - payload = self.redis_client.get(PENDING_CONTROL_KEY) - if payload: - self.redis_client.delete(PENDING_CONTROL_KEY) + for key in pending_keys: + payload = self.redis_client.get(key) + if payload: + self.redis_client.delete(key) + payload_key = key + break except Exception as e: logger.debug(f"读取待执行控制指令失败: {e}") return if not payload: return - logger.info(f"发现待执行 Worker 控制指令: {payload}") + logger.info(f"发现待执行 Worker 控制指令: key={payload_key or 'unknown'} payload={payload}") self._handle_control_message(payload) + def _active_detect_job_refresh_interval_seconds(self, reason: str) -> float: + if reason == "heartbeat_autoresume": + raw_value = os.getenv("DOMAINCHECK_AUTORESUME_ACTIVE_JOB_MIN_REFRESH_SECONDS", "5") + else: + raw_value = os.getenv("DOMAINCHECK_ACTIVE_JOB_MIN_REFRESH_SECONDS", "1") + try: + return max(0.0, float(raw_value or 0.0)) + except (TypeError, ValueError): + return 5.0 if reason == "heartbeat_autoresume" else 1.0 + + def _load_active_detect_job_snapshot(self, reason: str, now_ts: float): + refresh_interval = self._active_detect_job_refresh_interval_seconds(reason) + cached_job = getattr(self, "_cached_active_detect_job", None) + last_refresh_at = float(getattr(self, "_last_active_detect_job_refresh_at", 0.0) or 0.0) + if refresh_interval > 0 and cached_job is not None and now_ts - last_refresh_at < refresh_interval: + return dict(cached_job or {}) + active_job = self.db.get_active_detect_job() or {} + self._cached_active_detect_job = dict(active_job or {}) + self._last_active_detect_job_refresh_at = now_ts + return dict(active_job or {}) + + def _autoresume_same_job_restart_grace_seconds(self) -> float: + return max( + 5.0, + float(os.getenv("DOMAINCHECK_AUTORESUME_SAME_JOB_RESTART_GRACE_SECONDS", "30") or 30), + ) + + def _bootstrap_autoresume_lock_ttl_seconds(self) -> int: + return max( + 5, + int(os.getenv("DOMAINCHECK_BOOTSTRAP_AUTORESUME_LOCK_TTL_SECONDS", "30") or 30), + ) + + def _acquire_bootstrap_autoresume_lock(self, *, job_id: int, job_code: str = "") -> bool: + if not self.use_redis or self.redis_client is None: + return True + try: + normalized_job_id = int(job_id or 0) + except Exception: + normalized_job_id = 0 + if normalized_job_id <= 0: + return True + lock_action = f"bootstrap-autoresume:{normalized_job_id}:{str(job_code or '').strip() or '-'}" + lock_key = self._shared_job_maintenance_lock_key(lock_action) + if not lock_key: + return True + owner_token = self._job_maintenance_owner_token(lock_action) + try: + acquired = self.redis_client.set( + lock_key, + owner_token, + nx=True, + ex=self._bootstrap_autoresume_lock_ttl_seconds(), + ) + return bool(acquired) + except Exception as exc: + logger.debug(f"获取 bootstrap 自动回挂锁失败: action={lock_action}, error={exc}") + return True + + def _should_skip_same_job_autoresume(self, *, active_job: dict, now_ts: float) -> tuple[bool, str]: + incoming_job_code = str(active_job.get("job_code") or "").strip() + incoming_job_id = active_job.get("id") + try: + normalized_incoming_job_id = int(incoming_job_id or 0) + except Exception: + normalized_incoming_job_id = 0 + + current_job_code = str(getattr(self, "current_job_code", "") or "").strip() + current_job_id = getattr(self, "current_job_id", None) + try: + normalized_current_job_id = int(current_job_id or 0) + except Exception: + normalized_current_job_id = 0 + + same_job = bool( + incoming_job_code + and current_job_code + and incoming_job_code == current_job_code + ) or bool( + normalized_incoming_job_id > 0 + and normalized_current_job_id > 0 + and normalized_incoming_job_id == normalized_current_job_id + ) + if not same_job: + return False, "" + + live_active_threads = max(0, int(self._get_active_domain_threads() or 0)) + if live_active_threads > 0: + return True, f"same_job_threads_active:{live_active_threads}" + + handoff_job_id = self._restart_release_handoff_job_id_active() + if ( + handoff_job_id not in (None, "", 0, "0") + and normalized_incoming_job_id > 0 + and int(handoff_job_id) == normalized_incoming_job_id + ): + return True, "same_job_restart_handoff_active" + + recent_start_at = float(getattr(self, "_last_detection_start_at", 0.0) or 0.0) + recent_start_grace_seconds = self._autoresume_same_job_restart_grace_seconds() + recent_start_elapsed = max(0.0, now_ts - recent_start_at) if recent_start_at > 0 else 0.0 + if recent_start_at > 0 and recent_start_elapsed < recent_start_grace_seconds: + return True, f"same_job_recent_start:{int(recent_start_elapsed)}s" + + return False, "" + + def _should_skip_explicit_scope_autoresume(self, *, active_job: dict) -> tuple[bool, str]: + explicit_job_id = self._explicit_claim_scope_job_id_active() + if explicit_job_id in (None, "", 0, "0"): + return False, "" + try: + normalized_explicit_job_id = int(explicit_job_id or 0) + except Exception: + return False, "" + try: + normalized_incoming_job_id = int(active_job.get("id") or 0) + except Exception: + normalized_incoming_job_id = 0 + if normalized_explicit_job_id <= 0 or normalized_incoming_job_id <= 0: + return False, "" + if normalized_explicit_job_id != normalized_incoming_job_id: + return False, "" + explicit_job_code = str(getattr(self, "_explicit_claim_scope_job_code", "") or "").strip() + explicit_source = str(getattr(self, "_explicit_claim_scope_source", "") or "").strip() or "unknown" + detail_parts = [f"explicit_scope_active:{normalized_explicit_job_id}"] + if explicit_job_code: + detail_parts.append(explicit_job_code) + detail_parts.append(explicit_source) + return True, ":".join(detail_parts) + + def _ignored_targeted_job_window_seconds(self) -> float: + return max( + 30.0, + float(os.getenv("DOMAINCHECK_IGNORED_TARGETED_JOB_WINDOW_SECONDS", "180") or 180), + ) + + def _remember_ignored_targeted_control(self, control_payload=None) -> None: + if not isinstance(control_payload, dict): + return + if control_targets_current_worker(control_payload): + return + target_node_codes = control_target_node_codes(control_payload) + if not target_node_codes: + return + job_id = self._incoming_job_id(control_payload) + job_code = self._incoming_job_code(control_payload) + if job_id in (None, "", 0, "0") or not job_code: + return + self._ignored_targeted_job_id = int(job_id) + self._ignored_targeted_job_code = str(job_code or "").strip() + self._ignored_targeted_job_until = time.time() + self._ignored_targeted_job_window_seconds() + self._ignored_targeted_job_targets = ",".join(target_node_codes) + + def _clear_ignored_targeted_job(self, *, reason: str = "") -> None: + self._ignored_targeted_job_id = None + self._ignored_targeted_job_code = "" + self._ignored_targeted_job_until = 0.0 + self._ignored_targeted_job_targets = "" + + def _should_skip_ignored_targeted_job_autoresume(self, *, active_job: dict) -> tuple[bool, str]: + ignored_job_id = getattr(self, "_ignored_targeted_job_id", None) + if ignored_job_id in (None, "", 0, "0"): + return False, "" + if time.time() >= float(getattr(self, "_ignored_targeted_job_until", 0.0) or 0.0): + self._clear_ignored_targeted_job(reason="expired") + return False, "" + try: + active_job_id = int(active_job.get("id") or 0) + except Exception: + active_job_id = 0 + try: + normalized_ignored_job_id = int(ignored_job_id or 0) + except Exception: + normalized_ignored_job_id = 0 + if active_job_id <= 0 or normalized_ignored_job_id <= 0: + return False, "" + if active_job_id != normalized_ignored_job_id: + return False, "" + ignored_job_code = str(getattr(self, "_ignored_targeted_job_code", "") or "").strip() + ignored_targets = str(getattr(self, "_ignored_targeted_job_targets", "") or "").strip() or "unknown" + detail_parts = [f"ignored_targeted_job:{normalized_ignored_job_id}"] + if ignored_job_code: + detail_parts.append(ignored_job_code) + detail_parts.append(ignored_targets) + return True, ":".join(detail_parts) + def _resume_active_detect_job_if_needed(self, reason: str = "worker_bootstrap"): now_ts = time.time() stale_detect = False + autoresume_reasons = { + "heartbeat_autoresume", + "service_runtime_bootstrap", + "redis_subscription_bootstrap", + } + node_role = str(getattr(config, "NODE_ROLE", "") or "").strip() + is_primary_worker = self._is_primary_job_maintenance_worker() + tail_handoff_probe_enabled = self._runtime_bool_override( + "tail_handoff_autoresume_enabled", + "DOMAINCHECK_TAIL_HANDOFF_AUTORESUME", + default=True, + ) def log_skip(detail: str): if now_ts - float(getattr(self, "_last_autoresume_skip_log_at", 0.0) or 0.0) >= 15.0: logger.info(f"自动回挂跳过: reason={reason}, detail={detail}") self._last_autoresume_skip_log_at = now_ts + if ( + node_role == "control" + and not self._runtime_bool_override( + "control_node_autoresume_enabled", + "DOMAINCHECK_CONTROL_NODE_AUTORESUME", + default=False, + ) + ): + log_skip("control_node_autoresume_disabled") + return False + if ( + reason in autoresume_reasons + and node_role != "control" + and not is_primary_worker + and not tail_handoff_probe_enabled + ): + log_skip("non_primary_autoresume_worker") + return False if not self.running: log_skip("worker_not_running") return False @@ -1484,10 +2042,23 @@ class DetectWorker: if self.stop_requested: log_skip("stop_requested") return False + explicit_start_grace_seconds = max( + 15.0, + float(os.getenv("DOMAINCHECK_AUTORESUME_EXPLICIT_START_GRACE_SECONDS", "120") or 120), + ) + last_explicit_start_at = float(getattr(self, "_last_explicit_start_signal_at", 0.0) or 0.0) + if ( + last_explicit_start_at > 0 + and now_ts - last_explicit_start_at < explicit_start_grace_seconds + ): + log_skip( + "recent_explicit_start_signal" + ) + return False if self.detecting: stale_detect, stale_reason = self._is_stale_detect_session() if not stale_detect: - log_skip("already_detecting") + log_skip(stale_reason or "already_detecting") return False logger.warning(f"检测运行态疑似空转，允许自动回挂接管: {stale_reason}") thread = self.detect_command_thread @@ -1495,7 +2066,7 @@ class DetectWorker: log_skip("detect_command_thread_alive") return False try: - active_job = self.db.get_active_detect_job() or {} + active_job = self._load_active_detect_job_snapshot(reason, now_ts) except Exception as e: logger.debug(f"读取当前检测任务失败，跳过自动回挂: {e}") return False @@ -1508,12 +2079,56 @@ class DetectWorker: items_pending = int(active_job.get("items_pending", 0) or 0) items_claimed = int(active_job.get("items_claimed", 0) or 0) items_running = int(active_job.get("items_running", 0) or 0) + selection_reason = str(active_job.get("selection_reason") or "").strip() + if selection_reason == "running_job_stalled" or bool(active_job.get("running_job_stalled")): + self._maybe_recycle_stalled_job_items( + job_id, + reason=f"autoresume:{reason or 'unspecified'}", + ) + log_skip( + "running_job_stalled" + ) + return False + tail_handoff_candidate = bool(active_job.get("tail_handoff_candidate")) or selection_reason == "tail_handoff_pending" if items_pending <= 0 and items_claimed <= 0 and items_running <= 0: log_skip("active_job_empty") return False + if reason == "heartbeat_autoresume": + should_skip_targeted_job, targeted_job_detail = self._should_skip_ignored_targeted_job_autoresume( + active_job=active_job, + ) + if should_skip_targeted_job: + log_skip(targeted_job_detail or "ignored_targeted_job") + return False + should_skip_explicit_scope, explicit_scope_detail = self._should_skip_explicit_scope_autoresume( + active_job=active_job, + ) + if should_skip_explicit_scope: + log_skip(explicit_scope_detail or "explicit_scope_autoresume_guard") + return False + should_skip_same_job, skip_detail = self._should_skip_same_job_autoresume( + active_job=active_job, + now_ts=now_ts, + ) + if should_skip_same_job: + log_skip(skip_detail or "same_job_autoresume_guard") + return False + if reason in {"service_runtime_bootstrap", "redis_subscription_bootstrap"}: + if not self._acquire_bootstrap_autoresume_lock(job_id=int(job_id), job_code=job_code): + log_skip("bootstrap_autoresume_lock_held") + return False + if ( + reason in autoresume_reasons + and node_role != "control" + and not is_primary_worker + and not tail_handoff_candidate + ): + log_skip("non_primary_autoresume_worker") + return False logger.warning( f"检测 Worker 空闲但发现活动任务，准备自动回挂: " - f"job_code={job_code}, pending={items_pending}, claimed={items_claimed}, running={items_running}, reason={reason}" + f"job_code={job_code}, pending={items_pending}, claimed={items_claimed}, " + f"running={items_running}, reason={reason}, selection={selection_reason or '-'}" ) started = self.start_detection_async( source="auto-resume", @@ -1525,12 +2140,15 @@ class DetectWorker: "target_job_id": int(job_id), "target_job_code": job_code, "target_task_mode": task_mode, + "selection_reason": selection_reason, + "tail_handoff_candidate": tail_handoff_candidate, }, ) if started and now_ts - float(getattr(self, "_last_autoresume_success_log_at", 0.0) or 0.0) >= 5.0: logger.warning( f"自动回挂已触发: reason={reason}, job_code={job_code}, " - f"pending={items_pending}, claimed={items_claimed}, running={items_running}" + f"pending={items_pending}, claimed={items_claimed}, running={items_running}, " + f"selection={selection_reason or '-'}" ) self._last_autoresume_success_log_at = now_ts return started @@ -1588,8 +2206,10 @@ class DetectWorker: effective_active_threads = max(int(explicit_active_threads or 0), live_active_threads) merged_extra["active_threads"] = effective_active_threads + waiting_for_dispatch = bool(self.detecting and effective_active_threads <= 0) + merged_extra["waiting_for_dispatch"] = waiting_for_dispatch explicit_current_load = (extra or {}).get("current_load") - default_current_load = effective_active_threads if effective_active_threads > 0 else (1 if self.detecting else 0) + default_current_load = effective_active_threads if effective_active_threads > 0 else 0 if explicit_current_load in (None, ""): merged_extra["current_load"] = default_current_load else: @@ -1656,8 +2276,6 @@ class DetectWorker: actual_active_threads = int(payload.get("active_threads", 0) or 0) max_threads = int(payload.get("max_threads", getattr(self, "thread_count", 0)) or 0) current_load = actual_active_threads - if bool(payload.get("detecting")) and current_load <= 0: - current_load = 1 self.db.register_cluster_node( config.NODE_CODE, config.NODE_REGION, @@ -1668,6 +2286,7 @@ class DetectWorker: "phase": phase, "detail": detail, "detecting": bool(payload.get("detecting")), + "waiting_for_dispatch": bool(payload.get("waiting_for_dispatch")), "available_proxy_count": int(payload.get("available_proxy_count", 0) or 0), "active_threads": actual_active_threads, "max_threads": max_threads, @@ -1704,17 +2323,164 @@ class DetectWorker: return int(self._active_domain_threads or 0) def _note_detection_activity(self): - self.detect_command_last_activity_at = time.time() + now_ts = time.time() + self.detect_command_last_activity_at = now_ts + self._last_worker_activity_at = now_ts def _note_domain_started(self): now_ts = time.time() self._last_domain_started_at = now_ts self.detect_command_last_activity_at = now_ts + self._last_worker_activity_at = now_ts def _note_domain_result(self): now_ts = time.time() self._last_domain_result_at = now_ts self.detect_command_last_activity_at = now_ts + self._last_worker_activity_at = now_ts + + def _idle_worker_recycle_enabled(self): + return self._env_flag_enabled("DOMAINCHECK_IDLE_WORKER_RECYCLE_ENABLED", default=False) + + def _idle_worker_recycle_after_seconds(self): + try: + return max(30.0, float(os.getenv("DOMAINCHECK_IDLE_WORKER_RECYCLE_AFTER_SECONDS", "180") or 180.0)) + except Exception: + return 180.0 + + def _idle_worker_recycle_min_uptime_seconds(self): + try: + return max(30.0, float(os.getenv("DOMAINCHECK_IDLE_WORKER_RECYCLE_MIN_UPTIME_SECONDS", "300") or 300.0)) + except Exception: + return 300.0 + + def _idle_worker_recycle_explicit_start_grace_seconds(self): + try: + return max(10.0, float(os.getenv("DOMAINCHECK_IDLE_WORKER_RECYCLE_START_GRACE_SECONDS", "90") or 90.0)) + except Exception: + return 90.0 + + def _idle_worker_recycle_jitter_seconds(self): + try: + return max(0.0, float(os.getenv("DOMAINCHECK_IDLE_WORKER_RECYCLE_JITTER_SECONDS", "45") or 45.0)) + except Exception: + return 45.0 + + def _idle_worker_recycle_jitter_offset(self): + jitter_window = float(self._idle_worker_recycle_jitter_seconds() or 0.0) + if jitter_window <= 0: + return 0.0 + node_code = str(getattr(config, "NODE_CODE", "") or "").strip() or str(os.getpid()) + spread = sum(ord(ch) for ch in node_code) % 1000 + return (spread / 1000.0) * jitter_window + + def _is_recyclable_idle_worker_instance(self): + if str(os.getenv("WORKER_MODE", "") or "").strip() != "linux-systemd": + return False + parent_node_code = self._worker_parent_node_code() + node_code = str(getattr(config, "NODE_CODE", "") or "").strip() + if not parent_node_code or not node_code: + return False + return node_code != parent_node_code and node_code.startswith(f"{parent_node_code}-") + + def _idle_worker_waiting_for_dispatch(self): + runtime_extra = dict(getattr(self, "_last_runtime_extra", {}) or {}) + if bool(runtime_extra.get("waiting_for_dispatch", False)): + return True + return bool(self.detecting and self._get_active_domain_threads() <= 0) + + def _idle_worker_seconds(self): + last_activity_at = max( + float(getattr(self, "_last_worker_activity_at", 0.0) or 0.0), + float(getattr(self, "detect_command_last_activity_at", 0.0) or 0.0), + float(getattr(self, "_last_domain_started_at", 0.0) or 0.0), + float(getattr(self, "_last_domain_result_at", 0.0) or 0.0), + ) + if last_activity_at <= 0: + last_activity_at = float(getattr(self, "_worker_started_at", 0.0) or time.time()) + return max(0.0, time.time() - last_activity_at) + + def _trigger_idle_worker_recycle(self, *, trigger_reason: str, idle_seconds: float): + if self._idle_recycle_requested: + return False + self._idle_recycle_requested = True + detail = ( + f"子 Worker 空闲过久，触发生命周期回收: idle={int(idle_seconds)}s, " + f"trigger={trigger_reason or 'unknown'}" + ) + logger.warning(detail) + try: + self._sync_worker_log_event( + "空闲 Worker 生命周期回收", + level='warning', + payload={ + "idle_seconds": int(idle_seconds), + "trigger_reason": str(trigger_reason or "").strip() or "unknown", + "node_code": str(getattr(config, "NODE_CODE", "") or "").strip(), + }, + mode='key', + ) + except Exception: + pass + try: + self._update_runtime_state( + "restarting", + detail, + active_threads=0, + current_load=0, + recycle_reason=str(trigger_reason or "").strip() or "idle", + ) + except Exception: + pass + try: + self._flush_pending_running_marks(force=True, batch_limit=2048) + self._flush_pending_job_releases(force=True, batch_limit=2048) + self._flush_pending_job_finalizations(force=True, batch_limit=2048) + self._flush_pending_domain_status_updates(force=True, batch_limit=2048) + self._flush_pending_domain_completions(force=True, batch_limit=2048) + self._flush_pending_review_status_updates(force=True, batch_limit=2048) + self._flush_completed_futures(force=True, batch_limit=2048) + except Exception as exc: + logger.debug(f"空闲 Worker 回收前 flush 失败: {exc}") + self.stop_requested = True + self.detecting = False + self.running = False + self._runtime_heartbeat_stop.set() + self._worker_log_sync_stop.set() + try: + self._worker_log_sync_queue.put_nowait(None) + except Exception: + pass + time.sleep(0.2) + os.kill(os.getpid(), signal.SIGTERM) + return True + + def _maybe_recycle_idle_worker(self, *, trigger_reason: str = ""): + if self._idle_recycle_requested: + return False + if not self._idle_worker_recycle_enabled(): + return False + if not self._is_recyclable_idle_worker_instance(): + return False + if not self.running or self.stop_requested: + return False + if self._get_active_domain_threads() > 0: + return False + if not self._idle_worker_waiting_for_dispatch(): + return False + if time.time() - float(getattr(self, "_worker_started_at", 0.0) or 0.0) < self._idle_worker_recycle_min_uptime_seconds(): + return False + explicit_start_at = float(getattr(self, "_last_explicit_start_signal_at", 0.0) or 0.0) + if explicit_start_at > 0 and time.time() - explicit_start_at < self._idle_worker_recycle_explicit_start_grace_seconds(): + return False + idle_seconds = self._idle_worker_seconds() + recycle_after = self._idle_worker_recycle_after_seconds() + self._idle_worker_recycle_jitter_offset() + if idle_seconds < recycle_after: + return False + return self._trigger_idle_worker_recycle( + trigger_reason=trigger_reason or "idle_waiting_for_dispatch", + idle_seconds=idle_seconds, + ) def _schedule_pending_restart(self, source: str, control_payload=None, reason: str = ""): self._pending_restart_source = str(source or "deferred-restart").strip() or "deferred-restart" @@ -1761,6 +2527,62 @@ class DetectWorker: ) return self.start_detection_async(source=source or "deferred-restart", control_payload=payload) + def _detection_session_abort_reason(self, session_id: int = 0) -> str: + if not self.running: + return "worker_stopped" + normalized_session_id = int(session_id or 0) + if normalized_session_id <= 0: + return "" + current_owner = int(getattr(self, "_detect_session_owner", 0) or 0) + if current_owner > 0 and current_owner != normalized_session_id: + return f"session_replaced:{current_owner}" + return "" + + def _release_job_item_for_session_abort(self, job_item_id, claim_token, *, session_id: int = 0, domain_name: str = "", detail: str = ""): + abort_reason = self._detection_session_abort_reason(session_id) + if not abort_reason: + return False + try: + normalized_job_item_id = int(job_item_id or 0) + except Exception: + normalized_job_item_id = 0 + normalized_claim_token = str(claim_token or "").strip() + if normalized_job_item_id <= 0 or not normalized_claim_token: + return False + normalized_detail = str(detail or "").strip() + release_reason = f"released after session abort: {abort_reason}" + if normalized_detail: + release_reason = f"{release_reason} | {normalized_detail[:240]}" + release_reason = release_reason[:1000] + released = self._enqueue_job_release( + normalized_job_item_id, + normalized_claim_token, + reason=release_reason, + ) + if released: + pending_release_count = 0 + with self._job_release_lock: + pending_release_count = len(self._pending_job_releases) + if pending_release_count >= 64: + self._flush_pending_job_releases(batch_limit=512) + logger.warning( + f"检测会话已切换，任务项已登记回队列: domain={domain_name or 'unknown'}, " + f"job_item_id={normalized_job_item_id}, reason={abort_reason}, " + f"detail={str(detail or '').strip() or 'none'}" + ) + return released + + def _assert_detection_session_active(self, session_id: int = 0, *, domain_name: str = "", stage: str = "") -> None: + abort_reason = self._detection_session_abort_reason(session_id) + if not abort_reason: + return + normalized_stage = str(stage or "unknown").strip() or "unknown" + normalized_domain_name = str(domain_name or "").strip() + detail = f"检测会话已失效: reason={abort_reason} stage={normalized_stage}" + if normalized_domain_name: + detail = f"{detail} domain={normalized_domain_name}" + raise RuntimeError(detail) + def _incoming_job_code(self, control_payload=None) -> str: payload = control_payload or {} job_code = str(payload.get("job_code") or "").strip() @@ -1768,6 +2590,59 @@ class DetectWorker: job_code = str(payload.get("target_job_code") or "").strip() return job_code + def _incoming_cycle_token(self, control_payload=None) -> str: + payload = control_payload or {} + return str(payload.get("cycle_token") or "").strip() + + def _incoming_control_source(self, control_payload=None) -> str: + payload = control_payload or {} + return str(payload.get("source") or "").strip() + + def _incoming_job_id(self, control_payload=None): + payload = control_payload or {} + # For domain_pipeline sync-pull commands, runtime/log identity may follow + # the upstream projection job_code/job_id, but local queue ownership must + # stay pinned to the ingested target_job_id. Claim/release/session scope + # logic therefore prefers target_job_id when it exists. + job_id = payload.get("target_job_id") + if job_id in (None, "", 0, "0"): + job_id = payload.get("job_id") + try: + return int(job_id) if job_id not in (None, "", 0, "0") else None + except Exception: + return None + + def _is_overlap_handoff_command(self, control_payload=None, *, source: str = "") -> bool: + payload = control_payload or {} + normalized_source = str(source or self._incoming_control_source(payload) or "").strip() + if normalized_source == "overlap-handoff": + return True + return bool(payload.get("tail_handoff_candidate")) + + def _matches_active_detect_session(self, control_payload=None) -> bool: + payload = control_payload or {} + incoming_job_code = self._incoming_job_code(payload) + incoming_job_id = self._incoming_job_id(payload) + incoming_cycle_token = self._incoming_cycle_token(payload) + + current_job_code = str(self.current_job_code or "").strip() + current_cycle_token = str(self.current_cycle_token or "").strip() + current_job_id = getattr(self, "current_job_id", None) + + matches_job_code = bool( + incoming_job_code and current_job_code and incoming_job_code == current_job_code + ) + matches_job_id = bool( + incoming_job_id not in (None, 0) + and current_job_id not in (None, 0) + and int(incoming_job_id) == int(current_job_id) + ) + if not (matches_job_code or matches_job_id): + return False + if incoming_cycle_token and current_cycle_token and incoming_cycle_token != current_cycle_token: + return False + return True + def _is_stale_detect_session(self, control_payload=None): thread = self.detect_command_thread if not thread or not thread.is_alive(): @@ -1791,14 +2666,49 @@ class DetectWorker: last_result_at = float(getattr(self, "_last_domain_result_at", 0.0) or 0.0) result_idle_seconds = max(0.0, time.time() - last_result_at) if last_result_at > 0 else idle_seconds incoming_job_code = self._incoming_job_code(control_payload) + incoming_cycle_token = self._incoming_cycle_token(control_payload) + incoming_task_mode = str( + (control_payload or {}).get("task_mode") + or (control_payload or {}).get("target_task_mode") + or "" + ).strip() current_job_code = str(self.current_job_code or "").strip() - job_switched = bool(incoming_job_code and current_job_code and incoming_job_code != current_job_code) - incoming_job_changed = bool(incoming_job_code and incoming_job_code != current_job_code) + current_cycle_token = str(self.current_cycle_token or "").strip() + if self._matches_active_detect_session(control_payload): + if idle_seconds < 180 and result_idle_seconds < 180: + return False, ( + "同任务同周期重复唤起已忽略: " + f"job={incoming_job_code or current_job_code or 'unknown'} " + f"cycle={incoming_cycle_token or current_cycle_token or 'none'}" + ) + session_scoped_to_job = bool( + self._is_single_step_session_active() + or incoming_task_mode == "single_step" + ) + job_switched = bool( + session_scoped_to_job + and incoming_job_code + and current_job_code + and incoming_job_code != current_job_code + ) + incoming_job_changed = bool( + session_scoped_to_job + and incoming_job_code + and incoming_job_code != current_job_code + ) low_activity_recoverable = live_active_threads <= 1 multi_thread_force_recover = incoming_job_changed and thread_age_seconds >= 30 and result_idle_seconds >= 30 if not low_activity_recoverable and not multi_thread_force_recover: return False, "" + hold_for_proxy_shortage, hold_reason = self._should_hold_stale_detect_for_proxy_shortage( + idle_seconds=idle_seconds, + low_activity_recoverable=low_activity_recoverable, + incoming_job_changed=job_switched or incoming_job_changed, + ) + if hold_for_proxy_shortage: + return False, hold_reason + if job_switched and idle_seconds >= 15: return True, ( f"检测指令线程仍停留在旧任务 {current_job_code}，" @@ -1848,6 +2758,7 @@ class DetectWorker: self.current_job_id = None self.current_job_code = "" self.current_job_task_mode = "" + self._clear_restart_release_handoff(reason="clear_active_cycle") self.detect_command_started_at = 0.0 self.detect_command_last_activity_at = 0.0 self._last_domain_started_at = 0.0 @@ -1859,35 +2770,531 @@ class DetectWorker: return True return str(getattr(self, "current_job_code", "") or "").strip().startswith("step-") + def _shared_job_maintenance_scope(self): + parent_node_code = str(os.getenv("WORKER_PARENT_NODE_CODE", "") or "").strip() + scope_node = parent_node_code or str(getattr(config, "NODE_CODE", "") or "").strip() or "unknown" + db_host = str(getattr(config, "DB_HOST", "") or "").strip() or "db" + db_port = str(getattr(config, "DB_PORT", "") or "").strip() or "5432" + db_name = str(getattr(config, "DB_DATABASE", "") or "").strip() or "domain" + return f"{db_host}:{db_port}:{db_name}:{scope_node}" + + def _shared_job_maintenance_lock_key(self, action: str) -> str: + if not self.use_redis or self.redis_client is None: + return "" + normalized_action = str(action or "").strip() + if not normalized_action: + return "" + return f"domaincheck:job-maintenance:{normalized_action}:{self._shared_job_maintenance_scope()}" + + def _job_maintenance_owner_token(self, action: str) -> str: + return ( + f"{str(getattr(config, 'NODE_CODE', '') or '').strip()}:" + f"{os.getpid()}:{threading.get_ident()}:{time.time_ns()}:{str(action or '').strip()}" + ) + + def _acquire_shared_job_maintenance_lock(self, action: str, *, ttl_seconds: int = 15) -> str: + redis_key = self._shared_job_maintenance_lock_key(action) + if not redis_key: + return "__local__" if self._is_primary_job_maintenance_worker() else "" + owner_token = self._job_maintenance_owner_token(action) + try: + acquired = self.redis_client.set( + redis_key, + owner_token, + nx=True, + ex=max(1, int(ttl_seconds or 15)), + ) + except Exception as exc: + logger.debug(f"获取共享任务维护锁失败: action={action}, error={exc}") + return "" + return owner_token if acquired else "" + + def _release_shared_job_maintenance_lock(self, action: str, owner_token: str) -> None: + if not owner_token or owner_token == "__local__": + return + redis_key = self._shared_job_maintenance_lock_key(action) + if not redis_key: + return + try: + current_owner = self.redis_client.get(redis_key) + if str(current_owner or "") == str(owner_token): + self.redis_client.delete(redis_key) + except Exception as exc: + logger.debug(f"释放共享任务维护锁失败: action={action}, error={exc}") + + def _stalled_job_recycle_interval_seconds(self) -> float: + return max( + 15.0, + float(os.getenv("DOMAINCHECK_STALLED_JOB_RECYCLE_INTERVAL_SECONDS", "60") or 60), + ) + + def _stalled_job_recycle_stall_seconds(self) -> int: + return max( + 300, + int(os.getenv("DOMAINCHECK_RUNNING_JOB_STALL_SECONDS", "900") or 900), + ) + + def _maybe_recycle_stalled_job_items(self, job_id, *, reason: str = "") -> int: + try: + normalized_job_id = int(job_id or 0) + except Exception: + normalized_job_id = 0 + if normalized_job_id <= 0: + return 0 + now_ts = time.time() + last_attempts = getattr(self, "_last_stalled_job_recycle_attempt_at", None) + if not isinstance(last_attempts, dict): + last_attempts = {} + self._last_stalled_job_recycle_attempt_at = last_attempts + last_attempt_at = float(last_attempts.get(normalized_job_id, 0.0) or 0.0) + if now_ts - last_attempt_at < self._stalled_job_recycle_interval_seconds(): + return 0 + last_attempts[normalized_job_id] = now_ts + lock_action = f"recycle-stalled-job-items:{normalized_job_id}" + lock_owner = self._acquire_shared_job_maintenance_lock(lock_action, ttl_seconds=15) + if not lock_owner: + return 0 + try: + recycled = int( + self.db.recycle_stalled_detect_job_items( + normalized_job_id, + stall_seconds=self._stalled_job_recycle_stall_seconds(), + ) + or 0 + ) + if recycled: + logger.warning( + f"已定向回收长挂任务项: job_id={normalized_job_id}, " + f"recycled={recycled}, reason={reason or 'unspecified'}" + ) + return recycled + finally: + self._release_shared_job_maintenance_lock(lock_action, lock_owner) + + def _is_primary_job_maintenance_worker(self) -> bool: + node_code = str(getattr(config, "NODE_CODE", "") or "").strip() + parent_node_code = str(os.getenv("WORKER_PARENT_NODE_CODE", "") or "").strip() + if parent_node_code and node_code == parent_node_code: + return True + if parent_node_code and node_code == f"{parent_node_code}-a": + return True + return bool(node_code and node_code.endswith("-a")) + + def _maybe_release_node_job_items(self, *, reason: str = "", min_interval_seconds: float = 300.0) -> int: + now_ts = time.time() + last_release_at = float(getattr(self, "_last_job_item_release_at", 0.0) or 0.0) + if min_interval_seconds > 0 and now_ts - last_release_at < float(min_interval_seconds): + return 0 + released = int(self.db.release_detect_job_items_for_node(config.NODE_CODE) or 0) + self._last_job_item_release_at = now_ts + if released: + self._schedule_restart_release_handoff( + released_count=released, + reason=reason or "node_item_release", + ) + logger.warning( + f"释放当前节点遗留任务项: count={released}, node={config.NODE_CODE}, " + f"reason={reason or 'unspecified'}" + ) + return released + + def _restart_release_handoff_window_seconds(self) -> float: + return max( + 30.0, + float(os.getenv("DOMAINCHECK_RESTART_RELEASE_HANDOFF_WINDOW_SECONDS", "240") or 240), + ) + + def _restart_release_handoff_batch_limit(self, requested_limit: int) -> int: + configured = int(os.getenv("DOMAINCHECK_RESTART_RELEASE_HANDOFF_BATCH_LIMIT", "128") or 128) + safe_requested_limit = max(1, int(requested_limit or 1)) + return max(1, min(safe_requested_limit, max(1, configured))) + + def _schedule_restart_release_handoff(self, *, released_count: int = 0, reason: str = "") -> None: + if int(released_count or 0) <= 0: + return + job_id = getattr(self, "current_job_id", None) + if job_id in (None, "", 0, "0"): + return + try: + normalized_job_id = int(job_id) + except Exception: + return + self._restart_release_handoff_job_id = normalized_job_id + self._restart_release_handoff_job_code = str(getattr(self, "current_job_code", "") or "").strip() + self._restart_release_handoff_until = time.time() + self._restart_release_handoff_window_seconds() + self._restart_release_handoff_reason = str(reason or "").strip() + logger.warning( + f"登记重启尾批优先回收窗口: job_id={normalized_job_id}, " + f"job_code={self._restart_release_handoff_job_code or '-'}, " + f"released={int(released_count or 0)}, reason={self._restart_release_handoff_reason or 'unspecified'}" + ) + + def _clear_restart_release_handoff(self, *, reason: str = "") -> None: + job_id = getattr(self, "_restart_release_handoff_job_id", None) + if job_id not in (None, "", 0, "0"): + logger.info( + f"清理重启尾批优先回收窗口: job_id={job_id}, " + f"job_code={str(getattr(self, '_restart_release_handoff_job_code', '') or '').strip() or '-'}, " + f"reason={str(reason or 'unspecified').strip() or 'unspecified'}" + ) + self._restart_release_handoff_job_id = None + self._restart_release_handoff_job_code = "" + self._restart_release_handoff_until = 0.0 + self._restart_release_handoff_reason = "" + + def _restart_release_handoff_job_id_active(self): + handoff_job_id = getattr(self, "_restart_release_handoff_job_id", None) + if handoff_job_id in (None, "", 0, "0"): + return None + if time.time() >= float(getattr(self, "_restart_release_handoff_until", 0.0) or 0.0): + self._clear_restart_release_handoff(reason="expired") + return None + current_job_id = getattr(self, "current_job_id", None) + if current_job_id not in (None, "", 0, "0"): + try: + if int(current_job_id) != int(handoff_job_id): + self._clear_restart_release_handoff(reason="job_changed") + return None + except Exception: + self._clear_restart_release_handoff(reason="job_changed") + return None + try: + return int(handoff_job_id) + except Exception: + self._clear_restart_release_handoff(reason="invalid_job_id") + return None + + def _explicit_claim_scope_window_seconds(self) -> float: + return max( + 30.0, + float(os.getenv("DOMAINCHECK_EXPLICIT_CLAIM_SCOPE_WINDOW_SECONDS", "180") or 180), + ) + + def _schedule_explicit_claim_scope(self, control_payload=None, *, source: str = "") -> None: + payload = dict(control_payload or {}) + task_mode = str( + payload.get("task_mode") + or payload.get("target_task_mode") + or "" + ).strip() + if task_mode == "single_step": + return + job_id = self._incoming_job_id(payload) + if job_id in (None, "", 0, "0"): + return + try: + normalized_job_id = int(job_id) + except Exception: + return + job_code = self._incoming_job_code(payload) + self._explicit_claim_scope_job_id = normalized_job_id + self._explicit_claim_scope_job_code = str(job_code or "").strip() + self._explicit_claim_scope_until = time.time() + self._explicit_claim_scope_window_seconds() + self._explicit_claim_scope_source = str(source or "").strip() + logger.warning( + f"登记显式启动定向领取窗口: job_id={normalized_job_id}, " + f"job_code={self._explicit_claim_scope_job_code or '-'}, " + f"source={self._explicit_claim_scope_source or 'unknown'}" + ) + + def _clear_explicit_claim_scope(self, *, reason: str = "") -> None: + job_id = getattr(self, "_explicit_claim_scope_job_id", None) + if job_id not in (None, "", 0, "0"): + logger.info( + f"清理显式启动定向领取窗口: job_id={job_id}, " + f"job_code={str(getattr(self, '_explicit_claim_scope_job_code', '') or '').strip() or '-'}, " + f"reason={str(reason or 'unspecified').strip() or 'unspecified'}" + ) + self._explicit_claim_scope_job_id = None + self._explicit_claim_scope_job_code = "" + self._explicit_claim_scope_until = 0.0 + self._explicit_claim_scope_source = "" + + def _explicit_claim_scope_job_id_active(self): + explicit_job_id = getattr(self, "_explicit_claim_scope_job_id", None) + if explicit_job_id in (None, "", 0, "0"): + return None + if time.time() >= float(getattr(self, "_explicit_claim_scope_until", 0.0) or 0.0): + self._clear_explicit_claim_scope(reason="expired") + return None + try: + return int(explicit_job_id) + except Exception: + self._clear_explicit_claim_scope(reason="invalid_job_id") + return None + def _should_scope_claims_to_current_job(self): """ single_step 会话必须严格绑定当前 job； domain_pipeline 会话则应持续从全局队列补位，避免旧 job 占住 worker、 新 job 长时间堆在 pending。 """ - return self._is_single_step_session_active() + if self._is_single_step_session_active(): + return True + if self._restart_release_handoff_job_id_active() not in (None, "", 0, "0"): + return True + return self._explicit_claim_scope_job_id_active() not in (None, "", 0, "0") + + def _current_scoped_claim_job_id(self): + if self._is_single_step_session_active(): + try: + return int(self.current_job_id) if self.current_job_id not in (None, "", 0, "0") else None + except Exception: + return None + handoff_job_id = self._restart_release_handoff_job_id_active() + if handoff_job_id not in (None, "", 0, "0"): + return handoff_job_id + explicit_job_id = self._explicit_claim_scope_job_id_active() + if explicit_job_id not in (None, "", 0, "0"): + return explicit_job_id + return None + + def _log_explicit_scope_claim_result(self, *, scoped_job_id, limit, rows_count, reason=""): + try: + normalized_job_id = int(scoped_job_id or 0) + except Exception: + normalized_job_id = 0 + if normalized_job_id <= 0: + return + normalized_rows = max(0, int(rows_count or 0)) + normalized_limit = max(0, int(limit or 0)) + detail_reason = str(reason or "").strip() or ("claimed" if normalized_rows > 0 else "empty") + log_key = f"{normalized_job_id}:{normalized_rows}:{detail_reason}" + now_ts = time.time() + min_interval_seconds = 1.0 if normalized_rows > 0 else 10.0 + if ( + log_key == str(getattr(self, "_last_explicit_scope_claim_log_key", "") or "") + and now_ts - float(getattr(self, "_last_explicit_scope_claim_log_at", 0.0) or 0.0) < min_interval_seconds + ): + return + self._last_explicit_scope_claim_log_at = now_ts + self._last_explicit_scope_claim_log_key = log_key + log_message = ( + f"显式定向领取{'命中' if normalized_rows > 0 else '未命中'}: " + f"scoped_job_id={normalized_job_id} | current_job_id={self.current_job_id or '-'} " + f"| current_job_code={self.current_job_code or '-'} | detecting={1 if self.detecting else 0} " + f"| limit={normalized_limit} | rows={normalized_rows} | reason={detail_reason}" + ) + if normalized_rows > 0: + logger.warning(log_message) + else: + logger.info(log_message) + + def _log_explicit_scope_probe(self, *, scoped_job_id, max_threads, refill_slots, claim_batch_size, pending_buffer): + try: + normalized_job_id = int(scoped_job_id or 0) + except Exception: + normalized_job_id = 0 + if normalized_job_id <= 0: + return + now_ts = time.time() + if now_ts - float(getattr(self, "_last_explicit_scope_probe_log_at", 0.0) or 0.0) < 5.0: + return + self._last_explicit_scope_probe_log_at = now_ts + logger.warning( + f"显式定向领取准备开始: scoped_job_id={normalized_job_id} | current_job_id={self.current_job_id or '-'} " + f"| current_job_code={self.current_job_code or '-'} | detecting={1 if self.detecting else 0} " + f"| max_threads={max(0, int(max_threads or 0))} | refill_slots={max(0, int(refill_slots or 0))} " + f"| claim_batch_size={max(0, int(claim_batch_size or 0))} | pending_buffer={max(0, int(pending_buffer or 0))}" + ) + + def _log_job_status_refresh_probe(self, job_id): + try: + normalized_job_id = int(job_id or 0) + except Exception: + normalized_job_id = 0 + if normalized_job_id <= 0: + return + now_ts = time.time() + if now_ts - float(getattr(self, "_last_job_status_refresh_probe_log_at", 0.0) or 0.0) < 10.0: + return + self._last_job_status_refresh_probe_log_at = now_ts + logger.info( + f"准备刷新检测任务聚合状态: job_id={normalized_job_id} | current_job_code={self.current_job_code or '-'} " + f"| detecting={1 if self.detecting else 0} | active_threads={self._get_active_domain_threads()}" + ) + + def _recent_duplicate_start_window_seconds(self) -> float: + return max( + 15.0, + float(os.getenv("DOMAINCHECK_RECENT_START_DUPLICATE_WINDOW_SECONDS", "120") or 120), + ) + + def _record_detection_start_request(self, *, source: str = "", control_payload=None) -> None: + now_ts = time.time() + payload = dict(control_payload or {}) + job_id = self._incoming_job_id(payload) + job_code = self._incoming_job_code(payload) + task_mode = str( + payload.get("task_mode") + or payload.get("target_task_mode") + or "" + ).strip() + self._last_detection_start_at = now_ts + self._last_detection_start_source = str(source or "").strip() + self._last_detection_start_job_id = int(job_id) if job_id not in (None, 0) else None + self._last_detection_start_job_code = job_code + self._last_detection_start_task_mode = task_mode + + def _mark_explicit_start_signal(self, control_payload=None, *, source: str = "") -> None: + self._last_explicit_start_signal_at = time.time() + self._last_explicit_start_payload = { + **dict(control_payload or {}), + "_source": str(source or "").strip(), + } + self._schedule_explicit_claim_scope(control_payload, source=source) + + def _should_coalesce_recent_start(self, control_payload=None) -> tuple[bool, str]: + now_ts = time.time() + recent_start_at = float(getattr(self, "_last_detection_start_at", 0.0) or 0.0) + if recent_start_at <= 0: + return False, "" + if now_ts - recent_start_at > self._recent_duplicate_start_window_seconds(): + return False, "" + if self.stop_requested: + return False, "" + thread = self.detect_command_thread + thread_alive = bool(thread and thread.is_alive()) + if not bool(self.detecting) and not thread_alive: + # A recent start alone is not enough to suppress a legitimate retry. + # Once the previous session has fully stopped, repeated start_detection + # for the same job must be allowed to re-open tail work. + return False, "" + stale_detect, _ = self._is_stale_detect_session(control_payload) + if stale_detect: + # A stale session must never suppress a restart for the same job. + # Otherwise auto-resume loops forever on "already running" while + # active_threads stays at zero. + return False, "" + + payload = dict(control_payload or {}) + incoming_job_code = self._incoming_job_code(payload) + incoming_job_id = self._incoming_job_id(payload) + incoming_task_mode = str( + payload.get("task_mode") + or payload.get("target_task_mode") + or "" + ).strip() + recent_job_code = str(getattr(self, "_last_detection_start_job_code", "") or "").strip() + recent_job_id = getattr(self, "_last_detection_start_job_id", None) + recent_task_mode = str(getattr(self, "_last_detection_start_task_mode", "") or "").strip() + + if incoming_job_code and recent_job_code and incoming_job_code != recent_job_code: + return False, "" + if ( + incoming_job_id not in (None, 0) + and recent_job_id not in (None, 0) + and int(incoming_job_id) != int(recent_job_id) + ): + return False, "" + if incoming_task_mode == "single_step" and recent_task_mode and incoming_task_mode != recent_task_mode: + return False, "" + + effective_job_code = incoming_job_code or recent_job_code or str(self.current_job_code or "").strip() + effective_job_id = incoming_job_id or recent_job_id or self.current_job_id + if not effective_job_code and effective_job_id in (None, 0): + return False, "" + return True, ( + "最近已启动同一轮检测，会话仍在稳定中，忽略重复启动: " + f"job={effective_job_code or effective_job_id or 'none'} " + f"source={str(getattr(self, '_last_detection_start_source', '') or '').strip() or 'unknown'}" + ) def start_detection_async(self, source: str = "remote", control_payload=None): """ 异步启动一次检测任务，避免阻塞 Redis 订阅线程。 """ with self.detect_lock: + recent_duplicate_start, recent_duplicate_message = self._should_coalesce_recent_start(control_payload) + if recent_duplicate_start: + logger.info(recent_duplicate_message) + self._update_runtime_state( + "running", + recent_duplicate_message, + source=source, + cycle_token=self.current_cycle_token or self._incoming_cycle_token(control_payload), + job_id=self.current_job_id or self._incoming_job_id(control_payload), + job_code=str(self.current_job_code or "").strip() or self._incoming_job_code(control_payload), + ) + return True if self.detect_command_thread and self.detect_command_thread.is_alive(): + duplicate_session_command = self._matches_active_detect_session(control_payload) stale_detect, stale_reason = self._is_stale_detect_session(control_payload) if not stale_detect: incoming_job_code = self._incoming_job_code(control_payload) + incoming_cycle_token = self._incoming_cycle_token(control_payload) + incoming_job_id = self._incoming_job_id(control_payload) + overlap_handoff_requested = self._is_overlap_handoff_command(control_payload, source=source) current_job_code = str(self.current_job_code or "").strip() incoming_task_mode = str( (control_payload or {}).get("task_mode") or (control_payload or {}).get("target_task_mode") or "" ).strip() + if duplicate_session_command: + message = ( + "收到同任务同周期重复启动指令，保持当前检测会话: " + f"job={current_job_code or incoming_job_code or 'none'} " + f"cycle={self.current_cycle_token or incoming_cycle_token or 'none'}" + ) + logger.info(message) + self._update_runtime_state( + "running", + message, + source=source, + cycle_token=self.current_cycle_token or incoming_cycle_token, + job_id=self.current_job_id or incoming_job_id, + job_code=current_job_code or incoming_job_code, + ) + return True if ( incoming_task_mode != "single_step" and not self._is_single_step_session_active() and incoming_job_code and incoming_job_code != current_job_code ): + explicit_scope_job_id = self._explicit_claim_scope_job_id_active() + explicit_switch_requested = bool( + explicit_scope_job_id not in (None, "", 0, "0") + and incoming_job_id not in (None, "", 0, "0") + and int(explicit_scope_job_id) == int(incoming_job_id) + ) + live_active_threads = max(0, int(self._get_active_domain_threads() or 0)) + if overlap_handoff_requested and live_active_threads > 0: + message = ( + "收到 overlap handoff 启动指令，但当前会话仍有活跃线程，忽略跨 job 抢占: " + f"current={current_job_code or 'none'} -> incoming={incoming_job_code} " + f"active_threads={live_active_threads}" + ) + logger.info(message) + if explicit_switch_requested: + self._clear_explicit_claim_scope(reason="overlap_busy_ignore") + self._update_runtime_state( + "running", + message, + source=source, + cycle_token=self.current_cycle_token or incoming_cycle_token, + job_id=self.current_job_id or incoming_job_id, + job_code=current_job_code or incoming_job_code, + ) + return True + if explicit_switch_requested: + message = ( + "检测任务已在运行，收到显式 pipeline 切换指令，" + f"登记安全重启接棒: current={current_job_code or 'none'} -> incoming={incoming_job_code}" + ) + logger.warning(message) + self.stop_requested = True + self._schedule_pending_restart(source, control_payload, message) + self._update_runtime_state( + "restarting", + message, + source=source, + cycle_token=incoming_cycle_token or self.current_cycle_token, + job_id=incoming_job_id, + job_code=incoming_job_code, + ) + return True message = ( "检测任务已在运行，已接收新的 pipeline 唤醒指令，" f"继续由当前 worker pool 补位处理: current={current_job_code or 'none'} -> incoming={incoming_job_code}" @@ -1921,6 +3328,7 @@ class DetectWorker: self._detect_session_seq = int(self._detect_session_seq or 0) + 1 session_id = int(self._detect_session_seq or 0) self._detect_session_owner = session_id + self._record_detection_start_request(source=source, control_payload=control_payload) self.detect_command_thread = threading.Thread( target=self._run_detection_session, kwargs={ @@ -1965,7 +3373,7 @@ class DetectWorker: job_id=self.current_job_id, job_code=self.current_job_code, ) - self.start_detection() + self.start_detection(session_id=session_id) except Exception as e: logger.error(f"远程检测任务执行失败: {e}") import traceback @@ -2005,15 +3413,18 @@ class DetectWorker: except Exception as restart_error: logger.warning(f"检测任务延期重启触发失败: {restart_error}") if self.running and not self.stop_requested and not restarted_from_pending: + self._clear_active_cycle_context() self._update_runtime_state( "idle", "检测任务结束，Worker 保持待命", source=source, - cycle_token=self.current_cycle_token, - job_id=self.current_job_id, - job_code=self.current_job_code, ) - self._clear_active_cycle_context() + try: + self._maybe_recycle_idle_worker(trigger_reason="session_finalize") + except Exception as recycle_error: + logger.debug(f"检测任务结束后空闲回收检查失败: {recycle_error}") + else: + self._clear_active_cycle_context() def request_stop_detection(self, source: str = "remote"): """ @@ -2041,25 +3452,37 @@ class DetectWorker: request_id = str((control_payload or {}).get("request_id", "")).strip() if not request_id: return - try: - raw_pending = self.redis_client.get(PENDING_CONTROL_KEY) - if not raw_pending: - return - pending_payload = json.loads(raw_pending) - pending_request_id = str((pending_payload or {}).get("request_id", "")).strip() - if pending_request_id and pending_request_id == request_id: - self.redis_client.delete(PENDING_CONTROL_KEY) - except Exception as e: - logger.debug(f"确认待执行控制指令失败: {e}") + pending_keys = [pending_control_key()] + if pending_keys[0] != PENDING_CONTROL_KEY: + pending_keys.append(PENDING_CONTROL_KEY) + for key in pending_keys: + try: + raw_pending = self.redis_client.get(key) + if not raw_pending: + continue + pending_payload = json.loads(raw_pending) + pending_request_id = str((pending_payload or {}).get("request_id", "")).strip() + if pending_request_id and pending_request_id == request_id: + self.redis_client.delete(key) + except Exception as e: + logger.debug(f"确认待执行控制指令失败: key={key}, error={e}") def _handle_control_message(self, payload): try: control_payload = json.loads(payload) if isinstance(payload, str) else payload except Exception: control_payload = {"action": str(payload)} + if not control_targets_current_worker(control_payload): + self._remember_ignored_targeted_control(control_payload) + target_node_codes = ",".join(control_target_node_codes(control_payload)) or "unknown" + logger.info( + f"忽略发往其他 Worker 实例的控制指令: current={config.NODE_CODE}, targets={target_node_codes}" + ) + return self._acknowledge_pending_control_command(control_payload) action = str(control_payload.get("action", "")).strip() if action == "start_detection": + self._mark_explicit_start_signal(control_payload, source="redis-control") self.start_detection_async(source="redis-control", control_payload=control_payload) elif action == "stop_detection": self.request_stop_detection(source="redis-control") @@ -2116,7 +3539,7 @@ class DetectWorker: } try: # 从Redis获取配置 - if self.use_redis: + if getattr(self, "use_redis", False): detect_options_str = self.redis_client.get('domain_tool:detect_options') if detect_options_str: detect_options = default_options.copy() @@ -2198,21 +3621,36 @@ class DetectWorker: logger.error(f"加载代理配置失败: {e}") logger.info(f"使用默认代理配置: {default_config}") return default_config + + def _worker_parent_node_code(self): + return str(os.getenv("WORKER_PARENT_NODE_CODE", "") or "").strip() + + def _worker_node_code_candidates(self): + candidates = [] + for candidate in ( + str(getattr(config, "NODE_CODE", "") or "").strip(), + self._worker_parent_node_code(), + ): + if candidate and candidate not in candidates: + candidates.append(candidate) + return candidates def load_thread_count(self): """ 加载检测线程数 """ try: - node_code = str(getattr(config, "NODE_CODE", "") or "").strip() + node_code_candidates = self._worker_node_code_candidates() + node_code = node_code_candidates[0] if node_code_candidates else "" node_thread_counts = {} redis_global_thread_count = None def _warn_missing_node_override(source_name, overrides, fallback_value): if not node_code or not isinstance(overrides, dict) or not overrides: return - if node_code in overrides: - return + for candidate in node_code_candidates: + if candidate in overrides: + return available_codes = ", ".join(sorted(str(code) for code in overrides.keys() if str(code).strip())) logger.warning( "节点线程覆盖未命中，回退到%s通用线程数: node_code=%s, fallback=%s, available_node_codes=%s" @@ -2224,6 +3662,23 @@ class DetectWorker: ) ) + def _resolve_node_override(overrides, source_name): + if not isinstance(overrides, dict): + return None + for candidate in node_code_candidates: + node_thread_count_raw = overrides.get(candidate) + if node_thread_count_raw is None: + continue + thread_count = max(1, int(node_thread_count_raw)) + if candidate == node_code: + logger.info(f"从{source_name}加载节点专属检测线程数成功: {candidate} -> {thread_count}") + else: + logger.info( + f"从{source_name}加载父节点检测线程覆盖成功: {candidate} -> {thread_count} (current={node_code})" + ) + return thread_count + return None + # 从Redis获取配置 if self.use_redis: node_thread_counts_raw = self.redis_client.get('domain_tool:node_thread_counts') @@ -2233,12 +3688,9 @@ class DetectWorker: except Exception as e: logger.warning(f"解析 Redis 节点线程覆盖配置失败: {e}") - if node_code and isinstance(node_thread_counts, dict): - node_thread_count_str = node_thread_counts.get(node_code) - if node_thread_count_str is not None: - thread_count = max(1, int(node_thread_count_str)) - logger.info(f"从Redis加载节点专属检测线程数成功: {node_code} -> {thread_count}") - return thread_count + resolved_node_thread_count = _resolve_node_override(node_thread_counts, "Redis") + if resolved_node_thread_count is not None: + return resolved_node_thread_count thread_count_str = self.redis_client.get('domain_tool:thread_count') if thread_count_str: @@ -2249,19 +3701,15 @@ class DetectWorker: return thread_count # 从本地文件获取配置 - if os.path.exists('node_thread_counts.json'): - with open('node_thread_counts.json', 'r', encoding='utf-8') as f: - node_thread_counts = json.load(f) - if node_code and isinstance(node_thread_counts, dict): - node_thread_count = node_thread_counts.get(node_code) - if node_thread_count is not None: - thread_count = max(1, int(node_thread_count)) - logger.info(f"从本地文件加载节点专属检测线程数成功: {node_code} -> {thread_count}") - return thread_count + node_thread_counts_payload = _read_worker_local_json_config('node_thread_counts.json') + if isinstance(node_thread_counts_payload, dict): + node_thread_counts = node_thread_counts_payload + resolved_node_thread_count = _resolve_node_override(node_thread_counts, "本地文件") + if resolved_node_thread_count is not None: + return resolved_node_thread_count - if os.path.exists('thread_count.json'): - with open('thread_count.json', 'r', encoding='utf-8') as f: - thread_config = json.load(f) + thread_config = _read_worker_local_json_config('thread_count.json') + if isinstance(thread_config, dict): thread_count = thread_config.get('thread_count', '10') thread_count = max(1, int(thread_count)) if redis_global_thread_count is None: @@ -2269,15 +3717,110 @@ class DetectWorker: logger.info(f"从本地文件加载检测线程数成功: {thread_count}") return thread_count else: - default_thread_count = 4 # 默认线程数，降低线程数量以减轻系统负担 + default_thread_count = 1000 logger.info(f"使用默认检测线程数: {default_thread_count}") return default_thread_count except Exception as e: logger.error(f"加载检测线程数失败: {e}") - default_thread_count = 4 # 默认线程数，降低线程数量以减轻系统负担 + default_thread_count = 1000 logger.info(f"使用默认检测线程数: {default_thread_count}") return default_thread_count + def load_process_count(self): + """ + 加载检测进程数 + """ + try: + node_code_candidates = self._worker_node_code_candidates() + node_code = node_code_candidates[0] if node_code_candidates else "" + node_process_counts = {} + redis_global_process_count = None + + def _warn_missing_node_override(source_name, overrides, fallback_value): + if not node_code or not isinstance(overrides, dict) or not overrides: + return + for candidate in node_code_candidates: + if candidate in overrides: + return + available_codes = ", ".join(sorted(str(code) for code in overrides.keys() if str(code).strip())) + logger.warning( + "节点进程覆盖未命中，回退到%s通用进程数: node_code=%s, fallback=%s, available_node_codes=%s" + % ( + source_name, + node_code, + fallback_value, + available_codes or "-", + ) + ) + + def _resolve_node_override(overrides, source_name): + if not isinstance(overrides, dict): + return None + for candidate in node_code_candidates: + node_process_count_raw = overrides.get(candidate) + if node_process_count_raw is None: + continue + process_count = max(1, int(node_process_count_raw)) + if candidate == node_code: + logger.info(f"从{source_name}加载节点专属检测进程数成功: {candidate} -> {process_count}") + else: + logger.info( + f"从{source_name}加载父节点检测进程覆盖成功: {candidate} -> {process_count} (current={node_code})" + ) + return process_count + return None + + if self.use_redis: + node_process_counts_raw = self.redis_client.get('domain_tool:node_process_counts') + if node_process_counts_raw: + try: + node_process_counts = json.loads(node_process_counts_raw) + except Exception as e: + logger.warning(f"解析 Redis 节点进程覆盖配置失败: {e}") + + resolved_node_process_count = _resolve_node_override(node_process_counts, "Redis") + if resolved_node_process_count is not None: + return resolved_node_process_count + + process_count_str = self.redis_client.get('domain_tool:process_count') + if process_count_str: + process_count = max(1, int(process_count_str)) + redis_global_process_count = process_count + _warn_missing_node_override("Redis", node_process_counts, process_count) + logger.info(f"从Redis加载检测进程数成功: {process_count}") + return process_count + + node_process_counts_payload = _read_worker_local_json_config('node_process_counts.json') + if isinstance(node_process_counts_payload, dict): + node_process_counts = node_process_counts_payload + resolved_node_process_count = _resolve_node_override(node_process_counts, "本地文件") + if resolved_node_process_count is not None: + return resolved_node_process_count + + process_config = _read_worker_local_json_config('process_count.json') + if isinstance(process_config, dict): + process_count = process_config.get('process_count', '1') + process_count = max(1, int(process_count)) + if redis_global_process_count is None: + _warn_missing_node_override("本地文件", node_process_counts, process_count) + logger.info(f"从本地文件加载检测进程数成功: {process_count}") + return process_count + + default_process_count = max( + 1, + int(os.getenv("DOMAINCHECK_NODE_PROCESS_COUNT", "1") or 1), + ) + logger.info(f"使用默认检测进程数: {default_process_count}") + return default_process_count + except Exception as e: + logger.error(f"加载检测进程数失败: {e}") + default_process_count = max( + 1, + int(os.getenv("DOMAINCHECK_NODE_PROCESS_COUNT", "1") or 1), + ) + logger.info(f"使用默认检测进程数: {default_process_count}") + return default_process_count + def refresh_thread_count_runtime(self, *, force=False, min_interval=2.0): """ 运行中热刷新线程数配置。 @@ -2299,7 +3842,314 @@ class DetectWorker: self.update_config_labels() return max(1, int(getattr(self, "thread_count", latest) or latest)) - def _pull_sync_tasks_until_available(self, *, thread_limit: int) -> list[dict]: + def refresh_process_count_runtime(self, *, force=False, min_interval=5.0): + """ + 运行中热刷新进程数配置。 + """ + now_ts = time.time() + last_refresh_at = float(getattr(self, "_last_process_count_refresh_at", 0.0) or 0.0) + if not force and now_ts - last_refresh_at < max(0.5, float(min_interval or 0.0)): + return max(1, int(getattr(self, "process_count", 1) or 1)) + + previous = max(1, int(getattr(self, "process_count", 1) or 1)) + latest = max(1, int(self.load_process_count() or previous)) + self._last_process_count_refresh_at = now_ts + if latest != previous: + self.process_count = latest + logger.info(f"检测进程数热更新: {previous} -> {latest}") + self.update_config_labels() + return max(1, int(getattr(self, "process_count", latest) or latest)) + + def _proxy_runtime_process_count(self): + current = max(1, int(getattr(self, "process_count", 1) or 1)) + try: + latest = self.refresh_process_count_runtime(min_interval=5.0) + except Exception: + latest = current + return max(1, int(latest or current)) + + def _proxy_multi_process_divisor(self): + process_count = self._proxy_runtime_process_count() + if process_count <= 1: + return 1.0 + return max(1.0, math.sqrt(float(process_count))) + + def _proxy_refresh_holdoff_seconds(self): + process_count = self._proxy_runtime_process_count() + if process_count < 4 or not self.detecting: + return 0.0 + pid_spread = (os.getpid() % 7) * 0.02 + return min(0.9, 0.12 + math.log2(float(process_count)) * 0.06 + pid_spread) + + def _proxy_coordination_node_code(self): + return self._worker_parent_node_code() or str(getattr(config, "NODE_CODE", "") or "").strip() + + def _proxy_should_coordinate_shared_refresh(self): + return bool( + self.use_redis + and self.redis_client is not None + and self._proxy_runtime_process_count() > 1 + and self._proxy_coordination_node_code() + ) + + def _shared_proxy_pool_key(self): + node_code = self._proxy_coordination_node_code() + if not node_code: + return "" + return f"{SHARED_PROXY_POOL_KEY_PREFIX}:{node_code}" + + def _shared_proxy_refresh_lock_key(self): + node_code = self._proxy_coordination_node_code() + if not node_code: + return "" + return f"{SHARED_PROXY_REFRESH_LOCK_KEY_PREFIX}:{node_code}" + + def _shared_proxy_snapshot_ttl_seconds(self): + configured = int(os.getenv("DOMAINCHECK_SHARED_PROXY_SNAPSHOT_TTL", "18") or 18) + return max(5, configured) + + def _shared_proxy_refresh_lock_seconds(self): + configured = int(os.getenv("DOMAINCHECK_SHARED_PROXY_REFRESH_LOCK_SECONDS", "8") or 8) + return max(2, configured) + + def _shared_proxy_wait_timeout_seconds(self): + configured = float(os.getenv("DOMAINCHECK_SHARED_PROXY_WAIT_TIMEOUT", "0.9") or 0.9) + return max(0.2, configured) + + def _shared_proxy_reuse_gap(self): + configured = int(os.getenv("DOMAINCHECK_SHARED_PROXY_REUSE_GAP", "0") or 0) + if configured > 0: + return max(8, configured) + refresh_threshold = max(1, int(self._proxy_refresh_threshold() or 1)) + return max(24, min(160, max(24, refresh_threshold // 3))) + + def _shared_proxy_snapshot_should_replace_local_pool(self, shared_count, local_count, refresh_threshold): + normalized_shared = max(0, int(shared_count or 0)) + normalized_local = max(0, int(local_count or 0)) + normalized_threshold = max(1, int(refresh_threshold or 1)) + if normalized_shared <= 0: + return False + if normalized_local <= 0: + return True + if normalized_shared >= normalized_threshold: + return True + if normalized_shared <= normalized_local: + return False + return (normalized_shared - normalized_local) >= self._shared_proxy_reuse_gap() + + def _config_update_proxy_refresh_debounce_seconds(self): + configured = float(os.getenv("DOMAINCHECK_PROXY_CONFIG_UPDATE_REFRESH_DEBOUNCE", "20") or 20) + return max(2.0, configured) + + def _shared_proxy_snapshot_owner(self): + return f"{config.NODE_CODE}:{os.getpid()}:{int(time.time() * 1000)}" + + def _rotate_proxy_entries_for_current_worker(self, proxy_entries): + entries = list(proxy_entries or []) + if len(entries) <= 1: + return entries + seed_source = str(getattr(config, "NODE_CODE", "") or "").strip() or str(os.getpid()) + seed = sum(ord(ch) for ch in seed_source) + int(os.getpid() or 0) + offset = seed % len(entries) + if offset <= 0: + return entries + return entries[offset:] + entries[:offset] + + def _shared_proxy_payload_from_entries(self, proxy_entries, *, source_count=0, raw_items=0): + now_ts = time.time() + shareable_entries = [] + for proxy_entry in list(proxy_entries or []): + if not isinstance(proxy_entry, dict) or not proxy_entry.get("proxy"): + continue + if not self._proxy_entry_can_be_reused(proxy_entry, now_ts=now_ts): + continue + shareable_entries.append(dict(proxy_entry)) + if not shareable_entries: + return None + return { + "node_code": self._proxy_coordination_node_code(), + "owner": str(getattr(config, "NODE_CODE", "") or "").strip(), + "refreshed_at_ts": now_ts, + "source_count": int(source_count or 0), + "raw_items": int(raw_items or 0), + "available_count": len(shareable_entries), + "proxy_pool": shareable_entries, + } + + def _load_shared_proxy_snapshot_payload(self, *, max_age_seconds=None): + if not self._proxy_should_coordinate_shared_refresh(): + return None + redis_key = self._shared_proxy_pool_key() + if not redis_key: + return None + try: + raw_payload = self.redis_client.get(redis_key) + except Exception as exc: + logger.debug(f"读取共享代理快照失败: {exc}") + return None + if not raw_payload: + return None + try: + payload = json.loads(raw_payload) + except Exception as exc: + logger.debug(f"解析共享代理快照失败: {exc}") + return None + if not isinstance(payload, dict): + return None + refreshed_at_ts = float(payload.get("refreshed_at_ts", 0.0) or 0.0) + allowed_age = max( + 1.0, + float(max_age_seconds if max_age_seconds is not None else self._shared_proxy_snapshot_ttl_seconds()), + ) + if refreshed_at_ts > 0 and (time.time() - refreshed_at_ts) > allowed_age: + return None + shareable_payload = self._shared_proxy_payload_from_entries( + payload.get("proxy_pool") or [], + source_count=payload.get("source_count", 0), + raw_items=payload.get("raw_items", 0), + ) + if not shareable_payload: + return None + shareable_payload["owner"] = str(payload.get("owner") or "").strip() + shareable_payload["refreshed_at_ts"] = refreshed_at_ts or shareable_payload.get("refreshed_at_ts", time.time()) + return shareable_payload + + def _apply_shared_proxy_snapshot(self, payload, *, status_prefix="复用共享代理快照"): + if not isinstance(payload, dict): + return False + proxy_entries = payload.get("proxy_pool") or [] + if not isinstance(proxy_entries, list) or not proxy_entries: + return False + rotated_entries = self._rotate_proxy_entries_for_current_worker(proxy_entries) + with self.proxy_pool_lock: + self.proxy_pool = list(rotated_entries) + pool_size = len(self.proxy_pool) + refreshed_at_ts = float(payload.get("refreshed_at_ts", 0.0) or 0.0) + self.proxy_last_refresh_time = ( + datetime.fromtimestamp(refreshed_at_ts) if refreshed_at_ts > 0 else datetime.now() + ) + self.proxy_last_refresh_source_count = int(payload.get("source_count", 0) or 0) + self.proxy_last_refresh_total_items = int(payload.get("raw_items", 0) or 0) + self.proxy_last_validated_count = 0 + self.proxy_last_available_count = pool_size + source_owner = str(payload.get("owner") or "").strip() + if source_owner and source_owner != str(getattr(config, "NODE_CODE", "") or "").strip(): + self.proxy_last_refresh_status = f"{status_prefix} {pool_size} 个（来源 {source_owner}）" + else: + self.proxy_last_refresh_status = f"{status_prefix} {pool_size} 个" + return True + + def _publish_shared_proxy_snapshot(self, proxy_entries, *, source_count=0, raw_items=0): + if not self._proxy_should_coordinate_shared_refresh(): + return False + payload = self._shared_proxy_payload_from_entries( + proxy_entries, + source_count=source_count, + raw_items=raw_items, + ) + if not payload: + return False + redis_key = self._shared_proxy_pool_key() + if not redis_key: + return False + try: + self.redis_client.set( + redis_key, + json.dumps(payload, ensure_ascii=False), + ex=self._shared_proxy_snapshot_ttl_seconds(), + ) + return True + except Exception as exc: + logger.debug(f"写入共享代理快照失败: {exc}") + return False + + def _acquire_shared_proxy_refresh_lock(self): + if not self._proxy_should_coordinate_shared_refresh(): + return "" + redis_key = self._shared_proxy_refresh_lock_key() + if not redis_key: + return "" + owner_token = self._shared_proxy_snapshot_owner() + try: + acquired = self.redis_client.set( + redis_key, + owner_token, + nx=True, + ex=self._shared_proxy_refresh_lock_seconds(), + ) + except Exception as exc: + logger.debug(f"获取共享代理刷新锁失败: {exc}") + return "" + return owner_token if acquired else "" + + def _release_shared_proxy_refresh_lock(self, owner_token): + if not owner_token or not self._proxy_should_coordinate_shared_refresh(): + return + redis_key = self._shared_proxy_refresh_lock_key() + if not redis_key: + return + try: + current_owner = self.redis_client.get(redis_key) + if str(current_owner or "") == str(owner_token): + self.redis_client.delete(redis_key) + except Exception as exc: + logger.debug(f"释放共享代理刷新锁失败: {exc}") + + def _wait_for_shared_proxy_snapshot(self, timeout_seconds=None): + if not self._proxy_should_coordinate_shared_refresh(): + return False + deadline = time.time() + max( + 0.2, + float(timeout_seconds if timeout_seconds is not None else self._shared_proxy_wait_timeout_seconds()), + ) + while time.time() < deadline: + payload = self._load_shared_proxy_snapshot_payload() + if payload and self._apply_shared_proxy_snapshot(payload): + return True + time.sleep(0.05) + return False + + def _should_trigger_proxy_refresh_for_config_update( + self, + *, + config_type="", + previous_proxy_config=None, + previous_thread_count=0, + current_thread_count=0, + ): + previous_proxy_config = dict(previous_proxy_config or {}) + previous_urls = tuple(previous_proxy_config.get('proxy_urls') or []) + current_urls = tuple((self.proxy_config or {}).get('proxy_urls') or []) + proxy_changed = ( + bool(previous_proxy_config.get('proxy_enable', False)) != bool((self.proxy_config or {}).get('proxy_enable', False)) + or bool(previous_proxy_config.get('allow_direct', False)) != bool((self.proxy_config or {}).get('allow_direct', False)) + or previous_urls != current_urls + ) + if proxy_changed: + return True, True + + normalized_config_type = str(config_type or "").strip() + if normalized_config_type not in {"thread_count", "node_thread_counts"}: + return False, False + if int(current_thread_count or 0) == int(previous_thread_count or 0): + return False, False + + with self.proxy_pool_lock: + local_available = len(self.proxy_pool) + shared_payload = self._load_shared_proxy_snapshot_payload() + shared_available = 0 + if isinstance(shared_payload, dict): + shared_available = len(list(shared_payload.get("proxy_pool") or [])) + effective_available = max(local_available, shared_available) + refresh_threshold = self._proxy_refresh_threshold() + if effective_available >= max(1, int(refresh_threshold or 1)): + return False, False + shortage = max(0, int(refresh_threshold or 0) - effective_available) + if effective_available <= 0: + return True, False + return shortage >= self._shared_proxy_reuse_gap(), False + + def _pull_sync_tasks_until_available(self, *, thread_limit: int, claim_after_pull: bool = True) -> list[dict]: """ mainland-controller 在本地任务吃空时，不要立即退出检测循环，而是主动向海外控制面 连续补货几轮，尽量把本地队列重新喂满。 @@ -2311,96 +4161,133 @@ class DetectWorker: max_rounds = max(1, int(os.getenv("DOMAINCHECK_SYNC_PULL_BURST_ROUNDS", "4") or 4)) current_thread_limit = max(1, int(thread_limit or getattr(self, "thread_count", 1) or 1)) - configured_pull_limit = int(os.getenv("DOMAINCHECK_SYNC_PULL_LIMIT", "0") or 0) - pull_limit = max( - 1000, - configured_pull_limit if configured_pull_limit > 0 else min(20000, current_thread_limit * 4), - ) + pull_limit = self._resolve_sync_pull_limit(current_thread_limit) claim_batch_size = max(20, current_thread_limit) claim_lease_seconds = max(300, min(1800, claim_batch_size * 30)) + maintenance_lock_owner = self._acquire_shared_job_maintenance_lock( + "pull-sync-tasks", + ttl_seconds=max(15, max_rounds * 12), + ) + if not maintenance_lock_owner: + return [] local_api_base_url = str( os.getenv("DOMAINCHECK_LOCAL_API_BASE_URL", "http://127.0.0.1:8100/api/v1") or "http://127.0.0.1:8100/api/v1" ).strip().rstrip("/") - for round_index in range(1, max_rounds + 1): - try: - request = urllib.request.Request( - f"{local_api_base_url}/runtime/actions/pull_tasks", - data=json.dumps({"limit": pull_limit}, ensure_ascii=False).encode("utf-8"), - headers={"Content-Type": "application/json"}, - method="POST", + try: + for round_index in range(1, max_rounds + 1): + try: + request = urllib.request.Request( + f"{local_api_base_url}/runtime/actions/pull_tasks", + data=json.dumps({"limit": pull_limit}, ensure_ascii=False).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(request, timeout=max(15, min(60, 10 + pull_limit // 100))) as response: + raw = response.read().decode("utf-8") + response_data = json.loads(raw) if raw else {} + pull_ok = int(response_data.get("code", 1) or 1) == 0 + pull_message = str(response_data.get("message") or "").strip() + pull_data = response_data.get("data") or {} + except Exception as exc: + logger.warning(f"本地队列补货失败: round={round_index}/{max_rounds}, error={exc}") + self._sync_worker_log_event( + "本地队列补货失败", + level='warning', + payload={ + "round": round_index, + "max_rounds": max_rounds, + "pull_limit": pull_limit, + "error": str(exc), + }, + mode='full', + ) + break + + pull_state = str((pull_data or {}).get("pull_state") or "").strip() + queued_count = int((pull_data or {}).get("queued_count", 0) or 0) + worker_start_message = str((pull_data or {}).get("worker_start_message") or "").strip() + refill_message = ( + f"本地队列主动补货: round={round_index}/{max_rounds} | " + f"pull_limit={pull_limit} | ok={1 if pull_ok else 0} | " + f"state={pull_state or 'unknown'} | queued_count={queued_count}" ) - with urllib.request.urlopen(request, timeout=max(15, min(60, 10 + pull_limit // 100))) as response: - raw = response.read().decode("utf-8") - response_data = json.loads(raw) if raw else {} - pull_ok = int(response_data.get("code", 1) or 1) == 0 - pull_message = str(response_data.get("message") or "").strip() - pull_data = response_data.get("data") or {} - except Exception as exc: - logger.warning(f"本地队列补货失败: round={round_index}/{max_rounds}, error={exc}") + if worker_start_message: + refill_message = f"{refill_message} | worker_start={worker_start_message}" + logger.info(f"{refill_message} | message={pull_message}") self._sync_worker_log_event( - "本地队列补货失败", - level='warning', + refill_message, payload={ "round": round_index, "max_rounds": max_rounds, "pull_limit": pull_limit, - "error": str(exc), + "pull_ok": bool(pull_ok), + "pull_state": pull_state, + "queued_count": queued_count, + "message": str(pull_message or "").strip(), + "worker_start_message": worker_start_message, }, mode='full', ) - break - pull_state = str((pull_data or {}).get("pull_state") or "").strip() - queued_count = int((pull_data or {}).get("queued_count", 0) or 0) - worker_start_message = str((pull_data or {}).get("worker_start_message") or "").strip() - refill_message = ( - f"本地队列主动补货: round={round_index}/{max_rounds} | " - f"pull_limit={pull_limit} | ok={1 if pull_ok else 0} | " - f"state={pull_state or 'unknown'} | queued_count={queued_count}" - ) - if worker_start_message: - refill_message = f"{refill_message} | worker_start={worker_start_message}" - logger.info(f"{refill_message} | message={pull_message}") - self._sync_worker_log_event( - refill_message, - payload={ - "round": round_index, - "max_rounds": max_rounds, - "pull_limit": pull_limit, - "pull_ok": bool(pull_ok), - "pull_state": pull_state, - "queued_count": queued_count, - "message": str(pull_message or "").strip(), - "worker_start_message": worker_start_message, - }, - mode='full', - ) + if not claim_after_pull: + if queued_count > 0: + logger.info( + f"主动补货已写入本地队列: round={round_index}/{max_rounds}, " + f"queued_count={queued_count}, claim_after_pull=0" + ) + return [] + else: + domains = self._claim_detect_job_items( + limit=claim_batch_size, + lease_seconds=claim_lease_seconds, + ) + if domains: + logger.info( + f"主动补货后已重新领取任务: round={round_index}/{max_rounds}, " + f"claim_batch_size={claim_batch_size}, domains={len(domains)}" + ) + return domains - domains = self.db.claim_detect_job_items( - config.NODE_CODE, - limit=claim_batch_size, - lease_seconds=claim_lease_seconds, - ) - if domains: - logger.info( - f"主动补货后已重新领取任务: round={round_index}/{max_rounds}, " - f"claim_batch_size={claim_batch_size}, domains={len(domains)}" - ) - return domains + if pull_state == "idle": + break + if not pull_ok and pull_state not in {"ack_warning", "worker_start_warning"}: + break + time.sleep(0.2) - if pull_state == "idle": - break - if not pull_ok and pull_state not in {"ack_warning", "worker_start_warning"}: - break - time.sleep(0.2) + return [] + finally: + self._release_shared_job_maintenance_lock("pull-sync-tasks", maintenance_lock_owner) - return [] + def _resolve_sync_pull_limit(self, current_thread_limit: int) -> int: + configured_pull_limit = int(os.getenv("DOMAINCHECK_SYNC_PULL_LIMIT", "0") or 0) + configured_pull_cap = int(os.getenv("DOMAINCHECK_SYNC_PULL_LIMIT_CAP", "200000") or 200000) + pull_cap = max(10000, configured_pull_cap) + if configured_pull_limit > 0: + return max(1000, min(configured_pull_limit, pull_cap)) + + normalized_thread_limit = max(1, int(current_thread_limit or getattr(self, "thread_count", 1) or 1)) + process_count = max(1, int(self._proxy_runtime_process_count() or 1)) + total_thread_capacity = max( + normalized_thread_limit, + normalized_thread_limit * process_count, + ) + default_pull_limit = max( + 4000, + min(pull_cap, total_thread_capacity * 2), + ) + return max(1000, int(default_pull_limit or 1000)) def _process_pipeline_tasks_until_available(self, *, thread_limit: int) -> list[dict]: """ 当本地任务队列空了时，主动让 controller 处理已完成步骤，尽量把下一步任务补出来。 """ + maintenance_lock_owner = self._acquire_shared_job_maintenance_lock( + "process-pipeline-tasks", + ttl_seconds=20, + ) + if not maintenance_lock_owner: + return [] local_api_base_url = str( os.getenv("DOMAINCHECK_LOCAL_API_BASE_URL", "http://127.0.0.1:8100/api/v1") or "http://127.0.0.1:8100/api/v1" ).strip().rstrip("/") @@ -2414,47 +4301,49 @@ class DetectWorker: claim_lease_seconds = max(300, min(1800, claim_batch_size * 30)) try: - request = urllib.request.Request( - f"{local_api_base_url}/runtime/actions/process_pipeline", - data=json.dumps({"limit": process_limit}, ensure_ascii=False).encode("utf-8"), - headers={"Content-Type": "application/json"}, - method="POST", - ) - with urllib.request.urlopen(request, timeout=20) as response: - raw = response.read().decode("utf-8") - response_data = json.loads(raw) if raw else {} - process_ok = int(response_data.get("code", 1) or 1) == 0 - process_message = str(response_data.get("message") or "").strip() - process_data = response_data.get("data") or {} - logger.info( - f"本地 pipeline 推进: ok={1 if process_ok else 0} | " - f"processed={int(process_data.get('processed_items', 0) or 0)} | " - f"advanced={int(process_data.get('advanced_items', 0) or 0)} | " - f"retried={int(process_data.get('retried_items', 0) or 0)} | " - f"message={process_message}" - ) - except Exception as exc: - logger.warning(f"本地 pipeline 推进失败: {exc}") - self._sync_worker_log_event( - "本地 pipeline 推进失败", - level='warning', - payload={"error": str(exc), "process_limit": process_limit}, - mode='full', - ) - return [] + try: + request = urllib.request.Request( + f"{local_api_base_url}/runtime/actions/process_pipeline", + data=json.dumps({"limit": process_limit}, ensure_ascii=False).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(request, timeout=20) as response: + raw = response.read().decode("utf-8") + response_data = json.loads(raw) if raw else {} + process_ok = int(response_data.get("code", 1) or 1) == 0 + process_message = str(response_data.get("message") or "").strip() + process_data = response_data.get("data") or {} + logger.info( + f"本地 pipeline 推进: ok={1 if process_ok else 0} | " + f"processed={int(process_data.get('processed_items', 0) or 0)} | " + f"advanced={int(process_data.get('advanced_items', 0) or 0)} | " + f"retried={int(process_data.get('retried_items', 0) or 0)} | " + f"message={process_message}" + ) + except Exception as exc: + logger.warning(f"本地 pipeline 推进失败: {exc}") + self._sync_worker_log_event( + "本地 pipeline 推进失败", + level='warning', + payload={"error": str(exc), "process_limit": process_limit}, + mode='full', + ) + return [] - domains = self.db.claim_detect_job_items( - config.NODE_CODE, - limit=claim_batch_size, - lease_seconds=claim_lease_seconds, - ) - if domains: - self._sync_worker_log_event( - "本地 pipeline 推进后已重新领取任务", - payload={"domains": len(domains), "claim_batch_size": claim_batch_size}, - mode='full', + domains = self._claim_detect_job_items( + limit=claim_batch_size, + lease_seconds=claim_lease_seconds, ) - return domains + if domains: + self._sync_worker_log_event( + "本地 pipeline 推进后已重新领取任务", + payload={"domains": len(domains), "claim_batch_size": claim_batch_size}, + mode='full', + ) + return domains + finally: + self._release_shared_job_maintenance_lock("process-pipeline-tasks", maintenance_lock_owner) def load_runtime_settings(self): default_settings = { @@ -2462,17 +4351,72 @@ class DetectWorker: "worker_log_sync_mode": "key", "worker_step_trace_enabled": True, "worker_step_trace_sync_full": True, + "control_node_autoresume_enabled": False, + "claim_recent_jobs_first": False, + "claim_recent_jobs_limit": 0, + "claim_recent_jobs_window_hours": 0, + "claim_batch_floor": 0, + "claim_batch_ceil": 0, + "submit_backlog_floor": 0, + "submit_backlog_ceil": 0, + "dispatch_cap_multiplier": 1, + "pending_buffer_cap_multiplier": 1, } + + def _normalize_bool(value, default=False): + if value is None: + return bool(default) + if isinstance(value, bool): + return value + return str(value or "").strip().lower() not in {"", "0", "false", "no", "off"} + + def _normalize_non_negative_int(value, default=0): + try: + normalized = int(value) + except (TypeError, ValueError): + normalized = int(default) + return max(0, normalized) + + def _normalize_positive_int(value, default=1): + try: + normalized = int(value) + except (TypeError, ValueError): + normalized = int(default) + return max(1, normalized) + + def _normalize_runtime_settings(payload): + runtime_settings = default_settings.copy() + runtime_settings.update(payload or {}) + runtime_settings["worker_log_sync_enabled"] = _normalize_bool(runtime_settings.get("worker_log_sync_enabled", False)) + runtime_settings["worker_log_sync_mode"] = "full" if str(runtime_settings.get("worker_log_sync_mode", "key")).strip().lower() == "full" else "key" + runtime_settings["worker_step_trace_enabled"] = _normalize_bool(runtime_settings.get("worker_step_trace_enabled", True), default=True) + runtime_settings["worker_step_trace_sync_full"] = _normalize_bool(runtime_settings.get("worker_step_trace_sync_full", True), default=True) + runtime_settings["control_node_autoresume_enabled"] = _normalize_bool( + runtime_settings.get("control_node_autoresume_enabled", False), + default=False, + ) + runtime_settings["claim_recent_jobs_first"] = _normalize_bool( + runtime_settings.get("claim_recent_jobs_first", False), + default=False, + ) + for key in ( + "claim_recent_jobs_limit", + "claim_recent_jobs_window_hours", + "claim_batch_floor", + "claim_batch_ceil", + "submit_backlog_floor", + "submit_backlog_ceil", + ): + runtime_settings[key] = _normalize_non_negative_int(runtime_settings.get(key), default_settings[key]) + for key in ("dispatch_cap_multiplier", "pending_buffer_cap_multiplier"): + runtime_settings[key] = _normalize_positive_int(runtime_settings.get(key), default_settings[key]) + return runtime_settings + try: if self.use_redis: runtime_settings_raw = self.redis_client.get(RUNTIME_SETTINGS_KEY) if runtime_settings_raw: - runtime_settings = default_settings.copy() - runtime_settings.update(json.loads(runtime_settings_raw)) - runtime_settings["worker_log_sync_enabled"] = bool(runtime_settings.get("worker_log_sync_enabled", False)) - runtime_settings["worker_log_sync_mode"] = "full" if str(runtime_settings.get("worker_log_sync_mode", "key")).strip().lower() == "full" else "key" - runtime_settings["worker_step_trace_enabled"] = bool(runtime_settings.get("worker_step_trace_enabled", True)) - runtime_settings["worker_step_trace_sync_full"] = bool(runtime_settings.get("worker_step_trace_sync_full", True)) + runtime_settings = _normalize_runtime_settings(json.loads(runtime_settings_raw)) logger.info(f"从Redis加载运行时设置成功: {runtime_settings}") return runtime_settings @@ -2484,18 +4428,129 @@ class DetectWorker: for candidate_path in candidate_paths: if os.path.exists(candidate_path): with open(candidate_path, 'r', encoding='utf-8') as f: - runtime_settings = default_settings.copy() - runtime_settings.update(json.load(f)) - runtime_settings["worker_log_sync_enabled"] = bool(runtime_settings.get("worker_log_sync_enabled", False)) - runtime_settings["worker_log_sync_mode"] = "full" if str(runtime_settings.get("worker_log_sync_mode", "key")).strip().lower() == "full" else "key" - runtime_settings["worker_step_trace_enabled"] = bool(runtime_settings.get("worker_step_trace_enabled", True)) - runtime_settings["worker_step_trace_sync_full"] = bool(runtime_settings.get("worker_step_trace_sync_full", True)) + runtime_settings = _normalize_runtime_settings(json.load(f)) logger.info(f"从本地文件加载运行时设置成功: {runtime_settings}") return runtime_settings except Exception as e: logger.error(f"加载运行时设置失败: {e}") return default_settings + def _runtime_optional_int_override(self, runtime_key, env_key): + runtime_settings = self.runtime_settings or {} + try: + runtime_value = int(runtime_settings.get(runtime_key, 0) or 0) + except (TypeError, ValueError): + runtime_value = 0 + if runtime_value > 0: + return runtime_value + try: + return max(0, int(os.getenv(env_key, "0") or 0)) + except (TypeError, ValueError): + return 0 + + def _runtime_bool_override(self, runtime_key, env_key, default=False): + env_value = os.getenv(env_key) + if env_value is not None: + return self._normalize_flag_value(env_value, default=default) + runtime_settings = self.runtime_settings or {} + if runtime_key in runtime_settings: + return self._normalize_flag_value(runtime_settings.get(runtime_key, default), default=default) + return bool(default) + + def _runtime_positive_int_override(self, runtime_key, env_key, default=1): + runtime_settings = self.runtime_settings or {} + try: + runtime_value = int(runtime_settings.get(runtime_key, 0) or 0) + except (TypeError, ValueError): + runtime_value = 0 + if runtime_value > 0: + return runtime_value + try: + return max(1, int(os.getenv(env_key, str(default)) or default)) + except (TypeError, ValueError): + return max(1, int(default)) + + def _claim_recent_jobs_first_enabled(self): + return self._runtime_bool_override( + "claim_recent_jobs_first", + "DOMAINCHECK_CLAIM_RECENT_JOBS_FIRST", + default=False, + ) + + def _preferred_claim_job_limit(self): + return max(1, self._runtime_optional_int_override( + "claim_recent_jobs_limit", + "DOMAINCHECK_CLAIM_RECENT_JOBS_LIMIT", + ) or 8) + + def _preferred_claim_job_window_hours(self): + return max(1, self._runtime_optional_int_override( + "claim_recent_jobs_window_hours", + "DOMAINCHECK_CLAIM_RECENT_JOBS_WINDOW_HOURS", + ) or 24) + + def _claim_detect_job_items(self, *, limit, lease_seconds, job_id=None): + if job_id in (None, "", 0, "0") and not self._is_single_step_session_active(): + handoff_job_id = self._restart_release_handoff_job_id_active() + if handoff_job_id: + if not self._is_primary_job_maintenance_worker(): + return [] + handoff_limit = self._restart_release_handoff_batch_limit(limit) + handoff_rows = self.db.claim_restart_released_detect_job_items( + config.NODE_CODE, + handoff_job_id, + limit=handoff_limit, + lease_seconds=lease_seconds, + ) + if handoff_rows: + logger.warning( + f"优先回收重启尾批任务成功: job_id={handoff_job_id}, " + f"claimed={len(handoff_rows)}, reason={self._restart_release_handoff_reason or 'worker_restart'}" + ) + remaining = max(0, int(limit or 0) - len(handoff_rows)) + if remaining <= 0: + return handoff_rows + prefer_recent_jobs = self._claim_recent_jobs_first_enabled() + fallback_rows = self.db.claim_detect_job_items( + config.NODE_CODE, + limit=remaining, + lease_seconds=lease_seconds, + job_id=None, + prefer_recent_jobs=prefer_recent_jobs, + preferred_recent_job_limit=self._preferred_claim_job_limit(), + preferred_recent_job_window_hours=self._preferred_claim_job_window_hours(), + ) + return list(handoff_rows or []) + list(fallback_rows or []) + self._clear_restart_release_handoff(reason="handoff_queue_drained") + explicit_scope_job_id = self._explicit_claim_scope_job_id_active() + if explicit_scope_job_id not in (None, "", 0, "0"): + scoped_rows = self.db.claim_detect_job_items( + config.NODE_CODE, + limit=limit, + lease_seconds=lease_seconds, + job_id=explicit_scope_job_id, + prefer_recent_jobs=False, + preferred_recent_job_limit=self._preferred_claim_job_limit(), + preferred_recent_job_window_hours=self._preferred_claim_job_window_hours(), + ) + self._log_explicit_scope_claim_result( + scoped_job_id=explicit_scope_job_id, + limit=limit, + rows_count=len(scoped_rows or []), + reason="explicit_scope", + ) + return scoped_rows + prefer_recent_jobs = bool(job_id in (None, "", 0, "0") and self._claim_recent_jobs_first_enabled()) + return self.db.claim_detect_job_items( + config.NODE_CODE, + limit=limit, + lease_seconds=lease_seconds, + job_id=job_id, + prefer_recent_jobs=prefer_recent_jobs, + preferred_recent_job_limit=self._preferred_claim_job_limit(), + preferred_recent_job_window_hours=self._preferred_claim_job_window_hours(), + ) + def _load_sensitive_words_runtime(self): try: if self.use_redis and self.redis_client is not None: @@ -2562,10 +4617,36 @@ class DetectWorker: return f"{base_url}/debug-ingest" return f"{base_url}/api/v1/runtime/debug-ingest" + @staticmethod + def _remote_debug_event_timeout_seconds(): + configured_timeout = float(os.getenv("DOMAINCHECK_REMOTE_DEBUG_EVENT_TIMEOUT", "0") or 0.0) + if configured_timeout > 0: + return max(0.2, min(configured_timeout, 10.0)) + if str(getattr(config, "NODE_ROLE", "") or "").strip() == "control": + return 1.2 + return 2.0 + + @staticmethod + def _remote_debug_event_cooldown_seconds(failure_streak): + normalized_streak = max(1, int(failure_streak or 1)) + base_seconds = max( + 5.0, + float(os.getenv("DOMAINCHECK_REMOTE_DEBUG_EVENT_COOLDOWN_BASE_SECONDS", "15") or 15.0), + ) + max_seconds = max( + base_seconds, + float(os.getenv("DOMAINCHECK_REMOTE_DEBUG_EVENT_COOLDOWN_MAX_SECONDS", "180") or 180.0), + ) + return min(max_seconds, base_seconds * (2 ** max(0, normalized_streak - 1))) + def _push_remote_debug_event(self, *, message, level='info', payload=None, event_type='worker_log'): ingest_url = self._remote_debug_ingest_url() if not ingest_url: return + now_ts = time.time() + cooldown_until = float(getattr(self, "_remote_debug_event_cooldown_until", 0.0) or 0.0) + if cooldown_until > now_ts: + return request_payload = { "source_region": config.NODE_REGION, "node_code": config.NODE_CODE, @@ -2589,19 +4670,81 @@ class DetectWorker: method="POST", ) try: - with urllib.request.urlopen(request, timeout=REMOTE_DEBUG_EVENT_TIMEOUT) as response: + timeout_seconds = self._remote_debug_event_timeout_seconds() + with urllib.request.urlopen(request, timeout=timeout_seconds) as response: raw = response.read().decode("utf-8", errors="ignore") if raw: parsed = json.loads(raw) if isinstance(parsed, dict) and parsed.get("code") not in (0, "0", None, ""): logger.debug(f"远端调试事件返回非成功: {parsed}") + self._remote_debug_event_failure_streak = 0 + self._remote_debug_event_cooldown_until = 0.0 except Exception as e: - logger.debug(f"远端调试事件回传失败: {e}") + failure_streak = int(getattr(self, "_remote_debug_event_failure_streak", 0) or 0) + 1 + cooldown_seconds = self._remote_debug_event_cooldown_seconds(failure_streak) + self._remote_debug_event_failure_streak = failure_streak + self._remote_debug_event_cooldown_until = now_ts + cooldown_seconds + last_notice_at = float(getattr(self, "_last_remote_debug_event_notice_at", 0.0) or 0.0) + if failure_streak <= 2 or now_ts - last_notice_at >= max(10.0, min(cooldown_seconds, 30.0)): + logger.debug( + f"远端调试事件回传失败，进入冷却 {int(cooldown_seconds)} 秒: {e}" + ) + self._last_remote_debug_event_notice_at = now_ts def _worker_log_sync_mode(self): if not bool((self.runtime_settings or {}).get("worker_log_sync_enabled", False)): return "off" - return "full" if str((self.runtime_settings or {}).get("worker_log_sync_mode", "key")).strip().lower() == "full" else "key" + configured_mode = ( + "full" + if str((self.runtime_settings or {}).get("worker_log_sync_mode", "key")).strip().lower() == "full" + else "key" + ) + if configured_mode != "full": + return configured_mode + degrade_reason = self._worker_log_sync_full_degrade_reason() + if not degrade_reason: + return configured_mode + now_ts = time.time() + last_notice_at = float(getattr(self, "_last_worker_log_sync_degrade_notice_at", 0.0) or 0.0) + if now_ts - last_notice_at >= 10.0: + logger.warning(f"Worker 全量日志同步已自动降级为 key: {degrade_reason}") + self._last_worker_log_sync_degrade_notice_at = now_ts + return "key" + + def _worker_log_sync_full_active_thread_threshold(self): + configured_threshold = int(os.getenv("DOMAINCHECK_WORKER_LOG_SYNC_FULL_MAX_ACTIVE_THREADS", "0") or 0) + if configured_threshold > 0: + return max(32, configured_threshold) + thread_limit = max(1, int(getattr(self, "thread_count", 0) or 0)) + return max(120, min(600, max(120, thread_limit // 3))) + + def _worker_log_sync_full_queue_threshold(self): + configured_threshold = int(os.getenv("DOMAINCHECK_WORKER_LOG_SYNC_FULL_MAX_QUEUE", "0") or 0) + if configured_threshold > 0: + return max(256, configured_threshold) + queue_maxsize = int(getattr(getattr(self, "_worker_log_sync_queue", None), "maxsize", 0) or 0) + if queue_maxsize > 0: + return max(1000, min(15000, queue_maxsize // 5)) + return 5000 + + def _worker_log_sync_full_degrade_reason(self): + queue_obj = getattr(self, "_worker_log_sync_queue", None) + queue_depth = 0 + if queue_obj is not None: + try: + queue_depth = max(0, int(queue_obj.qsize() or 0)) + except Exception: + queue_depth = 0 + queue_threshold = self._worker_log_sync_full_queue_threshold() + if queue_depth >= queue_threshold: + return f"queue_backlog={queue_depth}, threshold={queue_threshold}" + + live_active_threads = max(0, int(self._get_active_domain_threads() or 0)) + active_threshold = self._worker_log_sync_full_active_thread_threshold() + if bool(getattr(self, "detecting", False)) and live_active_threads >= active_threshold: + return f"active_threads={live_active_threads}, threshold={active_threshold}" + + return "" def _worker_step_trace_enabled(self): return bool((self.runtime_settings or {}).get("worker_step_trace_enabled", True)) @@ -2697,6 +4840,15 @@ class DetectWorker: return False return bool(force_map.pop(normalized_step_name, False)) + def _peek_step_force_direct_once(self, step_name): + normalized_step_name = str(step_name or "").strip() + if not normalized_step_name: + return False + force_map = getattr(self._task_local, "step_force_direct_once", None) + if not isinstance(force_map, dict): + return False + return bool(force_map.get(normalized_step_name, False)) + def _cache_single_step_result_payload(self, domain_id, field_name, payload): normalized_field_name = str(field_name or "").strip() if not normalized_field_name: @@ -2855,7 +5007,7 @@ class DetectWorker: if state == "blacklisted": return "blacklisted", message or "步骤命中黑名单" if state == "degraded": - return "failed", message or "步骤外部依赖异常，建议重试" + return "completed", message or "步骤外部依赖异常，建议重试" if state in {"failed", "error", "rejected"}: return "failed", message or "步骤执行失败" if status is False: @@ -3067,20 +5219,214 @@ class DetectWorker: ensure_ascii=False, ) - def _is_proxy_item_expired(self, proxy_item): + def _proxy_item_expire_timestamp(self, proxy_item): if not isinstance(proxy_item, dict): - return True - now_ts = time.time() + return 0.0 expire_at_ms = proxy_item.get("expire_at_ms") expire_at = proxy_item.get("expire_at") try: if expire_at_ms not in (None, "", 0, "0"): - return float(expire_at_ms) / 1000.0 <= now_ts + return float(expire_at_ms) / 1000.0 if expire_at not in (None, "", 0, "0"): - return float(expire_at) <= now_ts + return float(expire_at) except Exception: + return 0.0 + return 0.0 + + def _proxy_entry_expire_timestamp(self, proxy_entry): + if isinstance(proxy_entry, dict): + expire_at_ts = proxy_entry.get("expire_at_ts") + try: + if expire_at_ts not in (None, "", 0, "0"): + return float(expire_at_ts) + except Exception: + return 0.0 + raw_item = proxy_entry.get("raw_item") + if isinstance(raw_item, dict): + return float(self._proxy_item_expire_timestamp(raw_item) or 0.0) + return float(self._proxy_item_expire_timestamp(proxy_entry) or 0.0) + + def _proxy_remaining_ttl_seconds(self, proxy_entry, *, now_ts=None): + expire_at_ts = float(self._proxy_entry_expire_timestamp(proxy_entry) or 0.0) + if expire_at_ts <= 0: + return float("inf") + observed_now_ts = float(now_ts or time.time()) + return max(0.0, expire_at_ts - observed_now_ts) + + def _proxy_min_reuse_ttl_seconds(self, step_name=""): + normalized_step_name = str(step_name or "").strip() + override_map = { + "注册状态检测": float(os.getenv("DOMAINCHECK_PROXY_REUSE_MIN_TTL_REGISTER", "6") or 6), + "百度site检测": float(os.getenv("DOMAINCHECK_PROXY_REUSE_MIN_TTL_BAIDU", "8") or 8), + "360检测": float(os.getenv("DOMAINCHECK_PROXY_REUSE_MIN_TTL_360", "8") or 8), + "站长之家检测": float(os.getenv("DOMAINCHECK_PROXY_REUSE_MIN_TTL_CHINAZ", "10") or 10), + "爱站网检测": float(os.getenv("DOMAINCHECK_PROXY_REUSE_MIN_TTL_AIZHAN", "10") or 10), + "聚查检测": float(os.getenv("DOMAINCHECK_PROXY_REUSE_MIN_TTL_JUCHA", "12") or 12), + "桔子检测": float(os.getenv("DOMAINCHECK_PROXY_REUSE_MIN_TTL_JUZISEO", "12") or 12), + } + if normalized_step_name in override_map: + return max(0.0, float(override_map.get(normalized_step_name, 0.0) or 0.0)) + configured_default = float(os.getenv("DOMAINCHECK_PROXY_REUSE_MIN_TTL_SECONDS", "8") or 8) + return max(0.0, configured_default) + + def _proxy_entry_can_be_reused(self, proxy_entry, step_name="", *, now_ts=None): + remaining_ttl_seconds = self._proxy_remaining_ttl_seconds(proxy_entry, now_ts=now_ts) + if remaining_ttl_seconds == float("inf"): + return True + return remaining_ttl_seconds >= self._proxy_min_reuse_ttl_seconds(step_name) + + def _store_proxy_active_lease(self, proxy_entry, *, step_name="", already_locked=False): + if not isinstance(proxy_entry, dict): + return + proxy = proxy_entry.get("proxy") + proxy_key = self._proxy_key(proxy) + if not proxy or not proxy_key: + return + + def _store(): + active_leases = getattr(self, "proxy_active_leases", None) + if active_leases is None: + self.proxy_active_leases = {} + active_leases = self.proxy_active_leases + lease_entry = dict(proxy_entry) + lease_entry["leased_at"] = time.time() + lease_entry["leased_step_name"] = str(step_name or proxy_entry.get("leased_step_name") or "").strip() + active_leases[proxy_key] = lease_entry + + if already_locked: + _store() + return + with self.proxy_pool_lock: + _store() + + def _pop_proxy_active_lease(self, proxy, *, already_locked=False): + proxy_key = self._proxy_key(proxy) + if not proxy_key: + return None + + def _pop(): + active_leases = getattr(self, "proxy_active_leases", None) + if not isinstance(active_leases, dict): + return None + return active_leases.pop(proxy_key, None) + + if already_locked: + return _pop() + with self.proxy_pool_lock: + return _pop() + + def release_proxy(self, proxy, *, step_name="", discard=False, reason=""): + if not proxy: return False - return False + + with self.proxy_pool_lock: + lease_entry = self._pop_proxy_active_lease(proxy, already_locked=True) + if not isinstance(lease_entry, dict): + return False + remaining_count = len(self.proxy_pool) + + normalized_step_name = str(step_name or lease_entry.get("leased_step_name") or "").strip() + source_tag = str(lease_entry.get("source_tag") or "").strip() + reusable = ( + not discard + and not self._is_proxy_quarantined(proxy) + and not self._is_proxy_source_quarantined(source_tag) + and self._proxy_entry_can_be_reused(lease_entry, normalized_step_name) + ) + if reusable: + lease_entry.pop("leased_at", None) + lease_entry.pop("leased_step_name", None) + with self.proxy_pool_lock: + self.proxy_pool.append(lease_entry) + remaining_count = len(self.proxy_pool) + + self._schedule_proxy_refresh_if_needed(remaining_count) + if not reusable: + remaining_ttl_seconds = self._proxy_remaining_ttl_seconds(lease_entry) + if remaining_ttl_seconds != float("inf"): + logger.debug( + f"代理归还时已丢弃: step={normalized_step_name or '-'} " + f"ttl={int(remaining_ttl_seconds)}s reason={reason or ('discard' if discard else 'ttl_low')}" + ) + return reusable + + def _is_proxy_item_expired(self, proxy_item, *, grace_seconds=0.0): + if not isinstance(proxy_item, dict): + return True + now_ts = time.time() + expire_at_ts = float(self._proxy_item_expire_timestamp(proxy_item) or 0.0) + if expire_at_ts <= 0: + return False + return expire_at_ts <= (now_ts - max(0.0, float(grace_seconds or 0.0))) + + def _summarize_proxy_expire_clock_skew(self, proxy_items, *, now_ts=None): + if not isinstance(proxy_items, list) or not proxy_items: + return None + observed_now_ts = float(now_ts or time.time()) + expire_timestamps = [] + for proxy_item in proxy_items: + expire_at_ts = float(self._proxy_item_expire_timestamp(proxy_item) or 0.0) + if expire_at_ts > 0: + expire_timestamps.append(expire_at_ts) + if not expire_timestamps: + return None + + latest_expire_at = max(expire_timestamps) + earliest_expire_at = min(expire_timestamps) + expired_timed_count = sum(1 for expire_at_ts in expire_timestamps if expire_at_ts <= observed_now_ts) + # 所有带过期时间的代理都比本机时间早很多时，优先提示“时钟异常/源数据陈旧”， + # 避免把问题误判成单纯的代理质量差。 + if latest_expire_at > observed_now_ts - 900: + return None + if expired_timed_count < max(12, int(len(expire_timestamps) * 0.8)): + return None + + return { + "observed_now_ts": observed_now_ts, + "observed_now_iso": datetime.fromtimestamp(observed_now_ts).strftime("%Y-%m-%d %H:%M:%S"), + "timed_proxy_count": len(expire_timestamps), + "expired_timed_count": expired_timed_count, + "latest_expire_at": latest_expire_at, + "latest_expire_at_iso": datetime.fromtimestamp(latest_expire_at).strftime("%Y-%m-%d %H:%M:%S"), + "earliest_expire_at": earliest_expire_at, + "earliest_expire_at_iso": datetime.fromtimestamp(earliest_expire_at).strftime("%Y-%m-%d %H:%M:%S"), + "clock_ahead_seconds": max(0.0, observed_now_ts - latest_expire_at), + } + + def _proxy_recently_expired_reuse_grace_seconds(self): + configured_grace = float(os.getenv("DOMAINCHECK_PROXY_STALE_REUSE_GRACE_SECONDS", "0") or 0.0) + if configured_grace > 0: + return max(0.0, configured_grace) + if not self.detecting: + return 0.0 + demand_threads = self._proxy_demand_threads() + if demand_threads >= 1600: + return 180.0 + if demand_threads >= 1000: + return 120.0 + return 90.0 + + def _proxy_recently_expired_backfill_floor(self): + configured_floor = int(os.getenv("DOMAINCHECK_PROXY_STALE_BACKFILL_FLOOR", "0") or 0) + if configured_floor > 0: + return max(8, configured_floor) + demand_threads = self._proxy_demand_threads() + return max(32, min(160, max(32, demand_threads // 12))) + + def _backfill_recently_expired_proxy_entries(self, current_entries, stale_entries): + merged_entries = list(current_entries or []) + normalized_stale_entries = list(stale_entries or []) + if not self.detecting or not normalized_stale_entries: + return merged_entries, 0, 0 + floor = self._proxy_recently_expired_backfill_floor() + if len(merged_entries) >= floor: + return merged_entries, 0, floor + before_count = len(merged_entries) + needed = max(0, floor - before_count) + if needed <= 0: + return merged_entries, 0, floor + merged_entries = self._merge_proxy_entries(normalized_stale_entries[:needed], merged_entries) + return merged_entries, max(0, len(merged_entries) - before_count), floor def _set_proxy_request_count(self, raw_url, count): normalized_url = str(raw_url or "").strip() @@ -3113,17 +5459,25 @@ class DetectWorker: } demand_threads = self._proxy_demand_threads() + process_count = self._proxy_runtime_process_count() source_count = len(source_urls) provider_batch_cap = max(20, int(os.getenv("DOMAINCHECK_PROXY_PROVIDER_BATCH_CAP", "140") or 140)) + provider_batch_floor = max( + 20, + min( + provider_batch_cap, + int(round(provider_batch_cap / self._proxy_multi_process_divisor())), + ), + ) current_pool_size = max(0, int(current_pool_size or 0)) refresh_threshold = self._proxy_refresh_threshold() shortage = max(0, refresh_threshold - current_pool_size) - headroom = max(source_count * 20, int(refresh_threshold * 0.15)) + headroom = max(source_count * 10, int(refresh_threshold * 0.15)) active_round_cap = max(1, int(os.getenv("DOMAINCHECK_PROXY_MAX_ROUNDS_ACTIVE", "2") or 2)) idle_round_cap = max(2, int(os.getenv("DOMAINCHECK_PROXY_MAX_ROUNDS_IDLE", "4") or 4)) round_cap = active_round_cap if self.detecting else idle_round_cap target_total = max( - source_count * provider_batch_cap, + source_count * provider_batch_floor, shortage + headroom, ) per_round_capacity = max(1, source_count * provider_batch_cap) @@ -3153,9 +5507,11 @@ class DetectWorker: "rounds": rounds, "source_count": source_count, "demand_threads": demand_threads, + "process_count": process_count, "current_pool_size": current_pool_size, "refresh_threshold": refresh_threshold, "shortage": shortage, + "provider_batch_floor": provider_batch_floor, } def _proxy_source_fetch_timeout_seconds(self): @@ -3295,6 +5651,8 @@ class DetectWorker: # 调用方可能已经持有 proxy_pool_lock；避免对同一把非重入锁再次加锁。 current_count = int(available_count or 0) threshold = self._proxy_refresh_threshold() + if float(getattr(self, "proxy_next_refresh_time", 0.0) or 0.0) > time.time(): + return if current_count < threshold and not self.proxy_refresh_lock.locked(): threading.Thread(target=self.refresh_proxy_pool, daemon=True).start() @@ -3304,7 +5662,7 @@ class DetectWorker: proxy_urls = [self.proxy_config.get('proxy_url', '')] return bool([url for url in proxy_urls if str(url or '').strip()]) - def trigger_proxy_refresh(self, reason="", reset_cooldown=False): + def trigger_proxy_refresh(self, reason="", reset_cooldown=False, debounce_seconds=None): """ 代理配置一旦下发，就允许在非检测态主动补一次刷新，避免页面长期停在“未刷新”。 """ @@ -3312,12 +5670,32 @@ class DetectWorker: return False if not self._has_proxy_sources(): return False + normalized_reason = str(reason or "manual").strip() or "manual" + if debounce_seconds is None and normalized_reason.startswith("config_update:"): + debounce_seconds = self._config_update_proxy_refresh_debounce_seconds() + debounce_seconds = max(0.0, float(debounce_seconds or 0.0)) + refresh_reason_timestamps = getattr(self, "_last_proxy_refresh_reason_at", None) + if not isinstance(refresh_reason_timestamps, dict): + refresh_reason_timestamps = {} + self._last_proxy_refresh_reason_at = refresh_reason_timestamps + trigger_key = "config_update" if normalized_reason.startswith("config_update:") else normalized_reason + now_ts = time.time() + last_trigger_at = float(refresh_reason_timestamps.get(trigger_key, 0.0) or 0.0) + if debounce_seconds > 0 and (now_ts - last_trigger_at) < debounce_seconds: + logger.debug( + f"代理池刷新触发去抖，跳过主动触发: {normalized_reason}, cooldown={debounce_seconds:.1f}s" + ) + return False if reset_cooldown: self.proxy_next_refresh_time = 0.0 - if self.proxy_refresh_lock.locked(): - logger.debug(f"代理池刷新已在进行中，跳过主动触发: {reason or 'manual'}") + elif float(getattr(self, "proxy_next_refresh_time", 0.0) or 0.0) > time.time(): + logger.debug(f"代理池仍在冷却窗口，跳过主动触发: {normalized_reason}") return False - logger.info(f"主动触发代理池刷新: {reason or 'manual'}") + if self.proxy_refresh_lock.locked(): + logger.debug(f"代理池刷新已在进行中，跳过主动触发: {normalized_reason}") + return False + refresh_reason_timestamps[trigger_key] = now_ts + logger.info(f"主动触发代理池刷新: {normalized_reason}") threading.Thread(target=self.refresh_proxy_pool, daemon=True).start() return True @@ -3326,7 +5704,13 @@ class DetectWorker: 代理池的补货阈值只用于触发后台刷新，不应反向限制并发。 """ demand_threads = self._proxy_demand_threads() - return max(840, demand_threads * 3) + base_threshold = max(840, demand_threads * 3) + process_count = self._proxy_runtime_process_count() + if process_count <= 1: + return base_threshold + scaled_threshold = int(round(base_threshold / self._proxy_multi_process_divisor())) + process_floor = max(120, min(480, max(120, demand_threads // 4))) + return max(process_floor, scaled_threshold) def _proxy_demand_threads(self): configured_threads = max(1, int(getattr(self, "thread_count", 0) or 0)) @@ -3367,6 +5751,32 @@ class DetectWorker: with self.proxy_pool_lock: return bool(self.proxy_pool) + def _prepare_proxy_pool_for_detection_start(self): + """ + 检测会话启动时，优先复用当前可用代理池/共享快照。 + 只有在本地和共享快照都为空时，才做同步刷新。 + 否则 large controller 会在每轮 auto-resume 前都卡一次 refresh_proxy_pool， + 线程池还没起就先停在 refreshing_proxy。 + """ + with self.proxy_pool_lock: + current_pool_size = len(self.proxy_pool) + if current_pool_size > 0: + self._schedule_proxy_refresh_if_needed(current_pool_size) + return True, current_pool_size, "local_pool_ready" + + if self._proxy_should_coordinate_shared_refresh(): + shared_payload = self._load_shared_proxy_snapshot_payload() + if shared_payload and self._apply_shared_proxy_snapshot( + shared_payload, + status_prefix="检测启动复用共享代理快照", + ): + with self.proxy_pool_lock: + shared_pool_size = len(self.proxy_pool) + self._schedule_proxy_refresh_if_needed(shared_pool_size) + return True, shared_pool_size, "shared_snapshot_ready" + + return False, 0, "proxy_pool_empty" + def refresh_proxy_pool(self): """ 刷新代理池 @@ -3375,6 +5785,7 @@ class DetectWorker: logger.debug("代理池刷新已在进行中，跳过本次重复刷新") return + shared_refresh_owner = "" if not self.proxy_config.get('proxy_enable', False): self.proxy_last_refresh_time = datetime.now() self.proxy_last_refresh_status = "代理未启用" @@ -3395,9 +5806,25 @@ class DetectWorker: try: now_ts = time.time() retry_delay_seconds = 0.15 if self.detecting else min(self.proxy_refresh_cooldown_seconds, 10) + holdoff_seconds = self._proxy_refresh_holdoff_seconds() + refresh_threshold = self._proxy_refresh_threshold() with self.proxy_pool_lock: has_cached_proxies = bool(self.proxy_pool) cached_proxy_pool = list(self.proxy_pool) + + if self._proxy_should_coordinate_shared_refresh(): + shared_payload = self._load_shared_proxy_snapshot_payload() + if shared_payload: + shared_entries = shared_payload.get("proxy_pool") or [] + if self._shared_proxy_snapshot_should_replace_local_pool( + len(shared_entries), + len(cached_proxy_pool), + refresh_threshold, + ): + if self._apply_shared_proxy_snapshot(shared_payload): + self.proxy_next_refresh_time = time.time() + holdoff_seconds + return + if not has_cached_proxies and self.proxy_next_refresh_time and now_ts < self.proxy_next_refresh_time: wait_seconds = int(max(1, self.proxy_next_refresh_time - now_ts)) self.proxy_last_refresh_status = f"冷却中，{wait_seconds} 秒后再试" @@ -3409,6 +5836,34 @@ class DetectWorker: self._sync_worker_log_event(f"代理池刷新冷却中，{wait_seconds} 秒后再试", mode='full') return + if self._proxy_should_coordinate_shared_refresh(): + shared_refresh_owner = self._acquire_shared_proxy_refresh_lock() + if not shared_refresh_owner: + if self._wait_for_shared_proxy_snapshot(): + self.proxy_next_refresh_time = time.time() + holdoff_seconds + return + if cached_proxy_pool: + with self.proxy_pool_lock: + self.proxy_pool = cached_proxy_pool + self.proxy_last_refresh_time = datetime.now() + self.proxy_last_refresh_total_items = 0 + self.proxy_last_validated_count = 0 + self.proxy_last_available_count = len(self.proxy_pool) + self.proxy_last_refresh_status = f"共享刷新进行中，继续沿用缓存 {len(cached_proxy_pool)} 个" + else: + self.proxy_last_refresh_time = datetime.now() + self.proxy_last_refresh_total_items = 0 + self.proxy_last_validated_count = 0 + self.proxy_last_available_count = 0 + self.proxy_last_refresh_status = "共享代理刷新进行中，等待补货" + self.proxy_next_refresh_time = time.time() + retry_delay_seconds + holdoff_seconds + self._update_runtime_state( + "refreshing_proxy" if self.detecting else "idle", + self.proxy_last_refresh_status, + ) + self._sync_worker_log_event(self.proxy_last_refresh_status, mode='key') + return + proxy_api_urls = self.proxy_config.get('proxy_urls') or [] if not proxy_api_urls and self.proxy_config.get('proxy_url'): proxy_api_urls = [self.proxy_config.get('proxy_url', '')] @@ -3421,6 +5876,7 @@ class DetectWorker: batch_size = int(fetch_plan_meta.get("batch_size", 0) or 0) target_total = int(fetch_plan_meta.get("target_total", 0) or 0) rounds = int(fetch_plan_meta.get("rounds", 0) or 0) + refresh_threshold = int(fetch_plan_meta.get("refresh_threshold", 0) or refresh_threshold) self.proxy_last_refresh_source_count = source_count self.proxy_last_source_stats = [] if proxy_fetch_plan: @@ -3567,6 +6023,7 @@ class DetectWorker: seen_proxy_strings = set() expired_count = 0 invalid_count = 0 + stale_reuse_grace_seconds = self._proxy_recently_expired_reuse_grace_seconds() for proxy_item in proxy_list: signature = self._proxy_signature_from_item(proxy_item) @@ -3591,16 +6048,53 @@ class DetectWorker: }, 'usage_count': 0, 'source_tag': source_tag, + 'expire_at_ts': float(self._proxy_item_expire_timestamp(proxy_item) or 0.0), + 'raw_item': dict(proxy_item), } if self._is_proxy_item_expired(proxy_item): expired_count += 1 - fallback_stale_proxies.append(proxy_entry) + if ( + stale_reuse_grace_seconds > 0 + and not self._is_proxy_item_expired( + proxy_item, + grace_seconds=stale_reuse_grace_seconds, + ) + ): + fallback_stale_proxies.append(proxy_entry) continue new_proxies.append(proxy_entry) + clock_skew_hint = self._summarize_proxy_expire_clock_skew(proxy_list) + if clock_skew_hint: + now_ts = time.time() + last_warning_at = float(getattr(self, "_last_proxy_clock_skew_warning_at", 0.0) or 0.0) + if now_ts - last_warning_at >= 30.0: + self._last_proxy_clock_skew_warning_at = now_ts + warning_message = ( + "疑似系统时钟异常或代理源返回陈旧数据: " + f"本机时间 {clock_skew_hint['observed_now_iso']}，" + f"代理最晚过期 {clock_skew_hint['latest_expire_at_iso']}，" + f"偏移约 {int(clock_skew_hint['clock_ahead_seconds'])} 秒" + ) + logger.warning( + warning_message + + ( + f"，带过期时间代理 {clock_skew_hint['timed_proxy_count']} 个，" + f"其中已过期 {clock_skew_hint['expired_timed_count']} 个" + ) + ) + self._sync_worker_log_event( + warning_message, + level='warning', + payload=clock_skew_hint, + mode='key', + ) + fallback_stale_count = 0 + stale_backfill_count = 0 + stale_backfill_floor = 0 if not new_proxies and fallback_stale_proxies and not cached_proxy_pool: new_proxies = fallback_stale_proxies fallback_stale_count = len(new_proxies) @@ -3611,6 +6105,16 @@ class DetectWorker: elif cached_proxy_pool: merged_proxy_pool = list(cached_proxy_pool) + if fallback_stale_proxies and stale_reuse_grace_seconds > 0: + ( + merged_proxy_pool, + stale_backfill_count, + stale_backfill_floor, + ) = self._backfill_recently_expired_proxy_entries( + merged_proxy_pool, + fallback_stale_proxies, + ) + with self.proxy_pool_lock: self.proxy_pool = merged_proxy_pool source_map = {} @@ -3621,25 +6125,45 @@ class DetectWorker: ).strip() with self.proxy_failure_lock: self.proxy_source_by_key = source_map - self.proxy_last_refresh_time = datetime.now() - self.proxy_last_available_count = len(self.proxy_pool) + self.proxy_last_refresh_time = datetime.now() + self.proxy_last_available_count = len(self.proxy_pool) + if new_proxies: + self._publish_shared_proxy_snapshot( + merged_proxy_pool, + source_count=source_count, + raw_items=self.proxy_last_refresh_total_items, + ) if new_proxies: if fallback_stale_count > 0: self.proxy_last_refresh_status = ( f"疑似过期回退增量入池 {len(new_proxies)} 个，当前池 {len(self.proxy_pool)} 个（跳过预验证）" ) + elif stale_backfill_count > 0: + self.proxy_last_refresh_status = ( + f"最近过期代理回补 {stale_backfill_count} 个，当前池 {len(self.proxy_pool)} 个" + ) else: self.proxy_last_refresh_status = ( f"增量入池 {len(new_proxies)} 个，当前池 {len(self.proxy_pool)} 个（跳过预验证）" ) - self.proxy_next_refresh_time = 0.0 + self.proxy_next_refresh_time = ( + 0.0 if len(self.proxy_pool) >= refresh_threshold else (time.time() + holdoff_seconds) + ) elif cached_proxy_pool: self.proxy_last_refresh_status = f"本轮未拿到新代理，继续沿用缓存 {len(cached_proxy_pool)} 个" - next_retry_delay = 0.5 if (self.detecting and aggressive_proxy_rotation and cooldown_detected) else (2 if cooldown_detected else retry_delay_seconds) + next_retry_delay = ( + 0.5 if (self.detecting and aggressive_proxy_rotation and cooldown_detected) + else (2 if cooldown_detected else retry_delay_seconds) + ) + next_retry_delay += holdoff_seconds self.proxy_next_refresh_time = time.time() + next_retry_delay else: self.proxy_last_refresh_status = "代理源暂时冷却中，稍后继续补货" if cooldown_detected else "未取到可用代理数据" - next_retry_delay = 0.5 if (self.detecting and aggressive_proxy_rotation and cooldown_detected) else (2 if cooldown_detected else retry_delay_seconds) + next_retry_delay = ( + 0.5 if (self.detecting and aggressive_proxy_rotation and cooldown_detected) + else (2 if cooldown_detected else retry_delay_seconds) + ) + next_retry_delay += holdoff_seconds self.proxy_next_refresh_time = time.time() + next_retry_delay logger.info( f"代理池刷新完成，共 {len(self.proxy_pool)} 个可用代理，来源链接 {source_count} 个，" @@ -3660,6 +6184,9 @@ class DetectWorker: "expired_count": expired_count, "invalid_count": invalid_count, "fallback_stale_count": fallback_stale_count, + "stale_backfill_count": stale_backfill_count, + "stale_backfill_floor": stale_backfill_floor, + "stale_reuse_grace_seconds": stale_reuse_grace_seconds, "target_total": target_total, "batch_size": batch_size, "rounds": rounds, @@ -3677,7 +6204,11 @@ class DetectWorker: self.proxy_last_refresh_status = f"未取到新代理，继续沿用缓存 {len(cached_proxy_pool)} 个" else: self.proxy_last_refresh_status = "代理源暂时冷却中，稍后继续补货" if cooldown_detected else "未取到可用代理数据" - next_retry_delay = 0.5 if (self.detecting and aggressive_proxy_rotation and cooldown_detected) else (2 if cooldown_detected else retry_delay_seconds) + next_retry_delay = ( + 0.5 if (self.detecting and aggressive_proxy_rotation and cooldown_detected) + else (2 if cooldown_detected else retry_delay_seconds) + ) + next_retry_delay += holdoff_seconds self.proxy_next_refresh_time = time.time() + next_retry_delay logger.warning("所有代理池链接均未返回可用代理数据") self._update_runtime_state( @@ -3693,7 +6224,10 @@ class DetectWorker: self.proxy_last_validated_count = 0 self.proxy_last_available_count = len(self.proxy_pool) self.proxy_last_refresh_status = "未配置代理池链接" - self.proxy_next_refresh_time = time.time() + (0.5 if (self.detecting and aggressive_proxy_rotation) else retry_delay_seconds) + self.proxy_next_refresh_time = time.time() + ( + (0.5 if (self.detecting and aggressive_proxy_rotation) else retry_delay_seconds) + + holdoff_seconds + ) self._update_runtime_state( "idle" if not self.detecting else "running", "未配置代理池链接", @@ -3711,7 +6245,7 @@ class DetectWorker: self.proxy_last_refresh_status = f"刷新失败，继续沿用缓存 {len(cached_proxy_pool)} 个: {e}" else: self.proxy_last_refresh_status = f"刷新失败: {e}" - self.proxy_next_refresh_time = time.time() + retry_delay_seconds + self.proxy_next_refresh_time = time.time() + retry_delay_seconds + holdoff_seconds logger.error(f"刷新代理池失败: {e}") self._update_runtime_state( "refreshing_proxy" if self.detecting else "failed", @@ -3719,6 +6253,7 @@ class DetectWorker: ) self._sync_worker_log_event(self.proxy_last_refresh_status, level='error', mode='key') finally: + self._release_shared_proxy_refresh_lock(shared_refresh_owner) self.proxy_refresh_lock.release() def remove_proxy(self, proxy): @@ -3731,6 +6266,8 @@ class DetectWorker: with self.proxy_pool_lock: remaining_count = len(self.proxy_pool) removed = 0 + if self._pop_proxy_active_lease(proxy, already_locked=True): + removed += 1 if self.proxy_pool and isinstance(self.proxy_pool[0], dict) and 'proxy' in self.proxy_pool[0]: filtered_pool = [] for proxy_item in self.proxy_pool: @@ -3759,7 +6296,48 @@ class DetectWorker: ) self._schedule_proxy_refresh_if_needed(remaining_count) - def get_proxies(self, excluded_proxy_keys=None): + def _proxy_pool_scan_batch_size(self): + configured_batch_size = int(os.getenv("DOMAINCHECK_PROXY_POOL_SCAN_BATCH_SIZE", "0") or 0) + if configured_batch_size > 0: + return max(8, configured_batch_size) + demand_threads = self._proxy_demand_threads() + if demand_threads >= 1600: + return 64 + if demand_threads >= 800: + return 48 + return 32 + + def _pop_proxy_candidate_batch(self, batch_size): + normalized_batch_size = max(1, int(batch_size or 1)) + with self.proxy_pool_lock: + if not self.proxy_pool: + return [] + actual_batch_size = min(len(self.proxy_pool), normalized_batch_size) + candidates = [] + for _ in range(actual_batch_size): + candidates.append(self.proxy_pool.pop(0)) + return candidates + + def _restore_proxy_candidate_batch(self, candidates, selected_entry=None, *, selected_step_name="", lease_selected=False): + restored_count = 0 + with self.proxy_pool_lock: + if candidates: + self.proxy_pool.extend(candidates) + restored_count += len(candidates) + if selected_entry is not None: + if lease_selected: + self._store_proxy_active_lease( + selected_entry, + step_name=selected_step_name, + already_locked=True, + ) + else: + self.proxy_pool.append(selected_entry) + restored_count += 1 + remaining_count = len(self.proxy_pool) + return remaining_count, restored_count + + def get_proxies(self, excluded_proxy_keys=None, step_name="", lease_selected=False): """ 获取代理配置 """ @@ -3771,60 +6349,95 @@ class DetectWorker: is_insufficient = current_pool_size < self._proxy_refresh_threshold() if is_empty: + if self._wait_for_shared_proxy_snapshot(timeout_seconds=min(0.25, self._shared_proxy_wait_timeout_seconds())): + with self.proxy_pool_lock: + current_pool_size = len(self.proxy_pool) + is_empty = current_pool_size == 0 + is_insufficient = current_pool_size < self._proxy_refresh_threshold() # 不要在检测线程里同步阻塞等待代理刷新，否则线程数上去了也会卡死在这里。 - self._schedule_proxy_refresh_if_needed(current_pool_size) + if is_empty: + self._schedule_proxy_refresh_if_needed(current_pool_size) elif is_insufficient: logger.debug( f"代理池低水位，触发后台补货: 当前 {current_pool_size}，阈值 {self._proxy_refresh_threshold()}" ) self._schedule_proxy_refresh_if_needed(current_pool_size) - with self.proxy_pool_lock: - if self.proxy_pool: - pool_size = len(self.proxy_pool) - for _ in range(pool_size): - if isinstance(self.proxy_pool[0], dict) and 'proxy' in self.proxy_pool[0]: - proxy_item = self.proxy_pool.pop(0) - proxy = proxy_item['proxy'] + if current_pool_size > 0: + scan_batch_size = self._proxy_pool_scan_batch_size() + scanned_count = 0 + while scanned_count < current_pool_size: + candidates = self._pop_proxy_candidate_batch( + min(scan_batch_size, max(1, current_pool_size - scanned_count)) + ) + if not candidates: + break + scanned_count += len(candidates) + deferred_candidates = [] + selected_proxy = None + selected_entry = None + observed_now_ts = time.time() + + for index, candidate in enumerate(candidates): + if isinstance(candidate, dict) and candidate.get('proxy'): + proxy = candidate['proxy'] proxy_key = self._proxy_key(proxy) if self._is_proxy_quarantined(proxy): - self.proxy_pool.append(proxy_item) + deferred_candidates.append(candidate) continue - source_tag = str(proxy_item.get("source_tag") or "").strip() + source_tag = str(candidate.get("source_tag") or "").strip() if self._is_proxy_source_quarantined(source_tag): - self.proxy_pool.append(proxy_item) + deferred_candidates.append(candidate) continue if proxy_key in excluded_proxy_keys: - self.proxy_pool.append(proxy_item) + deferred_candidates.append(candidate) + continue + if lease_selected and not self._proxy_entry_can_be_reused(candidate, step_name, now_ts=observed_now_ts): continue - usage_count = int(proxy_item.get('usage_count', 0)) + 1 - if usage_count >= self.proxy_max_reuse_count: - proxy_item['usage_count'] = 0 - else: - proxy_item['usage_count'] = usage_count - self.proxy_pool.append(proxy_item) + usage_count = int(candidate.get('usage_count', 0)) + 1 + candidate['usage_count'] = 0 if usage_count >= self.proxy_max_reuse_count else usage_count + selected_proxy = proxy + selected_entry = candidate with self.proxy_failure_lock: source_map = getattr(self, "proxy_source_by_key", None) if source_map is None: self.proxy_source_by_key = {} source_map = self.proxy_source_by_key source_map[proxy_key] = source_tag - self._schedule_proxy_refresh_if_needed(len(self.proxy_pool)) - return proxy + deferred_candidates.extend(candidates[index + 1 :]) + break - proxy = self.proxy_pool.pop(0) + proxy = candidate proxy_key = self._proxy_key(proxy) if self._is_proxy_quarantined(proxy): - self.proxy_pool.append(proxy) + deferred_candidates.append(candidate) continue if proxy_key in excluded_proxy_keys: - self.proxy_pool.append(proxy) + deferred_candidates.append(candidate) continue logger.info(f"从代理池选择代理: {proxy}") - self.proxy_pool.append(proxy) - self._schedule_proxy_refresh_if_needed(len(self.proxy_pool)) - return proxy + selected_proxy = proxy + selected_entry = candidate if isinstance(candidate, dict) else { + "proxy": proxy, + "usage_count": 0, + "source_tag": "", + } + deferred_candidates.extend(candidates[index + 1 :]) + break + + if selected_proxy is None: + deferred_candidates = candidates + + remaining_count, _ = self._restore_proxy_candidate_batch( + deferred_candidates, + selected_entry=selected_entry, + selected_step_name=step_name, + lease_selected=lease_selected, + ) + if selected_proxy is not None: + self._schedule_proxy_refresh_if_needed(remaining_count) + return selected_proxy if self.proxy_config.get('proxy_enable', False): now_ts = time.time() if now_ts - float(getattr(self, "_last_no_proxy_notice_at", 0.0) or 0.0) >= 5.0: @@ -3846,12 +6459,219 @@ class DetectWorker: def allow_direct_connection(self): return bool(self.proxy_config.get('allow_direct', False)) + @staticmethod + def _normalize_flag_value(value, default=False): + if value is None: + return bool(default) + if isinstance(value, bool): + return value + return str(value or "").strip().lower() not in {"", "0", "false", "no", "off"} + + @staticmethod + def _env_flag_enabled(env_name, default=False): + value = os.getenv(env_name) + return DetectWorker._normalize_flag_value(value, default=default) + + def _register_single_machine_mode_enabled(self): + cached = getattr(self, "register_single_machine_mode_enabled", None) + if cached is not None: + return bool(cached) + return self._env_flag_enabled("DOMAINCHECK_REGISTER_SINGLE_MACHINE_MODE") + + def _register_single_machine_direct_streak_attempts(self): + cached = getattr(self, "register_single_machine_direct_streak_attempts", None) + if cached is not None: + return max(0, int(cached or 0)) + if not self._register_single_machine_mode_enabled(): + return 0 + return max(0, int(os.getenv("DOMAINCHECK_REGISTER_DIRECT_STREAK_ATTEMPTS", "2") or 2)) + + def _single_machine_site_direct_fallback_enabled(self): + cached = getattr(self, "single_machine_site_direct_fallback_enabled", None) + if cached is not None: + return bool(cached) + return self._env_flag_enabled( + "DOMAINCHECK_SINGLE_MACHINE_SITE_DIRECT_FALLBACK", + default=self._register_single_machine_mode_enabled(), + ) + + def _single_machine_aizhan_direct_first_enabled(self): + cached = getattr(self, "single_machine_aizhan_direct_first_enabled", None) + if cached is not None: + return bool(cached) + return self._env_flag_enabled( + "DOMAINCHECK_SINGLE_MACHINE_AIZHAN_DIRECT_FIRST", + default=False, + ) + + def _single_machine_baidu_direct_first_enabled(self): + cached = getattr(self, "single_machine_baidu_direct_first_enabled", None) + if cached is not None: + return bool(cached) + return self._env_flag_enabled( + "DOMAINCHECK_SINGLE_MACHINE_BAIDU_DIRECT_FIRST", + default=False, + ) + + def _single_machine_chinaz_direct_first_enabled(self): + cached = getattr(self, "single_machine_chinaz_direct_first_enabled", None) + if cached is not None: + return bool(cached) + return self._env_flag_enabled( + "DOMAINCHECK_SINGLE_MACHINE_CHINAZ_DIRECT_FIRST", + default=False, + ) + + def _aizhan_remote_disconnect_degrade_enabled(self): + cached = getattr(self, "aizhan_remote_disconnect_degrade_enabled", None) + if cached is not None: + return bool(cached) + return self._env_flag_enabled( + "DOMAINCHECK_AIZHAN_REMOTE_DISCONNECT_DEGRADE", + default=False, + ) + + def _aizhan_external_fast_degrade_enabled(self): + cached = getattr(self, "aizhan_external_fast_degrade_enabled", None) + if cached is not None: + return bool(cached) + return self._env_flag_enabled( + "DOMAINCHECK_AIZHAN_EXTERNAL_FAST_DEGRADE", + default=False, + ) + + def _should_force_direct_first_attempt(self, step_name, attempt_count): + if int(attempt_count or 0) != 0: + return False + if ( + step_name == "爱站网检测" + and self._single_machine_aizhan_direct_first_enabled() + and self._allow_direct_connection_for_step(step_name) + ): + return True + if ( + step_name == "百度site检测" + and self._single_machine_baidu_direct_first_enabled() + and self._allow_direct_connection_for_step(step_name) + ): + return True + if ( + step_name == "站长之家检测" + and self._single_machine_chinaz_direct_first_enabled() + and self._allow_direct_connection_for_step(step_name) + ): + return True + return False + + def _should_degrade_remote_disconnect_immediately(self, step_name, reason): + if step_name != "爱站网检测" or not self._aizhan_remote_disconnect_degrade_enabled(): + return False + lowered = str(reason or "").strip().lower() + if not lowered: + return False + return ( + "remotedisconnected" in lowered + or "remote end closed connection without response" in lowered + ) + + def _should_degrade_external_issue_immediately(self, step_name, reason): + if step_name != "爱站网检测": + return False + if self._should_degrade_remote_disconnect_immediately(step_name, reason): + return True + if not self._aizhan_external_fast_degrade_enabled(): + return False + return self._is_external_dependency_issue(reason) + + def _allow_direct_connection_for_step(self, step_name): + if step_name == "注册状态检测" and self._register_single_machine_mode_enabled(): + return True + if ( + step_name in {"360检测", "百度site检测", "站长之家检测", "爱站网检测"} + and self._single_machine_site_direct_fallback_enabled() + ): + return True + return self.allow_direct_connection() + + def _proxy_status_indicates_shortage(self): + status = str(getattr(self, "proxy_last_refresh_status", "") or "").strip() + if not status: + return False + keywords = ( + "冷却中", + "暂时冷却", + "无可用代理", + "未取到可用代理", + "继续沿用缓存 0", + "稍后继续补货", + ) + return any(keyword in status for keyword in keywords) + + def _proxy_pool_available_count(self): + try: + with self.proxy_pool_lock: + return len(list(getattr(self, "proxy_pool", []) or [])) + except Exception: + return len(list(getattr(self, "proxy_pool", []) or [])) + + def _should_hold_stale_detect_for_proxy_shortage( + self, + *, + idle_seconds: float, + low_activity_recoverable: bool, + incoming_job_changed: bool, + ): + if incoming_job_changed or not low_activity_recoverable: + return False, "" + if not bool(self.proxy_config.get('proxy_enable', False)): + return False, "" + if not self._proxy_status_indicates_shortage(): + return False, "" + if self._proxy_pool_available_count() > 0: + return False, "" + grace_seconds = 120.0 if self.allow_direct_connection() else 180.0 + if idle_seconds >= grace_seconds: + return False, "" + return True, ( + f"代理池仍处于短缺/冷却窗口，当前空转 {int(idle_seconds)} 秒，" + f"小于保护阈值 {int(grace_seconds)} 秒，最近代理状态：{self.proxy_last_refresh_status}" + ) + + def _proxy_direct_fallback_grace_for_step(self, step_name): + default_grace = max(0.0, float(getattr(self, "proxy_direct_fallback_grace_seconds", 0.0) or 0.0)) + override_map = getattr(self, "proxy_direct_fallback_grace_overrides", None) or {} + step_grace = max(0.0, float(override_map.get(step_name, default_grace) or 0.0)) + if step_name == "注册状态检测" and self._register_single_machine_mode_enabled(): + return 0.0 + if not self._allow_direct_connection_for_step(step_name): + return step_grace + if ( + step_name in {"注册状态检测", "360检测", "百度site检测", "站长之家检测", "爱站网检测"} + and self._proxy_status_indicates_shortage() + ): + return min(step_grace, 0.05) + return step_grace + + def _proxy_direct_retry_wait_for_step(self, step_name): + default_wait = max(0.0, float(getattr(self, "proxy_direct_retry_wait_seconds", 0.0) or 0.0)) + override_map = getattr(self, "proxy_direct_retry_wait_overrides", None) or {} + step_wait = max(0.0, float(override_map.get(step_name, default_wait) or 0.0)) + if step_name == "注册状态检测" and self._register_single_machine_mode_enabled(): + return 0.0 + if ( + step_name in {"注册状态检测", "360检测", "百度site检测", "站长之家检测", "爱站网检测"} + and self._proxy_status_indicates_shortage() + ): + return min(step_wait, 0.25) + return step_wait + def _get_proxy_for_step(self, domain_id, domain_name, step_name, excluded_proxy_keys=None): started_at = time.perf_counter() excluded_proxy_keys = set(excluded_proxy_keys or []) proxy_enabled = bool(self.proxy_config.get('proxy_enable', False)) - direct_allowed = self.allow_direct_connection() + direct_allowed = self._allow_direct_connection_for_step(step_name) force_direct_once = self._consume_step_force_direct_once(step_name) + step_direct_grace_seconds = self._proxy_direct_fallback_grace_for_step(step_name) if proxy_enabled and direct_allowed and force_direct_once: self._emit_step_trace( domain_name, @@ -3869,13 +6689,17 @@ class DetectWorker: if proxy_enabled and not direct_allowed: wait_window_seconds = self.proxy_step_wait_timeout_seconds elif proxy_enabled and direct_allowed: - wait_window_seconds = max(self.proxy_direct_fallback_grace_seconds, direct_retry_wait_seconds) + wait_window_seconds = max(step_direct_grace_seconds, direct_retry_wait_seconds) wait_deadline = time.time() + wait_window_seconds wait_logged_at = 0.0 attempt = 0 while True: - proxy = self.get_proxies(excluded_proxy_keys=excluded_proxy_keys) + proxy = self.get_proxies( + excluded_proxy_keys=excluded_proxy_keys, + step_name=step_name, + lease_selected=True, + ) elapsed_ms = int((time.perf_counter() - started_at) * 1000) if proxy: self._emit_step_trace( @@ -3970,7 +6794,7 @@ class DetectWorker: normalized_reason = str(reason or "") if not self._is_external_dependency_issue(normalized_reason): return False - if proxy and self.allow_direct_connection(): + if proxy and self._allow_direct_connection_for_step(step_name): if self._is_proxy_auth_issue(normalized_reason): self._mark_proxy_source_failure(proxy, normalized_reason) self.remove_proxy(proxy) @@ -3987,13 +6811,14 @@ class DetectWorker: if self._should_rotate_proxy_on_error(proxy, normalized_reason, step_name, domain_name): return True if self.proxy_config.get('proxy_enable', False): - if not proxy and self.allow_direct_connection(): + if not proxy and self._allow_direct_connection_for_step(step_name): + retry_proxy_wait_seconds = self._proxy_direct_retry_wait_for_step(step_name) forced_wait_seconds = max( - self.proxy_direct_fallback_grace_seconds, + self._proxy_direct_fallback_grace_for_step(step_name), min( self.proxy_step_wait_timeout_seconds or 0.0, - float(os.getenv("DOMAINCHECK_DIRECT_RETRY_PROXY_WAIT", "1.2") or 1.2), - ) if self.proxy_step_wait_timeout_seconds > 0 else float(os.getenv("DOMAINCHECK_DIRECT_RETRY_PROXY_WAIT", "1.2") or 1.2), + retry_proxy_wait_seconds, + ) if self.proxy_step_wait_timeout_seconds > 0 else retry_proxy_wait_seconds, ) self._set_step_proxy_retry_wait(step_name, forced_wait_seconds) try: @@ -4167,7 +6992,12 @@ class DetectWorker: def _check_proxy_step_retry_budget(self, step_name, domain_name, started_at, attempt_count, tried_proxy_keys, last_reason=""): elapsed_seconds = max(0.0, time.perf_counter() - float(started_at or 0.0)) max_attempts, max_seconds = self._resolve_proxy_step_retry_budget(step_name) - if attempt_count < max_attempts and elapsed_seconds < max_seconds: + pending_direct_fallback = ( + attempt_count < max_attempts + and self._allow_direct_connection_for_step(step_name) + and self._peek_step_force_direct_once(step_name) + ) + if attempt_count < max_attempts and (elapsed_seconds < max_seconds or pending_direct_fallback): return False, "" reason = ( @@ -4197,8 +7027,8 @@ class DetectWorker: def _resolve_submit_backlog_limit(self, max_threads, live_active): normalized_threads = max(1, int(max_threads or 1)) live_active = max(0, int(live_active or 0)) - configured_backlog_floor = int(os.getenv("DOMAINCHECK_SUBMIT_BACKLOG_FLOOR", "0") or 0) - configured_backlog_ceiling = int(os.getenv("DOMAINCHECK_SUBMIT_BACKLOG_CEIL", "0") or 0) + configured_backlog_floor = self._runtime_optional_int_override("submit_backlog_floor", "DOMAINCHECK_SUBMIT_BACKLOG_FLOOR") + configured_backlog_ceiling = self._runtime_optional_int_override("submit_backlog_ceil", "DOMAINCHECK_SUBMIT_BACKLOG_CEIL") # Default to a full-thread warm window. Large machines should be allowed # to keep enough local backlog to actually fill the configured worker # pool instead of self-throttling to a few hundred runnable items. @@ -4218,9 +7048,10 @@ class DetectWorker: normalized_threads = max(1, int(max_threads or 1)) live_active = max(0, int(live_active or 0)) backlog_limit = max(1, int(backlog_limit or 1)) - dispatch_cap_multiplier = max( - 1, - int(os.getenv("DOMAINCHECK_DISPATCH_CAP_MULTIPLIER", "1") or 1), + dispatch_cap_multiplier = self._runtime_positive_int_override( + "dispatch_cap_multiplier", + "DOMAINCHECK_DISPATCH_CAP_MULTIPLIER", + default=1, ) dispatch_capacity_cap = max(1, normalized_threads * dispatch_cap_multiplier) return min(dispatch_capacity_cap, max(backlog_limit, live_active + backlog_limit)) @@ -4236,8 +7067,8 @@ class DetectWorker: 1, int(os.getenv("DOMAINCHECK_CLAIM_CAP_MULTIPLIER", "1") or 1), ) - configured_claim_floor = int(os.getenv("DOMAINCHECK_CLAIM_BATCH_FLOOR", "0") or 0) - configured_claim_ceiling = int(os.getenv("DOMAINCHECK_CLAIM_BATCH_CEIL", "0") or 0) + configured_claim_floor = self._runtime_optional_int_override("claim_batch_floor", "DOMAINCHECK_CLAIM_BATCH_FLOOR") + configured_claim_ceiling = self._runtime_optional_int_override("claim_batch_ceil", "DOMAINCHECK_CLAIM_BATCH_CEIL") # Default to a full-thread claim window so large worker pools can # actually fill their slots. Operators can still shrink this via env # overrides if they explicitly want a tighter rolling buffer. @@ -4260,6 +7091,32 @@ class DetectWorker: ), ) + def _should_proactively_top_up_pipeline(self, claimed_count, refill_slots, max_threads): + claimed_count = max(0, int(claimed_count or 0)) + refill_slots = max(0, int(refill_slots or 0)) + normalized_threads = max(1, int(max_threads or 1)) + remaining_gap = max(0, refill_slots - claimed_count) + if remaining_gap <= 0: + return False + configured_threshold = int(os.getenv("DOMAINCHECK_PIPELINE_TOPUP_THRESHOLD", "0") or 0) + threshold = max( + 1, + configured_threshold if configured_threshold > 0 else max(128, normalized_threads // 4), + ) + return remaining_gap >= threshold + + def _should_prefetch_sync_tasks(self, pending_buffer_count, inflight_count, max_threads): + pending_buffer_count = max(0, int(pending_buffer_count or 0)) + inflight_count = max(0, int(inflight_count or 0)) + normalized_threads = max(1, int(max_threads or 1)) + local_work = pending_buffer_count + inflight_count + configured_threshold = int(os.getenv("DOMAINCHECK_SYNC_PULL_PREFETCH_THRESHOLD", "0") or 0) + threshold = max( + 32, + configured_threshold if configured_threshold > 0 else max(64, min(normalized_threads, normalized_threads // 2)), + ) + return local_work <= threshold + def _complete_detection(self, domain_id, domain_name, domain): register_status = int((domain or {}).get('register_status', 0) or 0) use_status = int((domain or {}).get('use_status', 0) or 0) @@ -4334,18 +7191,32 @@ class DetectWorker: started_at = time.perf_counter() attempt_count = 0 last_reason = "" + step_name = '注册状态检测' + direct_allowed = self._allow_direct_connection_for_step(step_name) + register_single_machine_mode = self._register_single_machine_mode_enabled() + direct_first_attempt_limit = 0 # 高并发下如果注册检测仍然直连优先，会先把 controller 本机的外连资源打爆， # 导致大量线程卡在 direct request error，而不是尽快让代理承接流量。 - # 这里默认改成“代理优先，直连兜底”；只有显式打开环境变量时才允许直连优先。 - register_direct_first = ( - self.allow_direct_connection() - and str(os.getenv("DOMAINCHECK_REGISTER_DIRECT_FIRST", "0") or "0").strip().lower() - not in {"", "0", "false", "no", "off"} - ) - direct_attempt_consumed = False + # 这里默认改成“代理优先，直连兜底”；只有显式打开环境变量，或最近已明确 + # 进入代理短缺/冷却态时，才允许注册检测直接优先走直连。 + register_direct_first = False + if direct_allowed: + register_direct_first = ( + self._env_flag_enabled("DOMAINCHECK_REGISTER_DIRECT_FIRST") + or self._proxy_status_indicates_shortage() + or register_single_machine_mode + ) + if register_direct_first and direct_allowed: + direct_first_attempt_limit = 1 + if register_single_machine_mode and direct_allowed: + direct_first_attempt_limit = max( + direct_first_attempt_limit, + self._register_single_machine_direct_streak_attempts(), + ) + direct_attempt_count = 0 while True: budget_exhausted, budget_reason = self._check_proxy_step_retry_budget( - '注册状态检测', + step_name, domain_name, started_at, attempt_count, @@ -4357,17 +7228,17 @@ class DetectWorker: return False proxy = None - if register_direct_first and not direct_attempt_consumed: - direct_attempt_consumed = True + if direct_attempt_count < direct_first_attempt_limit: + direct_attempt_count += 1 self._emit_step_trace( domain_name, - '注册状态检测', + step_name, 'proxy_direct_fallback', - proxy_mode='direct_first', + proxy_mode='direct_perf' if register_single_machine_mode else 'direct_first', retry_attempts=attempt_count, ) else: - proxy = self._get_proxy_for_step(domain_id, domain_name, '注册状态检测', excluded_proxy_keys=tried_proxy_keys) + proxy = self._get_proxy_for_step(domain_id, domain_name, step_name, excluded_proxy_keys=tried_proxy_keys) if proxy == '__NO_PROXY__': return False if proxy: @@ -4376,7 +7247,7 @@ class DetectWorker: try: tld = domain_name.split('.')[-1] remaining_budget_seconds = self._remaining_proxy_step_retry_budget_seconds( - '注册状态检测', + step_name, started_at, reserve_seconds=0.2, ) @@ -4390,6 +7261,7 @@ class DetectWorker: if proxy: self._clear_proxy_failure(proxy) self._clear_proxy_source_failure(proxy) + self.release_proxy(proxy, step_name='注册状态检测') normalized_expire_date = expire_date if expire_date: from datetime import datetime, timedelta @@ -4400,6 +7272,33 @@ class DetectWorker: except Exception as e: logger.error(f"处理过期日期失败: {e}") normalized_expire_date = expire_date + if status in {REGISTER_STATUS_CLIENT_HOLD, REGISTER_STATUS_SERVER_HOLD}: + hold_label = ( + "clientHold" + if int(status) == int(REGISTER_STATUS_CLIENT_HOLD) + else "serverHold" + ) + blacklist_reason = f"注册状态命中 {hold_label}" + if self._is_current_single_step_task(): + self._cache_single_step_result_payload( + domain_id, + 'register_status', + self._step_result_payload( + ok=False, + state="blacklisted", + message=blacklist_reason, + register_status=int(status), + expire_date=normalized_expire_date or "", + ), + ) + else: + self.db.update_domain_register_result(domain_id, status, normalized_expire_date) + domain['register_status'] = status + if normalized_expire_date: + domain['expire_date'] = normalized_expire_date + self._mark_blacklisted(domain_id, domain_name, blacklist_reason) + logger.warning(f"注册状态检测命中黑名单: {domain_name}, 原因: {blacklist_reason}") + return False if self._is_current_single_step_task(): self._cache_single_step_result_payload( domain_id, @@ -4421,19 +7320,33 @@ class DetectWorker: return True logger.warning(f"注册状态检测失败，不更新数据: {domain_name}") last_reason = 'register lookup failed' - if self._should_rotate_proxy_on_error(proxy, 'register lookup failed', '注册状态检测', domain_name): + if self._should_rotate_proxy_on_error(proxy, 'register lookup failed', step_name, domain_name): continue - self._mark_detection_failed(domain_id, domain_name, '注册状态检测', '注册状态检测返回未知状态') + if proxy: + self.release_proxy(proxy, step_name=step_name, reason='register lookup failed') + self._mark_detection_failed(domain_id, domain_name, step_name, '注册状态检测返回未知状态') return False except Exception as e: logger.error(f"注册状态检测失败: {domain_name}, 错误: {e}") last_reason = str(e) if proxy is None and self.proxy_config.get('proxy_enable', False): - logger.warning(f"注册状态检测直连失败，切换代理继续: {domain_name}, 原因: {e}") + if direct_allowed and self._proxy_status_indicates_shortage(): + self._set_step_force_direct_once(step_name, True) + self._set_step_proxy_retry_wait( + step_name, + self._proxy_direct_fallback_grace_for_step(step_name), + ) + logger.warning( + f"注册状态检测直连失败，代理源仍短缺，下一次继续快速直连: {domain_name}, 原因: {e}" + ) + else: + logger.warning(f"注册状态检测直连失败，切换代理继续: {domain_name}, 原因: {e}") continue - if self._should_rotate_proxy_on_error(proxy, str(e), '注册状态检测', domain_name): + if self._should_rotate_proxy_on_error(proxy, str(e), step_name, domain_name): continue - self._mark_detection_failed(domain_id, domain_name, '注册状态检测', str(e)) + if proxy: + self.release_proxy(proxy, step_name=step_name, reason=str(e)) + self._mark_detection_failed(domain_id, domain_name, step_name, str(e)) return False return True @@ -4527,9 +7440,22 @@ class DetectWorker: if budget_exhausted: self._mark_detection_degraded(domain_id, domain_name, '站长之家检测', budget_reason, 'chinaz_info') return True - proxy = self._get_proxy_for_step(domain_id, domain_name, '站长之家检测', excluded_proxy_keys=tried_proxy_keys) - if proxy == '__NO_PROXY__': - return False + if self._should_force_direct_first_attempt('站长之家检测', attempt_count): + self._emit_step_trace( + domain_name, + '站长之家检测', + 'proxy_direct_fallback', + elapsed_ms=0, + proxy_mode='direct', + recent_proxy_status=self.proxy_last_refresh_status, + retry_attempts=int(attempt_count or 0), + reason='single_machine_direct_first', + ) + proxy = None + else: + proxy = self._get_proxy_for_step(domain_id, domain_name, '站长之家检测', excluded_proxy_keys=tried_proxy_keys) + if proxy == '__NO_PROXY__': + return False if proxy: tried_proxy_keys.add(self._proxy_key(proxy)) attempt_count += 1 @@ -4557,16 +7483,26 @@ class DetectWorker: logger.warning(f"站长之家检测未通过: {domain_name}, 原因: {message}") last_reason = str(message or "") if self._should_blacklist_result(message): + if proxy: + self.release_proxy(proxy, step_name='站长之家检测', reason=message or 'blacklisted') self._record_step_result(domain_id, 'chinaz_info', ok=False, state="blacklisted", message=message) self._mark_blacklisted(domain_id, domain_name, message) return False if self._should_rotate_proxy_on_error(proxy, message, '站长之家检测', domain_name): continue if self._is_external_dependency_issue(message): + if proxy: + self.release_proxy(proxy, step_name='站长之家检测', reason=message or 'external dependency issue') self._mark_detection_degraded(domain_id, domain_name, '站长之家检测', message or 'external dependency issue', 'chinaz_info') return True + if proxy: + self.release_proxy(proxy, step_name='站长之家检测', reason=message or 'business rejected') self._mark_detection_rejected(domain_id, domain_name, '站长之家检测', message or 'business rejected', 'chinaz_info') return False + if proxy: + self._clear_proxy_failure(proxy) + self._clear_proxy_source_failure(proxy) + self.release_proxy(proxy, step_name='站长之家检测') self._record_step_result(domain_id, 'chinaz_info', ok=True, state="passed", message=message or "success", seo=seo_data) logger.info(f"站长之家检测完成: {domain_name}") return True @@ -4576,8 +7512,12 @@ class DetectWorker: if self._should_rotate_proxy_on_error(proxy, str(e), '站长之家检测', domain_name): continue if self._is_external_dependency_issue(e): + if proxy: + self.release_proxy(proxy, step_name='站长之家检测', reason=str(e)) self._mark_detection_degraded(domain_id, domain_name, '站长之家检测', str(e), 'chinaz_info') return True + if proxy: + self.release_proxy(proxy, step_name='站长之家检测', reason=str(e)) self._mark_detection_failed(domain_id, domain_name, '站长之家检测', str(e), 'chinaz_info') return False return True @@ -4600,6 +7540,8 @@ class DetectWorker: if budget_exhausted: self._mark_detection_degraded(domain_id, domain_name, '爱站网检测', budget_reason, 'aizhan_info') return True + if self._should_force_direct_first_attempt('爱站网检测', attempt_count): + self._set_step_force_direct_once('爱站网检测', True) proxy = self._get_proxy_for_step(domain_id, domain_name, '爱站网检测', excluded_proxy_keys=tried_proxy_keys) if proxy == '__NO_PROXY__': return False @@ -4622,27 +7564,51 @@ class DetectWorker: logger.warning(f"爱站网检测未通过: {domain_name}, 原因: {message}") last_reason = str(message or "") if self._should_blacklist_result(message): + if proxy: + self.release_proxy(proxy, step_name='爱站网检测', reason=message or 'blacklisted') self._record_step_result(domain_id, 'aizhan_info', ok=False, state="blacklisted", message=message) self._mark_blacklisted(domain_id, domain_name, message) return False - if self._should_rotate_proxy_on_error(proxy, message, '爱站网检测', domain_name): - continue - if self._is_external_dependency_issue(message): + if self._should_degrade_external_issue_immediately('爱站网检测', message): + if proxy: + self.release_proxy(proxy, step_name='爱站网检测', reason=message or 'external dependency issue') self._mark_detection_degraded(domain_id, domain_name, '爱站网检测', message or 'external dependency issue', 'aizhan_info') return True + if self._should_retry_external_issue(proxy, message, '爱站网检测', domain_name): + continue + if self._is_external_dependency_issue(message): + if proxy: + self.release_proxy(proxy, step_name='爱站网检测', reason=message or 'external dependency issue') + self._mark_detection_degraded(domain_id, domain_name, '爱站网检测', message or 'external dependency issue', 'aizhan_info') + return True + if proxy: + self.release_proxy(proxy, step_name='爱站网检测', reason=message or 'business rejected') self._mark_detection_rejected(domain_id, domain_name, '爱站网检测', message or 'business rejected', 'aizhan_info') return False + if proxy: + self._clear_proxy_failure(proxy) + self._clear_proxy_source_failure(proxy) + self.release_proxy(proxy, step_name='爱站网检测') self._record_step_result(domain_id, 'aizhan_info', ok=True, state="passed", message=message or "success") logger.info(f"爱站网检测完成: {domain_name}") return True except Exception as e: logger.error(f"爱站网检测失败: {domain_name}, 错误: {e}") last_reason = str(e) - if self._should_rotate_proxy_on_error(proxy, str(e), '爱站网检测', domain_name): - continue - if self._is_external_dependency_issue(e): + if self._should_degrade_external_issue_immediately('爱站网检测', e): + if proxy: + self.release_proxy(proxy, step_name='爱站网检测', reason=str(e)) self._mark_detection_degraded(domain_id, domain_name, '爱站网检测', str(e), 'aizhan_info') return True + if self._should_retry_external_issue(proxy, str(e), '爱站网检测', domain_name): + continue + if self._is_external_dependency_issue(e): + if proxy: + self.release_proxy(proxy, step_name='爱站网检测', reason=str(e)) + self._mark_detection_degraded(domain_id, domain_name, '爱站网检测', str(e), 'aizhan_info') + return True + if proxy: + self.release_proxy(proxy, step_name='爱站网检测', reason=str(e)) self._mark_detection_failed(domain_id, domain_name, '爱站网检测', str(e), 'aizhan_info') return False return True @@ -4665,6 +7631,8 @@ class DetectWorker: if budget_exhausted: self._mark_detection_degraded(domain_id, domain_name, '百度site检测', budget_reason, 'baidu_site') return True + if self._should_force_direct_first_attempt('百度site检测', attempt_count): + self._set_step_force_direct_once('百度site检测', True) proxy = self._get_proxy_for_step(domain_id, domain_name, '百度site检测', excluded_proxy_keys=tried_proxy_keys) if proxy == '__NO_PROXY__': return False @@ -4687,16 +7655,26 @@ class DetectWorker: logger.warning(f"百度site检测未通过: {domain_name}, 原因: {message}") last_reason = str(message or "") if self._should_blacklist_result(message): + if proxy: + self.release_proxy(proxy, step_name='百度site检测', reason=message or 'blacklisted') self._record_step_result(domain_id, 'baidu_site', ok=False, state="blacklisted", message=message) self._mark_blacklisted(domain_id, domain_name, message) return False if self._should_retry_external_issue(proxy, message, '百度site检测', domain_name): continue if self._is_external_dependency_issue(message): + if proxy: + self.release_proxy(proxy, step_name='百度site检测', reason=message or 'external dependency issue') self._mark_detection_degraded(domain_id, domain_name, '百度site检测', message, 'baidu_site') return True + if proxy: + self.release_proxy(proxy, step_name='百度site检测', reason=message or 'business rejected') self._mark_detection_rejected(domain_id, domain_name, '百度site检测', message or 'business rejected', 'baidu_site') return False + if proxy: + self._clear_proxy_failure(proxy) + self._clear_proxy_source_failure(proxy) + self.release_proxy(proxy, step_name='百度site检测') self._record_step_result(domain_id, 'baidu_site', ok=True, state="passed", message=message or "success") logger.info(f"百度site检测完成: {domain_name}") return True @@ -4706,8 +7684,12 @@ class DetectWorker: if self._should_retry_external_issue(proxy, str(e), '百度site检测', domain_name): continue if self._is_external_dependency_issue(e): + if proxy: + self.release_proxy(proxy, step_name='百度site检测', reason=str(e)) self._mark_detection_degraded(domain_id, domain_name, '百度site检测', str(e), 'baidu_site') return True + if proxy: + self.release_proxy(proxy, step_name='百度site检测', reason=str(e)) self._mark_detection_failed(domain_id, domain_name, '百度site检测', str(e), 'baidu_site') return False return True @@ -4752,27 +7734,41 @@ class DetectWorker: logger.warning(f"360检测未通过: {domain_name}, 原因: {message}") last_reason = str(message or "") if self._should_blacklist_result(message): + if proxy: + self.release_proxy(proxy, step_name='360检测', reason=message or 'blacklisted') self._record_step_result(domain_id, 'qihu360_site', ok=False, state="blacklisted", message=message) self._mark_blacklisted(domain_id, domain_name, message) return False - if self._should_rotate_proxy_on_error(proxy, message, '360检测', domain_name): + if self._should_retry_external_issue(proxy, message, '360检测', domain_name): continue if self._is_external_dependency_issue(message): + if proxy: + self.release_proxy(proxy, step_name='360检测', reason=message or 'external dependency issue') self._mark_detection_degraded(domain_id, domain_name, '360检测', message or 'external dependency issue', 'qihu360_site') return True + if proxy: + self.release_proxy(proxy, step_name='360检测', reason=message or 'business rejected') self._mark_detection_rejected(domain_id, domain_name, '360检测', message or 'business rejected', 'qihu360_site') return False + if proxy: + self._clear_proxy_failure(proxy) + self._clear_proxy_source_failure(proxy) + self.release_proxy(proxy, step_name='360检测') self._record_step_result(domain_id, 'qihu360_site', ok=True, state="passed", message=message or "success") logger.info(f"360检测完成: {domain_name}") return True except Exception as e: logger.error(f"360检测失败: {domain_name}, 错误: {e}") last_reason = str(e) - if self._should_rotate_proxy_on_error(proxy, str(e), '360检测', domain_name): + if self._should_retry_external_issue(proxy, str(e), '360检测', domain_name): continue if self._is_external_dependency_issue(e): + if proxy: + self.release_proxy(proxy, step_name='360检测', reason=str(e)) self._mark_detection_degraded(domain_id, domain_name, '360检测', str(e), 'qihu360_site') return True + if proxy: + self.release_proxy(proxy, step_name='360检测', reason=str(e)) self._mark_detection_failed(domain_id, domain_name, '360检测', str(e), 'qihu360_site') return False return True @@ -4862,6 +7858,10 @@ class DetectWorker: has_beian=beian_update_payload.get("has_beian"), beian_year=beian_update_payload.get("beian_year"), ) + if proxy: + self._clear_proxy_failure(proxy) + self._clear_proxy_source_failure(proxy) + self.release_proxy(proxy, step_name='聚查检测') return True def _run_detect_juziseo(self, domain_id, domain, domain_name, sensitive_words): @@ -4898,6 +7898,10 @@ class DetectWorker: logger.error(f"桔子外链检测失败: {domain_name}, 错误: {e}") finally: self.db.mark_juziseo_detected(domain_id) + if proxy: + self._clear_proxy_failure(proxy) + self._clear_proxy_source_failure(proxy) + self.release_proxy(proxy, step_name='桔子检测') return True def _upsert_json_detection(self, domain_id, **payload): @@ -4942,6 +7946,7 @@ class DetectWorker: is_step_task = bool(requested_step_code) cycle_token = str(job_context.get('cycle_token') or self.current_cycle_token or '').strip() job_code = str(job_context.get('job_code') or self.current_job_code or '').strip() + session_id = int(job_context.get('session_id') or 0) domain_started_at = time.perf_counter() self._set_current_task_context( domain_id=domain_id, @@ -4949,6 +7954,30 @@ class DetectWorker: task_mode=task_mode, job_item_id=job_item_id, ) + early_abort_reason = self._detection_session_abort_reason(session_id) + if early_abort_reason: + self._release_job_item_for_session_abort( + job_item_id, + claim_token, + session_id=session_id, + domain_name=domain_name, + detail="queued_before_start", + ) + logger.info( + f"检测会话已切换，跳过旧域名任务派发: {domain_name}, " + f"job_item_id={job_item_id or 0}, reason={early_abort_reason}" + ) + self._emit_step_trace( + domain_name, + "domain", + "aborted", + job_item_id=job_item_id, + elapsed_ms=0, + ok=False, + error=f"检测会话已失效: reason={early_abort_reason} stage=queued_before_start domain={domain_name}"[:300], + ) + self._clear_current_task_context() + return current_active_threads = self._change_active_domain_threads(1) self._note_domain_started() if self.detecting and ( @@ -4976,7 +8005,23 @@ class DetectWorker: self._note_domain_result() if not job_item_id or not claim_token: return + abort_reason = self._detection_session_abort_reason(session_id) + if abort_reason: + self._release_job_item_for_session_abort( + job_item_id, + claim_token, + session_id=session_id, + domain_name=domain_name, + detail=f"finalize:{final_status}", + ) + logger.warning( + f"检测会话已切换，跳过任务项完成态回写并释放回队列: domain={domain_name}, " + f"job_item_id={job_item_id}, status={final_status}, reason={abort_reason}" + ) + return try: + db_host = str(getattr(self.db, "host", "") or getattr(config, "DB_HOST", "") or "").strip() or "db" + db_name = str(getattr(self.db, "database", "") or getattr(config, "DB_DATABASE", "") or "").strip() or "-" event_payload = { "domain_id": domain_id, "domain": domain_name, @@ -4992,17 +8037,40 @@ class DetectWorker: result_payload=result_payload, job_id=job_id, node_code=config.NODE_CODE, - event_type="" if is_step_task else f"domain_{final_status}", + event_type=f"domain_{final_status}", event_level='error' if final_status == 'failed' else 'info', event_message=message or f"{domain_name} -> {final_status}", event_payload=event_payload, ) + logger.info( + "登记 domain 终态事件入队: " + f"domain={domain_name} job_id={job_id or 0} job_item_id={job_item_id or 0} " + f"event_type=domain_{final_status} db={db_host}:{db_name}" + ) except Exception as finalize_error: - logger.warning(f"回写任务项状态失败: {domain_name}, 错误: {finalize_error}") + logger.warning( + f"回写任务项状态失败: {domain_name}, 错误: {finalize_error}, " + f"job_id={job_id or 0}, job_item_id={job_item_id or 0}, " + f"event_type=domain_{final_status}" + ) def renew_job_item_lease(detail=""): if is_step_task or not job_item_id or not claim_token: return + abort_reason = self._detection_session_abort_reason(session_id) + if abort_reason: + self._release_job_item_for_session_abort( + job_item_id, + claim_token, + session_id=session_id, + domain_name=domain_name, + detail=f"renew:{detail}", + ) + logger.warning( + f"检测会话已切换，跳过任务项续租并释放回队列: domain={domain_name}, " + f"job_item_id={job_item_id}, detail={detail}, reason={abort_reason}" + ) + return try: self.db.renew_detect_job_item_lease( job_item_id, @@ -5020,6 +8088,7 @@ class DetectWorker: logger.info(f"检测已停止，跳过域名: {domain_name}") finalize_job_item('failed', 'Worker 已停止，任务未执行') return + self._assert_detection_session_active(session_id, domain_name=domain_name, stage="prepare") logger.info(f"开始检测域名: {domain_name}") if not is_step_task: @@ -5028,20 +8097,29 @@ class DetectWorker: if job_item_id and claim_token: self._enqueue_running_mark(job_item_id, claim_token) renew_job_item_lease("domain_started") - if not is_step_task: - self.db.append_detect_run_event( - job_id, - job_item_id, - config.NODE_CODE, - event_type='domain_started', - message=f"开始检测域名: {domain_name}", - payload={ - "domain_id": domain_id, - "domain": domain_name, - "cycle_token": cycle_token, - "job_code": job_code, - }, - ) + db_host = str(getattr(self.db, "host", "") or getattr(config, "DB_HOST", "") or "").strip() or "db" + db_name = str(getattr(self.db, "database", "") or getattr(config, "DB_DATABASE", "") or "").strip() or "-" + self.db.append_detect_run_event( + job_id, + job_item_id, + config.NODE_CODE, + event_type='domain_started', + message=f"开始检测域名: {domain_name}", + payload={ + "domain_id": domain_id, + "domain": domain_name, + "cycle_token": cycle_token, + "job_code": job_code, + "task_mode": task_mode, + "step_code": requested_step_code if is_step_task else "", + }, + ) + logger.info( + "写入 domain_started 事件成功: " + f"domain={domain_name} job_id={job_id or 0} job_item_id={job_item_id or 0} " + f"event_type=domain_started db={db_host}:{db_name} " + f"task_mode={task_mode} step_code={requested_step_code if is_step_task else '-'}" + ) self._sync_worker_log_event( f"开始检测域名: {domain_name}", payload={ @@ -5073,6 +8151,7 @@ class DetectWorker: def execute_detect_step(detect_key): step_name = self._step_display_name(detect_key) + self._assert_detection_session_active(session_id, domain_name=domain_name, stage=f"{detect_key}:prepare") renew_job_item_lease(detect_key) step_started_at = time.perf_counter() self._emit_step_trace( @@ -5109,6 +8188,7 @@ class DetectWorker: step_ok = False step_elapsed_ms = int((time.perf_counter() - step_started_at) * 1000) + self._assert_detection_session_active(session_id, domain_name=domain_name, stage=f"{detect_key}:finished") self._emit_step_trace( domain_name, step_name, @@ -5137,6 +8217,7 @@ class DetectWorker: task_mode=task_mode, ) step_result = execute_detect_step(requested_step_code) + self._assert_detection_session_active(session_id, domain_name=domain_name, stage=f"{requested_step_code}:finalize") result_payload = self._build_single_step_job_result( domain_id=domain_id, detect_key=requested_step_code, @@ -5207,6 +8288,11 @@ class DetectWorker: "step_ok": False, "step_elapsed_ms": 0, } + self._assert_detection_session_active( + session_id, + domain_name=domain_name, + stage=f"{ordered_key}:parallel_finalize", + ) if not step_result["step_ok"]: final_status = self._resolve_current_task_final_status() self._emit_step_trace( @@ -5251,6 +8337,11 @@ class DetectWorker: ) for ordered_key in parallel_group: step_result = execute_detect_step(ordered_key) + self._assert_detection_session_active( + session_id, + domain_name=domain_name, + stage=f"{ordered_key}:serial_finalize", + ) if not step_result["step_ok"]: final_status = self._resolve_current_task_final_status() self._emit_step_trace( @@ -5279,6 +8370,7 @@ class DetectWorker: step_index += 1 step_result = execute_detect_step(detect_key) + self._assert_detection_session_active(session_id, domain_name=domain_name, stage=f"{detect_key}:finalize") if not step_result["step_ok"]: final_status = self._resolve_current_task_final_status() self._emit_step_trace( @@ -5304,6 +8396,7 @@ class DetectWorker: finalize_job_item(final_status, failure_reason_map.get(detect_key, f'{step_result["step_name"]} 未通过')) return + self._assert_detection_session_active(session_id, domain_name=domain_name, stage="completion") completion_tail_started_at = time.perf_counter() complete_detection_started_at = time.perf_counter() self._complete_detection(domain_id, domain_name, domain) @@ -5351,7 +8444,27 @@ class DetectWorker: f"| trace_emit_ms={trace_emit_elapsed_ms} " f"| completion_log_enqueue_ms={completion_log_sync_elapsed_ms}" ) - + except RuntimeError as e: + if str(e).startswith("检测会话已失效:"): + self._release_job_item_for_session_abort( + job_item_id, + claim_token, + session_id=session_id, + domain_name=domain_name, + detail=str(e), + ) + logger.info(f"检测会话切换，终止旧域名任务: {domain_name}, 详情: {e}") + self._emit_step_trace( + domain_name, + "domain", + "aborted", + job_item_id=job_item_id, + elapsed_ms=int((time.perf_counter() - domain_started_at) * 1000), + ok=False, + error=str(e)[:300], + ) + return + raise except Exception as e: logger.error(f"检测域名出错: {domain_name}, 错误: {e}") if not is_step_task: @@ -5389,7 +8502,7 @@ class DetectWorker: max_threads=self.thread_count, ) - def start_detection(self): + def start_detection(self, session_id: int = 0): """ 开始检测 """ @@ -5415,7 +8528,7 @@ class DetectWorker: recycled = self.db.recycle_running_domains(DETECT_STATUS_FAILED) if recycled: logger.warning(f"检测启动前回收了 {recycled} 个遗留的检测中任务，已标记为检测失败待重试") - released_job_items = self.db.release_detect_job_items_for_node(config.NODE_CODE) + released_job_items = self._maybe_release_node_job_items(reason="start_detection") if released_job_items: logger.warning(f"检测启动前释放了 {released_job_items} 个当前节点遗留任务项，已重新回到 pending") except Exception as e: @@ -5445,7 +8558,22 @@ class DetectWorker: logger.info("开始检测，刷新代理池") self._mark_detection_phase("refreshing_proxy", "开始检测，正在刷新代理池") self._sync_worker_log_event("开始检测，正在刷新代理池", mode='key') - if self.proxy_refresh_lock.locked(): + proxy_ready, proxy_count, proxy_ready_reason = self._prepare_proxy_pool_for_detection_start() + if proxy_ready: + logger.info( + f"检测启动前代理池已就绪，跳过同步刷新: " + f"count={proxy_count}, reason={proxy_ready_reason}" + ) + self._update_runtime_state( + "running", + f"检测启动前代理池已就绪，直接进入任务派发: count={proxy_count}, reason={proxy_ready_reason}", + ) + self._sync_worker_log_event( + "检测启动前代理池已就绪，直接进入任务派发", + payload={"proxy_count": proxy_count, "reason": proxy_ready_reason}, + mode='full', + ) + elif self.proxy_refresh_lock.locked(): if self._wait_for_proxy_refresh_settle(timeout_seconds=2.0): logger.info("检测启动前等待到了可用代理，直接复用当前代理池") else: @@ -5498,12 +8626,34 @@ class DetectWorker: last_job_status_refresh_at = 0.0 last_expired_job_recycle_at = 0.0 last_future_fallback_scan_at = 0.0 - worker_pool_size = max(1, int(self.refresh_thread_count_runtime(force=True, min_interval=0.0) or 1)) + thread_count_refresh_interval = max( + 0.5, + float(os.getenv("DOMAINCHECK_THREAD_COUNT_REFRESH_INTERVAL", "2.0") or 2.0), + ) + job_status_refresh_interval = max( + 5.0, + float(os.getenv("DOMAINCHECK_JOB_STATUS_REFRESH_INTERVAL", "15.0") or 15.0), + ) + worker_pool_size = max( + 1, + int( + self.refresh_thread_count_runtime( + force=True, + min_interval=thread_count_refresh_interval, + ) + or 1 + ), + ) + configured_thread_stack_size = _configure_worker_thread_stack_size() worker_pool = ThreadPoolExecutor( max_workers=worker_pool_size, thread_name_prefix="detect-domain", ) last_worker_pool_size = worker_pool_size + if configured_thread_stack_size > 0: + logger.info( + f"检测线程池初始化: max_workers={worker_pool_size}, thread_stack_size={configured_thread_stack_size}" + ) def prune_finished_futures(): nonlocal inflight_futures, last_future_fallback_scan_at @@ -5649,6 +8799,8 @@ class DetectWorker: int(os.getenv("DOMAINCHECK_ENABLE_LEGACY_DOMAIN_FALLBACK", "0") or 0) ) last_legacy_fallback_skip_log_at = 0.0 + last_proactive_pipeline_topup_at = 0.0 + last_proactive_sync_topup_at = 0.0 def has_dispatchable_step_items(): nonlocal last_dispatchable_step_items_check_at, dispatchable_step_items_cached @@ -5667,15 +8819,21 @@ class DetectWorker: domain_id = domain['id'] try: job_context = { - 'job_item_id': domain.get('job_item_id'), - 'job_id': domain.get('job_id'), - 'claim_token': domain.get('claim_token'), - 'task_mode': domain.get('task_mode'), - 'step_code': domain.get('step_code'), - 'step_payload': domain.get('step_payload'), 'cycle_token': self.current_cycle_token, 'job_code': domain.get('job_code') or self.current_job_code, - } if using_job_queue else None + 'session_id': session_id, + } + if using_job_queue: + job_context.update( + { + 'job_item_id': domain.get('job_item_id'), + 'job_id': domain.get('job_id'), + 'claim_token': domain.get('claim_token'), + 'task_mode': domain.get('task_mode'), + 'step_code': domain.get('step_code'), + 'step_payload': domain.get('step_payload'), + } + ) future = worker_pool.submit(self.detect_domain, domain_id, domain, job_context) future.add_done_callback(self._enqueue_completed_future) inflight_futures.add(future) @@ -5697,6 +8855,18 @@ class DetectWorker: try: while self.running: + abort_reason = self._detection_session_abort_reason(session_id) + if abort_reason: + logger.warning(f"检测会话已切换，停止旧会话主循环: {abort_reason}") + self._mark_detection_phase("restarting", f"检测会话已切换，停止旧会话主循环: {abort_reason}") + self._sync_worker_log_event( + f"检测会话已切换，停止旧会话主循环: {abort_reason}", + level='warning', + mode='key', + ) + final_phase = "restarting" + final_detail = f"检测会话已切换，停止旧会话主循环: {abort_reason}" + break if self.stop_requested: logger.info("检测任务收到停止请求，停止继续领取任务") self._mark_detection_phase("stopping", "检测任务收到停止请求，准备安全退出") @@ -5706,25 +8876,41 @@ class DetectWorker: break prune_finished_futures() self._flush_pending_running_marks() + self._flush_pending_job_releases() self._flush_pending_job_finalizations() self._flush_pending_domain_completions() self._flush_pending_domain_status_updates() self._flush_pending_review_status_updates() - current_thread_limit = self.refresh_thread_count_runtime(force=True, min_interval=0.0) + current_thread_limit = self.refresh_thread_count_runtime( + min_interval=thread_count_refresh_interval + ) max_threads = sync_worker_pool_size(current_thread_limit) live_active, submitted_total, queued_backlog, backlog_limit, dispatch_capacity, available_slots = inflight_submission_metrics(max_threads) now_ts = time.time() if now_ts - last_expired_job_recycle_at >= 2.0: - recycled_job_items = self.db.recycle_expired_detect_job_items() - last_expired_job_recycle_at = now_ts - if recycled_job_items: - logger.warning(f"已回收 {recycled_job_items} 个租约过期的任务项，重新回到 pending") - if self.current_job_id and now_ts - last_job_status_refresh_at >= 2.0: + recycle_lock_owner = self._acquire_shared_job_maintenance_lock( + "recycle-expired-job-items", + ttl_seconds=10, + ) + if recycle_lock_owner: + try: + recycled_job_items = self.db.recycle_expired_detect_job_items() + last_expired_job_recycle_at = now_ts + if recycled_job_items: + logger.warning(f"已回收 {recycled_job_items} 个租约过期的任务项，重新回到 pending") + finally: + self._release_shared_job_maintenance_lock( + "recycle-expired-job-items", + recycle_lock_owner, + ) + if self.current_job_id and now_ts - last_job_status_refresh_at >= job_status_refresh_interval: + self._log_job_status_refresh_probe(self.current_job_id) self.db.refresh_detect_job_status(self.current_job_id) last_job_status_refresh_at = now_ts - pending_buffer_cap_multiplier = max( - 1, - int(os.getenv("DOMAINCHECK_PENDING_BUFFER_CAP_MULTIPLIER", "1") or 1), + pending_buffer_cap_multiplier = self._runtime_positive_int_override( + "pending_buffer_cap_multiplier", + "DOMAINCHECK_PENDING_BUFFER_CAP_MULTIPLIER", + default=1, ) pending_buffer_target = min( max(4, backlog_limit), @@ -5737,20 +8923,27 @@ class DetectWorker: claim_lease_seconds = max(300, min(1800, claim_batch_size * 30)) # single_step 必须绑定当前 job；domain_pipeline 则持续从全局任务队列补位， # 否则旧 job 未完全结束时，新 job 会长期堆在 pending。 - scoped_job_id = ( - self.current_job_id - if self.current_job_id and self._should_scope_claims_to_current_job() - else None - ) + scoped_job_id = self._current_scoped_claim_job_id() + if scoped_job_id not in (None, "", 0, "0"): + self._log_explicit_scope_probe( + scoped_job_id=scoped_job_id, + max_threads=max_threads, + refill_slots=refill_slots, + claim_batch_size=claim_batch_size, + pending_buffer=len(pending_domains), + ) claim_started_at = time.perf_counter() - domains = self.db.claim_detect_job_items( - config.NODE_CODE, + domains = self._claim_detect_job_items( limit=claim_batch_size, lease_seconds=claim_lease_seconds, job_id=scoped_job_id, ) claim_elapsed_ms = int((time.perf_counter() - claim_started_at) * 1000) legacy_fetch_elapsed_ms = 0 + proactive_topup_needed = ( + not single_step_session_active + and self._should_proactively_top_up_pipeline(len(domains), refill_slots, max_threads) + ) if domains: append_pending_domains( domains, @@ -5762,6 +8955,47 @@ class DetectWorker: claim_batch_size, claim_lease_seconds, ) + if proactive_topup_needed: + proactive_now = time.time() + if proactive_now - float(last_proactive_pipeline_topup_at or 0.0) >= 0.5: + pipeline_domains = self._process_pipeline_tasks_until_available(thread_limit=max_threads) + last_proactive_pipeline_topup_at = proactive_now + if pipeline_domains: + append_pending_domains( + pipeline_domains, + True, + "任务队列(pipeline推进补位)", + claim_elapsed_ms, + legacy_fetch_elapsed_ms, + dispatch_capacity, + claim_batch_size, + claim_lease_seconds, + ) + elif proactive_now - float(last_proactive_sync_topup_at or 0.0) >= 1.0: + refill_domains = self._pull_sync_tasks_until_available( + thread_limit=max_threads, + claim_after_pull=False, + ) + last_proactive_sync_topup_at = proactive_now + if refill_domains: + append_pending_domains( + refill_domains, + True, + "任务队列(主动补货补位)", + claim_elapsed_ms, + legacy_fetch_elapsed_ms, + dispatch_capacity, + claim_batch_size, + claim_lease_seconds, + ) + if self._should_prefetch_sync_tasks(len(pending_domains), len(inflight_futures), max_threads): + proactive_now = time.time() + if proactive_now - float(last_proactive_sync_topup_at or 0.0) >= 1.0: + self._pull_sync_tasks_until_available( + thread_limit=max_threads, + claim_after_pull=False, + ) + last_proactive_sync_topup_at = proactive_now elif not single_step_session_active: pipeline_domains = self._process_pipeline_tasks_until_available(thread_limit=max_threads) if pipeline_domains: @@ -5880,13 +9114,21 @@ class DetectWorker: time.sleep(0.05 if pending_domains or inflight_futures else 0.2) finally: self._flush_pending_running_marks(force=True, batch_limit=5000) + self._flush_pending_job_releases(force=True, batch_limit=5000) self._flush_pending_job_finalizations(force=True, batch_limit=5000) self._flush_pending_domain_completions(force=True, batch_limit=5000) self._flush_pending_domain_status_updates(force=True, batch_limit=5000) self._flush_pending_review_status_updates(force=True, batch_limit=5000) if self.current_job_id: self.db.refresh_detect_job_status(self.current_job_id) - worker_pool.shutdown(wait=False, cancel_futures=False) + can_blocking_shutdown = ( + not pending_domains + and not inflight_futures + and self._get_active_domain_threads() <= 0 + ) + worker_pool.shutdown(wait=can_blocking_shutdown, cancel_futures=True) + if can_blocking_shutdown: + gc.collect() # 完成进度 if self.detect_thread: @@ -6083,17 +9325,7 @@ class DetectWorker: pubsub = None try: # 创建新的Redis客户端用于订阅 - redis_sub_client = redis.Redis( - host=config.REDIS_HOST, - port=config.REDIS_PORT, - password=config.REDIS_PASSWORD, - db=config.REDIS_DB, - decode_responses=True, - socket_connect_timeout=30, # 增加连接超时时间 - socket_timeout=60, # 增加读取超时时间 - retry_on_timeout=True, - health_check_interval=30 - ) + redis_sub_client = get_redis_client(role="pubsub") # 订阅配置更新频道 pubsub = redis_sub_client.pubsub() @@ -6118,23 +9350,23 @@ class DetectWorker: previous_proxy_config = dict(self.proxy_config or {}) self.detect_options = self.load_detect_options() self.proxy_config = self.load_proxy_config() + previous_thread_count = int(getattr(self, "thread_count", 0) or 0) self.thread_count = self.load_thread_count() self.sensitive_words = self._load_sensitive_words_runtime() self.proxy_max_reuse_count = max(8, self.thread_count * 4) self.runtime_settings = self.load_runtime_settings() self.load_cookies_from_remote() self.update_config_labels() - previous_urls = tuple(previous_proxy_config.get('proxy_urls') or []) - current_urls = tuple(self.proxy_config.get('proxy_urls') or []) - proxy_changed = ( - bool(previous_proxy_config.get('proxy_enable', False)) != bool(self.proxy_config.get('proxy_enable', False)) - or bool(previous_proxy_config.get('allow_direct', False)) != bool(self.proxy_config.get('allow_direct', False)) - or previous_urls != current_urls + should_refresh_proxy, reset_proxy_cooldown = self._should_trigger_proxy_refresh_for_config_update( + config_type=config_type, + previous_proxy_config=previous_proxy_config, + previous_thread_count=previous_thread_count, + current_thread_count=self.thread_count, ) - if config_type in {"proxy_config", "thread_count", "node_thread_counts", "runtime_settings"} or proxy_changed: + if should_refresh_proxy: self.trigger_proxy_refresh( reason=f"config_update:{config_type}", - reset_cooldown=proxy_changed or config_type == "proxy_config", + reset_cooldown=reset_proxy_cooldown, ) logger.debug("配置已更新") continue diff --git a/domainCheck/node_process_counts.json b/domainCheck/node_process_counts.json new file mode 100644 index 0000000..95fd5d7 --- /dev/null +++ b/domainCheck/node_process_counts.json @@ -0,0 +1,4 @@ +{ + "mainland-controller-01": 160, + "mainland-worker-01": 1 +} diff --git a/domainCheck/process_count.json b/domainCheck/process_count.json new file mode 100644 index 0000000..818793e --- /dev/null +++ b/domainCheck/process_count.json @@ -0,0 +1,3 @@ +{ + "process_count": "160" +} diff --git a/domainCheck/tests/test_database_claim_priority.py b/domainCheck/tests/test_database_claim_priority.py index 818cf35..444bc55 100644 --- a/domainCheck/tests/test_database_claim_priority.py +++ b/domainCheck/tests/test_database_claim_priority.py @@ -1,5 +1,6 @@ -import unittest import sys +import unittest +from datetime import datetime, timedelta sys.path.insert(0, "/www/wwwroot/getDomain/domainCheck") @@ -7,6 +8,7 @@ sys.path.insert(0, "/www/wwwroot/getDomain/domainCheck") from app.utils.database import _detect_job_item_step_priority from app.utils.database import _ordered_step_claim_codes from app.utils.database import _resolve_step_claim_quota +from app.utils.database import _select_preferred_claim_job_ids class DetectJobItemClaimPriorityTestCase(unittest.TestCase): @@ -46,9 +48,42 @@ class DetectJobItemClaimPriorityTestCase(unittest.TestCase): _detect_job_item_step_priority("detect_register"), ) - def test_step_claim_quota_defaults_to_quarter_window_with_floor(self): + def test_step_claim_quota_defaults_to_full_window_for_large_pools(self): self.assertEqual(64, _resolve_step_claim_quota(120)) - self.assertEqual(300, _resolve_step_claim_quota(1200)) + self.assertEqual(1200, _resolve_step_claim_quota(1200)) + + def test_select_preferred_claim_job_ids_prioritizes_running_jobs_even_if_older(self): + now = datetime(2026, 4, 24, 18, 30, 0) + + selected = _select_preferred_claim_job_ids( + [ + (19, "pending", now - timedelta(hours=60)), + (57, "running", now - timedelta(hours=72)), + (58, "pending", now - timedelta(hours=2)), + ], + limit=3, + recent_hours=24, + now=now, + ) + + self.assertEqual([57, 58], selected) + + def test_select_preferred_claim_job_ids_filters_out_stale_pending_jobs(self): + now = datetime(2026, 4, 24, 18, 30, 0) + + selected = _select_preferred_claim_job_ids( + [ + (19, "pending", now - timedelta(hours=60)), + (20, "pending", now - timedelta(hours=40)), + (58, "pending", now - timedelta(hours=2)), + (59, "pending", now - timedelta(minutes=30)), + ], + limit=4, + recent_hours=24, + now=now, + ) + + self.assertEqual([59, 58], selected) if __name__ == "__main__": diff --git a/domainCheck/tests/test_database_connection_pool.py b/domainCheck/tests/test_database_connection_pool.py new file mode 100644 index 0000000..7c6fc1f --- /dev/null +++ b/domainCheck/tests/test_database_connection_pool.py @@ -0,0 +1,440 @@ +import threading +import time +import sys +import unittest +from unittest.mock import MagicMock, patch + + +sys.path.insert(0, "/www/wwwroot/getDomain/domainCheck") + +from psycopg2 import extensions # noqa: E402 + +from app.utils.database import Database, _build_claim_token # noqa: E402 + + +class _FakeCursor: + def __init__(self): + self.closed = False + self.executed = [] + self.fetchone_queue = [(1,)] + self.fetchall_queue = [] + + def execute(self, sql, params=None): + self.executed.append((sql, params)) + + def fetchone(self): + if self.fetchone_queue: + return self.fetchone_queue.pop(0) + return (1,) + + def fetchall(self): + if self.fetchall_queue: + return self.fetchall_queue.pop(0) + return [] + + def close(self): + self.closed = True + + +class _HealthyConn: + def __init__(self): + self.closed = False + self.cursor_calls = 0 + + def get_transaction_status(self): + return extensions.TRANSACTION_STATUS_IDLE + + def rollback(self): + return None + + def cursor(self): + self.cursor_calls += 1 + return _FakeCursor() + + def close(self): + self.closed = True + + +class _BrokenRollbackConn: + def __init__(self): + self.closed = False + + def get_transaction_status(self): + return extensions.TRANSACTION_STATUS_INTRANS + + def rollback(self): + raise RuntimeError("rollback failed") + + def close(self): + self.closed = True + + +class DatabaseConnectionPoolTests(unittest.TestCase): + def _build_db(self): + db = Database.__new__(Database) + db.connection_pool = [] + db.pool_size = 2 + db.pool_idle_keep_max = 1 + db.pool_healthcheck_interval = 30.0 + db.pool_acquire_timeout = 1.0 + db.pool_lock = threading.Lock() + db.pool_condition = threading.Condition(db.pool_lock) + db._pool_initialized = True + db.total_connections = 0 + db._connection_last_healthcheck = {} + db.close = Database.close.__get__(db, Database) + db.connect = Database.connect.__get__(db, Database) + db._prepare_pooled_connection = Database._prepare_pooled_connection.__get__(db, Database) + db._discard_connection = Database._discard_connection.__get__(db, Database) + return db + + def test_claim_token_is_unique_even_for_same_thread_and_instant(self): + token_a = _build_claim_token("mainland-controller-01", thread_id=123) + token_b = _build_claim_token("mainland-controller-01", thread_id=123) + + self.assertNotEqual(token_a, token_b) + self.assertLessEqual(len(token_a), 64) + self.assertLessEqual(len(token_b), 64) + self.assertTrue(token_a.startswith("mainland-controller-01-7b-")) + self.assertTrue(token_b.startswith("mainland-controller-01-7b-")) + + def test_connect_discards_bad_pooled_connection_and_creates_new_one(self): + db = self._build_db() + bad_conn = _BrokenRollbackConn() + good_conn = _HealthyConn() + db.connection_pool = [bad_conn] + db.total_connections = 1 + db._connection_last_healthcheck[id(bad_conn)] = time.monotonic() + db._create_connection = MagicMock(return_value=good_conn) + + conn, cur = db.connect(thread_id=123) + + self.assertIs(conn, good_conn) + self.assertIsInstance(cur, _FakeCursor) + self.assertTrue(bad_conn.closed) + self.assertEqual(1, db.total_connections) + db._create_connection.assert_called_once() + + def test_close_discards_connection_when_rollback_fails(self): + db = self._build_db() + conn = _BrokenRollbackConn() + db.total_connections = 1 + db._connection_last_healthcheck[id(conn)] = time.monotonic() + + db.close(conn, None) + + self.assertTrue(conn.closed) + self.assertEqual(0, db.total_connections) + self.assertEqual([], db.connection_pool) + + def test_ensure_cluster_runtime_tables_skips_ddl_when_schema_is_ready(self): + db = Database.__new__(Database) + db._cluster_runtime_schema_ready = MagicMock(return_value=True) + db.execute = MagicMock() + db.ensure_cluster_runtime_tables = Database.ensure_cluster_runtime_tables.__get__(db, Database) + + self.assertTrue(db.ensure_cluster_runtime_tables()) + db.execute.assert_not_called() + + def test_ensure_cluster_runtime_tables_executes_ddl_when_schema_is_missing(self): + db = Database.__new__(Database) + db._cluster_runtime_schema_ready = MagicMock(return_value=False) + db._cluster_runtime_schema_basics_present = MagicMock(return_value=False) + db.execute = MagicMock(return_value=True) + db.ensure_cluster_runtime_tables = Database.ensure_cluster_runtime_tables.__get__(db, Database) + + self.assertTrue(db.ensure_cluster_runtime_tables()) + db.execute.assert_called_once() + + def test_ensure_cluster_runtime_tables_skips_index_repair_without_explicit_enable(self): + db = Database.__new__(Database) + db._cluster_runtime_schema_ready = MagicMock(return_value=False) + db._cluster_runtime_schema_basics_present = MagicMock(return_value=True) + db._ensure_cluster_runtime_indexes = MagicMock(return_value=True) + db.execute = MagicMock() + db.ensure_cluster_runtime_tables = Database.ensure_cluster_runtime_tables.__get__(db, Database) + db._runtime_index_repair_enabled = Database._runtime_index_repair_enabled.__get__(db, Database) + + with patch.dict("os.environ", {}, clear=False): + self.assertFalse(db.ensure_cluster_runtime_tables()) + + db._ensure_cluster_runtime_indexes.assert_not_called() + db.execute.assert_not_called() + + def test_cluster_runtime_schema_ready_requires_claim_step_indexes(self): + db = Database.__new__(Database) + db._cluster_runtime_schema_basics_present = MagicMock(return_value=True) + db._cluster_runtime_missing_indexes = MagicMock(return_value=["idx_detect_job_items_claim_step_ready"]) + db._cluster_runtime_schema_ready = Database._cluster_runtime_schema_ready.__get__(db, Database) + + self.assertFalse(db._cluster_runtime_schema_ready()) + + def test_cluster_runtime_missing_indexes_treats_invalid_indexes_as_missing(self): + db = Database.__new__(Database) + db.fetch_all = MagicMock( + return_value=[ + {"index_name": "idx_detect_job_items_job_domain_step", "is_valid": True}, + {"index_name": "idx_detect_job_items_claim_step_ready", "is_valid": False}, + ] + ) + db._cluster_runtime_missing_indexes = Database._cluster_runtime_missing_indexes.__get__(db, Database) + + missing = db._cluster_runtime_missing_indexes() + + self.assertIn("idx_detect_job_items_claim_step_ready", missing) + self.assertIn("idx_detect_job_items_claim_job_step_ready", missing) + self.assertNotIn("idx_detect_job_items_job_domain_step", missing) + + def test_ensure_cluster_runtime_indexes_skips_invalid_rebuild_by_default(self): + db = Database.__new__(Database) + conn = MagicMock() + cur = _FakeCursor() + cur.fetchone_queue = [(True,)] + db._cluster_runtime_missing_indexes = MagicMock(return_value=["idx_detect_job_items_stalled_job_activity"]) + db._cluster_runtime_invalid_indexes = MagicMock(return_value=["idx_detect_job_items_stalled_job_activity"]) + db._create_connection = MagicMock(return_value=conn) + conn.cursor.return_value = cur + db._ensure_cluster_runtime_indexes = Database._ensure_cluster_runtime_indexes.__get__(db, Database) + db._runtime_index_repair_enabled = Database._runtime_index_repair_enabled.__get__(db, Database) + + with patch.dict("os.environ", {}, clear=False): + self.assertTrue(db._ensure_cluster_runtime_indexes()) + + executed_sql = [sql for sql, _ in cur.executed] + self.assertFalse(any("DROP INDEX CONCURRENTLY IF EXISTS idx_detect_job_items_stalled_job_activity" in str(sql) for sql in executed_sql)) + self.assertFalse(any("CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_detect_job_items_stalled_job_activity" in str(sql) for sql in executed_sql)) + + def test_ensure_cluster_runtime_indexes_rebuilds_invalid_index_when_enabled(self): + db = Database.__new__(Database) + conn = MagicMock() + cur = _FakeCursor() + cur.fetchone_queue = [(True,)] + db._cluster_runtime_missing_indexes = MagicMock(return_value=["idx_detect_job_items_stalled_job_activity"]) + db._cluster_runtime_invalid_indexes = MagicMock(return_value=["idx_detect_job_items_stalled_job_activity"]) + db._create_connection = MagicMock(return_value=conn) + conn.cursor.return_value = cur + db._ensure_cluster_runtime_indexes = Database._ensure_cluster_runtime_indexes.__get__(db, Database) + db._runtime_index_repair_enabled = Database._runtime_index_repair_enabled.__get__(db, Database) + + with patch.dict("os.environ", {"DOMAINCHECK_RUNTIME_INDEX_REPAIR_ENABLED": "1"}, clear=False): + self.assertTrue(db._ensure_cluster_runtime_indexes()) + + executed_sql = [str(sql) for sql, _ in cur.executed] + self.assertTrue(any("DROP INDEX CONCURRENTLY IF EXISTS idx_detect_job_items_stalled_job_activity" in sql for sql in executed_sql)) + self.assertTrue(any("CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_detect_job_items_stalled_job_activity" in sql for sql in executed_sql)) + + def test_release_detect_job_items_is_enabled_by_default(self): + db = Database.__new__(Database) + conn = MagicMock() + cur = _FakeCursor() + cur.fetchone_queue = [(0, [])] + db.connect = MagicMock(return_value=(conn, cur)) + db.close = MagicMock() + db.release_detect_job_items_for_node = Database.release_detect_job_items_for_node.__get__(db, Database) + + self.assertEqual(0, db.release_detect_job_items_for_node("mainland-controller-01-a")) + db.connect.assert_called_once() + executed_sql = [sql for sql, _ in cur.executed] + self.assertTrue(any("FOR UPDATE SKIP LOCKED" in sql for sql in executed_sql)) + + def test_claim_restart_released_detect_job_items_targets_restart_release_rows(self): + db = Database.__new__(Database) + conn = MagicMock() + cur = _FakeCursor() + cur.fetchall_queue = [[ + ( + 101, + 880, + 501, + "token-a", + "detect_register", + "domain_pipeline", + "sync-overseas-20249", + "detect_register", + None, + "example.com", + 0, + 0, + 0, + 0, + None, + 0, + 0, + ) + ]] + db.connect = MagicMock(return_value=(conn, cur)) + db.close = MagicMock() + db.claim_restart_released_detect_job_items = Database.claim_restart_released_detect_job_items.__get__(db, Database) + + rows = db.claim_restart_released_detect_job_items("mainland-controller-01-a", 880, limit=16, lease_seconds=900) + + self.assertEqual(1, len(rows)) + executed_sql = [sql for sql, _ in cur.executed] + self.assertTrue(any("released after worker restart" in sql for sql in executed_sql)) + self.assertTrue(any("released before execution after worker restart" in sql for sql in executed_sql)) + self.assertTrue(any("FOR UPDATE SKIP LOCKED" in sql for sql in executed_sql)) + + def test_release_detect_job_items_can_be_disabled_explicitly(self): + db = Database.__new__(Database) + db.connect = MagicMock() + db.release_detect_job_items_for_node = Database.release_detect_job_items_for_node.__get__(db, Database) + + with patch.dict("os.environ", {"DOMAINCHECK_ENABLE_NODE_ITEM_RELEASE": "0"}, clear=False): + self.assertEqual(0, db.release_detect_job_items_for_node("mainland-controller-01-a")) + + db.connect.assert_not_called() + + def test_release_detect_job_items_for_node_job_targets_single_job(self): + db = Database.__new__(Database) + conn = MagicMock() + cur = _FakeCursor() + cur.fetchone_queue = [(4, [876])] + db.connect = MagicMock(return_value=(conn, cur)) + db.close = MagicMock() + db._refresh_detect_job_status_with_cursor = MagicMock() + db.release_detect_job_items_for_node_job = ( + Database.release_detect_job_items_for_node_job.__get__(db, Database) + ) + + self.assertEqual(4, db.release_detect_job_items_for_node_job("mainland-controller-01-ba", 876)) + executed_sql = [sql for sql, _ in cur.executed] + self.assertTrue(any("WHERE claimed_by = %s" in sql and "AND job_id = %s" in sql for sql in executed_sql)) + self.assertTrue(any("FOR UPDATE SKIP LOCKED" in sql for sql in executed_sql)) + db._refresh_detect_job_status_with_cursor.assert_called_once_with(cur, 876) + + def test_release_single_detect_job_item_skips_job_refresh(self): + db = Database.__new__(Database) + conn = MagicMock() + cur = _FakeCursor() + cur.fetchone_queue = [(11,)] + db.connect = MagicMock(return_value=(conn, cur)) + db.close = MagicMock() + db._refresh_detect_job_status_with_cursor = MagicMock() + db.release_detect_job_item = Database.release_detect_job_item.__get__(db, Database) + + self.assertTrue(db.release_detect_job_item(101, "token-a", reason="session_replaced:7")) + executed_sql = [sql for sql, _ in cur.executed] + self.assertTrue(any("UPDATE detect_job_items" in sql for sql in executed_sql)) + db._refresh_detect_job_status_with_cursor.assert_not_called() + + def test_release_detect_job_items_batch_skips_job_refresh(self): + db = Database.__new__(Database) + conn = MagicMock() + cur = _FakeCursor() + cur.fetchall_queue = [[(101,), (102,)]] + cur.mogrify = MagicMock(side_effect=lambda sql, params: str(tuple(params)).encode("utf-8")) + db.connect = MagicMock(return_value=(conn, cur)) + db.close = MagicMock() + db._refresh_detect_job_status_with_cursor = MagicMock() + db.release_detect_job_items_batch = Database.release_detect_job_items_batch.__get__(db, Database) + + released = db.release_detect_job_items_batch( + [ + (101, "token-a", "session_replaced:7"), + (102, "token-b", "queued_before_start"), + ] + ) + + self.assertEqual(2, released) + executed_sql = [sql for sql, _ in cur.executed] + self.assertTrue(any("UPDATE detect_job_items" in sql for sql in executed_sql)) + db._refresh_detect_job_status_with_cursor.assert_not_called() + + def test_recycle_expired_detect_job_items_skips_when_advisory_lock_is_busy(self): + db = Database.__new__(Database) + conn = MagicMock() + cur = _FakeCursor() + cur.fetchone_queue = [(False,)] + db.connect = MagicMock(return_value=(conn, cur)) + db.close = MagicMock() + db.recycle_expired_detect_job_items = Database.recycle_expired_detect_job_items.__get__(db, Database) + + self.assertEqual(0, db.recycle_expired_detect_job_items()) + executed_sql = [sql for sql, _ in cur.executed] + self.assertIn("SELECT pg_try_advisory_lock(%s)", executed_sql) + self.assertNotIn( + "WITH recycled AS (", + " ".join(executed_sql), + ) + + def test_recycle_expired_detect_job_items_runs_under_advisory_lock(self): + db = Database.__new__(Database) + conn = MagicMock() + cur = _FakeCursor() + cur.fetchone_queue = [(True,), (3, [11, 12])] + db.connect = MagicMock(return_value=(conn, cur)) + db.close = MagicMock() + db._refresh_detect_job_status_with_cursor = MagicMock() + db.recycle_expired_detect_job_items = Database.recycle_expired_detect_job_items.__get__(db, Database) + + self.assertEqual(3, db.recycle_expired_detect_job_items()) + executed_sql = [sql for sql, _ in cur.executed] + self.assertIn("SELECT pg_try_advisory_lock(%s)", executed_sql) + self.assertTrue(any("expired_candidates" in sql and "WITH" in sql for sql in executed_sql)) + self.assertTrue(any("FOR UPDATE SKIP LOCKED" in sql for sql in executed_sql)) + self.assertIn("SELECT pg_advisory_unlock(%s)", executed_sql) + db._refresh_detect_job_status_with_cursor.assert_any_call(cur, 11) + db._refresh_detect_job_status_with_cursor.assert_any_call(cur, 12) + + def test_recycle_stalled_detect_job_items_skips_when_advisory_lock_is_busy(self): + db = Database.__new__(Database) + conn = MagicMock() + cur = _FakeCursor() + cur.fetchone_queue = [(False,)] + db.connect = MagicMock(return_value=(conn, cur)) + db.close = MagicMock() + db.recycle_stalled_detect_job_items = Database.recycle_stalled_detect_job_items.__get__(db, Database) + + self.assertEqual(0, db.recycle_stalled_detect_job_items(876, stall_seconds=1800)) + executed_sql = [sql for sql, _ in cur.executed] + self.assertIn("SELECT pg_try_advisory_lock(%s)", executed_sql) + self.assertNotIn( + "WITH stalled_candidates AS (", + " ".join(executed_sql), + ) + + def test_recycle_stalled_detect_job_items_runs_under_advisory_lock(self): + db = Database.__new__(Database) + conn = MagicMock() + cur = _FakeCursor() + cur.fetchone_queue = [(True,), (5, [876])] + db.connect = MagicMock(return_value=(conn, cur)) + db.close = MagicMock() + db._refresh_detect_job_status_with_cursor = MagicMock() + db.recycle_stalled_detect_job_items = Database.recycle_stalled_detect_job_items.__get__(db, Database) + + self.assertEqual(5, db.recycle_stalled_detect_job_items(876, stall_seconds=1800, batch_size=64)) + executed_sql = [sql for sql, _ in cur.executed] + self.assertIn("SELECT pg_try_advisory_lock(%s)", executed_sql) + self.assertTrue(any("stalled_candidates" in sql and "WITH" in sql for sql in executed_sql)) + self.assertTrue(any("FOR UPDATE SKIP LOCKED" in sql for sql in executed_sql)) + self.assertIn("SELECT pg_advisory_unlock(%s)", executed_sql) + db._refresh_detect_job_status_with_cursor.assert_called_once_with(cur, 876) + + def test_refresh_detect_job_status_uses_exists_queries_for_final_status(self): + db = Database.__new__(Database) + cur = _FakeCursor() + cur.fetchone_queue = [ + ("domain_pipeline",), + (False,), # dispatch_active_exists + (False,), # unprocessed_terminal_exists + (False,), # pending_exists + (True,), # failed_exists + (True,), # done_exists + ] + + Database._refresh_detect_job_status_with_cursor(db, cur, 876) + + executed_sql = [sql for sql, _ in cur.executed] + self.assertTrue(any("SELECT COALESCE(task_mode, '')" in sql for sql in executed_sql)) + self.assertTrue(any("SELECT EXISTS" in sql and "item.status IN ('claimed', 'running')" in sql for sql in executed_sql)) + self.assertTrue(any("SELECT EXISTS" in sql and "controller_processed" in sql for sql in executed_sql)) + self.assertTrue(any("SELECT EXISTS" in sql and "item.status = 'pending'" in sql for sql in executed_sql)) + self.assertTrue(any("SELECT EXISTS" in sql and "item.status = 'failed'" in sql for sql in executed_sql)) + self.assertTrue(any("SELECT EXISTS" in sql and "item.status IN ('completed', 'blacklisted')" in sql for sql in executed_sql)) + self.assertTrue(any("UPDATE detect_jobs" in sql for sql in executed_sql)) + self.assertTrue(any(params == ('partial_failed', 876) for _, params in cur.executed)) + + +if __name__ == "__main__": + unittest.main() diff --git a/domainCheck/tests/test_database_job_status_refresh.py b/domainCheck/tests/test_database_job_status_refresh.py new file mode 100644 index 0000000..da56ff4 --- /dev/null +++ b/domainCheck/tests/test_database_job_status_refresh.py @@ -0,0 +1,200 @@ +import sys +import unittest +from unittest.mock import patch +from unittest.mock import MagicMock +import json +import time + + +sys.path.insert(0, "/www/wwwroot/getDomain/domainCheck") + +from app.utils.database import Database # noqa: E402 + + +class _FakeConn: + def __init__(self): + self.committed = False + self.rolled_back = False + + def commit(self): + self.committed = True + + def rollback(self): + self.rolled_back = True + + +class _FakeCursor: + def __init__(self, fetchall_sequence): + self.fetchall_sequence = list(fetchall_sequence or []) + self.executed_sql = [] + + def mogrify(self, template, params): + rendered = [] + for value in params: + if isinstance(value, str): + rendered.append(f"'{value}'") + else: + rendered.append(str(value)) + return f"({', '.join(rendered)})".encode("utf-8") + + def execute(self, sql, params=None): + self.executed_sql.append((sql, params)) + + def fetchall(self): + if self.fetchall_sequence: + return self.fetchall_sequence.pop(0) + return [] + + +class DatabaseJobStatusRefreshTests(unittest.TestCase): + def test_mark_running_batch_skips_job_status_refresh(self): + db = Database.__new__(Database) + conn = _FakeConn() + cur = _FakeCursor( + fetchall_sequence=[ + [(101,), (101,), (102,)], + ] + ) + db.connect = MagicMock(return_value=(conn, cur)) + db.close = MagicMock() + db._refresh_detect_job_status_with_cursor = MagicMock() + + updated = db.mark_detect_job_items_running_batch( + [(1, "token-a"), (2, "token-b"), (3, "token-c")] + ) + + self.assertEqual(3, updated) + self.assertTrue(conn.committed) + db._refresh_detect_job_status_with_cursor.assert_not_called() + + def test_release_single_item_skips_job_status_refresh(self): + db = Database.__new__(Database) + conn = _FakeConn() + cur = _FakeCursor(fetchall_sequence=[[(101,)]] ) + cur.fetchone = lambda: cur.fetchall_sequence.pop(0)[0] if cur.fetchall_sequence else None + db.connect = MagicMock(return_value=(conn, cur)) + db.close = MagicMock() + db._refresh_detect_job_status_with_cursor = MagicMock() + + released = db.release_detect_job_item(1, "token-a", reason="session replaced") + + self.assertTrue(released) + self.assertTrue(conn.committed) + db._refresh_detect_job_status_with_cursor.assert_not_called() + + def test_get_active_detect_job_uses_lightweight_candidate_query(self): + db = Database.__new__(Database) + db.fetch_one = MagicMock(return_value={"id": 9, "job_code": "sync-overseas-9"}) + db.redis_client = None + + with patch.dict( + "os.environ", + { + "DOMAINCHECK_TAIL_HANDOFF_ENABLED": "1", + "DOMAINCHECK_TAIL_HANDOFF_MAX_ACTIVE_ITEMS": "64", + "DOMAINCHECK_TAIL_HANDOFF_MAX_PENDING_ITEMS": "128", + "DOMAINCHECK_TAIL_HANDOFF_MIN_PENDING_ITEMS": "1", + "DOMAINCHECK_RUNNING_JOB_STALL_SECONDS": "1800", + }, + clear=False, + ): + result = db.get_active_detect_job() + + self.assertEqual({"id": 9, "job_code": "sync-overseas-9"}, result) + sql, params = db.fetch_one.call_args[0] + self.assertIn("WITH tail_config AS", sql) + self.assertIn("candidate_jobs AS", sql) + self.assertIn("tail_handoff_candidate", sql) + self.assertIn("selection_reason", sql) + self.assertIn("LEFT JOIN LATERAL", sql) + self.assertIn("item.status IN ('pending', 'claimed', 'running')", sql) + self.assertIn("running_job_stalled", sql) + self.assertIn("latest_unfinished_activity_at", sql) + self.assertEqual((True, 64, 128, 1, 1800), params) + + def test_get_active_detect_job_uses_cache_when_available(self): + db = Database.__new__(Database) + db.redis_client = MagicMock() + db.redis_client.get.return_value = json.dumps({"id": 12, "job_code": "sync-overseas-12"}) + db.fetch_one = MagicMock() + + result = db.get_active_detect_job() + + self.assertEqual({"id": 12, "job_code": "sync-overseas-12"}, result) + db.fetch_one.assert_not_called() + + def test_get_active_detect_job_writes_cache_after_query(self): + db = Database.__new__(Database) + db.redis_client = MagicMock() + db.redis_client.get.return_value = None + db.redis_client.set.return_value = True + db.fetch_one = MagicMock(return_value={"id": 13, "job_code": "sync-overseas-13", "items_pending": 5}) + + result = db.get_active_detect_job() + + self.assertEqual({"id": 13, "job_code": "sync-overseas-13", "items_pending": 5}, result) + db.redis_client.setex.assert_called_once() + cache_key, ttl, payload = db.redis_client.setex.call_args[0] + self.assertEqual("domaincheck:active_detect_job_summary:v1", cache_key) + self.assertEqual(3, ttl) + self.assertEqual({"id": 13, "job_code": "sync-overseas-13", "items_pending": 5}, json.loads(payload)) + + def test_get_active_detect_job_uses_local_cache_after_first_query(self): + db = Database.__new__(Database) + db.redis_client = None + db.fetch_one = MagicMock(return_value={"id": 14, "job_code": "sync-overseas-14"}) + + first = db.get_active_detect_job() + second = db.get_active_detect_job() + + self.assertEqual({"id": 14, "job_code": "sync-overseas-14"}, first) + self.assertEqual({"id": 14, "job_code": "sync-overseas-14"}, second) + db.fetch_one.assert_called_once() + + def test_get_active_detect_job_uses_stale_local_cache_when_refresh_lock_is_busy(self): + db = Database.__new__(Database) + db.redis_client = MagicMock() + db.redis_client.get.return_value = None + db.redis_client.set.return_value = False + db.fetch_one = MagicMock() + now_ts = time.time() + db._set_local_active_detect_job_cache( + {"id": 15, "job_code": "sync-overseas-15"}, + now_ts=now_ts - 4, + fresh_ttl_seconds=3, + stale_ttl_seconds=10, + ) + + with patch("time.sleep", return_value=None): + result = db.get_active_detect_job() + + self.assertEqual({"id": 15, "job_code": "sync-overseas-15"}, result) + db.fetch_one.assert_not_called() + + def test_finalize_batch_skips_job_status_refresh(self): + db = Database.__new__(Database) + conn = _FakeConn() + cur = _FakeCursor( + fetchall_sequence=[ + [(1, 101), (2, 101), (3, 102)], + ] + ) + db.connect = MagicMock(return_value=(conn, cur)) + db.close = MagicMock() + db._refresh_detect_job_status_with_cursor = MagicMock() + + updated = db.finalize_detect_job_items_batch( + [ + {"job_item_id": 1, "claim_token": "token-a", "final_status": "completed"}, + {"job_item_id": 2, "claim_token": "token-b", "final_status": "failed"}, + {"job_item_id": 3, "claim_token": "token-c", "final_status": "completed"}, + ] + ) + + self.assertEqual(3, updated) + self.assertTrue(conn.committed) + db._refresh_detect_job_status_with_cursor.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/domainCheck/tests/test_database_pool_limits.py b/domainCheck/tests/test_database_pool_limits.py new file mode 100644 index 0000000..644913a --- /dev/null +++ b/domainCheck/tests/test_database_pool_limits.py @@ -0,0 +1,171 @@ +import json +import os +import sys +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + + +sys.path.insert(0, "/www/wwwroot/getDomain/domainCheck") + +from app.config import config # noqa: E402 +from app.utils.database import _local_config_search_roots, _resolve_db_pool_limits # noqa: E402 + + +class DatabasePoolLimitTests(unittest.TestCase): + def _write_json(self, root: str, name: str, payload: dict) -> None: + Path(root, name).write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8") + + def test_explicit_total_budget_override_keeps_previous_pool_shape(self): + with tempfile.TemporaryDirectory() as temp_dir, patch.object( + config, "NODE_CODE", "mainland-controller-01-a" + ), patch.dict( + os.environ, + { + "WORKER_PARENT_NODE_CODE": "mainland-controller-01", + "DOMAINCHECK_DB_POOL_TOTAL_BUDGET": "480", + "DB_POOL_SIZE": "", + "DB_POOL_WARM_SIZE": "", + "DB_POOL_IDLE_KEEP_MAX": "", + }, + clear=False, + ): + self._write_json(temp_dir, "thread_count.json", {"thread_count": "1000"}) + self._write_json(temp_dir, "process_count.json", {"process_count": "80"}) + self._write_json(temp_dir, "node_thread_counts.json", {"mainland-controller-01": 1200}) + self._write_json(temp_dir, "node_process_counts.json", {"mainland-controller-01": 80}) + + previous_cwd = os.getcwd() + try: + os.chdir(temp_dir) + limits = _resolve_db_pool_limits() + finally: + os.chdir(previous_cwd) + + self.assertEqual(6, limits["pool_size"]) + self.assertEqual(1, limits["pool_warm_size"]) + self.assertEqual(3, limits["pool_idle_keep_max"]) + self.assertEqual(1200, limits["scaling_hints"]["thread_count"]) + self.assertEqual(80, limits["scaling_hints"]["process_count"]) + + def test_pool_budget_scales_down_for_mid_sized_multi_process_workers(self): + with tempfile.TemporaryDirectory() as temp_dir, patch.object( + config, "NODE_CODE", "mainland-worker-01-a" + ), patch.dict( + os.environ, + { + "WORKER_PARENT_NODE_CODE": "mainland-worker-01", + "DB_POOL_SIZE": "", + "DB_POOL_WARM_SIZE": "", + "DB_POOL_IDLE_KEEP_MAX": "", + "DOMAINCHECK_DB_POOL_TOTAL_BUDGET": "", + }, + clear=False, + ): + self._write_json(temp_dir, "thread_count.json", {"thread_count": "1000"}) + self._write_json(temp_dir, "process_count.json", {"process_count": "60"}) + self._write_json(temp_dir, "node_thread_counts.json", {"mainland-worker-01": 1200}) + self._write_json(temp_dir, "node_process_counts.json", {"mainland-worker-01": 60}) + + previous_cwd = os.getcwd() + try: + os.chdir(temp_dir) + limits = _resolve_db_pool_limits() + finally: + os.chdir(previous_cwd) + + self.assertEqual(4, limits["pool_size"]) + self.assertEqual(1, limits["pool_warm_size"]) + self.assertEqual(2, limits["pool_idle_keep_max"]) + self.assertEqual(60, limits["scaling_hints"]["process_count"]) + + def test_explicit_pool_env_overrides_win_over_auto_scaling(self): + with tempfile.TemporaryDirectory() as temp_dir, patch.object( + config, "NODE_CODE", "mainland-controller-01" + ), patch.dict( + os.environ, + { + "DB_POOL_SIZE": "16", + "DB_POOL_WARM_SIZE": "4", + "DB_POOL_IDLE_KEEP_MAX": "8", + }, + clear=False, + ): + self._write_json(temp_dir, "thread_count.json", {"thread_count": "1000"}) + self._write_json(temp_dir, "process_count.json", {"process_count": "80"}) + + previous_cwd = os.getcwd() + try: + os.chdir(temp_dir) + limits = _resolve_db_pool_limits() + finally: + os.chdir(previous_cwd) + + self.assertEqual(16, limits["pool_size"]) + self.assertEqual(4, limits["pool_warm_size"]) + self.assertEqual(8, limits["pool_idle_keep_max"]) + + def test_pool_limits_can_resolve_parent_process_override_from_absolute_config_root(self): + with tempfile.TemporaryDirectory() as temp_dir, patch.object( + config, "NODE_CODE", "mainland-controller-01-u" + ), patch.dict( + os.environ, + { + "WORKER_PARENT_NODE_CODE": "mainland-controller-01", + "DB_POOL_SIZE": "", + "DB_POOL_WARM_SIZE": "", + "DB_POOL_IDLE_KEEP_MAX": "", + "DOMAINCHECK_DB_POOL_TOTAL_BUDGET": "", + }, + clear=False, + ): + self._write_json(temp_dir, "thread_count.json", {"thread_count": "1000"}) + self._write_json(temp_dir, "process_count.json", {"process_count": "80"}) + self._write_json(temp_dir, "node_thread_counts.json", {"mainland-controller-01": 1000}) + self._write_json(temp_dir, "node_process_counts.json", {"mainland-controller-01": 80}) + + with patch( + "app.utils.database._local_config_search_roots", + return_value=[Path("/nonexistent/domainCheck"), Path(temp_dir)], + ): + limits = _resolve_db_pool_limits() + + self.assertEqual(80, limits["scaling_hints"]["process_count"]) + self.assertEqual("mainland-controller-01", limits["scaling_hints"]["parent_node_code"]) + self.assertEqual(9, limits["pool_size"]) + + def test_controller_high_process_pool_keeps_ten_connections_at_hundred_processes(self): + with tempfile.TemporaryDirectory() as temp_dir, patch.object( + config, "NODE_CODE", "mainland-controller-01-aa" + ), patch.dict( + os.environ, + { + "WORKER_PARENT_NODE_CODE": "mainland-controller-01", + "DB_POOL_SIZE": "", + "DB_POOL_WARM_SIZE": "", + "DB_POOL_IDLE_KEEP_MAX": "", + "DOMAINCHECK_DB_POOL_TOTAL_BUDGET": "", + }, + clear=False, + ): + self._write_json(temp_dir, "thread_count.json", {"thread_count": "1000"}) + self._write_json(temp_dir, "process_count.json", {"process_count": "100"}) + self._write_json(temp_dir, "node_thread_counts.json", {"mainland-controller-01": 1000}) + self._write_json(temp_dir, "node_process_counts.json", {"mainland-controller-01": 100}) + + previous_cwd = os.getcwd() + try: + os.chdir(temp_dir) + limits = _resolve_db_pool_limits() + finally: + os.chdir(previous_cwd) + + self.assertEqual(100, limits["scaling_hints"]["process_count"]) + self.assertEqual(10, limits["pool_size"]) + self.assertEqual(2, limits["pool_warm_size"]) + self.assertEqual(5, limits["pool_idle_keep_max"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/domainCheck/tests/test_detect_engine_outcomes.py b/domainCheck/tests/test_detect_engine_outcomes.py new file mode 100644 index 0000000..88dd15a --- /dev/null +++ b/domainCheck/tests/test_detect_engine_outcomes.py @@ -0,0 +1,170 @@ +import sys +import unittest +from pathlib import Path +from unittest.mock import MagicMock, patch + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +DOMAINCHECK_ROOT = PROJECT_ROOT / "domainCheck" +if str(DOMAINCHECK_ROOT) not in sys.path: + sys.path.insert(0, str(DOMAINCHECK_ROOT)) + +from app.core.detect_engine import DetectEngine +from app.detectors.rdap_detector import RDAPDetector + + +class DetectEngineOutcomeTests(unittest.TestCase): + def _build_engine(self): + patchers = [ + patch("app.core.detect_engine.Database"), + patch("app.core.detect_engine.RDAPDetector"), + patch("app.core.detect_engine.WaybackDetector"), + patch("app.core.detect_engine.BaiduDetector"), + patch("app.core.detect_engine.Qihu360Detector"), + patch("app.core.detect_engine.GoogleDetector"), + patch("app.core.detect_engine.ChinazDetector"), + patch("app.core.detect_engine.AizhanDetector"), + patch("app.core.detect_engine.JuziseoDetector"), + patch("app.core.detect_engine.JuchaDetector"), + ] + started = [patcher.start() for patcher in patchers] + self.addCleanup(lambda: [patcher.stop() for patcher in reversed(patchers)]) + engine = DetectEngine() + for started_mock in started: + started_mock.return_value = MagicMock() + return engine + + def test_process_task_blacklisted_is_completed_without_retry(self): + engine = self._build_engine() + engine.db.get_task_by_id.return_value = {"id": 7, "domain_id": 42, "retry_count": 1} + engine._detect_domain_with_outcome = MagicMock(return_value=engine.OUTCOME_BLACKLISTED) + + success = engine.process_task(7) + + self.assertTrue(success) + self.assertEqual( + [(7, 1), (7, 2)], + [call.args for call in engine.db.update_task_status.call_args_list], + ) + engine.db.update_task_retry_count.assert_not_called() + + def test_process_task_failed_requeues_for_retry(self): + engine = self._build_engine() + engine.db.get_task_by_id.return_value = {"id": 9, "domain_id": 99, "retry_count": 0} + engine._detect_domain_with_outcome = MagicMock(return_value=engine.OUTCOME_FAILED) + + success = engine.process_task(9) + + self.assertFalse(success) + self.assertEqual( + [(9, 1), (9, 0)], + [call.args for call in engine.db.update_task_status.call_args_list], + ) + engine.db.update_task_retry_count.assert_called_once_with(9, 1) + + def test_deep_detect_fails_when_any_detector_returns_error(self): + engine = self._build_engine() + engine.baidu_detector.check_history.return_value = {"error": "timeout"} + engine.baidu_detector.check_site.return_value = {"has_收录": False, "subdomains": []} + engine.qihu360_detector.check_site.return_value = {"has_收录": False, "subdomains": []} + engine.google_detector.check_site.return_value = {"has_收录": False} + engine.chinaz_detector.check_domain.return_value = {"title": "", "category": "", "has_sensitive": False} + engine.aizhan_detector.check_domain.return_value = {"title": "", "risk": "", "has_sensitive": False} + engine.juziseo_detector.check_domain.return_value = {"history": {}, "backlink": {}} + engine.jucha_detector.check_domain.return_value = {"whois": {}, "beian": {}, "intercept": {"normal": True}} + engine.db.add_detection_result.return_value = True + + outcome = engine._deep_detect(1, "example.com") + + self.assertEqual(engine.OUTCOME_FAILED, outcome) + engine.db.add_detection_result.assert_not_called() + + def test_deep_detect_fails_when_result_persistence_fails(self): + engine = self._build_engine() + engine.baidu_detector.check_history.return_value = {"has_history": False, "has_gray": False} + engine.baidu_detector.check_site.return_value = {"has_收录": False, "subdomains": []} + engine.qihu360_detector.check_site.return_value = {"has_收录": False, "subdomains": []} + engine.google_detector.check_site.return_value = {"has_收录": False} + engine.chinaz_detector.check_domain.return_value = {"title": "", "category": "", "has_sensitive": False} + engine.aizhan_detector.check_domain.return_value = {"title": "", "risk": "", "has_sensitive": False} + engine.juziseo_detector.check_domain.return_value = {"history": {}, "backlink": {}} + engine.jucha_detector.check_domain.return_value = {"whois": {}, "beian": {}, "intercept": {"normal": True}} + engine.db.add_detection_result.return_value = False + + outcome = engine._deep_detect(1, "example.com") + + self.assertEqual(engine.OUTCOME_FAILED, outcome) + engine.db.add_detection_result.assert_called_once() + + def test_deep_detect_normalizes_results_before_persisting(self): + engine = self._build_engine() + engine.baidu_detector.check_history.return_value = {"has_history": True, "has_gray": False} + engine.baidu_detector.check_site.return_value = {"has_收录": True, "subdomains": ["www"]} + engine.qihu360_detector.check_site.return_value = {"has_收录": False, "subdomains": []} + engine.google_detector.check_site.return_value = {"has_收录": False} + engine.chinaz_detector.check_domain.return_value = {"title": "Example", "category": "", "has_sensitive": False} + engine.aizhan_detector.check_domain.return_value = {"title": "Example", "risk": "", "has_sensitive": False} + engine.juziseo_detector.check_domain.return_value = { + "history": {"has_sensitive": False, "has_baidu_history": True, "has_subdomains": False, "is_simplified": True}, + "backlink": {"has_sensitive": False, "has_subdomains": False}, + } + engine.jucha_detector.check_domain.return_value = { + "whois": {"status": ""}, + "beian": {"has_beian": True, "beian_year": "2024", "is_enterprise": True, "beian_match": True}, + "intercept": {"normal": True}, + } + engine.db.add_detection_result.return_value = True + + outcome = engine._deep_detect(1, "example.com") + + self.assertEqual(engine.OUTCOME_SUCCESS, outcome) + persisted_args = engine.db.add_detection_result.call_args.args + self.assertTrue(persisted_args[1]["status"]) + self.assertTrue(persisted_args[1]["has_history"]) + self.assertTrue(persisted_args[2]["status"]) + self.assertTrue(persisted_args[2]["has_收录"]) + self.assertIn("state", persisted_args[7]["history"]) + self.assertTrue(persisted_args[8]["beian"]["status"]) + + def test_deep_detect_stops_early_after_blacklist_hit(self): + engine = self._build_engine() + engine.baidu_detector.check_history.return_value = {"has_history": True, "has_gray": True} + engine.baidu_detector.check_site.return_value = {"has_收录": False, "subdomains": []} + engine.db.add_detection_result.return_value = True + + outcome = engine._deep_detect(1, "example.com") + + self.assertEqual(engine.OUTCOME_BLACKLISTED, outcome) + engine.qihu360_detector.check_site.assert_not_called() + engine.google_detector.check_site.assert_not_called() + engine.db.add_detection_result.assert_called_once() + + def test_deep_detect_fails_when_nested_detector_returns_error(self): + engine = self._build_engine() + engine.baidu_detector.check_history.return_value = {"has_history": False, "has_gray": False} + engine.baidu_detector.check_site.return_value = {"has_收录": False, "subdomains": []} + engine.qihu360_detector.check_site.return_value = {"has_收录": False, "subdomains": []} + engine.google_detector.check_site.return_value = {"has_收录": False} + engine.chinaz_detector.check_domain.return_value = {"title": "", "category": "", "has_sensitive": False} + engine.aizhan_detector.check_domain.return_value = {"title": "", "risk": "", "has_sensitive": False} + engine.juziseo_detector.check_domain.return_value = { + "history": {"error": "HTTP 429"}, + "backlink": {"has_sensitive": False, "has_subdomains": False}, + } + engine.jucha_detector.check_domain.return_value = {"whois": {}, "beian": {}, "intercept": {"normal": True}} + + outcome = engine._deep_detect(1, "example.com") + + self.assertEqual(engine.OUTCOME_FAILED, outcome) + engine.db.add_detection_result.assert_not_called() + + +class RDAPDetectorStatusMappingTests(unittest.TestCase): + def test_check_register_status_uses_statuses_field(self): + detector = RDAPDetector() + with patch.object(detector, "check_domain", return_value={"statuses": ["clientHold"]}): + status = detector.check_register_status("example.com") + self.assertEqual(7, status) + + +if __name__ == "__main__": + unittest.main() diff --git a/domainCheck/tests/test_detect_worker_flush_retry.py b/domainCheck/tests/test_detect_worker_flush_retry.py index fef0b4a..cc45a0f 100644 --- a/domainCheck/tests/test_detect_worker_flush_retry.py +++ b/domainCheck/tests/test_detect_worker_flush_retry.py @@ -1,14 +1,22 @@ import collections +import json +import os import threading +import tempfile import unittest import sys +import time +from queue import Queue from unittest.mock import MagicMock from unittest.mock import patch sys.path.insert(0, "/www/wwwroot/getDomain/domainCheck") -from detect_worker import DetectWorker # noqa: E402 +import detect_worker as detect_worker_module # noqa: E402 +from app.config import config # noqa: E402 +from detect_worker import DetectWorker, control_targets_current_worker, pending_control_key # noqa: E402 +from app.utils.status_codes import REGISTER_STATUS_CLIENT_HOLD, REGISTER_STATUS_SERVER_HOLD # noqa: E402 class _FakeDb: @@ -23,10 +31,46 @@ class _FakeDb: def mark_detect_job_items_running_batch(self, batch): return self._run(batch) + def release_detect_job_items_batch(self, batch): + return self._run(batch) + def finalize_detect_job_items_batch(self, batch): return self._run(batch) +class _FakeRedis: + def __init__(self): + self._store = {} + self._expires = {} + + def _cleanup(self, key): + expires_at = self._expires.get(key) + if expires_at is not None and expires_at <= time.time(): + self._store.pop(key, None) + self._expires.pop(key, None) + + def get(self, key): + self._cleanup(key) + return self._store.get(key) + + def set(self, key, value, nx=False, ex=None): + self._cleanup(key) + if nx and key in self._store: + return False + self._store[key] = value + if ex is not None: + self._expires[key] = time.time() + float(ex) + else: + self._expires.pop(key, None) + return True + + def delete(self, key): + existed = key in self._store + self._store.pop(key, None) + self._expires.pop(key, None) + return 1 if existed else 0 + + class DetectWorkerFlushRetryTest(unittest.TestCase): def _build_worker(self, db): worker = DetectWorker.__new__(DetectWorker) @@ -35,24 +79,110 @@ class DetectWorkerFlushRetryTest(unittest.TestCase): worker._running_mark_lock = threading.Lock() worker._pending_running_marks = collections.deque() worker._last_running_mark_flush_at = 0.0 + worker._job_release_lock = threading.Lock() + worker._pending_job_releases = collections.deque() + worker._pending_job_release_reasons = {} + worker._last_job_release_flush_at = 0.0 worker._job_finalize_lock = threading.Lock() worker._pending_job_finalizations = collections.deque() worker._last_job_finalize_flush_at = 0.0 worker.proxy_pool = [] worker.proxy_pool_lock = threading.Lock() + worker.proxy_refresh_lock = threading.Lock() + worker.proxy_refresh_cooldown_seconds = 2 + worker.proxy_next_refresh_time = 0.0 + worker._last_proxy_refresh_reason_at = {} worker.proxy_failure_lock = threading.Lock() worker.proxy_failure_counts = {} worker.proxy_quarantine_until = {} worker.proxy_source_by_key = {} worker.proxy_source_failure_counts = {} worker.proxy_source_quarantine_until = {} + worker.proxy_active_leases = {} worker.proxy_config = {"proxy_enable": True, "allow_direct": True} + worker.proxy_last_refresh_time = None worker.proxy_last_refresh_status = "test" + worker.proxy_last_refresh_source_count = 0 + worker.proxy_last_refresh_total_items = 0 + worker.proxy_last_validated_count = 0 + worker.proxy_last_available_count = 0 + worker.proxy_last_source_stats = [] + worker._last_proxy_clock_skew_warning_at = 0.0 + worker.proxy_max_reuse_count = 1000 + worker.proxy_step_wait_timeout_seconds = 1.5 + worker.proxy_direct_fallback_grace_seconds = 0.35 + worker.proxy_direct_fallback_grace_overrides = {} + worker.proxy_direct_retry_wait_seconds = 1.2 + worker.proxy_direct_retry_wait_overrides = {} + worker.register_single_machine_mode_enabled = False + worker.register_single_machine_direct_streak_attempts = 0 worker.trigger_proxy_refresh = MagicMock() + worker._update_runtime_state = MagicMock() + worker._sync_worker_log_event = MagicMock() + worker._emit_step_trace = MagicMock() worker.runtime_settings = {} worker.thread_count = 600 + worker.process_count = 1 + worker.refresh_process_count_runtime = MagicMock(side_effect=lambda *args, **kwargs: worker.process_count) + worker.use_redis = False + worker.redis_client = None + worker.detect_lock = threading.Lock() + worker.running = True worker.detecting = True + worker.stop_requested = False + worker.detect_command_thread = None + worker.detect_command_started_at = 0.0 + worker.detect_command_last_activity_at = 0.0 + worker._last_domain_started_at = 0.0 + worker._last_domain_result_at = 0.0 + worker._last_job_item_release_at = 0.0 + worker._active_domain_threads = 0 + worker._domain_thread_counter_lock = threading.Lock() + worker.current_cycle_token = "" + worker.current_job_id = None + worker.current_job_code = "" + worker.current_job_task_mode = "" + worker._detect_session_owner = 0 + worker._detect_session_seq = 0 + worker._pending_restart_source = "" + worker._pending_restart_payload = None + worker._pending_restart_reason = "" + worker._worker_started_at = time.time() + worker._last_worker_activity_at = worker._worker_started_at + worker._idle_recycle_requested = False + worker._last_explicit_start_signal_at = 0.0 + worker._last_explicit_start_payload = {} + worker._last_detection_start_at = 0.0 + worker._last_detection_start_source = "" + worker._last_detection_start_job_id = None + worker._last_detection_start_job_code = "" + worker._last_detection_start_task_mode = "" + worker._cached_active_detect_job = None + worker._last_active_detect_job_refresh_at = 0.0 + worker._explicit_claim_scope_job_id = None + worker._explicit_claim_scope_job_code = "" + worker._explicit_claim_scope_until = 0.0 + worker._explicit_claim_scope_source = "" + worker._ignored_targeted_job_id = None + worker._ignored_targeted_job_code = "" + worker._ignored_targeted_job_until = 0.0 + worker._ignored_targeted_job_targets = "" worker._get_active_domain_threads = MagicMock(return_value=0) + worker._worker_log_sync_queue = Queue(maxsize=50000) + worker._last_worker_log_sync_degrade_notice_at = 0.0 + worker._remote_debug_event_failure_streak = 0 + worker._remote_debug_event_cooldown_until = 0.0 + worker._last_remote_debug_event_notice_at = 0.0 + worker._last_runtime_extra = {} + worker._last_runtime_phase = "idle" + worker._last_runtime_detail = "" + worker._runtime_heartbeat_stop = threading.Event() + worker._worker_log_sync_stop = threading.Event() + worker._flush_pending_domain_status_updates = MagicMock(return_value=0) + worker._flush_pending_domain_completions = MagicMock(return_value=0) + worker._flush_pending_review_status_updates = MagicMock(return_value=0) + worker._flush_completed_futures = MagicMock(return_value=0) + worker.single_machine_site_direct_fallback_enabled = False return worker def test_running_mark_flush_requeues_batch_when_db_write_fails(self): @@ -84,6 +214,1011 @@ class DetectWorkerFlushRetryTest(unittest.TestCase): self.assertEqual(2, updated) self.assertEqual(0, len(worker._pending_job_finalizations)) + def test_job_release_flush_requeues_batch_when_db_write_fails(self): + worker = self._build_worker(_FakeDb("release")) + worker._enqueue_job_release(201, "token-a", "session_replaced:7") + worker._enqueue_job_release(202, "token-b", "queued_before_start") + + updated = worker._flush_pending_job_releases(force=True, batch_limit=10) + self.assertEqual(0, updated) + self.assertEqual(2, len(worker._pending_job_releases)) + + updated = worker._flush_pending_job_releases(force=True, batch_limit=10) + self.assertEqual(2, updated) + self.assertEqual(0, len(worker._pending_job_releases)) + + def test_detection_session_abort_reason_rejects_replaced_owner(self): + worker = self._build_worker(_FakeDb("noop")) + worker._detect_session_owner = 7 + + self.assertEqual("session_replaced:7", worker._detection_session_abort_reason(6)) + self.assertEqual("", worker._detection_session_abort_reason(7)) + + def test_release_job_item_for_session_abort_enqueues_current_item(self): + db = MagicMock() + worker = self._build_worker(db) + worker._detect_session_owner = 7 + + released = worker._release_job_item_for_session_abort( + 201, + "token-a", + session_id=6, + domain_name="example.com", + detail="prepare", + ) + + self.assertTrue(released) + self.assertEqual(1, len(worker._pending_job_releases)) + job_item_id, claim_token = worker._pending_job_releases[0] + reason = worker._pending_job_release_reasons[(job_item_id, claim_token)] + self.assertEqual(201, job_item_id) + self.assertEqual("token-a", claim_token) + self.assertIn("session_replaced:7", reason) + self.assertIn("prepare", reason) + + def test_enqueue_job_release_deduplicates_same_job_item_and_claim_token(self): + worker = self._build_worker(_FakeDb("release")) + + self.assertTrue(worker._enqueue_job_release(201, "token-a", "queued_before_start")) + self.assertTrue(worker._enqueue_job_release(201, "token-a", "session_replaced:9")) + self.assertEqual(1, len(worker._pending_job_releases)) + self.assertEqual((201, "token-a"), worker._pending_job_releases[0]) + self.assertEqual( + "session_replaced:9", + worker._pending_job_release_reasons[(201, "token-a")], + ) + + def test_incoming_job_id_prefers_target_job_id_for_local_queue_scope(self): + worker = self._build_worker(_FakeDb("noop")) + + payload = { + "job_id": 26025, + "job_code": "sync-overseas-26025", + "target_job_id": 1, + "target_job_code": "sync-overseas-26025-local", + "task_mode": "domain_pipeline", + } + + self.assertEqual(1, worker._incoming_job_id(payload)) + + def test_schedule_explicit_claim_scope_uses_target_job_id_when_present(self): + worker = self._build_worker(_FakeDb("noop")) + + payload = { + "job_id": 26025, + "job_code": "sync-overseas-26025", + "target_job_id": 1, + "target_job_code": "sync-overseas-26025-local", + "task_mode": "domain_pipeline", + } + + worker._schedule_explicit_claim_scope(payload, source="sync-pull") + + self.assertEqual(1, worker._explicit_claim_scope_job_id) + self.assertEqual("sync-overseas-26025", worker._explicit_claim_scope_job_code) + self.assertEqual("sync-pull", worker._explicit_claim_scope_source) + + def test_detect_domain_releases_job_item_when_session_replaced(self): + db = MagicMock() + worker = self._build_worker(db) + worker._detect_session_owner = 7 + worker._set_current_task_context = MagicMock() + worker._clear_current_task_context = MagicMock() + worker._mark_detection_phase = MagicMock() + worker._release_job_item_for_session_abort = MagicMock(return_value=True) + + worker.detect_domain( + 101, + {"id": 101, "domain": "example.com"}, + { + "job_item_id": 301, + "job_id": 1, + "claim_token": "token-x", + "task_mode": "domain_pipeline", + "session_id": 6, + }, + ) + + worker._release_job_item_for_session_abort.assert_called_once() + args, kwargs = worker._release_job_item_for_session_abort.call_args + self.assertEqual(301, args[0]) + self.assertEqual("token-x", args[1]) + self.assertEqual(6, kwargs["session_id"]) + self.assertEqual("example.com", kwargs["domain_name"]) + + def test_detect_domain_step_task_writes_domain_started_event(self): + db = MagicMock() + worker = self._build_worker(db) + worker.sensitive_words = [] + worker._set_current_task_context = MagicMock() + worker._clear_current_task_context = MagicMock() + worker._assert_detection_session_active = MagicMock() + worker._change_active_domain_threads = MagicMock(return_value=1) + worker._mark_detection_phase = MagicMock() + worker._enqueue_running_mark = MagicMock() + worker._run_detect_register = MagicMock(return_value=True) + worker._get_detect_execution_order = MagicMock(return_value=["detect_register"]) + worker._build_single_step_job_result = MagicMock(return_value={"state": "completed", "message": "ok"}) + worker._resolve_single_step_finalization = MagicMock(return_value=("completed", "ok")) + worker._enqueue_job_finalization = MagicMock() + + worker.detect_domain( + 101, + {"id": 101, "domain": "example.com"}, + { + "job_item_id": 301, + "job_id": 1, + "job_code": "sync-overseas-26118", + "claim_token": "token-x", + "task_mode": "domain_pipeline", + "step_code": "detect_register", + "step_payload": {"step_code": "detect_register"}, + "session_id": 7, + }, + ) + + db.append_detect_run_event.assert_called_once() + self.assertEqual("domain_started", db.append_detect_run_event.call_args.kwargs["event_type"]) + payload = db.append_detect_run_event.call_args.kwargs["payload"] + self.assertEqual("domain_pipeline", payload["task_mode"]) + self.assertEqual("detect_register", payload["step_code"]) + + def test_detect_domain_step_task_finalize_emits_domain_terminal_event(self): + db = MagicMock() + worker = self._build_worker(db) + worker.sensitive_words = [] + worker._set_current_task_context = MagicMock() + worker._clear_current_task_context = MagicMock() + worker._assert_detection_session_active = MagicMock() + worker._change_active_domain_threads = MagicMock(return_value=1) + worker._mark_detection_phase = MagicMock() + worker._enqueue_running_mark = MagicMock() + worker._run_detect_register = MagicMock(return_value=True) + worker._get_detect_execution_order = MagicMock(return_value=["detect_register"]) + worker._build_single_step_job_result = MagicMock( + return_value={"state": "blocked", "message": "client hold", "field_name": "register_status"} + ) + worker._resolve_single_step_finalization = MagicMock(return_value=("blacklisted", "client hold")) + worker._enqueue_job_finalization = MagicMock() + + worker.detect_domain( + 101, + {"id": 101, "domain": "example.com"}, + { + "job_item_id": 301, + "job_id": 1, + "job_code": "sync-overseas-26118", + "claim_token": "token-x", + "task_mode": "domain_pipeline", + "step_code": "detect_register", + "step_payload": {"step_code": "detect_register"}, + "session_id": 7, + }, + ) + + self.assertTrue(worker._enqueue_job_finalization.called) + self.assertEqual( + "domain_blacklisted", + worker._enqueue_job_finalization.call_args.kwargs["event_type"], + ) + event_payload = worker._enqueue_job_finalization.call_args.kwargs["event_payload"] + self.assertEqual("example.com", event_payload["domain"]) + self.assertEqual("blacklisted", event_payload["status"]) + + def test_start_detection_async_ignores_duplicate_start_for_same_job_and_cycle(self): + worker = self._build_worker(_FakeDb("noop")) + live_thread = MagicMock() + live_thread.is_alive.return_value = True + worker.detect_command_thread = live_thread + worker.current_job_id = 321 + worker.current_job_code = "sync-overseas-321" + worker.current_cycle_token = "cycle-abc" + worker.detect_command_started_at = time.time() - 75 + worker.detect_command_last_activity_at = time.time() - 60 + worker._last_domain_result_at = time.time() - 60 + + started = worker.start_detection_async( + source="redis-control", + control_payload={ + "job_id": 321, + "job_code": "sync-overseas-321", + "cycle_token": "cycle-abc", + }, + ) + + self.assertTrue(started) + self.assertIs(worker.detect_command_thread, live_thread) + self.assertFalse(worker.stop_requested) + worker._update_runtime_state.assert_called() + + def test_start_detection_async_does_not_coalesce_recent_duplicate_without_live_thread(self): + worker = self._build_worker(_FakeDb("noop")) + worker.detect_command_thread = None + worker.detecting = False + worker.current_job_id = 321 + worker.current_job_code = "sync-overseas-321" + worker.current_job_task_mode = "domain_pipeline" + worker._last_detection_start_at = time.time() - 10 + worker._last_detection_start_source = "auto-resume" + worker._last_detection_start_job_id = 321 + worker._last_detection_start_job_code = "sync-overseas-321" + worker._last_detection_start_task_mode = "domain_pipeline" + + with patch("detect_worker.threading.Thread") as mock_thread: + started = worker.start_detection_async( + source="redis-control", + control_payload={}, + ) + + self.assertTrue(started) + mock_thread.assert_called_once() + worker._update_runtime_state.assert_called() + + def test_start_detection_async_does_not_coalesce_recent_duplicate_for_stale_session(self): + worker = self._build_worker(_FakeDb("noop")) + live_thread = MagicMock() + live_thread.is_alive.return_value = True + worker.detect_command_thread = live_thread + worker.detecting = True + worker.current_job_id = 874 + worker.current_job_code = "sync-overseas-19897" + worker.current_job_task_mode = "domain_pipeline" + worker._last_detection_start_at = time.time() - 10 + worker._last_detection_start_source = "auto-resume" + worker._last_detection_start_job_id = 874 + worker._last_detection_start_job_code = "sync-overseas-19897" + worker._last_detection_start_task_mode = "domain_pipeline" + worker._is_stale_detect_session = MagicMock(return_value=(True, "stale")) + worker._should_force_takeover_stale_session = MagicMock(return_value=True) + + with patch("detect_worker.threading.Thread") as mock_thread: + started = worker.start_detection_async( + source="auto-resume", + control_payload={ + "job_id": 874, + "job_code": "sync-overseas-19897", + "task_mode": "domain_pipeline", + }, + ) + + self.assertTrue(started) + mock_thread.assert_called_once() + worker._update_runtime_state.assert_called() + + def test_run_detection_session_clears_context_before_idle_runtime_update(self): + worker = self._build_worker(_FakeDb("noop")) + worker.start_detection = MagicMock(return_value=None) + worker._consume_pending_restart_request = MagicMock(return_value=False) + worker._maybe_recycle_idle_worker = MagicMock(return_value=False) + control_payload = { + "job_id": 876, + "job_code": "sync-overseas-19961", + "task_mode": "domain_pipeline", + "cycle_token": "cycle-876", + } + + worker._run_detection_session(source="redis-control", control_payload=control_payload, session_id=1) + + phase, detail = worker._update_runtime_state.call_args_list[-1].args[:2] + kwargs = worker._update_runtime_state.call_args_list[-1].kwargs + self.assertEqual("idle", phase) + self.assertEqual("检测任务结束，Worker 保持待命", detail) + self.assertNotIn("job_id", kwargs) + self.assertNotIn("job_code", kwargs) + self.assertEqual("", worker.current_cycle_token) + self.assertIsNone(worker.current_job_id) + self.assertEqual("", worker.current_job_code) + + def test_start_detection_async_schedules_pending_restart_for_explicit_pipeline_switch(self): + worker = self._build_worker(_FakeDb("noop")) + live_thread = MagicMock() + live_thread.is_alive.return_value = True + worker.detect_command_thread = live_thread + worker.current_job_id = 867 + worker.current_job_code = "sync-overseas-19835" + worker.current_job_task_mode = "domain_pipeline" + worker._explicit_claim_scope_job_id = 884 + worker._explicit_claim_scope_job_code = "sync-overseas-20554" + worker._explicit_claim_scope_until = time.time() + 120 + + started = worker.start_detection_async( + source="redis-control", + control_payload={ + "job_id": 884, + "job_code": "sync-overseas-20554", + "task_mode": "domain_pipeline", + }, + ) + + self.assertTrue(started) + self.assertTrue(worker.stop_requested) + self.assertEqual("redis-control", worker._pending_restart_source) + self.assertEqual(884, worker._pending_restart_payload.get("job_id")) + worker._update_runtime_state.assert_called() + + def test_start_detection_async_ignores_overlap_handoff_when_current_session_has_active_threads(self): + worker = self._build_worker(_FakeDb("noop")) + live_thread = MagicMock() + live_thread.is_alive.return_value = True + worker.detect_command_thread = live_thread + worker.current_job_id = 884 + worker.current_job_code = "sync-overseas-20554" + worker.current_job_task_mode = "domain_pipeline" + worker._explicit_claim_scope_job_id = 885 + worker._explicit_claim_scope_job_code = "sync-overseas-20592" + worker._explicit_claim_scope_until = time.time() + 120 + worker._get_active_domain_threads.return_value = 128 + + started = worker.start_detection_async( + source="redis-control", + control_payload={ + "job_id": 885, + "job_code": "sync-overseas-20592", + "task_mode": "domain_pipeline", + "source": "overlap-handoff", + "tail_handoff_candidate": True, + }, + ) + + self.assertTrue(started) + self.assertFalse(worker.stop_requested) + self.assertEqual("", worker._pending_restart_source) + self.assertIsNone(worker._explicit_claim_scope_job_id) + worker._update_runtime_state.assert_called() + + def test_single_machine_mode_allows_direct_fallback_for_360_and_baidu(self): + worker = self._build_worker(_FakeDb("noop")) + worker.proxy_config["allow_direct"] = False + worker.single_machine_site_direct_fallback_enabled = True + + self.assertTrue(worker._allow_direct_connection_for_step("360检测")) + self.assertTrue(worker._allow_direct_connection_for_step("百度site检测")) + self.assertFalse(worker._allow_direct_connection_for_step("时光机检测")) + + def test_retry_budget_allows_one_pending_direct_fallback_after_proxy_failure(self): + worker = self._build_worker(_FakeDb("noop")) + worker.proxy_config["allow_direct"] = False + worker.single_machine_site_direct_fallback_enabled = True + worker.proxy_step_retry_budget_overrides = {"360检测": {"max_attempts": 2, "max_seconds": 5}} + worker._set_step_force_direct_once("360检测", True) + + exhausted, reason = worker._check_proxy_step_retry_budget( + "360检测", + "example.com", + time.perf_counter() - 15, + 1, + {"proxy-a"}, + last_reason="proxy timeout", + ) + + self.assertFalse(exhausted) + self.assertEqual("", reason) + + def test_retry_budget_still_exhausts_when_no_direct_fallback_is_pending(self): + worker = self._build_worker(_FakeDb("noop")) + worker.proxy_config["allow_direct"] = False + worker.single_machine_site_direct_fallback_enabled = True + worker.proxy_step_retry_budget_overrides = {"360检测": {"max_attempts": 2, "max_seconds": 5}} + + exhausted, reason = worker._check_proxy_step_retry_budget( + "360检测", + "example.com", + time.perf_counter() - 15, + 1, + {"proxy-a"}, + last_reason="proxy timeout", + ) + + self.assertTrue(exhausted) + self.assertIn("attempts=1", reason) + + def test_auto_resume_skips_when_recent_explicit_start_signal_exists(self): + worker = self._build_worker(MagicMock()) + worker.db.get_active_detect_job = MagicMock( + return_value={ + "id": 1, + "job_code": "sync-overseas-123", + "task_mode": "domain_pipeline", + "items_pending": 30, + "items_claimed": 4, + "items_running": 2, + } + ) + worker.detecting = False + worker.detect_command_thread = None + worker._last_explicit_start_signal_at = time.time() + worker.start_detection_async = MagicMock(return_value=True) + + started = worker._resume_active_detect_job_if_needed(reason="heartbeat_autoresume") + + self.assertFalse(started) + worker.db.get_active_detect_job.assert_not_called() + worker.start_detection_async.assert_not_called() + + def test_auto_resume_skips_for_non_primary_worker_on_heartbeat(self): + worker = self._build_worker(MagicMock()) + worker.db.get_active_detect_job = MagicMock(return_value={}) + worker.detecting = False + worker.detect_command_thread = None + + with patch.object(config, "NODE_CODE", "mainland-controller-01-b"), patch.dict( + os.environ, + {"WORKER_PARENT_NODE_CODE": "mainland-controller-01"}, + clear=False, + ): + started = worker._resume_active_detect_job_if_needed(reason="heartbeat_autoresume") + + self.assertFalse(started) + worker.db.get_active_detect_job.assert_called_once() + + def test_auto_resume_heartbeat_reuses_recent_active_job_snapshot(self): + worker = self._build_worker(MagicMock()) + worker.db.get_active_detect_job = MagicMock( + return_value={ + "id": 1, + "job_code": "sync-overseas-123", + "task_mode": "domain_pipeline", + "items_pending": 30, + "items_claimed": 4, + "items_running": 2, + } + ) + worker.detecting = False + worker.detect_command_thread = None + worker.start_detection_async = MagicMock(return_value=True) + + with patch.dict( + os.environ, + { + "DOMAINCHECK_AUTORESUME_ACTIVE_JOB_MIN_REFRESH_SECONDS": "60", + "WORKER_PARENT_NODE_CODE": "mainland-controller-01", + }, + clear=False, + ), patch.object(config, "NODE_CODE", "mainland-controller-01-a"): + first = worker._resume_active_detect_job_if_needed(reason="heartbeat_autoresume") + second = worker._resume_active_detect_job_if_needed(reason="heartbeat_autoresume") + + self.assertTrue(first) + self.assertTrue(second) + worker.db.get_active_detect_job.assert_called_once() + self.assertEqual(2, worker.start_detection_async.call_count) + + def test_auto_resume_skips_same_job_when_threads_already_active(self): + worker = self._build_worker(MagicMock()) + worker.db.get_active_detect_job = MagicMock( + return_value={ + "id": 895, + "job_code": "sync-overseas-25520", + "task_mode": "domain_pipeline", + "items_pending": 73298, + "items_claimed": 89865, + "items_running": 31396, + "selection_reason": "running_job_active", + } + ) + worker.detecting = False + worker.detect_command_thread = None + worker.current_job_id = 895 + worker.current_job_code = "sync-overseas-25520" + worker.current_job_task_mode = "domain_pipeline" + worker._get_active_domain_threads.return_value = 800 + worker.start_detection_async = MagicMock(return_value=True) + + started = worker._resume_active_detect_job_if_needed(reason="heartbeat_autoresume") + + self.assertFalse(started) + worker.db.get_active_detect_job.assert_called_once() + worker.start_detection_async.assert_not_called() + + def test_auto_resume_skips_same_job_during_restart_handoff_window(self): + worker = self._build_worker(MagicMock()) + worker.db.get_active_detect_job = MagicMock( + return_value={ + "id": 895, + "job_code": "sync-overseas-25520", + "task_mode": "domain_pipeline", + "items_pending": 73298, + "items_claimed": 89865, + "items_running": 31396, + "selection_reason": "running_job_active", + } + ) + worker.detecting = False + worker.detect_command_thread = None + worker.current_job_id = 895 + worker.current_job_code = "sync-overseas-25520" + worker.current_job_task_mode = "domain_pipeline" + worker._restart_release_handoff_job_id = 895 + worker._restart_release_handoff_job_code = "sync-overseas-25520" + worker._restart_release_handoff_until = time.time() + 120 + worker.start_detection_async = MagicMock(return_value=True) + + started = worker._resume_active_detect_job_if_needed(reason="heartbeat_autoresume") + + self.assertFalse(started) + worker.db.get_active_detect_job.assert_called_once() + worker.start_detection_async.assert_not_called() + + def test_auto_resume_skips_when_explicit_claim_scope_matches_active_job(self): + worker = self._build_worker(MagicMock()) + worker.db.get_active_detect_job = MagicMock( + return_value={ + "id": 871, + "job_code": "sync-overseas-19873", + "task_mode": "domain_pipeline", + "items_pending": 2134, + "items_claimed": 1936, + "items_running": 3402, + "selection_reason": "running_job_active", + } + ) + worker.detecting = False + worker.detect_command_thread = None + worker._explicit_claim_scope_job_id = 871 + worker._explicit_claim_scope_job_code = "sync-overseas-19873" + worker._explicit_claim_scope_until = time.time() + 120 + worker._explicit_claim_scope_source = "redis-control" + worker.start_detection_async = MagicMock(return_value=True) + + started = worker._resume_active_detect_job_if_needed(reason="heartbeat_autoresume") + + self.assertFalse(started) + worker.db.get_active_detect_job.assert_called_once() + worker.start_detection_async.assert_not_called() + + def test_auto_resume_allows_non_primary_worker_for_tail_handoff_pending_job(self): + worker = self._build_worker(MagicMock()) + worker.db.get_active_detect_job = MagicMock( + return_value={ + "id": 885, + "job_code": "sync-overseas-20592", + "task_mode": "domain_pipeline", + "items_pending": 64000, + "items_claimed": 0, + "items_running": 0, + "tail_handoff_candidate": True, + "selection_reason": "tail_handoff_pending", + } + ) + worker.detecting = False + worker.detect_command_thread = None + worker.start_detection_async = MagicMock(return_value=True) + + with patch.object(config, "NODE_CODE", "mainland-controller-01-b"), patch.dict( + os.environ, + {"WORKER_PARENT_NODE_CODE": "mainland-controller-01"}, + clear=False, + ): + started = worker._resume_active_detect_job_if_needed(reason="heartbeat_autoresume") + + self.assertTrue(started) + worker.db.get_active_detect_job.assert_called_once() + worker.start_detection_async.assert_called_once() + + def test_auto_resume_skips_stalled_running_job(self): + worker = self._build_worker(MagicMock()) + worker.db.get_active_detect_job = MagicMock( + return_value={ + "id": 876, + "job_code": "sync-overseas-19961", + "task_mode": "domain_pipeline", + "items_pending": 4, + "items_claimed": 235, + "items_running": 266, + "selection_reason": "running_job_stalled", + "running_job_stalled": True, + } + ) + worker.db.recycle_stalled_detect_job_items = MagicMock(return_value=7) + worker.detecting = False + worker.detect_command_thread = None + worker.start_detection_async = MagicMock(return_value=True) + worker._acquire_shared_job_maintenance_lock = MagicMock(return_value="lock-token") + worker._release_shared_job_maintenance_lock = MagicMock() + + started = worker._resume_active_detect_job_if_needed(reason="heartbeat_autoresume") + + self.assertFalse(started) + worker.db.get_active_detect_job.assert_called_once() + worker.db.recycle_stalled_detect_job_items.assert_called_once_with(876, stall_seconds=900) + worker._release_shared_job_maintenance_lock.assert_called_once() + worker.start_detection_async.assert_not_called() + + def test_pull_sync_tasks_skips_when_shared_lock_not_acquired(self): + worker = self._build_worker(MagicMock()) + worker._acquire_shared_job_maintenance_lock = MagicMock(return_value="") + worker._release_shared_job_maintenance_lock = MagicMock() + + with patch.object(config, "NODE_REGION", "mainland"), patch.object(config, "NODE_ROLE", "control"), patch( + "detect_worker.urllib.request.urlopen" + ) as mock_urlopen: + domains = worker._pull_sync_tasks_until_available(thread_limit=1000) + + self.assertEqual([], domains) + mock_urlopen.assert_not_called() + worker._release_shared_job_maintenance_lock.assert_not_called() + + def test_process_pipeline_skips_when_shared_lock_not_acquired(self): + worker = self._build_worker(MagicMock()) + worker._acquire_shared_job_maintenance_lock = MagicMock(return_value="") + worker._release_shared_job_maintenance_lock = MagicMock() + + with patch.object(config, "NODE_REGION", "mainland"), patch.object(config, "NODE_ROLE", "control"), patch( + "detect_worker.urllib.request.urlopen" + ) as mock_urlopen: + domains = worker._process_pipeline_tasks_until_available(thread_limit=1000) + + self.assertEqual([], domains) + mock_urlopen.assert_not_called() + worker._release_shared_job_maintenance_lock.assert_not_called() + + def test_runtime_heartbeat_does_not_fake_current_load_when_detecting_but_idle(self): + class _SpyRedis: + def __init__(self): + self.calls = [] + + def set(self, *args, **kwargs): + self.calls.append((args, kwargs)) + return True + + db = MagicMock() + worker = self._build_worker(db) + worker.use_redis = True + worker.redis_client = _SpyRedis() + worker.detecting = True + worker.running = True + worker.stop_requested = False + worker.proxy_last_refresh_status = "ok" + worker.proxy_last_refresh_time = None + worker.proxy_last_source_stats = [] + worker._active_domain_threads = 0 + worker._last_runtime_state_push_phase = "" + worker._last_runtime_state_push_at = 0.0 + + DetectWorker._update_runtime_state(worker, "running", "等待线程派发") + + register_calls = [call for call in db.mock_calls if call[0] == "register_cluster_node"] + self.assertEqual(1, len(register_calls)) + _, args, kwargs = register_calls[0] + self.assertEqual(config.NODE_CODE, args[0]) + self.assertEqual("busy", kwargs["status"]) + self.assertEqual(0, kwargs["current_load"]) + self.assertTrue(kwargs["metadata"]["detecting"]) + self.assertTrue(kwargs["metadata"]["waiting_for_dispatch"]) + self.assertEqual(0, kwargs["metadata"]["active_threads"]) + + def test_release_node_job_items_is_throttled_between_detection_restarts(self): + db = MagicMock() + db.release_detect_job_items_for_node.return_value = 7 + worker = self._build_worker(db) + + first = worker._maybe_release_node_job_items(reason="test", min_interval_seconds=300) + second = worker._maybe_release_node_job_items(reason="test", min_interval_seconds=300) + + self.assertEqual(7, first) + self.assertEqual(0, second) + db.release_detect_job_items_for_node.assert_called_once_with(config.NODE_CODE) + + def test_control_node_can_release_node_job_items(self): + db = MagicMock() + db.release_detect_job_items_for_node.return_value = 7 + worker = self._build_worker(db) + + with patch.object(config, "NODE_ROLE", "control"): + released = worker._maybe_release_node_job_items(reason="control_start") + + self.assertEqual(7, released) + db.release_detect_job_items_for_node.assert_called_once_with(config.NODE_CODE) + + def test_release_node_job_items_records_restart_release_handoff_for_current_job(self): + db = MagicMock() + db.release_detect_job_items_for_node.return_value = 7 + worker = self._build_worker(db) + worker.current_job_id = 880 + worker.current_job_code = "sync-overseas-20249" + + released = worker._maybe_release_node_job_items(reason="start_detection", min_interval_seconds=0) + + self.assertEqual(7, released) + self.assertEqual(880, worker._restart_release_handoff_job_id) + self.assertEqual("sync-overseas-20249", worker._restart_release_handoff_job_code) + self.assertGreater(worker._restart_release_handoff_until, time.time()) + + def test_claim_detect_job_items_prioritizes_restart_released_handoff_before_global_queue(self): + db = MagicMock() + db.claim_restart_released_detect_job_items.return_value = [{"job_item_id": 1}] + db.claim_detect_job_items.return_value = [{"job_item_id": 2}] + worker = self._build_worker(db) + worker.current_job_id = 880 + worker.current_job_code = "sync-overseas-20249" + worker.current_job_task_mode = "domain_pipeline" + worker._restart_release_handoff_job_id = 880 + worker._restart_release_handoff_job_code = "sync-overseas-20249" + worker._restart_release_handoff_until = time.time() + 120 + worker._restart_release_handoff_reason = "start_detection" + + with patch.object(config, "NODE_CODE", "mainland-controller-01-a"), patch.dict( + os.environ, + {"WORKER_PARENT_NODE_CODE": "mainland-controller-01"}, + clear=False, + ): + rows = worker._claim_detect_job_items(limit=320, lease_seconds=900, job_id=None) + + self.assertEqual([{"job_item_id": 1}, {"job_item_id": 2}], rows) + db.claim_restart_released_detect_job_items.assert_called_once_with( + "mainland-controller-01-a", + 880, + limit=128, + lease_seconds=900, + ) + db.claim_detect_job_items.assert_called_once_with( + "mainland-controller-01-a", + limit=319, + lease_seconds=900, + job_id=None, + prefer_recent_jobs=False, + preferred_recent_job_limit=8, + preferred_recent_job_window_hours=24, + ) + + def test_claim_detect_job_items_skips_restart_handoff_query_for_non_primary_worker(self): + db = MagicMock() + worker = self._build_worker(db) + worker.current_job_id = 880 + worker.current_job_code = "sync-overseas-20249" + worker.current_job_task_mode = "domain_pipeline" + worker._restart_release_handoff_job_id = 880 + worker._restart_release_handoff_job_code = "sync-overseas-20249" + worker._restart_release_handoff_until = time.time() + 120 + worker._restart_release_handoff_reason = "start_detection" + + with patch.object(config, "NODE_CODE", "mainland-controller-01-b"), patch.dict( + os.environ, + {"WORKER_PARENT_NODE_CODE": "mainland-controller-01"}, + clear=False, + ): + rows = worker._claim_detect_job_items(limit=320, lease_seconds=900, job_id=None) + + self.assertEqual([], rows) + db.claim_restart_released_detect_job_items.assert_not_called() + db.claim_detect_job_items.assert_not_called() + + def test_mark_explicit_start_signal_records_pipeline_claim_scope(self): + db = MagicMock() + worker = self._build_worker(db) + + worker._mark_explicit_start_signal( + { + "job_id": 884, + "job_code": "sync-overseas-20554", + "task_mode": "domain_pipeline", + }, + source="redis-control", + ) + + self.assertEqual(884, worker._explicit_claim_scope_job_id) + self.assertEqual("sync-overseas-20554", worker._explicit_claim_scope_job_code) + self.assertEqual("redis-control", worker._explicit_claim_scope_source) + self.assertGreater(worker._explicit_claim_scope_until, time.time()) + + def test_current_scoped_claim_job_id_prefers_explicit_pipeline_scope(self): + db = MagicMock() + worker = self._build_worker(db) + worker.current_job_id = 867 + worker.current_job_code = "sync-overseas-19835" + worker.current_job_task_mode = "domain_pipeline" + worker._explicit_claim_scope_job_id = 884 + worker._explicit_claim_scope_job_code = "sync-overseas-20554" + worker._explicit_claim_scope_until = time.time() + 120 + + self.assertEqual(884, worker._current_scoped_claim_job_id()) + + def test_explicit_claim_scope_window_expires(self): + db = MagicMock() + worker = self._build_worker(db) + worker._explicit_claim_scope_job_id = 884 + worker._explicit_claim_scope_job_code = "sync-overseas-20554" + worker._explicit_claim_scope_until = time.time() - 1 + + self.assertIsNone(worker._explicit_claim_scope_job_id_active()) + self.assertIsNone(worker._explicit_claim_scope_job_id) + + def test_claim_detect_job_items_honors_explicit_pipeline_scope(self): + db = MagicMock() + db.claim_detect_job_items.return_value = [{"job_item_id": 9}] + worker = self._build_worker(db) + worker._explicit_claim_scope_job_id = 884 + worker._explicit_claim_scope_job_code = "sync-overseas-20554" + worker._explicit_claim_scope_until = time.time() + 120 + + with patch.object(config, "NODE_CODE", "mainland-controller-01-da"): + rows = worker._claim_detect_job_items(limit=250, lease_seconds=900, job_id=None) + + self.assertEqual([{"job_item_id": 9}], rows) + db.claim_detect_job_items.assert_called_once_with( + "mainland-controller-01-da", + limit=250, + lease_seconds=900, + job_id=884, + prefer_recent_jobs=False, + preferred_recent_job_limit=8, + preferred_recent_job_window_hours=24, + ) + + def test_shared_job_maintenance_lock_uses_single_worker_with_same_scope(self): + redis_client = _FakeRedis() + db = MagicMock() + worker_a = self._build_worker(db) + worker_a.use_redis = True + worker_a.redis_client = redis_client + worker_b = self._build_worker(db) + worker_b.use_redis = True + worker_b.redis_client = redis_client + + with patch.object(config, "NODE_CODE", "mainland-controller-01-a"), patch.dict( + os.environ, + {"WORKER_PARENT_NODE_CODE": "mainland-controller-01"}, + clear=False, + ): + token_a = worker_a._acquire_shared_job_maintenance_lock("recycle-expired-job-items", ttl_seconds=10) + with patch.object(config, "NODE_CODE", "mainland-controller-01-b"), patch.dict( + os.environ, + {"WORKER_PARENT_NODE_CODE": "mainland-controller-01"}, + clear=False, + ): + token_b = worker_b._acquire_shared_job_maintenance_lock("recycle-expired-job-items", ttl_seconds=10) + + self.assertTrue(token_a) + self.assertEqual("", token_b) + + def test_control_node_skips_local_auto_resume(self): + worker = self._build_worker(MagicMock()) + worker.db.get_active_detect_job = MagicMock(return_value={"id": 1, "job_code": "sync-overseas-123"}) + worker.detecting = False + worker.detect_command_thread = None + + with patch.object(config, "NODE_ROLE", "control"): + started = worker._resume_active_detect_job_if_needed(reason="service_runtime_bootstrap") + + self.assertFalse(started) + worker.db.get_active_detect_job.assert_not_called() + + def test_control_node_can_auto_resume_when_override_enabled(self): + worker = self._build_worker(MagicMock()) + worker.runtime_settings = {"control_node_autoresume_enabled": True} + worker.db.get_active_detect_job = MagicMock( + return_value={ + "id": 1, + "job_code": "sync-overseas-123", + "task_mode": "domain_pipeline", + "items_pending": 30, + "items_claimed": 4, + "items_running": 2, + } + ) + worker.detecting = False + worker.detect_command_thread = None + worker.start_detection_async = MagicMock(return_value=True) + + with patch.object(config, "NODE_ROLE", "control"): + started = worker._resume_active_detect_job_if_needed(reason="service_runtime_bootstrap") + + self.assertTrue(started) + worker.db.get_active_detect_job.assert_called_once() + worker.start_detection_async.assert_called_once() + + def test_bootstrap_autoresume_lock_allows_only_one_worker_for_same_job(self): + redis_client = _FakeRedis() + db = MagicMock() + active_job = { + "id": 895, + "job_code": "sync-overseas-25520", + "task_mode": "domain_pipeline", + "items_pending": 73298, + "items_claimed": 89865, + "items_running": 31396, + "selection_reason": "running_job_active", + } + worker_a = self._build_worker(db) + worker_a.use_redis = True + worker_a.redis_client = redis_client + worker_a.runtime_settings = {"control_node_autoresume_enabled": True} + worker_a.db.get_active_detect_job = MagicMock(return_value=active_job) + worker_a.detecting = False + worker_a.start_detection_async = MagicMock(return_value=True) + worker_b = self._build_worker(db) + worker_b.use_redis = True + worker_b.redis_client = redis_client + worker_b.runtime_settings = {"control_node_autoresume_enabled": True} + worker_b.db.get_active_detect_job = MagicMock(return_value=active_job) + worker_b.detecting = False + worker_b.start_detection_async = MagicMock(return_value=True) + + with patch.object(config, "NODE_ROLE", "control"), patch.object(config, "NODE_CODE", "mainland-controller-01-a"), patch.dict( + os.environ, + {"WORKER_PARENT_NODE_CODE": "mainland-controller-01"}, + clear=False, + ): + started_a = worker_a._resume_active_detect_job_if_needed(reason="service_runtime_bootstrap") + with patch.object(config, "NODE_ROLE", "control"), patch.object(config, "NODE_CODE", "mainland-controller-01-b"), patch.dict( + os.environ, + {"WORKER_PARENT_NODE_CODE": "mainland-controller-01"}, + clear=False, + ): + started_b = worker_b._resume_active_detect_job_if_needed(reason="service_runtime_bootstrap") + + self.assertTrue(started_a) + self.assertFalse(started_b) + worker_a.start_detection_async.assert_called_once() + worker_b.start_detection_async.assert_not_called() + + def test_same_job_recent_start_skip_detail_never_goes_negative(self): + worker = self._build_worker(MagicMock()) + worker.current_job_id = 895 + worker.current_job_code = "sync-overseas-25520" + worker._last_detection_start_at = time.time() + 900 + + should_skip, detail = worker._should_skip_same_job_autoresume( + active_job={"id": 895, "job_code": "sync-overseas-25520"}, + now_ts=time.time(), + ) + + self.assertTrue(should_skip) + self.assertEqual("same_job_recent_start:0s", detail) + + def test_claim_detect_job_items_prefers_recent_jobs_when_runtime_enabled(self): + db = MagicMock() + db.claim_detect_job_items.return_value = [] + worker = self._build_worker(db) + worker.runtime_settings = { + "claim_recent_jobs_first": True, + "claim_recent_jobs_limit": 6, + "claim_recent_jobs_window_hours": 72, + } + + worker._claim_detect_job_items(limit=320, lease_seconds=900, job_id=None) + + db.claim_detect_job_items.assert_called_once_with( + config.NODE_CODE, + limit=320, + lease_seconds=900, + job_id=None, + prefer_recent_jobs=True, + preferred_recent_job_limit=6, + preferred_recent_job_window_hours=72, + ) + + def test_claim_detect_job_items_does_not_enable_recent_priority_for_scoped_job(self): + db = MagicMock() + db.claim_detect_job_items.return_value = [] + worker = self._build_worker(db) + worker.runtime_settings = { + "claim_recent_jobs_first": True, + "claim_recent_jobs_limit": 6, + "claim_recent_jobs_window_hours": 72, + } + + worker._claim_detect_job_items(limit=320, lease_seconds=900, job_id=99) + + db.claim_detect_job_items.assert_called_once_with( + config.NODE_CODE, + limit=320, + lease_seconds=900, + job_id=99, + prefer_recent_jobs=False, + preferred_recent_job_limit=6, + preferred_recent_job_window_hours=72, + ) + + def test_resolve_sync_pull_limit_scales_with_process_pool_capacity(self): + worker = self._build_worker(MagicMock()) + worker.process_count = 60 + worker.refresh_process_count_runtime = MagicMock(return_value=60) + + pull_limit = worker._resolve_sync_pull_limit(1000) + + self.assertEqual(120000, pull_limit) + def test_proxy_auth_failure_quarantines_whole_source_group(self): worker = self._build_worker(_FakeDb("noop")) bad_proxy = {"http": "http://bad-proxy", "https": "http://bad-proxy"} @@ -110,6 +1245,47 @@ class DetectWorkerFlushRetryTest(unittest.TestCase): worker.proxy_pool, ) + def test_get_proxies_skips_quarantined_entries_without_dropping_pool_items(self): + worker = self._build_worker(_FakeDb("noop")) + bad_proxy = {"http": "http://bad-proxy", "https": "http://bad-proxy"} + good_proxy = {"http": "http://good-proxy", "https": "http://good-proxy"} + worker.proxy_pool = [ + {"proxy": bad_proxy, "usage_count": 0, "source_tag": "B"}, + {"proxy": good_proxy, "usage_count": 0, "source_tag": "C"}, + ] + worker.proxy_quarantine_until[worker._proxy_key(bad_proxy)] = time.time() + 60 + worker._schedule_proxy_refresh_if_needed = MagicMock() + + selected = worker.get_proxies() + + self.assertEqual(good_proxy, selected) + self.assertEqual(2, len(worker.proxy_pool)) + self.assertEqual( + [entry["proxy"] for entry in worker.proxy_pool], + [bad_proxy, good_proxy], + ) + self.assertEqual(1, worker.proxy_pool[1]["usage_count"]) + + def test_get_proxies_scans_multiple_batches_until_it_finds_usable_proxy(self): + worker = self._build_worker(_FakeDb("noop")) + worker._proxy_pool_scan_batch_size = MagicMock(return_value=2) + proxies = [ + {"http": f"http://proxy-{idx}", "https": f"http://proxy-{idx}"} + for idx in range(5) + ] + worker.proxy_pool = [ + {"proxy": proxy, "usage_count": 0, "source_tag": f"S{idx}"} + for idx, proxy in enumerate(proxies) + ] + excluded_keys = {worker._proxy_key(proxy) for proxy in proxies[:4]} + worker._schedule_proxy_refresh_if_needed = MagicMock() + + selected = worker.get_proxies(excluded_proxy_keys=excluded_keys) + + self.assertEqual(proxies[4], selected) + self.assertEqual(5, len(worker.proxy_pool)) + self.assertEqual(1, worker.proxy_pool[-1]["usage_count"]) + def test_prepare_proxy_fetch_plan_respects_pool_shortage_instead_of_overfetching(self): worker = self._build_worker(_FakeDb("noop")) @@ -148,6 +1324,705 @@ class DetectWorkerFlushRetryTest(unittest.TestCase): self.assertLessEqual(meta["rounds"], 2) self.assertEqual(meta["rounds"] * 6, len(plan)) + def test_prepare_proxy_fetch_plan_scales_down_target_when_process_count_is_high(self): + worker = self._build_worker(_FakeDb("noop")) + worker.thread_count = 1000 + worker.process_count = 80 + worker.refresh_process_count_runtime = MagicMock(return_value=80) + + plan, meta = worker._prepare_proxy_fetch_plan( + [ + "http://provider.local/get?group=B", + "http://provider.local/get?group=C", + "http://provider.local/get?group=D", + "http://provider.local/get?group=E", + "http://provider.local/get?group=F", + "http://provider.local/get?group=G", + ], + current_pool_size=150, + ) + + self.assertEqual(80, meta["process_count"]) + self.assertLessEqual(meta["target_total"], 360) + self.assertLessEqual(meta["batch_size"], 60) + self.assertEqual(6, len(plan)) + + def test_refresh_proxy_pool_reuses_shared_snapshot_when_refresh_lock_is_busy(self): + worker = self._build_worker(_FakeDb("noop")) + worker.use_redis = True + worker.redis_client = _FakeRedis() + worker.process_count = 80 + worker.refresh_process_count_runtime = MagicMock(return_value=80) + worker.proxy_config = { + "proxy_enable": True, + "allow_direct": True, + "proxy_urls": ["http://provider.local/get?group=B"], + } + local_proxy = {"http": "http://local-proxy", "https": "http://local-proxy"} + shared_proxy = {"http": "http://shared-proxy", "https": "http://shared-proxy"} + worker.proxy_pool = [ + {"proxy": local_proxy, "usage_count": 0, "source_tag": "L", "expire_at_ts": time.time() + 120} + ] + + with patch.object(config, "NODE_CODE", "mainland-controller-01-u"): + with patch.dict(os.environ, {"WORKER_PARENT_NODE_CODE": "mainland-controller-01"}, clear=False): + worker.redis_client.set( + worker._shared_proxy_pool_key(), + json.dumps( + { + "node_code": "mainland-controller-01", + "owner": "mainland-controller-01-k", + "refreshed_at_ts": time.time(), + "source_count": 6, + "raw_items": 180, + "available_count": 1, + "proxy_pool": [ + { + "proxy": shared_proxy, + "usage_count": 0, + "source_tag": "S", + "expire_at_ts": time.time() + 120, + } + ], + }, + ensure_ascii=False, + ), + ex=30, + ) + worker.redis_client.set(worker._shared_proxy_refresh_lock_key(), "other-owner", ex=10) + + worker.refresh_proxy_pool() + + self.assertEqual(1, len(worker.proxy_pool)) + self.assertEqual(shared_proxy, worker.proxy_pool[0]["proxy"]) + self.assertIn("共享代理快照", worker.proxy_last_refresh_status) + + def test_refresh_proxy_pool_publishes_shared_snapshot_after_success(self): + worker = self._build_worker(_FakeDb("noop")) + worker.use_redis = True + worker.redis_client = _FakeRedis() + worker.process_count = 80 + worker.refresh_process_count_runtime = MagicMock(return_value=80) + worker.proxy_config = { + "proxy_enable": True, + "allow_direct": True, + "proxy_urls": ["http://provider.local/get?group=B"], + } + + class _FakeResponse: + status_code = 200 + + def json(self): + return { + "list": [ + { + "ip": "1.1.1.1", + "port": "8080", + "expire_at": time.time() + 120, + } + ] + } + + with patch.object(config, "NODE_CODE", "mainland-controller-01-u"): + with patch.dict(os.environ, {"WORKER_PARENT_NODE_CODE": "mainland-controller-01"}, clear=False): + with patch("requests.get", return_value=_FakeResponse()): + worker.refresh_proxy_pool() + raw_payload = worker.redis_client.get(worker._shared_proxy_pool_key()) + + self.assertTrue(raw_payload) + payload = json.loads(raw_payload) + self.assertEqual("mainland-controller-01-u", payload["owner"]) + self.assertEqual(1, payload["available_count"]) + + def test_refresh_proxy_pool_prefers_shared_snapshot_when_local_pool_lags(self): + worker = self._build_worker(_FakeDb("noop")) + worker.use_redis = True + worker.redis_client = _FakeRedis() + worker.process_count = 80 + worker.refresh_process_count_runtime = MagicMock(return_value=80) + worker.thread_count = 1000 + worker.proxy_config = { + "proxy_enable": True, + "allow_direct": True, + "proxy_urls": ["http://provider.local/get?group=B"], + } + worker.proxy_pool = [ + {"proxy": {"http": f"http://local-{idx}", "https": f"http://local-{idx}"}, "usage_count": 0, "source_tag": "L", "expire_at_ts": time.time() + 120} + for idx in range(20) + ] + shared_entries = [ + {"proxy": {"http": f"http://shared-{idx}", "https": f"http://shared-{idx}"}, "usage_count": 0, "source_tag": "S", "expire_at_ts": time.time() + 120} + for idx in range(80) + ] + + with patch.object(config, "NODE_CODE", "mainland-controller-01-u"): + with patch.dict(os.environ, {"WORKER_PARENT_NODE_CODE": "mainland-controller-01"}, clear=False): + worker.redis_client.set( + worker._shared_proxy_pool_key(), + json.dumps( + { + "node_code": "mainland-controller-01", + "owner": "mainland-controller-01-k", + "refreshed_at_ts": time.time(), + "source_count": 6, + "raw_items": 120, + "available_count": len(shared_entries), + "proxy_pool": shared_entries, + }, + ensure_ascii=False, + ), + ex=30, + ) + + worker.refresh_proxy_pool() + + self.assertEqual(len(shared_entries), len(worker.proxy_pool)) + self.assertIn("共享代理快照", worker.proxy_last_refresh_status) + + def test_schedule_proxy_refresh_skips_empty_pool_during_cooldown(self): + worker = self._build_worker(_FakeDb("noop")) + worker.proxy_next_refresh_time = time.time() + 10 + + with patch("detect_worker.threading.Thread") as mock_thread: + worker._schedule_proxy_refresh_if_needed(0) + + mock_thread.assert_not_called() + + def test_schedule_proxy_refresh_skips_low_water_pool_during_cooldown(self): + worker = self._build_worker(_FakeDb("noop")) + worker.process_count = 80 + worker.refresh_process_count_runtime = MagicMock(return_value=80) + worker.proxy_next_refresh_time = time.time() + 10 + + with patch("detect_worker.threading.Thread") as mock_thread: + worker._schedule_proxy_refresh_if_needed(150) + + mock_thread.assert_not_called() + + def test_proxy_refresh_threshold_scales_down_for_multi_process_node(self): + worker = self._build_worker(_FakeDb("noop")) + worker.thread_count = 1000 + worker._get_active_domain_threads = MagicMock(return_value=900) + + self.assertEqual(2700, worker._proxy_refresh_threshold()) + + worker.process_count = 80 + worker.refresh_process_count_runtime = MagicMock(return_value=80) + + self.assertLess(worker._proxy_refresh_threshold(), 400) + + def test_trigger_proxy_refresh_respects_cooldown_without_reset(self): + worker = self._build_worker(_FakeDb("noop")) + worker.proxy_config["proxy_urls"] = ["http://provider.local/get?group=B&count=1"] + worker.proxy_next_refresh_time = time.time() + 10 + worker.trigger_proxy_refresh = DetectWorker.trigger_proxy_refresh.__get__(worker, DetectWorker) + + with patch("detect_worker.threading.Thread") as mock_thread: + scheduled = worker.trigger_proxy_refresh(reason="cooldown-check") + + self.assertFalse(scheduled) + mock_thread.assert_not_called() + + def test_trigger_proxy_refresh_debounces_config_update_burst(self): + worker = self._build_worker(_FakeDb("noop")) + worker.proxy_config["proxy_urls"] = ["http://provider.local/get?group=B&count=1"] + worker.trigger_proxy_refresh = DetectWorker.trigger_proxy_refresh.__get__(worker, DetectWorker) + + with patch("detect_worker.threading.Thread") as mock_thread: + first = worker.trigger_proxy_refresh(reason="config_update:thread_count") + second = worker.trigger_proxy_refresh(reason="config_update:runtime_settings") + + self.assertTrue(first) + self.assertFalse(second) + mock_thread.assert_called_once() + + def test_config_update_proxy_refresh_ignores_runtime_settings_only_changes(self): + worker = self._build_worker(_FakeDb("noop")) + worker.proxy_config = { + "proxy_enable": True, + "allow_direct": True, + "proxy_urls": ["http://provider.local/get?group=B"], + } + worker.runtime_settings = {"worker_log_sync_mode": "key"} + worker.proxy_pool = [ + {"proxy": {"http": "http://cached", "https": "http://cached"}, "usage_count": 0, "source_tag": "L", "expire_at_ts": time.time() + 120} + for _ in range(160) + ] + should_refresh, reset_cooldown = worker._should_trigger_proxy_refresh_for_config_update( + config_type="runtime_settings", + previous_proxy_config=dict(worker.proxy_config), + previous_thread_count=worker.thread_count, + current_thread_count=worker.thread_count, + ) + + self.assertFalse(should_refresh) + self.assertFalse(reset_cooldown) + + def test_proxy_item_expiry_respects_recent_grace_window(self): + worker = self._build_worker(_FakeDb("noop")) + expired_recently = {"expire_at": time.time() - 30} + + self.assertTrue(worker._is_proxy_item_expired(expired_recently)) + self.assertFalse(worker._is_proxy_item_expired(expired_recently, grace_seconds=90)) + self.assertTrue(worker._is_proxy_item_expired(expired_recently, grace_seconds=10)) + + def test_proxy_expire_clock_skew_hint_detects_large_positive_offset(self): + worker = self._build_worker(_FakeDb("noop")) + base_now = 1_700_000_000.0 + proxy_items = [ + {"ip": "1.1.1.1", "port": "8001", "expire_at": base_now - 46000}, + {"ip": "1.1.1.2", "port": "8002", "expire_at": base_now - 45990}, + {"ip": "1.1.1.3", "port": "8003", "expire_at": base_now - 45980}, + {"ip": "1.1.1.4", "port": "8004", "expire_at": base_now - 45970}, + {"ip": "1.1.1.5", "port": "8005", "expire_at": base_now - 45960}, + {"ip": "1.1.1.6", "port": "8006", "expire_at": base_now - 45950}, + {"ip": "1.1.1.7", "port": "8007", "expire_at": base_now - 45940}, + {"ip": "1.1.1.8", "port": "8008", "expire_at": base_now - 45930}, + {"ip": "1.1.1.9", "port": "8009", "expire_at": base_now - 45920}, + {"ip": "1.1.1.10", "port": "8010", "expire_at": base_now - 45910}, + {"ip": "1.1.1.11", "port": "8011", "expire_at": base_now - 45900}, + {"ip": "1.1.1.12", "port": "8012", "expire_at": base_now - 45890}, + ] + + hint = worker._summarize_proxy_expire_clock_skew(proxy_items, now_ts=base_now) + + self.assertIsNotNone(hint) + self.assertEqual(12, hint["timed_proxy_count"]) + self.assertEqual(12, hint["expired_timed_count"]) + self.assertGreater(hint["clock_ahead_seconds"], 45_000) + + def test_proxy_expire_clock_skew_hint_ignores_normally_fresh_entries(self): + worker = self._build_worker(_FakeDb("noop")) + base_now = 1_700_000_000.0 + proxy_items = [ + {"ip": "2.2.2.1", "port": "9001", "expire_at": base_now + 120}, + {"ip": "2.2.2.2", "port": "9002", "expire_at": base_now + 180}, + {"ip": "2.2.2.3", "port": "9003", "expire_at": base_now + 240}, + {"ip": "2.2.2.4", "port": "9004", "expire_at": base_now + 300}, + {"ip": "2.2.2.5", "port": "9005", "expire_at": base_now + 360}, + {"ip": "2.2.2.6", "port": "9006", "expire_at": base_now + 420}, + {"ip": "2.2.2.7", "port": "9007", "expire_at": base_now + 480}, + {"ip": "2.2.2.8", "port": "9008", "expire_at": base_now + 540}, + {"ip": "2.2.2.9", "port": "9009", "expire_at": base_now + 600}, + {"ip": "2.2.2.10", "port": "9010", "expire_at": base_now + 660}, + {"ip": "2.2.2.11", "port": "9011", "expire_at": base_now + 720}, + {"ip": "2.2.2.12", "port": "9012", "expire_at": base_now + 780}, + ] + + hint = worker._summarize_proxy_expire_clock_skew(proxy_items, now_ts=base_now) + + self.assertIsNone(hint) + + def test_recently_expired_backfill_only_kicks_in_when_pool_is_too_small(self): + worker = self._build_worker(_FakeDb("noop")) + worker.thread_count = 2000 + worker._get_active_domain_threads = MagicMock(return_value=1800) + stale_entries = [ + { + "proxy": {"http": f"http://stale-{idx}", "https": f"http://stale-{idx}"}, + "usage_count": 0, + "source_tag": f"S{idx % 3}", + } + for idx in range(200) + ] + + merged_entries, added_count, floor = worker._backfill_recently_expired_proxy_entries( + [ + { + "proxy": {"http": "http://fresh-1", "https": "http://fresh-1"}, + "usage_count": 0, + "source_tag": "F", + } + ], + stale_entries, + ) + + self.assertGreater(floor, 1) + self.assertEqual(floor, len(merged_entries)) + self.assertEqual(floor - 1, added_count) + + healthy_pool = [ + { + "proxy": {"http": f"http://fresh-{idx}", "https": f"http://fresh-{idx}"}, + "usage_count": 0, + "source_tag": "F", + } + for idx in range(floor + 5) + ] + merged_entries, added_count, reused_floor = worker._backfill_recently_expired_proxy_entries( + healthy_pool, + stale_entries, + ) + + self.assertEqual(0, added_count) + self.assertEqual(reused_floor, floor) + self.assertEqual(len(healthy_pool), len(merged_entries)) + + def test_refresh_proxy_pool_can_reuse_recently_expired_entries_without_unbound_local_error(self): + worker = self._build_worker(_FakeDb("noop")) + worker.thread_count = 1200 + worker.proxy_config = { + "proxy_enable": True, + "allow_direct": True, + "proxy_urls": ["http://provider.local/get?group=B&count=1"], + } + + class _Response: + status_code = 200 + + @staticmethod + def json(): + return [ + { + "ip": "1.1.1.1", + "port": "8080", + "expire_at": time.time() - 30, + } + ] + + with patch("requests.get", return_value=_Response()): + worker.refresh_proxy_pool() + + def test_pending_control_key_scopes_to_current_node_code(self): + with patch.object(config, "NODE_CODE", "mainland-controller-01-a"): + self.assertEqual( + "domain_tool:worker_pending_command:mainland-controller-01-a", + pending_control_key(), + ) + + def test_control_targets_current_worker_rejects_other_instance(self): + with patch.object(config, "NODE_CODE", "mainland-controller-01-a"): + self.assertTrue(control_targets_current_worker({"action": "start_detection"})) + self.assertTrue(control_targets_current_worker({"target_node_code": "mainland-controller-01-a"})) + self.assertFalse(control_targets_current_worker({"target_node_code": "mainland-controller-01-b"})) + + def test_consume_pending_control_command_prefers_instance_key(self): + worker = DetectWorker.__new__(DetectWorker) + worker.use_redis = True + worker._handle_control_message = MagicMock() + + class _FakeRedis: + def __init__(self): + self.values = { + "domain_tool:worker_pending_command:mainland-controller-01-a": '{"action":"start_detection","target_node_code":"mainland-controller-01-a"}', + "domain_tool:worker_pending_command": '{"action":"start_detection"}', + } + + def getdel(self, key): + return self.values.pop(key, None) + + def get(self, key): + return self.values.get(key) + + def delete(self, key): + self.values.pop(key, None) + + worker.redis_client = _FakeRedis() + + with patch.object(config, "NODE_CODE", "mainland-controller-01-a"): + worker._consume_pending_control_command() + + worker._handle_control_message.assert_called_once_with( + '{"action":"start_detection","target_node_code":"mainland-controller-01-a"}' + ) + + def test_handle_control_message_ignores_command_for_other_instance(self): + worker = DetectWorker.__new__(DetectWorker) + worker.use_redis = False + worker.redis_client = None + worker._acknowledge_pending_control_command = MagicMock() + worker._remember_ignored_targeted_control = MagicMock() + worker.start_detection_async = MagicMock() + worker.request_stop_detection = MagicMock() + + with patch.object(config, "NODE_CODE", "mainland-controller-01-a"): + worker._handle_control_message( + '{"action":"start_detection","target_node_code":"mainland-controller-01-b"}' + ) + + worker._acknowledge_pending_control_command.assert_not_called() + worker._remember_ignored_targeted_control.assert_called_once() + worker.start_detection_async.assert_not_called() + + def test_handle_control_message_records_ignored_targeted_job_for_other_instance(self): + worker = self._build_worker(MagicMock()) + + with patch.object(config, "NODE_CODE", "mainland-controller-01-a"): + worker._handle_control_message( + { + "action": "start_detection", + "job_id": 870, + "job_code": "sync-overseas-19868", + "target_node_codes": ["mainland-controller-01-do", "mainland-controller-01-cz"], + } + ) + + self.assertEqual(870, worker._ignored_targeted_job_id) + self.assertEqual("sync-overseas-19868", worker._ignored_targeted_job_code) + self.assertEqual( + "mainland-controller-01-do,mainland-controller-01-cz", + worker._ignored_targeted_job_targets, + ) + + def test_auto_resume_skips_when_active_job_matches_recently_ignored_targeted_job(self): + worker = self._build_worker(MagicMock()) + worker._load_active_detect_job_snapshot = MagicMock( + return_value={ + "id": 870, + "job_code": "sync-overseas-19868", + "task_mode": "domain_pipeline", + "items_pending": 1200, + "items_claimed": 949, + "items_running": 6315, + "selection_reason": "running_job_active", + } + ) + worker.start_detection_async = MagicMock(return_value=True) + worker._ignored_targeted_job_id = 870 + worker._ignored_targeted_job_code = "sync-overseas-19868" + worker._ignored_targeted_job_until = time.time() + 120 + worker._ignored_targeted_job_targets = "mainland-controller-01-do,mainland-controller-01-cz" + + started = worker._resume_active_detect_job_if_needed(reason="heartbeat_autoresume") + + self.assertFalse(started) + worker.start_detection_async.assert_not_called() + + def test_proxy_direct_fallback_grace_shrinks_for_shortage_steps(self): + worker = self._build_worker(_FakeDb("noop")) + worker.proxy_direct_fallback_grace_overrides = { + "注册状态检测": 0.08, + "360检测": 0.12, + "站长之家检测": 0.12, + } + worker.proxy_last_refresh_status = "代理源暂时冷却中，稍后继续补货" + + self.assertEqual(0.05, worker._proxy_direct_fallback_grace_for_step("注册状态检测")) + self.assertEqual(0.05, worker._proxy_direct_fallback_grace_for_step("360检测")) + self.assertEqual(0.05, worker._proxy_direct_fallback_grace_for_step("站长之家检测")) + + def test_run_detect_register_prefers_direct_when_proxy_source_is_short(self): + worker = self._build_worker(_FakeDb("noop")) + worker.proxy_last_refresh_status = "代理源暂时冷却中，稍后继续补货" + worker._check_proxy_step_retry_budget = MagicMock(return_value=(False, "")) + worker._remaining_proxy_step_retry_budget_seconds = MagicMock(return_value=2.0) + worker._is_current_single_step_task = MagicMock(return_value=False) + worker._clear_proxy_failure = MagicMock() + worker._clear_proxy_source_failure = MagicMock() + worker._get_proxy_for_step = MagicMock(return_value={"http": "http://proxy", "https": "http://proxy"}) + worker.db.update_domain_register_result = MagicMock() + + with patch("detect_worker.register.check_register", return_value=(3, "")) as mock_check: + result = worker._run_detect_register(1, {"source_type": 0}, "example.com") + + self.assertTrue(result) + worker._get_proxy_for_step.assert_not_called() + self.assertIsNone(mock_check.call_args.args[2]) + + def test_run_detect_register_single_machine_mode_allows_direct_even_when_global_direct_disabled(self): + worker = self._build_worker(_FakeDb("noop")) + worker.proxy_config = {"proxy_enable": True, "allow_direct": False} + worker.register_single_machine_mode_enabled = True + worker.register_single_machine_direct_streak_attempts = 2 + worker._check_proxy_step_retry_budget = MagicMock(return_value=(False, "")) + worker._remaining_proxy_step_retry_budget_seconds = MagicMock(return_value=2.0) + worker._is_current_single_step_task = MagicMock(return_value=False) + worker._clear_proxy_failure = MagicMock() + worker._clear_proxy_source_failure = MagicMock() + worker._get_proxy_for_step = MagicMock(return_value={"http": "http://proxy", "https": "http://proxy"}) + worker.db.update_domain_register_result = MagicMock() + + with patch("detect_worker.register.check_register", return_value=(3, "")) as mock_check: + result = worker._run_detect_register(1, {"source_type": 0}, "example.com") + + self.assertTrue(result) + worker._get_proxy_for_step.assert_not_called() + self.assertIsNone(mock_check.call_args.args[2]) + + def test_run_detect_register_single_machine_mode_retries_direct_before_proxy(self): + worker = self._build_worker(_FakeDb("noop")) + worker.proxy_config = {"proxy_enable": True, "allow_direct": False} + worker.register_single_machine_mode_enabled = True + worker.register_single_machine_direct_streak_attempts = 2 + worker._check_proxy_step_retry_budget = MagicMock(return_value=(False, "")) + worker._remaining_proxy_step_retry_budget_seconds = MagicMock(return_value=2.0) + worker._is_current_single_step_task = MagicMock(return_value=False) + worker._clear_proxy_failure = MagicMock() + worker._clear_proxy_source_failure = MagicMock() + worker._get_proxy_for_step = MagicMock(return_value={"http": "http://proxy", "https": "http://proxy"}) + worker.db.update_domain_register_result = MagicMock() + + with patch( + "detect_worker.register.check_register", + side_effect=[RuntimeError("direct timeout"), (3, "")], + ) as mock_check: + result = worker._run_detect_register(1, {"source_type": 0}, "example.com") + + self.assertTrue(result) + self.assertEqual(2, mock_check.call_count) + self.assertTrue(all(call.args[2] is None for call in mock_check.call_args_list)) + worker._get_proxy_for_step.assert_not_called() + + def test_run_detect_register_direct_error_keeps_fast_direct_retry_during_shortage(self): + worker = self._build_worker(_FakeDb("noop")) + worker.proxy_last_refresh_status = "代理源暂时冷却中，稍后继续补货" + worker._check_proxy_step_retry_budget = MagicMock(side_effect=[(False, ""), (True, "budget exhausted")]) + worker._get_proxy_for_step = MagicMock(return_value=None) + worker._mark_detection_failed = MagicMock() + + with patch("detect_worker.register.check_register", side_effect=RuntimeError("direct timeout")): + result = worker._run_detect_register(1, {"source_type": 0}, "example.com") + + self.assertFalse(result) + self.assertTrue(worker._consume_step_force_direct_once("注册状态检测")) + self.assertAlmostEqual(0.05, worker._consume_step_proxy_retry_wait("注册状态检测")) + worker._mark_detection_failed.assert_called_once() + + def test_run_detect_register_blacklists_client_hold_for_single_step_task(self): + worker = self._build_worker(_FakeDb("noop")) + worker._check_proxy_step_retry_budget = MagicMock(return_value=(False, "")) + worker._remaining_proxy_step_retry_budget_seconds = MagicMock(return_value=2.0) + worker._is_current_single_step_task = MagicMock(return_value=True) + worker._cache_single_step_result_payload = MagicMock() + worker._mark_blacklisted = MagicMock() + worker._clear_proxy_failure = MagicMock() + worker._clear_proxy_source_failure = MagicMock() + worker._get_proxy_for_step = MagicMock(return_value={"http": "http://proxy", "https": "http://proxy"}) + worker.db.update_domain_register_result = MagicMock() + + with patch("detect_worker.register.check_register", return_value=(REGISTER_STATUS_CLIENT_HOLD, "2026-05-01")): + result = worker._run_detect_register(1, {"source_type": 0}, "example.com") + + self.assertFalse(result) + worker._cache_single_step_result_payload.assert_called_once() + payload = worker._cache_single_step_result_payload.call_args.args[2] + self.assertEqual("blacklisted", payload["state"]) + self.assertEqual(REGISTER_STATUS_CLIENT_HOLD, payload["register_status"]) + worker._mark_blacklisted.assert_called_once() + worker.db.update_domain_register_result.assert_not_called() + + def test_run_detect_register_blacklists_server_hold_for_domain_task(self): + worker = self._build_worker(_FakeDb("noop")) + worker._check_proxy_step_retry_budget = MagicMock(return_value=(False, "")) + worker._remaining_proxy_step_retry_budget_seconds = MagicMock(return_value=2.0) + worker._is_current_single_step_task = MagicMock(return_value=False) + worker._mark_blacklisted = MagicMock() + worker._clear_proxy_failure = MagicMock() + worker._clear_proxy_source_failure = MagicMock() + worker._get_proxy_for_step = MagicMock(return_value={"http": "http://proxy", "https": "http://proxy"}) + worker.db.update_domain_register_result = MagicMock() + + domain = {"source_type": 0} + with patch("detect_worker.register.check_register", return_value=(REGISTER_STATUS_SERVER_HOLD, "2026-05-01")): + result = worker._run_detect_register(1, domain, "example.com") + + self.assertFalse(result) + worker.db.update_domain_register_result.assert_called_once() + self.assertEqual(REGISTER_STATUS_SERVER_HOLD, domain["register_status"]) + worker._mark_blacklisted.assert_called_once() + + def test_get_proxy_for_step_fast_direct_fallback_when_proxy_source_is_cooling_down(self): + worker = self._build_worker(_FakeDb("noop")) + worker.proxy_last_refresh_status = "代理源暂时冷却中，稍后继续补货" + worker.proxy_direct_fallback_grace_overrides = {"360检测": 0.12} + worker.get_proxies = MagicMock(return_value=None) + + with patch("detect_worker.time.time", side_effect=[100.0, 100.2]), patch("detect_worker.time.sleep") as mock_sleep: + proxy = worker._get_proxy_for_step(1, "example.com", "360检测") + + self.assertIsNone(proxy) + mock_sleep.assert_not_called() + + def test_get_proxy_for_step_register_single_machine_mode_skips_proxy_wait(self): + worker = self._build_worker(_FakeDb("noop")) + worker.proxy_config = {"proxy_enable": True, "allow_direct": False} + worker.register_single_machine_mode_enabled = True + worker.register_single_machine_direct_streak_attempts = 2 + worker.get_proxies = MagicMock(return_value=None) + + with patch("detect_worker.time.time", side_effect=[100.0, 100.0]), patch("detect_worker.time.sleep") as mock_sleep: + proxy = worker._get_proxy_for_step(1, "example.com", "注册状态检测") + + self.assertEqual(0.0, worker._proxy_direct_fallback_grace_for_step("注册状态检测")) + self.assertEqual(0.0, worker._proxy_direct_retry_wait_for_step("注册状态检测")) + self.assertIsNone(proxy) + mock_sleep.assert_not_called() + + def test_external_issue_retry_wait_uses_step_specific_short_window(self): + worker = self._build_worker(_FakeDb("noop")) + worker._set_current_task_context(domain_id=1, is_step_task=True, task_mode="single_step") + worker.proxy_direct_retry_wait_overrides = {"360检测": 0.25} + worker.proxy_last_refresh_status = "代理源暂时冷却中，稍后继续补货" + + should_retry = worker._should_retry_external_issue( + None, + "360搜索请求超时", + "360检测", + "example.com", + ) + + self.assertTrue(should_retry) + self.assertAlmostEqual(0.25, worker._consume_step_proxy_retry_wait("360检测")) + + def test_stale_detect_session_holds_during_proxy_shortage_for_same_job(self): + worker = self._build_worker(_FakeDb("noop")) + worker.detect_command_thread = MagicMock() + worker.detect_command_thread.is_alive.return_value = True + worker.current_job_code = "sync-overseas-51" + worker.proxy_last_refresh_status = "代理源暂时冷却中，稍后继续补货" + worker.proxy_pool = [] + + with patch("detect_worker.time.time", return_value=200.0): + worker.detect_command_started_at = 160.0 + worker.detect_command_last_activity_at = 165.0 + worker._last_domain_started_at = 165.0 + worker._last_domain_result_at = 165.0 + stale_detect, stale_reason = worker._is_stale_detect_session({"job_code": "sync-overseas-51"}) + + self.assertFalse(stale_detect) + self.assertIn("同任务同周期重复唤起已忽略", stale_reason) + + def test_stale_detect_session_still_recovers_when_new_job_arrives(self): + worker = self._build_worker(_FakeDb("noop")) + worker.detect_command_thread = MagicMock() + worker.detect_command_thread.is_alive.return_value = True + worker.current_job_code = "sync-overseas-51" + worker.current_job_task_mode = "single_step" + worker.proxy_last_refresh_status = "代理源暂时冷却中，稍后继续补货" + worker.proxy_pool = [] + + with patch("detect_worker.time.time", return_value=200.0): + worker.detect_command_started_at = 150.0 + worker.detect_command_last_activity_at = 170.0 + worker._last_domain_started_at = 170.0 + worker._last_domain_result_at = 170.0 + stale_detect, stale_reason = worker._is_stale_detect_session({"job_code": "sync-overseas-99"}) + + self.assertTrue(stale_detect) + self.assertIn("新任务", stale_reason) + + def test_stale_detect_session_does_not_replace_busy_domain_pipeline_for_new_job(self): + worker = self._build_worker(_FakeDb("noop")) + worker.detect_command_thread = MagicMock() + worker.detect_command_thread.is_alive.return_value = True + worker.current_job_code = "sync-overseas-51" + worker.current_job_task_mode = "domain_pipeline" + worker.proxy_last_refresh_status = "代理正常" + worker.proxy_pool = [{}] + worker._active_domain_threads = 6 + + with patch("detect_worker.time.time", return_value=200.0): + worker.detect_command_started_at = 150.0 + worker.detect_command_last_activity_at = 170.0 + worker._last_domain_started_at = 170.0 + worker._last_domain_result_at = 170.0 + stale_detect, stale_reason = worker._is_stale_detect_session( + {"job_code": "sync-overseas-99", "task_mode": "domain_pipeline"} + ) + + self.assertFalse(stale_detect) + self.assertEqual("", stale_reason) + def test_submit_backlog_limit_defaults_to_thread_count_window(self): worker = self._build_worker(_FakeDb("noop")) @@ -166,6 +2041,93 @@ class DetectWorkerFlushRetryTest(unittest.TestCase): self.assertEqual(2000, worker._resolve_claim_batch_size(2000, 2000, live_active=0)) self.assertEqual(1500, worker._resolve_claim_batch_size(2000, 1500, live_active=800)) + def test_runtime_settings_can_shrink_claim_and_backlog_windows(self): + worker = self._build_worker(_FakeDb("noop")) + worker.runtime_settings = { + "claim_batch_floor": 200, + "claim_batch_ceil": 800, + "submit_backlog_floor": 500, + "submit_backlog_ceil": 1500, + "dispatch_cap_multiplier": 2, + "pending_buffer_cap_multiplier": 2, + } + + self.assertEqual(500, worker._resolve_submit_backlog_limit(2000, 0)) + self.assertEqual(500, worker._resolve_claim_batch_size(2000, 2000, live_active=0)) + self.assertEqual(1500, worker._resolve_dispatch_capacity(2000, 0, 1500)) + self.assertEqual(2, worker._runtime_positive_int_override("pending_buffer_cap_multiplier", "DOMAINCHECK_PENDING_BUFFER_CAP_MULTIPLIER")) + + def test_runtime_settings_override_environment_claim_and_backlog_windows(self): + worker = self._build_worker(_FakeDb("noop")) + worker.runtime_settings = { + "claim_batch_floor": 200, + "claim_batch_ceil": 800, + "submit_backlog_floor": 500, + "submit_backlog_ceil": 1500, + "dispatch_cap_multiplier": 2, + } + + with patch.dict( + os.environ, + { + "DOMAINCHECK_CLAIM_BATCH_FLOOR": "4000", + "DOMAINCHECK_CLAIM_BATCH_CEIL": "10000", + "DOMAINCHECK_SUBMIT_BACKLOG_FLOOR": "5000", + "DOMAINCHECK_SUBMIT_BACKLOG_CEIL": "10000", + "DOMAINCHECK_DISPATCH_CAP_MULTIPLIER": "5", + }, + clear=False, + ): + self.assertEqual(500, worker._resolve_submit_backlog_limit(2000, 0)) + self.assertEqual(500, worker._resolve_claim_batch_size(2000, 2000, live_active=0)) + self.assertEqual(1500, worker._resolve_dispatch_capacity(2000, 0, 1500)) + + def test_pipeline_topup_triggers_when_partial_claim_leaves_large_gap(self): + worker = self._build_worker(_FakeDb("noop")) + + self.assertTrue(worker._should_proactively_top_up_pipeline(280, 1800, 2000)) + self.assertFalse(worker._should_proactively_top_up_pipeline(1700, 1800, 2000)) + + def test_sync_prefetch_triggers_when_local_work_falls_below_threshold(self): + worker = self._build_worker(_FakeDb("noop")) + + self.assertTrue(worker._should_prefetch_sync_tasks(24, 18, 1000)) + self.assertFalse(worker._should_prefetch_sync_tasks(420, 180, 1000)) + + def test_pull_sync_tasks_can_prefetch_without_claiming_after_pull(self): + worker = self._build_worker(_FakeDb("noop")) + worker._acquire_shared_job_maintenance_lock = MagicMock(return_value="lock-owner") + worker._release_shared_job_maintenance_lock = MagicMock() + worker._claim_detect_job_items = MagicMock(return_value=[{"domain_name": "should-not-claim.com"}]) + + class _FakeResponse: + def __enter__(self_inner): + return self_inner + + def __exit__(self_inner, exc_type, exc, tb): + return False + + @staticmethod + def read(): + return json.dumps( + { + "code": 0, + "message": "ok", + "data": {"pull_state": "queued", "queued_count": 500}, + }, + ensure_ascii=False, + ).encode("utf-8") + + with patch.object(config, "NODE_REGION", "mainland"), patch.object(config, "NODE_ROLE", "control"), patch( + "detect_worker.urllib.request.urlopen", + return_value=_FakeResponse(), + ): + domains = worker._pull_sync_tasks_until_available(thread_limit=1000, claim_after_pull=False) + + self.assertEqual([], domains) + worker._claim_detect_job_items.assert_not_called() + worker._release_shared_job_maintenance_lock.assert_called_once() + def test_get_proxy_for_step_can_force_single_direct_fallback_once(self): worker = self._build_worker(_FakeDb("noop")) worker._set_current_task_context(domain_id=1, is_step_task=True, task_mode="single_step") @@ -176,6 +2138,94 @@ class DetectWorkerFlushRetryTest(unittest.TestCase): self.assertIsNone(proxy) self.assertFalse(worker._consume_step_force_direct_once("百度site检测")) + def test_load_thread_count_can_inherit_parent_node_override(self): + worker = DetectWorker.__new__(DetectWorker) + worker.use_redis = True + + class _FakeRedis: + def get(self, key): + if key == "domain_tool:node_thread_counts": + return json.dumps({"mainland-controller-01": 1200}, ensure_ascii=False) + if key == "domain_tool:thread_count": + return "1000" + return None + + worker.redis_client = _FakeRedis() + + with patch.object(config, "NODE_CODE", "mainland-controller-01-a"), patch.dict( + os.environ, + {"WORKER_PARENT_NODE_CODE": "mainland-controller-01"}, + clear=False, + ): + thread_count = DetectWorker.load_thread_count(worker) + + self.assertEqual(1200, thread_count) + + def test_load_process_count_can_inherit_parent_node_override_from_local_file_root(self): + worker = DetectWorker.__new__(DetectWorker) + worker.use_redis = False + + with patch.object(config, "NODE_CODE", "mainland-controller-01-u"), patch.dict( + os.environ, + {"WORKER_PARENT_NODE_CODE": "mainland-controller-01"}, + clear=False, + ), patch( + "detect_worker._read_worker_local_json_config", + side_effect=lambda name: ( + {"mainland-controller-01": 80} + if name == "node_process_counts.json" + else ({"process_count": "80"} if name == "process_count.json" else None) + ), + ): + process_count = DetectWorker.load_process_count(worker) + + self.assertEqual(80, process_count) + + def test_get_proxies_with_lease_holds_entry_until_release(self): + worker = self._build_worker(_FakeDb("noop")) + good_proxy = {"http": "http://good-proxy", "https": "http://good-proxy"} + worker.proxy_pool = [ + { + "proxy": good_proxy, + "usage_count": 0, + "source_tag": "C", + "expire_at_ts": time.time() + 60, + } + ] + worker._schedule_proxy_refresh_if_needed = MagicMock() + + selected = worker.get_proxies(step_name="百度site检测", lease_selected=True) + + self.assertEqual(good_proxy, selected) + self.assertEqual(0, len(worker.proxy_pool)) + self.assertIn(worker._proxy_key(good_proxy), worker.proxy_active_leases) + + reused = worker.release_proxy(good_proxy, step_name="百度site检测") + + self.assertTrue(reused) + self.assertEqual(1, len(worker.proxy_pool)) + self.assertNotIn(worker._proxy_key(good_proxy), worker.proxy_active_leases) + + def test_release_proxy_discards_entry_when_remaining_ttl_is_too_low(self): + worker = self._build_worker(_FakeDb("noop")) + proxy = {"http": "http://soon-expire", "https": "http://soon-expire"} + worker.proxy_active_leases = { + worker._proxy_key(proxy): { + "proxy": proxy, + "usage_count": 1, + "source_tag": "D", + "expire_at_ts": time.time() + 2, + "leased_step_name": "百度site检测", + } + } + worker._schedule_proxy_refresh_if_needed = MagicMock() + + reused = worker.release_proxy(proxy, step_name="百度site检测") + + self.assertFalse(reused) + self.assertEqual([], worker.proxy_pool) + self.assertNotIn(worker._proxy_key(proxy), worker.proxy_active_leases) + def test_retry_external_issue_with_proxy_forces_next_attempt_direct_when_allowed(self): worker = self._build_worker(_FakeDb("noop")) worker._set_current_task_context(domain_id=1, is_step_task=True, task_mode="single_step") @@ -209,7 +2259,7 @@ class DetectWorkerFlushRetryTest(unittest.TestCase): worker._mark_detection_degraded = MagicMock() worker._should_blacklist_result = MagicMock(return_value=False) worker._record_step_result = MagicMock() - worker._should_rotate_proxy_on_error = MagicMock(return_value=False) + worker._should_retry_external_issue = MagicMock(return_value=False) worker._is_external_dependency_issue = MagicMock(return_value=False) worker._mark_detection_rejected = MagicMock() worker._mark_detection_failed = MagicMock() @@ -220,6 +2270,419 @@ class DetectWorkerFlushRetryTest(unittest.TestCase): self.assertTrue(result) self.assertEqual(1.7, mock_check.call_args.kwargs["budget_seconds"]) + def test_run_detect_aizhan_uses_external_issue_retry_path(self): + worker = self._build_worker(_FakeDb("noop")) + worker._check_proxy_step_retry_budget = MagicMock(side_effect=[(False, ""), (False, ""), (True, "budget exhausted")]) + worker._remaining_proxy_step_retry_budget_seconds = MagicMock(return_value=2.0) + worker._get_proxy_for_step = MagicMock(side_effect=[ + {"http": "http://proxy-1", "https": "http://proxy-1"}, + None, + ]) + worker._mark_detection_degraded = MagicMock() + worker._should_blacklist_result = MagicMock(return_value=False) + worker._record_step_result = MagicMock() + worker._should_retry_external_issue = MagicMock(side_effect=[True, False]) + worker._is_external_dependency_issue = MagicMock(return_value=False) + worker._mark_detection_rejected = MagicMock() + worker._mark_detection_failed = MagicMock() + worker.release_proxy = MagicMock() + + with patch( + "detect_worker.aizhan.check_aizhan", + side_effect=[(False, "proxy timeout"), (True, "ok")], + ) as mock_check: + result = worker._run_detect_aizhan(1, "example.com", []) + + self.assertTrue(result) + self.assertEqual(2, mock_check.call_count) + worker._should_retry_external_issue.assert_called_with( + {"http": "http://proxy-1", "https": "http://proxy-1"}, + "proxy timeout", + "爱站网检测", + "example.com", + ) + + def test_single_machine_aizhan_direct_first_enabled_only_on_first_attempt(self): + worker = self._build_worker(_FakeDb("noop")) + worker.single_machine_aizhan_direct_first_enabled = True + worker.single_machine_site_direct_fallback_enabled = True + worker.allow_direct_connection = MagicMock(return_value=False) + + self.assertTrue(worker._should_force_direct_first_attempt("爱站网检测", 0)) + self.assertFalse(worker._should_force_direct_first_attempt("爱站网检测", 1)) + self.assertFalse(worker._should_force_direct_first_attempt("百度site检测", 0)) + + def test_single_machine_baidu_direct_first_enabled_only_on_first_attempt(self): + worker = self._build_worker(_FakeDb("noop")) + worker.single_machine_baidu_direct_first_enabled = True + worker.single_machine_site_direct_fallback_enabled = True + worker.allow_direct_connection = MagicMock(return_value=False) + + self.assertTrue(worker._should_force_direct_first_attempt("百度site检测", 0)) + self.assertFalse(worker._should_force_direct_first_attempt("百度site检测", 1)) + self.assertFalse(worker._should_force_direct_first_attempt("爱站网检测", 0)) + + def test_single_machine_chinaz_direct_first_enabled_only_on_first_attempt(self): + worker = self._build_worker(_FakeDb("noop")) + worker.single_machine_chinaz_direct_first_enabled = True + worker.single_machine_site_direct_fallback_enabled = True + worker.allow_direct_connection = MagicMock(return_value=False) + + self.assertTrue(worker._should_force_direct_first_attempt("站长之家检测", 0)) + self.assertFalse(worker._should_force_direct_first_attempt("站长之家检测", 1)) + self.assertFalse(worker._should_force_direct_first_attempt("百度site检测", 0)) + + def test_run_detect_aizhan_forces_direct_first_when_single_machine_flag_enabled(self): + worker = self._build_worker(_FakeDb("noop")) + worker.single_machine_aizhan_direct_first_enabled = True + worker.single_machine_site_direct_fallback_enabled = True + worker.allow_direct_connection = MagicMock(return_value=False) + worker._check_proxy_step_retry_budget = MagicMock(return_value=(False, "")) + worker._remaining_proxy_step_retry_budget_seconds = MagicMock(return_value=2.0) + worker._get_proxy_for_step = MagicMock(return_value=None) + worker._mark_detection_degraded = MagicMock() + worker._should_blacklist_result = MagicMock(return_value=False) + worker._record_step_result = MagicMock() + worker._should_retry_external_issue = MagicMock(return_value=False) + worker._is_external_dependency_issue = MagicMock(return_value=False) + worker._mark_detection_rejected = MagicMock() + worker._mark_detection_failed = MagicMock() + worker._set_step_force_direct_once = MagicMock() + + with patch("detect_worker.aizhan.check_aizhan", return_value=(True, "")): + result = worker._run_detect_aizhan(1, "example.com", []) + + self.assertTrue(result) + worker._set_step_force_direct_once.assert_called_once_with("爱站网检测", True) + + def test_run_detect_baidu_forces_direct_first_when_single_machine_flag_enabled(self): + worker = self._build_worker(_FakeDb("noop")) + worker.single_machine_baidu_direct_first_enabled = True + worker.single_machine_site_direct_fallback_enabled = True + worker.allow_direct_connection = MagicMock(return_value=False) + worker._check_proxy_step_retry_budget = MagicMock(return_value=(False, "")) + worker._remaining_proxy_step_retry_budget_seconds = MagicMock(return_value=2.0) + worker._get_proxy_for_step = MagicMock(return_value=None) + worker._mark_detection_degraded = MagicMock() + worker._should_blacklist_result = MagicMock(return_value=False) + worker._record_step_result = MagicMock() + worker._should_retry_external_issue = MagicMock(return_value=False) + worker._is_external_dependency_issue = MagicMock(return_value=False) + worker._mark_detection_rejected = MagicMock() + worker._mark_detection_failed = MagicMock() + worker._set_step_force_direct_once = MagicMock() + + with patch("detect_worker.baidu.check_site", return_value=(True, "")): + result = worker._run_detect_baidu(1, "example.com", []) + + self.assertTrue(result) + worker._set_step_force_direct_once.assert_called_once_with("百度site检测", True) + + def test_run_detect_chinaz_forces_direct_first_when_single_machine_flag_enabled(self): + worker = self._build_worker(_FakeDb("noop")) + worker.single_machine_chinaz_direct_first_enabled = True + worker.single_machine_site_direct_fallback_enabled = True + worker.allow_direct_connection = MagicMock(return_value=False) + worker._check_proxy_step_retry_budget = MagicMock(return_value=(False, "")) + worker._remaining_proxy_step_retry_budget_seconds = MagicMock(return_value=2.0) + worker._get_proxy_for_step = MagicMock(return_value={"http": "http://proxy-1", "https": "http://proxy-1"}) + worker._mark_detection_degraded = MagicMock() + worker._should_blacklist_result = MagicMock(return_value=False) + worker._record_step_result = MagicMock() + worker._should_rotate_proxy_on_error = MagicMock(return_value=False) + worker._is_external_dependency_issue = MagicMock(return_value=False) + worker._mark_detection_rejected = MagicMock() + worker._mark_detection_failed = MagicMock() + worker._emit_step_trace = MagicMock() + + with patch("detect_worker.chinaz.check_title", return_value=(True, "", {})) as mock_check_title: + result = worker._run_detect_chinaz(1, "example.com", []) + + self.assertTrue(result) + worker._get_proxy_for_step.assert_not_called() + worker._emit_step_trace.assert_called_once() + self.assertIsNone(mock_check_title.call_args.args[2]) + + def test_run_detect_aizhan_degrades_remote_disconnected_early_when_flag_enabled(self): + worker = self._build_worker(_FakeDb("noop")) + worker.aizhan_remote_disconnect_degrade_enabled = True + worker._check_proxy_step_retry_budget = MagicMock(return_value=(False, "")) + worker._remaining_proxy_step_retry_budget_seconds = MagicMock(return_value=2.0) + proxy = {"http": "http://proxy-1", "https": "http://proxy-1"} + worker._get_proxy_for_step = MagicMock(return_value=proxy) + worker._mark_detection_degraded = MagicMock() + worker._should_blacklist_result = MagicMock(return_value=False) + worker._record_step_result = MagicMock() + worker._should_retry_external_issue = MagicMock(return_value=True) + worker._is_external_dependency_issue = MagicMock(return_value=True) + worker._mark_detection_rejected = MagicMock() + worker._mark_detection_failed = MagicMock() + worker.release_proxy = MagicMock() + + with patch( + "detect_worker.aizhan.check_aizhan", + return_value=(False, "('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))"), + ): + result = worker._run_detect_aizhan(1, "example.com", []) + + self.assertTrue(result) + worker._should_retry_external_issue.assert_not_called() + worker.release_proxy.assert_called_once() + worker._mark_detection_degraded.assert_called_once() + + def test_run_detect_aizhan_degrades_external_issue_early_when_fast_flag_enabled(self): + worker = self._build_worker(_FakeDb("noop")) + worker.aizhan_external_fast_degrade_enabled = True + worker._check_proxy_step_retry_budget = MagicMock(return_value=(False, "")) + worker._remaining_proxy_step_retry_budget_seconds = MagicMock(return_value=2.0) + proxy = {"http": "http://proxy-1", "https": "http://proxy-1"} + worker._get_proxy_for_step = MagicMock(return_value=proxy) + worker._mark_detection_degraded = MagicMock() + worker._should_blacklist_result = MagicMock(return_value=False) + worker._record_step_result = MagicMock() + worker._should_retry_external_issue = MagicMock(return_value=True) + worker._is_external_dependency_issue = MagicMock(return_value=True) + worker._mark_detection_rejected = MagicMock() + worker._mark_detection_failed = MagicMock() + worker.release_proxy = MagicMock() + + with patch( + "detect_worker.aizhan.check_aizhan", + return_value=(False, "Read timed out. (read timeout=1.8)"), + ): + result = worker._run_detect_aizhan(1, "example.com", []) + + self.assertTrue(result) + worker._should_retry_external_issue.assert_not_called() + worker.release_proxy.assert_called_once() + worker._mark_detection_degraded.assert_called_once() + + def test_resolve_single_step_finalization_keeps_degraded_as_completed(self): + status, message = DetectWorker._resolve_single_step_finalization( + { + "state": "degraded", + "message": "外部依赖异常，建议重试", + "retry_recommended": True, + } + ) + + self.assertEqual("completed", status) + self.assertEqual("外部依赖异常，建议重试", message) + + def test_worker_log_sync_mode_keeps_full_when_load_is_small(self): + worker = self._build_worker(_FakeDb("noop")) + worker.runtime_settings = {"worker_log_sync_enabled": True, "worker_log_sync_mode": "full"} + worker.thread_count = 1200 + worker.detecting = True + worker._get_active_domain_threads = MagicMock(return_value=80) + + self.assertEqual("full", worker._worker_log_sync_mode()) + + def test_worker_log_sync_mode_downgrades_full_when_threads_are_busy(self): + worker = self._build_worker(_FakeDb("noop")) + worker.runtime_settings = {"worker_log_sync_enabled": True, "worker_log_sync_mode": "full"} + worker.thread_count = 1200 + worker.detecting = True + worker._get_active_domain_threads = MagicMock(return_value=650) + + self.assertEqual("key", worker._worker_log_sync_mode()) + + def test_worker_log_sync_mode_downgrades_full_when_queue_backlog_is_high(self): + worker = self._build_worker(_FakeDb("noop")) + worker.runtime_settings = {"worker_log_sync_enabled": True, "worker_log_sync_mode": "full"} + worker.thread_count = 1200 + worker.detecting = False + worker._worker_log_sync_queue = MagicMock() + worker._worker_log_sync_queue.maxsize = 50000 + worker._worker_log_sync_queue.qsize.return_value = 12000 + + self.assertEqual("key", worker._worker_log_sync_mode()) + + def test_push_remote_debug_event_enters_cooldown_after_failure(self): + worker = self._build_worker(_FakeDb("noop")) + + with patch.object(worker, "_remote_debug_ingest_url", return_value="http://example.invalid/api/v1/runtime/debug-ingest"): + with patch("detect_worker.urllib.request.urlopen", side_effect=TimeoutError("timed out")) as mock_urlopen: + worker._push_remote_debug_event(message="test") + + self.assertEqual(1, worker._remote_debug_event_failure_streak) + self.assertGreater(worker._remote_debug_event_cooldown_until, time.time()) + self.assertEqual(1, mock_urlopen.call_count) + + worker._push_remote_debug_event(message="test") + self.assertEqual(1, mock_urlopen.call_count) + + def test_push_remote_debug_event_resets_failure_streak_after_success(self): + worker = self._build_worker(_FakeDb("noop")) + worker._remote_debug_event_failure_streak = 3 + + class _FakeResponse: + def __enter__(self_inner): + return self_inner + + def __exit__(self_inner, exc_type, exc, tb): + return False + + @staticmethod + def read(): + return b'{"code":0}' + + with patch.object(worker, "_remote_debug_ingest_url", return_value="http://example.invalid/api/v1/runtime/debug-ingest"): + with patch("detect_worker.urllib.request.urlopen", return_value=_FakeResponse()): + worker._push_remote_debug_event(message="test") + + self.assertEqual(0, worker._remote_debug_event_failure_streak) + self.assertEqual(0.0, worker._remote_debug_event_cooldown_until) + + def test_idle_worker_recycle_skips_parent_control_worker(self): + worker = self._build_worker(_FakeDb("noop")) + worker.running = True + worker.detecting = True + worker._get_active_domain_threads = MagicMock(return_value=0) + worker._last_runtime_extra = {"waiting_for_dispatch": True} + worker._last_worker_activity_at = time.time() - 600 + worker._worker_started_at = time.time() - 900 + + with patch.dict( + os.environ, + { + "WORKER_MODE": "linux-systemd", + "WORKER_PARENT_NODE_CODE": "mainland-controller-01", + "DOMAINCHECK_IDLE_WORKER_RECYCLE_ENABLED": "1", + }, + clear=False, + ): + with patch.object(config, "NODE_CODE", "mainland-controller-01"): + with patch.object(worker, "_trigger_idle_worker_recycle") as mock_recycle: + recycled = worker._maybe_recycle_idle_worker(trigger_reason="heartbeat") + + self.assertFalse(recycled) + mock_recycle.assert_not_called() + + def test_configure_worker_thread_stack_size_uses_linux_systemd_default(self): + state = {"size": 0} + + def fake_stack_size(value=None): + if value is None: + return state["size"] + state["size"] = int(value) + return 0 + + with patch.dict(os.environ, {"WORKER_MODE": "linux-systemd"}, clear=False): + with patch.object(detect_worker_module.threading, "stack_size", side_effect=fake_stack_size): + applied = detect_worker_module._configure_worker_thread_stack_size() + + self.assertEqual(1024 * 1024, applied) + self.assertEqual(1024 * 1024, state["size"]) + + def test_configure_worker_thread_stack_size_skips_non_service_default(self): + state = {"size": 0} + + def fake_stack_size(value=None): + if value is None: + return state["size"] + state["size"] = int(value) + return 0 + + with patch.dict(os.environ, {"WORKER_MODE": "desktop"}, clear=False): + with patch.object(detect_worker_module.threading, "stack_size", side_effect=fake_stack_size): + applied = detect_worker_module._configure_worker_thread_stack_size() + + self.assertEqual(0, applied) + self.assertEqual(0, state["size"]) + + def test_idle_worker_recycle_triggers_for_idle_child_worker(self): + worker = self._build_worker(_FakeDb("noop")) + worker.running = True + worker.detecting = True + worker._get_active_domain_threads = MagicMock(return_value=0) + worker._last_runtime_extra = {"waiting_for_dispatch": True} + worker._last_worker_activity_at = time.time() - 600 + worker._worker_started_at = time.time() - 900 + + with patch.dict( + os.environ, + { + "WORKER_MODE": "linux-systemd", + "WORKER_PARENT_NODE_CODE": "mainland-controller-01", + "DOMAINCHECK_IDLE_WORKER_RECYCLE_ENABLED": "1", + "DOMAINCHECK_IDLE_WORKER_RECYCLE_AFTER_SECONDS": "120", + "DOMAINCHECK_IDLE_WORKER_RECYCLE_MIN_UPTIME_SECONDS": "180", + "DOMAINCHECK_IDLE_WORKER_RECYCLE_JITTER_SECONDS": "0", + }, + clear=False, + ): + with patch.object(config, "NODE_CODE", "mainland-controller-01-b"): + with patch.object(worker, "_trigger_idle_worker_recycle", return_value=True) as mock_recycle: + recycled = worker._maybe_recycle_idle_worker(trigger_reason="heartbeat") + + self.assertTrue(recycled) + mock_recycle.assert_called_once() + + def test_idle_worker_recycle_respects_recent_explicit_start_grace(self): + worker = self._build_worker(_FakeDb("noop")) + worker.running = True + worker.detecting = True + worker._get_active_domain_threads = MagicMock(return_value=0) + worker._last_runtime_extra = {"waiting_for_dispatch": True} + worker._last_worker_activity_at = time.time() - 600 + worker._worker_started_at = time.time() - 900 + worker._last_explicit_start_signal_at = time.time() - 10 + + with patch.dict( + os.environ, + { + "WORKER_MODE": "linux-systemd", + "WORKER_PARENT_NODE_CODE": "mainland-controller-01", + "DOMAINCHECK_IDLE_WORKER_RECYCLE_ENABLED": "1", + "DOMAINCHECK_IDLE_WORKER_RECYCLE_AFTER_SECONDS": "120", + "DOMAINCHECK_IDLE_WORKER_RECYCLE_MIN_UPTIME_SECONDS": "180", + "DOMAINCHECK_IDLE_WORKER_RECYCLE_START_GRACE_SECONDS": "90", + "DOMAINCHECK_IDLE_WORKER_RECYCLE_JITTER_SECONDS": "0", + }, + clear=False, + ): + with patch.object(config, "NODE_CODE", "mainland-controller-01-c"): + with patch.object(worker, "_trigger_idle_worker_recycle") as mock_recycle: + recycled = worker._maybe_recycle_idle_worker(trigger_reason="heartbeat") + + self.assertFalse(recycled) + mock_recycle.assert_not_called() + + def test_prepare_proxy_pool_for_detection_start_reuses_local_pool(self): + worker = self._build_worker(_FakeDb("noop")) + worker.proxy_pool = [{"proxy": "http://1.1.1.1:8080"}] + worker._schedule_proxy_refresh_if_needed = MagicMock() + + ready, count, reason = worker._prepare_proxy_pool_for_detection_start() + + self.assertTrue(ready) + self.assertEqual(1, count) + self.assertEqual("local_pool_ready", reason) + worker._schedule_proxy_refresh_if_needed.assert_called_once_with(1) + + def test_prepare_proxy_pool_for_detection_start_reuses_shared_snapshot(self): + worker = self._build_worker(_FakeDb("noop")) + worker._schedule_proxy_refresh_if_needed = MagicMock() + worker._proxy_should_coordinate_shared_refresh = MagicMock(return_value=True) + worker._load_shared_proxy_snapshot_payload = MagicMock( + return_value={"proxy_pool": [{"proxy": "http://2.2.2.2:8080"}]} + ) + + def apply_snapshot(payload, status_prefix=""): + worker.proxy_pool = list(payload.get("proxy_pool") or []) + return True + + worker._apply_shared_proxy_snapshot = MagicMock(side_effect=apply_snapshot) + + ready, count, reason = worker._prepare_proxy_pool_for_detection_start() + + self.assertTrue(ready) + self.assertEqual(1, count) + self.assertEqual("shared_snapshot_ready", reason) + worker._apply_shared_proxy_snapshot.assert_called_once() + worker._schedule_proxy_refresh_if_needed.assert_called_once_with(1) + if __name__ == "__main__": unittest.main() diff --git a/domainCheck/tests/test_detection_result_schema.py b/domainCheck/tests/test_detection_result_schema.py new file mode 100644 index 0000000..e14de95 --- /dev/null +++ b/domainCheck/tests/test_detection_result_schema.py @@ -0,0 +1,44 @@ +import sys +import unittest +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +DOMAINCHECK_ROOT = PROJECT_ROOT / "domainCheck" +if str(DOMAINCHECK_ROOT) not in sys.path: + sys.path.insert(0, str(DOMAINCHECK_ROOT)) + +from app.utils.detection_results import ( + build_manual_detection_result, + normalize_detector_result, + resolve_detection_status, +) + + +class DetectionResultSchemaTests(unittest.TestCase): + def test_resolve_detection_status_supports_legacy_key(self): + self.assertTrue(resolve_detection_status({"has_收录": True}, "has_收录")) + self.assertFalse(resolve_detection_status({"has_history": False}, "has_history")) + + def test_manual_detection_result_preserves_legacy_key(self): + result = build_manual_detection_result(True, legacy_key="has_收录") + + self.assertTrue(result["status"]) + self.assertTrue(result["has_收录"]) + self.assertEqual("manual", result["state"]) + + def test_normalize_jucha_info_bubbles_nested_error(self): + normalized = normalize_detector_result( + "jucha_info", + { + "whois": {"error": "HTTP 403"}, + "beian": {"has_beian": False, "beian_year": "", "is_enterprise": False, "beian_match": False}, + "intercept": {"normal": True}, + }, + ) + + self.assertEqual("HTTP 403", normalized["error"]) + self.assertEqual("error", normalized["state"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/domainCheck/tests/test_geetest_cookie_safety.py b/domainCheck/tests/test_geetest_cookie_safety.py new file mode 100644 index 0000000..253b9ed --- /dev/null +++ b/domainCheck/tests/test_geetest_cookie_safety.py @@ -0,0 +1,82 @@ +import sys +import tempfile +import unittest +from pathlib import Path +from unittest.mock import MagicMock, patch + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +DOMAINCHECK_ROOT = PROJECT_ROOT / "domainCheck" +if str(DOMAINCHECK_ROOT) not in sys.path: + sys.path.insert(0, str(DOMAINCHECK_ROOT)) + +import requests + +from domainCheck.detect import geetest2, jucha, juming, juziseo +from domainCheck.detect.locked_pickle import load_pickle_locked, save_pickle_atomic + + +class GeetestCookieSafetyTests(unittest.TestCase): + def test_geetest_slide_asset_fetch_applies_timeout(self): + response = MagicMock() + response.content = b"binary" + with patch("domainCheck.detect.geetest2.requests.get", return_value=response) as mock_get: + slider = geetest2.slide() + with patch.object(slider, "tp_huanyuan", return_value=b"bg-bytes"): + with patch("domainCheck.detect.geetest2.quekou") as mock_quekou: + mock_quekou.return_value.get_distance.return_value = 12 + slider.huak({"bg": "bg.png", "slice": "slice.png"}) + + self.assertGreaterEqual(mock_get.call_count, 2) + for call in mock_get.call_args_list: + self.assertEqual(slider.asset_timeout, call.kwargs["timeout"]) + + def test_locked_pickle_roundtrip_is_atomic(self): + with tempfile.TemporaryDirectory() as temp_dir: + target = Path(temp_dir) / "cookies.pkl" + save_pickle_atomic(str(target), {"sid": "abc"}) + loaded = load_pickle_locked(str(target), default_factory=dict) + self.assertEqual({"sid": "abc"}, loaded) + + def test_juziseo_cookie_roundtrip_uses_locked_pickle(self): + with tempfile.TemporaryDirectory() as temp_dir: + target = Path(temp_dir) / "juziseo.pkl" + detector = juziseo.Juziseo() + jar = requests.cookies.RequestsCookieJar() + jar.set("sid", "value") + detector.cookie = jar + detector.save_cookies(str(target)) + + loaded = juziseo.Juziseo() + loaded.load_cookies(str(target)) + + self.assertEqual("value", loaded.cookie.get("sid")) + + def test_jucha_and_juming_cookie_roundtrip_use_locked_pickle(self): + with tempfile.TemporaryDirectory() as temp_dir: + jucha_path = Path(temp_dir) / "jucha.pkl" + juming_path = Path(temp_dir) / "juming.pkl" + + jc = jucha.JC() + jc_jar = requests.cookies.RequestsCookieJar() + jc_jar.set("jc", "cookie") + jc.cookie = jc_jar + jc.save_cookies(str(jucha_path)) + + jm = juming.JM() + jm_jar = requests.cookies.RequestsCookieJar() + jm_jar.set("jm", "cookie") + jm.cookie = jm_jar + jm.save_cookies(str(juming_path)) + + loaded_jc = jucha.JC() + loaded_jc.load_cookies(str(jucha_path)) + + loaded_jm = juming.JM() + loaded_jm.load_cookies(str(juming_path)) + + self.assertEqual("cookie", loaded_jc.cookie.get("jc")) + self.assertEqual("cookie", loaded_jm.cookie.get("jm")) + + +if __name__ == "__main__": + unittest.main() diff --git a/domainCheck/tests/test_legacy_detector_timeouts.py b/domainCheck/tests/test_legacy_detector_timeouts.py new file mode 100644 index 0000000..de048e9 --- /dev/null +++ b/domainCheck/tests/test_legacy_detector_timeouts.py @@ -0,0 +1,60 @@ +import unittest +import sys +from pathlib import Path +from unittest.mock import MagicMock + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +DOMAINCHECK_ROOT = PROJECT_ROOT / "domainCheck" +if str(DOMAINCHECK_ROOT) not in sys.path: + sys.path.insert(0, str(DOMAINCHECK_ROOT)) + +from domainCheck.detect import jucha, juming, juziseo + + +class LegacyDetectorTimeoutTests(unittest.TestCase): + def test_juziseo_request_helper_applies_default_timeout(self): + client = juziseo.Juziseo() + client.session = MagicMock() + client._request("get", "https://example.com/api", headers={"x": "1"}) + + client.session.get.assert_called_once_with( + "https://example.com/api", + headers={"x": "1"}, + timeout=client.request_timeout, + ) + + def test_jucha_request_helper_applies_default_timeout(self): + client = jucha.JC() + client.session = MagicMock() + client._request("post", "https://example.com/api", data={"a": 1}) + + client.session.post.assert_called_once_with( + "https://example.com/api", + data={"a": 1}, + timeout=client.request_timeout, + ) + + def test_juming_request_helper_applies_default_timeout(self): + client = juming.JM() + client.session = MagicMock() + client._request("post", "https://example.com/api", json={"a": 1}) + + client.session.post.assert_called_once_with( + "https://example.com/api", + json={"a": 1}, + timeout=client.request_timeout, + ) + + def test_request_helper_preserves_explicit_timeout_override(self): + client = juming.JM() + client.session = MagicMock() + client._request("get", "https://example.com/download", timeout=60) + + client.session.get.assert_called_once_with( + "https://example.com/download", + timeout=60, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/domainCheck/tests/test_redis_client.py b/domainCheck/tests/test_redis_client.py new file mode 100644 index 0000000..aa8dfcd --- /dev/null +++ b/domainCheck/tests/test_redis_client.py @@ -0,0 +1,39 @@ +import unittest +from unittest.mock import patch + +from app.utils.redis_client import get_redis_client, reset_redis_clients_for_tests + + +class DomainCheckRedisClientTests(unittest.TestCase): + def tearDown(self) -> None: + reset_redis_clients_for_tests() + + @patch("app.utils.redis_client.redis.Redis") + @patch("app.utils.redis_client.redis.BlockingConnectionPool") + def test_standard_client_is_cached_per_process(self, mock_pool, mock_redis) -> None: + client = object() + mock_redis.return_value = client + + first = get_redis_client() + second = get_redis_client() + + self.assertIs(first, client) + self.assertIs(second, client) + mock_pool.assert_called_once() + mock_redis.assert_called_once() + + @patch("app.utils.redis_client.redis.Redis") + @patch("app.utils.redis_client.redis.BlockingConnectionPool") + def test_pubsub_role_uses_separate_cached_client(self, mock_pool, mock_redis) -> None: + mock_redis.side_effect = [object(), object()] + + standard_client = get_redis_client() + pubsub_client = get_redis_client(role="pubsub") + + self.assertIsNot(standard_client, pubsub_client) + self.assertEqual(2, mock_pool.call_count) + self.assertEqual(2, mock_redis.call_count) + + +if __name__ == "__main__": + unittest.main() diff --git a/domainCheck/tests/test_register_timeout_config.py b/domainCheck/tests/test_register_timeout_config.py index ac1ad87..d9ea6f8 100644 --- a/domainCheck/tests/test_register_timeout_config.py +++ b/domainCheck/tests/test_register_timeout_config.py @@ -7,6 +7,9 @@ from domainCheck.detect import register class RegisterTimeoutConfigTests(unittest.TestCase): + def tearDown(self): + register._PROXY_MANAGERS.clear() + def test_proxy_timeout_uses_bounded_total(self): with patch.dict("os.environ", {}, clear=False): timeout = register._resolve_register_timeout({"http": "http://127.0.0.1:8080"}) @@ -59,6 +62,19 @@ class RegisterTimeoutConfigTests(unittest.TestCase): allow_redirects=True, ) + def test_proxy_manager_cache_evicts_oldest_entry(self): + with patch.dict("os.environ", {"DOMAINCHECK_REGISTER_PROXY_SESSION_CACHE_SIZE": "2"}, clear=False): + first = register._get_http_manager({"http": "http://127.0.0.1:8080"}) + second = register._get_http_manager({"http": "http://127.0.0.1:8081"}) + third = register._get_http_manager({"http": "http://127.0.0.1:8082"}) + + self.assertEqual(2, len(register._PROXY_MANAGERS)) + self.assertIsNotNone(second) + self.assertIsNotNone(third) + + recreated_first = register._get_http_manager({"http": "http://127.0.0.1:8080"}) + self.assertIsNot(first, recreated_first) + if __name__ == "__main__": unittest.main() diff --git a/domainCheck/tests/test_step_timeout_budgets.py b/domainCheck/tests/test_step_timeout_budgets.py index 7f778f2..44489b0 100644 --- a/domainCheck/tests/test_step_timeout_budgets.py +++ b/domainCheck/tests/test_step_timeout_budgets.py @@ -5,6 +5,9 @@ from domainCheck.detect import aizhan, baidu, c360, chinaz class StepTimeoutBudgetTests(unittest.TestCase): + def tearDown(self): + c360._PROXY_SESSIONS.clear() + def test_baidu_timeout_respects_remaining_budget(self): with patch.dict("os.environ", {}, clear=False): timeout = baidu._resolve_baidu_timeout({"http": "http://127.0.0.1:8080"}, budget_seconds=1.1) @@ -17,6 +20,33 @@ class StepTimeoutBudgetTests(unittest.TestCase): self.assertLessEqual(timeout, 0.9) self.assertGreaterEqual(timeout, 0.6) + def test_360_direct_session_disables_env_proxy_and_is_reused(self): + first = c360._get_session() + second = c360._get_session() + self.assertIs(first, second) + self.assertFalse(first.trust_env) + + def test_360_proxy_session_is_reused_per_proxy_url(self): + first = c360._get_session({"http": "http://127.0.0.1:8080"}) + second = c360._get_session({"https": "http://127.0.0.1:8080"}) + third = c360._get_session({"http": "http://127.0.0.1:8081"}) + self.assertIs(first, second) + self.assertIsNot(first, third) + self.assertFalse(first.trust_env) + + def test_360_proxy_session_cache_evicts_oldest_entry(self): + with patch.dict("os.environ", {"DOMAINCHECK_360_PROXY_SESSION_CACHE_SIZE": "2"}, clear=False): + first = c360._get_session({"http": "http://127.0.0.1:8080"}) + second = c360._get_session({"http": "http://127.0.0.1:8081"}) + third = c360._get_session({"http": "http://127.0.0.1:8082"}) + + self.assertEqual(2, len(c360._PROXY_SESSIONS)) + self.assertIsNotNone(second) + self.assertIsNotNone(third) + + recreated_first = c360._get_session({"http": "http://127.0.0.1:8080"}) + self.assertIsNot(first, recreated_first) + def test_chinaz_timeout_respects_remaining_budget(self): timeout = chinaz._resolve_chinaz_timeout({"http": "http://127.0.0.1:8080"}, budget_seconds=1.3) self.assertLessEqual(timeout, 1.3) diff --git a/domainCheck/tests/test_wayback_detector_recent_years.py b/domainCheck/tests/test_wayback_detector_recent_years.py index 5c5ef62..2a62062 100644 --- a/domainCheck/tests/test_wayback_detector_recent_years.py +++ b/domainCheck/tests/test_wayback_detector_recent_years.py @@ -92,7 +92,47 @@ class WaybackDetectorRecentYearsTest(unittest.TestCase): # 最新快照会先独立尝试一次，再进入裁剪后的扫描窗口。 self.assertEqual(4, result["fetched_snapshot_count"]) - def test_scan_snapshots_fast_degrades_when_latest_cdx_is_transient_failure(self): + def test_scan_snapshots_fetches_recent_cdx_window_instead_of_full_history(self): + detector = WaybackDetector.__new__(WaybackDetector) + fetch_limits = [] + + def fake_fetch(domain, limit=None, fast_latest=False): + fetch_limits.append((limit, fast_latest)) + if fast_latest: + return { + "records": [{"timestamp": "20260101000000", "digest": "latest"}], + "error": None, + } + return { + "records": [ + {"timestamp": "20260101000000", "digest": "latest"}, + {"timestamp": "20251201000000", "digest": "older-1"}, + {"timestamp": "20251101000000", "digest": "older-2"}, + {"timestamp": "20251001000000", "digest": "older-3"}, + ], + "error": None, + } + + detector._fetch_cdx_records_with_meta = fake_fetch + detector._load_cached_records = lambda domain: None + detector._save_cached_records = lambda domain, records: None + detector._save_cached_timestamps = lambda domain, timestamps: None + detector._fetch_snapshot_title = lambda domain, timestamp: { + "timestamp": timestamp, + "title": f"title-{timestamp}", + "ok": True, + } + detector._normalize_title = lambda title: title + detector._find_sensitive_word = lambda title, words: None + detector._log_info = lambda message: None + detector._handle_exception = lambda exc, domain: None + + with patch("app.detectors.wayback_detector.config.WAYBACK_MAX_RECORDS", 3): + detector.scan_snapshots("example.com", sensitive_words=[], recent_years=5) + + self.assertEqual((-18, False), fetch_limits[1]) + + def test_scan_snapshots_continues_with_cached_records_when_latest_cdx_is_transient_failure(self): detector = WaybackDetector.__new__(WaybackDetector) detector._fetch_cdx_records_with_meta = lambda domain, limit=None, fast_latest=False: { "records": [], @@ -104,19 +144,55 @@ class WaybackDetectorRecentYearsTest(unittest.TestCase): ] detector._save_cached_records = lambda domain, records: None detector._save_cached_timestamps = lambda domain, timestamps: None + detector._fetch_snapshot_title = lambda domain, timestamp: { + "timestamp": timestamp, + "title": f"title-{timestamp}", + "ok": True, + } + detector._normalize_title = lambda title: title + detector._find_sensitive_word = lambda title, words: None + detector._log_info = lambda message: None + detector._handle_exception = lambda exc, domain: None + detector._trip_transient_backoff = lambda: self.fail("single latest_cdx timeout should not trigger global backoff") + + result = detector.scan_snapshots("example.com", sensitive_words=[], recent_years=5) + + self.assertEqual(2, result["checked_snapshot_count"]) + self.assertEqual(2, result["fetched_snapshot_count"]) + self.assertGreaterEqual(result["request_error_count"], 1) + + def test_scan_snapshots_skips_records_cdx_when_latest_cdx_transient_fails_without_cache(self): + detector = WaybackDetector.__new__(WaybackDetector) + fetch_calls = [] + + def fake_fetch(domain, limit=None, fast_latest=False): + fetch_calls.append((limit, fast_latest)) + return { + "records": [], + "error": "HTTPSConnectionPool(host='web.archive.org', port=443): Max retries exceeded", + } + + detector._fetch_cdx_records_with_meta = fake_fetch + detector._load_cached_records = lambda domain: None + detector._save_cached_records = lambda domain, records: None + detector._save_cached_timestamps = lambda domain, timestamps: None detector._fetch_snapshot_title = lambda domain, timestamp: self.fail("should not fetch snapshot titles") detector._normalize_title = lambda title: title detector._find_sensitive_word = lambda title, words: None detector._log_info = lambda message: None detector._handle_exception = lambda exc, domain: None + trip_calls = [] + detector._trip_transient_backoff = lambda: trip_calls.append("trip") result = detector.scan_snapshots("example.com", sensitive_words=[], recent_years=5) + self.assertEqual([(-1, True)], fetch_calls) + self.assertEqual(1, len(trip_calls)) self.assertEqual(0, result["checked_snapshot_count"]) - self.assertGreaterEqual(result["failed_snapshot_count"], 1) + self.assertEqual(0, result["fetched_snapshot_count"]) self.assertGreaterEqual(result["request_error_count"], 1) - def test_scan_snapshots_fast_degrades_when_latest_snapshot_is_transient_failure(self): + def test_scan_snapshots_continues_when_latest_snapshot_is_transient_failure(self): detector = WaybackDetector.__new__(WaybackDetector) detector._fetch_cdx_records_with_meta = lambda domain, limit=None, fast_latest=False: { "records": [{"timestamp": "20260101000000", "digest": "latest"}] if fast_latest else [ @@ -128,21 +204,31 @@ class WaybackDetectorRecentYearsTest(unittest.TestCase): detector._load_cached_records = lambda domain: None detector._save_cached_records = lambda domain, records: None detector._save_cached_timestamps = lambda domain, timestamps: None - detector._fetch_snapshot_title = lambda domain, timestamp: { - "timestamp": timestamp, - "title": "", - "ok": False, - "error": "HTTPSConnectionPool(host='web.archive.org', port=443): Max retries exceeded", - } + def fake_fetch(domain, timestamp): + if timestamp == "20260101000000": + return { + "timestamp": timestamp, + "title": "", + "ok": False, + "error": "HTTPSConnectionPool(host='web.archive.org', port=443): Max retries exceeded", + } + return { + "timestamp": timestamp, + "title": "older-title", + "ok": True, + } + + detector._fetch_snapshot_title = fake_fetch detector._normalize_title = lambda title: title detector._find_sensitive_word = lambda title, words: None detector._log_info = lambda message: None detector._handle_exception = lambda exc, domain: None + detector._trip_transient_backoff = lambda: self.fail("single latest_snapshot timeout should not trigger global backoff") result = detector.scan_snapshots("example.com", sensitive_words=[], recent_years=5) - self.assertEqual(1, result["checked_snapshot_count"]) - self.assertEqual(0, result["fetched_snapshot_count"]) + self.assertEqual(2, result["checked_snapshot_count"]) + self.assertEqual(1, result["fetched_snapshot_count"]) self.assertGreaterEqual(result["failed_snapshot_count"], 1) self.assertTrue( any("latest_snapshot:" in item for item in result["request_errors"]) @@ -168,6 +254,87 @@ class WaybackDetectorRecentYearsTest(unittest.TestCase): self.assertEqual(1, result["request_error_count"]) self.assertTrue(any("wayback_backoff_active:" in item for item in result["request_errors"])) + def test_scan_snapshots_does_not_trip_global_backoff_on_single_snapshot_timeout(self): + detector = WaybackDetector.__new__(WaybackDetector) + detector._fetch_cdx_records_with_meta = lambda domain, limit=None, fast_latest=False: { + "records": [{"timestamp": "20260101000000", "digest": "latest"}] if fast_latest else [ + {"timestamp": "20260101000000", "digest": "latest"}, + {"timestamp": "20250101000000", "digest": "older"}, + ], + "error": None, + } + detector._load_cached_records = lambda domain: None + detector._save_cached_records = lambda domain, records: None + detector._save_cached_timestamps = lambda domain, timestamps: None + + def fake_fetch(domain, timestamp): + if timestamp == "20260101000000": + return {"timestamp": timestamp, "title": "latest", "ok": True} + return { + "timestamp": timestamp, + "title": "", + "ok": False, + "error": "HTTPSConnectionPool(host='web.archive.org', port=443): Max retries exceeded", + } + + detector._fetch_snapshot_title = fake_fetch + detector._normalize_title = lambda title: title + detector._find_sensitive_word = lambda title, words: None + detector._log_info = lambda message: None + detector._handle_exception = lambda exc, domain: None + detector._trip_transient_backoff = lambda: self.fail("single snapshot timeout should not trigger global backoff") + + result = detector.scan_snapshots("example.com", sensitive_words=[], recent_years=5) + + self.assertEqual(2, result["checked_snapshot_count"]) + self.assertEqual(1, result["fetched_snapshot_count"]) + self.assertEqual(1, result["failed_snapshot_count"]) + + def test_scan_snapshots_trips_global_backoff_after_threshold_transient_failures(self): + detector = WaybackDetector.__new__(WaybackDetector) + detector._fetch_cdx_records_with_meta = lambda domain, limit=None, fast_latest=False: { + "records": [{"timestamp": "20260101000000", "digest": "latest"}] if fast_latest else [ + {"timestamp": "20260101000000", "digest": "latest"}, + {"timestamp": "20250101000000", "digest": "older-1"}, + {"timestamp": "20240101000000", "digest": "older-2"}, + ], + "error": None, + } + detector._load_cached_records = lambda domain: None + detector._save_cached_records = lambda domain, records: None + detector._save_cached_timestamps = lambda domain, timestamps: None + + def fake_fetch(domain, timestamp): + if timestamp == "20260101000000": + return {"timestamp": timestamp, "title": "latest", "ok": True} + return { + "timestamp": timestamp, + "title": "", + "ok": False, + "error": "HTTPSConnectionPool(host='web.archive.org', port=443): Max retries exceeded", + } + + detector._fetch_snapshot_title = fake_fetch + detector._normalize_title = lambda title: title + detector._find_sensitive_word = lambda title, words: None + detector._log_info = lambda message: None + detector._handle_exception = lambda exc, domain: None + trip_calls = [] + detector._trip_transient_backoff = lambda: trip_calls.append("trip") + + result = detector.scan_snapshots("example.com", sensitive_words=[], recent_years=5) + + self.assertEqual(1, len(trip_calls)) + self.assertEqual(3, result["checked_snapshot_count"]) + self.assertEqual(2, result["failed_snapshot_count"]) + + def test_build_session_disables_env_proxy(self): + detector = WaybackDetector.__new__(WaybackDetector) + + session = detector._build_session() + + self.assertFalse(session.trust_env) + if __name__ == "__main__": unittest.main() diff --git a/package_domain_release.ps1 b/package_domain_release.ps1 index 406f333..8127021 100644 --- a/package_domain_release.ps1 +++ b/package_domain_release.ps1 @@ -142,6 +142,8 @@ if (Test-Path -LiteralPath $domainCheckTarget) { "proxy_config.json", "thread_count.json", "node_thread_counts.json", + "process_count.json", + "node_process_counts.json", "runtime_settings.json", "detect_worker.log" ) diff --git a/package_domain_release.sh b/package_domain_release.sh index b359880..2464438 100755 --- a/package_domain_release.sh +++ b/package_domain_release.sh @@ -164,6 +164,8 @@ if [[ -d "${DOMAINCHECK_TARGET}" ]]; then "${DOMAINCHECK_TARGET}/proxy_config.json" \ "${DOMAINCHECK_TARGET}/thread_count.json" \ "${DOMAINCHECK_TARGET}/node_thread_counts.json" \ + "${DOMAINCHECK_TARGET}/process_count.json" \ + "${DOMAINCHECK_TARGET}/node_process_counts.json" \ "${DOMAINCHECK_TARGET}/runtime_settings.json" \ "${DOMAINCHECK_TARGET}/detect_worker.log" find "${DOMAINCHECK_TARGET}" -type d -name "__pycache__" -prune -exec rm -rf {} + diff --git a/tools/collect_chinaz_gray_metrics.py b/tools/collect_chinaz_gray_metrics.py new file mode 100644 index 0000000..9fd01d9 --- /dev/null +++ b/tools/collect_chinaz_gray_metrics.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +import argparse +import json +import os +import re +import subprocess +import time +from datetime import datetime, timezone +from typing import Dict, List, Optional + + +def run_ssh(remote_cmd: str, timeout: int) -> str: + proc = subprocess.run( + [ + "python3", + "-c", + ( + "import paramiko,sys;" + "host,user,password,cmd,timeout=sys.argv[1:6];" + "client=paramiko.SSHClient();" + "client.set_missing_host_key_policy(paramiko.AutoAddPolicy());" + "client.connect(host, username=user, password=password, timeout=int(timeout));" + "stdin,stdout,stderr=client.exec_command(cmd, timeout=int(timeout));" + "sys.stdout.write(stdout.read().decode('utf-8','ignore'));" + "err=stderr.read().decode('utf-8','ignore');" + "client.close();" + "sys.stderr.write(err)" + ), + os.environ["REMOTE_HOST"], + os.environ["REMOTE_USER"], + os.environ["REMOTE_PASS"], + remote_cmd, + str(timeout), + ], + capture_output=True, + text=True, + timeout=timeout + 10, + ) + if proc.returncode != 0 and proc.stderr.strip(): + raise RuntimeError(proc.stderr.strip()) + return proc.stdout + + +def parse_cpu_busy(text: str) -> Optional[float]: + match = re.search(r"([0-9]+(?:\.[0-9]+)?)\s*id", text) + if not match: + return None + idle = float(match.group(1)) + return round(max(0.0, 100.0 - idle), 2) + + +def collect_snapshot(gray_workers: List[str], since: str) -> dict: + units = " ".join(f"-u domaincheck-worker@{worker}.service" for worker in gray_workers) + remote_cmd = ( + "echo '###CPU'; " + "top -bn1 | grep '%Cpu' || true; " + "echo '###GRAY'; " + f"journalctl {units} --since '{since}' --no-pager " + "| grep -E 'proxy_direct_fallback|站长之家调用边界|single_step_finalized|重试预算耗尽' " + "| tail -n 1200 || true; " + "echo '###GLOBAL'; " + "journalctl -u 'domaincheck-worker@*.service' " + f"--since '{since}' --no-pager " + "| grep 'detect_order=detect_chinaz' | tail -n 1200 || true" + ) + out = run_ssh(remote_cmd, timeout=40) + cpu_part = "" + gray_part = "" + global_part = "" + if "###GRAY" in out: + cpu_part, rest = out.split("###GRAY", 1) + if "###GLOBAL" in rest: + gray_part, global_part = rest.split("###GLOBAL", 1) + else: + gray_part = rest + else: + cpu_part = out + cpu_busy = parse_cpu_busy(cpu_part) + + counts = { + "proxy_direct_fallback": 0, + "boundary_proxy_yes": 0, + "boundary_proxy_no": 0, + "single_step_finalized": 0, + "retry_budget_exhausted": 0, + } + per_worker: Dict[str, Dict[str, int]] = {} + for line in gray_part.splitlines(): + worker_match = re.search(r"domaincheck-worker@([a-z0-9]+)\[", line) + worker = worker_match.group(1) if worker_match else "-" + bucket = per_worker.setdefault( + worker, + { + "proxy_direct_fallback": 0, + "boundary_proxy_yes": 0, + "boundary_proxy_no": 0, + "single_step_finalized": 0, + "retry_budget_exhausted": 0, + }, + ) + if "proxy_direct_fallback" in line: + counts["proxy_direct_fallback"] += 1 + bucket["proxy_direct_fallback"] += 1 + if "站长之家调用边界" in line and "proxy=yes" in line: + counts["boundary_proxy_yes"] += 1 + bucket["boundary_proxy_yes"] += 1 + if "站长之家调用边界" in line and "proxy=no" in line: + counts["boundary_proxy_no"] += 1 + bucket["boundary_proxy_no"] += 1 + if "single_step_finalized" in line and "detect_key=detect_chinaz" in line: + counts["single_step_finalized"] += 1 + bucket["single_step_finalized"] += 1 + if "重试预算耗尽" in line and "站长之家检测" in line: + counts["retry_budget_exhausted"] += 1 + bucket["retry_budget_exhausted"] += 1 + + global_worker_counts: Dict[str, int] = {} + if global_part.strip(): + top_counts: Dict[str, int] = {} + for line in global_part.splitlines(): + worker_match = re.search(r"domaincheck-worker@([a-z0-9]+)\[", line) + if not worker_match: + continue + worker = worker_match.group(1) + top_counts[worker] = top_counts.get(worker, 0) + 1 + global_worker_counts = dict( + sorted(top_counts.items(), key=lambda item: item[1], reverse=True)[:12] + ) + + return { + "timestamp": datetime.now(timezone.utc).isoformat(), + "cpu_busy_pct": cpu_busy, + "counts": counts, + "per_worker": per_worker, + "global_detect_chinaz_orders": sum(global_worker_counts.values()), + "global_detect_chinaz_top_workers": global_worker_counts, + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--output", required=True) + parser.add_argument("--workers", required=True, help="comma-separated worker suffixes") + parser.add_argument("--duration-seconds", type=int, default=3600) + parser.add_argument("--interval-seconds", type=int, default=30) + parser.add_argument("--since-window-seconds", type=int, default=45) + args = parser.parse_args() + + gray_workers = [item.strip() for item in args.workers.split(",") if item.strip()] + os.makedirs(os.path.dirname(args.output), exist_ok=True) + deadline = time.time() + max(0, args.duration_seconds) + while time.time() <= deadline: + since_ts = datetime.fromtimestamp( + time.time() - max(1, args.since_window_seconds), + tz=timezone.utc, + ).astimezone().strftime("%Y-%m-%d %H:%M:%S") + snapshot = collect_snapshot(gray_workers, since_ts) + with open(args.output, "a", encoding="utf-8") as fh: + fh.write(json.dumps(snapshot, ensure_ascii=False) + "\n") + time.sleep(max(1, args.interval_seconds)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/collect_overnight_metrics.py b/tools/collect_overnight_metrics.py new file mode 100644 index 0000000..26f95bf --- /dev/null +++ b/tools/collect_overnight_metrics.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import time +from datetime import datetime, timedelta +from pathlib import Path + +import paramiko +import psycopg2 +import requests + + +def _now() -> datetime: + return datetime.now() + + +def _safe_json_response(url: str) -> dict: + try: + response = requests.get(url, timeout=8) + payload = response.json() + if isinstance(payload, dict): + return payload.get("data") or payload + except Exception as exc: + return {"error": str(exc), "url": url} + return {} + + +def _query_local_db() -> dict: + try: + conn = psycopg2.connect( + host="127.0.0.1", + port=5432, + dbname="domain", + user="postgres", + password="Qazwe123,./", + ) + with conn: + with conn.cursor() as cur: + cur.execute( + """ + select node_code, status, current_load, last_heartbeat_at + from detect_worker_nodes + where node_code in ('mainland-controller-01', 'mainland-worker-01') + order by node_code + """ + ) + worker_nodes = [ + { + "node_code": row[0], + "status": row[1], + "current_load": int(row[2] or 0), + "last_heartbeat_at": row[3].isoformat(sep=" ", timespec="seconds") if row[3] else "", + } + for row in cur.fetchall() + ] + conn.close() + return {"worker_nodes": worker_nodes} + except Exception as exc: + return {"error": str(exc)} + + +def _run_remote_sample(host: str, user: str, password: str) -> dict: + if not str(host or "").strip() or not str(user or "").strip() or not str(password or "").strip(): + return {"skipped": True, "reason": "remote ssh credentials not provided"} + cli = paramiko.SSHClient() + cli.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + try: + cli.connect(hostname=host, username=user, password=password, timeout=10) + command = r"""printf 'proc='; pgrep -fc '[d]etect_worker.py' +printf 'load='; ps -eo pcpu,cmd | grep '[d]etect_worker.py' | awk '{s+=$1} END {print s+0}' +printf 'db_err_recent='; journalctl -u domaincheck-worker -u 'domaincheck-worker@*' --since '5 min ago' --no-pager | egrep -c '初始化数据库连接池失败|连接池耗尽|等待超时|数据库连接失败' || true +cd /www/wwwroot/getDomain/domain-api && /opt/domaincheck/domainCheck/.venv/bin/python - <<'INNER' +import json +from app.services.detect_service import get_detect_status +result = get_detect_status() or {} +print('detect_status=' + json.dumps({ + 'job_code': result.get('job_code'), + 'active_thread_count': result.get('active_thread_count'), + 'max_thread_count': result.get('max_thread_count'), + 'aggregate_process_count': result.get('aggregate_process_count'), + 'display_items_running': result.get('display_items_running'), + 'display_active_threads': result.get('display_active_threads'), + 'display_max_threads': result.get('display_max_threads'), +}, ensure_ascii=False)) +INNER""" + _, stdout, stderr = cli.exec_command(command, timeout=45) + output = stdout.read().decode("utf-8", errors="replace") + error = stderr.read().decode("utf-8", errors="replace").strip() + sample: dict[str, object] = {"raw": output.strip()} + if error: + sample["stderr"] = error + for line in output.splitlines(): + if "=" not in line: + continue + key, value = line.split("=", 1) + if key == "detect_status": + try: + sample[key] = json.loads(value) + except Exception: + sample[key] = value + continue + try: + sample[key] = int(value) + except Exception: + try: + sample[key] = float(value) + except Exception: + sample[key] = value + return sample + except Exception as exc: + return {"error": str(exc)} + finally: + try: + cli.close() + except Exception: + pass + + +def main() -> int: + parser = argparse.ArgumentParser(description="Collect overnight worker metrics into JSONL.") + parser.add_argument("--worker-host", default="") + parser.add_argument("--worker-user", default="") + parser.add_argument("--worker-password", default="") + parser.add_argument("--interval-seconds", type=int, default=60) + parser.add_argument("--duration-hours", type=float, default=6.0) + parser.add_argument("--label", default="mainland-worker-01-night") + args = parser.parse_args() + + start_time = _now() + end_time = start_time + timedelta(hours=max(0.25, float(args.duration_hours))) + run_dir = Path("/www/wwwroot/getDomain/docs/ops_center_runtime/night_runs") / ( + f"night_run_{start_time.strftime('%Y%m%d_%H%M%S')}_{args.label}" + ) + run_dir.mkdir(parents=True, exist_ok=True) + metrics_path = run_dir / "metrics.jsonl" + meta_path = run_dir / "meta.json" + meta_path.write_text( + json.dumps( + { + "label": args.label, + "started_at": start_time.isoformat(sep=" ", timespec="seconds"), + "planned_end_at": end_time.isoformat(sep=" ", timespec="seconds"), + "interval_seconds": int(args.interval_seconds), + "worker_host": args.worker_host, + "remote_sampling_enabled": bool( + str(args.worker_host or "").strip() + and str(args.worker_user or "").strip() + and str(args.worker_password or "").strip() + ), + }, + ensure_ascii=False, + indent=2, + ), + encoding="utf-8", + ) + + base_url = "http://127.0.0.1:8100/api/v1" + sample_index = 0 + while _now() < end_time: + sample_index += 1 + timestamp = _now().isoformat(sep=" ", timespec="seconds") + row = { + "timestamp": timestamp, + "sample_index": sample_index, + "local": { + "detect_status": _safe_json_response(f"{base_url}/detect/status"), + "runtime_status": _safe_json_response(f"{base_url}/runtime/status"), + "dashboard_overview": _safe_json_response(f"{base_url}/dashboard/overview"), + "db": _query_local_db(), + }, + "remote_worker": _run_remote_sample( + host=args.worker_host, + user=args.worker_user, + password=args.worker_password, + ), + } + with metrics_path.open("a", encoding="utf-8") as fp: + fp.write(json.dumps(row, ensure_ascii=False) + "\n") + time.sleep(max(10, int(args.interval_seconds))) + + (run_dir / "done.flag").write_text(_now().isoformat(sep=" ", timespec="seconds"), encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/collect_remote_step_mix.py b/tools/collect_remote_step_mix.py new file mode 100644 index 0000000..f7ef670 --- /dev/null +++ b/tools/collect_remote_step_mix.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import re +import time +from datetime import datetime, timedelta +from pathlib import Path + +import paramiko + + +def _now() -> datetime: + return datetime.now() + + +def _parse_key_value_lines(raw: str) -> dict: + result: dict[str, object] = {"raw": raw.strip()} + for line in raw.splitlines(): + if "=" not in line: + continue + key, value = line.split("=", 1) + key = key.strip() + value = value.strip() + if not key: + continue + if key in {"running_jobs", "step_counts"}: + try: + result[key] = json.loads(value) + except Exception: + result[key] = value + continue + for caster in (int, float): + try: + result[key] = caster(value) + break + except Exception: + continue + else: + result[key] = value + return result + + +def _remote_sample(host: str, user: str, password: str, recent_minutes: int) -> dict: + cli = paramiko.SSHClient() + cli.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + command = """ +python3 - <<'PY' +import json, re, subprocess + +def sh(cmd: str) -> str: + return subprocess.check_output(cmd, shell=True, text=True, stderr=subprocess.DEVNULL) + +cpu_line = sh("top -bn1 | sed -n '3p'").strip() +idle_match = re.search(r'([0-9]+(?:\.[0-9]+)?)\s+id', cpu_line) +busy = round(100.0 - float(idle_match.group(1)), 2) if idle_match else None +print(f"cpu_busy_pct={busy if busy is not None else ''}") + +worker_stats = sh(r'''ps -eo pcpu,cmd | awk '/[d]etect_worker.py/ {cpu=$1+0; total++; sum+=cpu; if(cpu>1) gt1++; if(cpu>2) gt2++; if(cpu>3) gt3++;} END {printf("workers_total=%d\\nworkers_gt_1pct=%d\\nworkers_gt_2pct=%d\\nworkers_gt_3pct=%d\\nworkers_cpu_sum=%.1f\\n", total, gt1, gt2, gt3, sum)}' ''') +print(worker_stats.strip()) + +running_jobs_raw = sh(r'''/www/server/pgsql/bin/psql -U postgres -d domain -At -c "select id, job_code, status from detect_jobs where status='running' order by id desc limit 16;" ''') +running_jobs = [] +for line in running_jobs_raw.splitlines(): + parts = line.split("|") + if len(parts) >= 3: + running_jobs.append({"job_id": int(parts[0]), "job_code": parts[1], "status": parts[2]}) +print("running_jobs=" + json.dumps(running_jobs, ensure_ascii=False)) + +recent = __RECENT_MINUTES__ +journal_cmd = f"journalctl -u domaincheck-worker -u 'domaincheck-worker@*' --since '{recent} min ago' --no-pager -o cat" +logs = sh(journal_cmd) +step_counts = {} +for match in re.finditer(r'detect_order=(detect_[a-zA-Z0-9_]+)', logs): + step = match.group(1) + step_counts[step] = step_counts.get(step, 0) + 1 +print("step_counts=" + json.dumps(step_counts, ensure_ascii=False, sort_keys=True)) +PY +""" + command = command.replace("__RECENT_MINUTES__", str(int(recent_minutes))) + try: + cli.connect(hostname=host, username=user, password=password, timeout=15, banner_timeout=15, auth_timeout=15) + _, stdout, stderr = cli.exec_command(command, timeout=90) + output = stdout.read().decode("utf-8", errors="replace") + error = stderr.read().decode("utf-8", errors="replace").strip() + row = _parse_key_value_lines(output) + if error: + row["stderr"] = error + return row + except Exception as exc: + return {"error": str(exc)} + finally: + try: + cli.close() + except Exception: + pass + + +def main() -> int: + parser = argparse.ArgumentParser(description="Collect remote CPU/worker/step-mix metrics into JSONL.") + parser.add_argument("--worker-host", required=True) + parser.add_argument("--worker-user", required=True) + parser.add_argument("--worker-password", required=True) + parser.add_argument("--interval-seconds", type=int, default=120) + parser.add_argument("--duration-hours", type=float, default=8.0) + parser.add_argument("--recent-minutes", type=int, default=3) + parser.add_argument("--label", default="remote-step-mix") + args = parser.parse_args() + + start_time = _now() + end_time = start_time + timedelta(hours=max(0.25, float(args.duration_hours))) + run_dir = Path("/www/wwwroot/getDomain/docs/ops_center_runtime/night_runs") / ( + f"step_mix_{start_time.strftime('%Y%m%d_%H%M%S')}_{args.label}" + ) + run_dir.mkdir(parents=True, exist_ok=True) + metrics_path = run_dir / "metrics.jsonl" + meta_path = run_dir / "meta.json" + meta_path.write_text( + json.dumps( + { + "label": args.label, + "started_at": start_time.isoformat(sep=" ", timespec="seconds"), + "planned_end_at": end_time.isoformat(sep=" ", timespec="seconds"), + "interval_seconds": int(args.interval_seconds), + "recent_minutes": int(args.recent_minutes), + "worker_host": args.worker_host, + }, + ensure_ascii=False, + indent=2, + ), + encoding="utf-8", + ) + + sample_index = 0 + while _now() < end_time: + sample_index += 1 + row = { + "timestamp": _now().isoformat(sep=" ", timespec="seconds"), + "sample_index": sample_index, + "remote_worker": _remote_sample( + host=args.worker_host, + user=args.worker_user, + password=args.worker_password, + recent_minutes=max(1, int(args.recent_minutes)), + ), + } + with metrics_path.open("a", encoding="utf-8") as fp: + fp.write(json.dumps(row, ensure_ascii=False) + "\n") + time.sleep(max(30, int(args.interval_seconds))) + + (run_dir / "done.flag").write_text(_now().isoformat(sep=" ", timespec="seconds"), encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/quiet_window_rebuild_release_index.py b/tools/quiet_window_rebuild_release_index.py new file mode 100644 index 0000000..89a27d9 --- /dev/null +++ b/tools/quiet_window_rebuild_release_index.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python3 +"""Quiet-window helper for rebuilding idx_detect_job_items_release_node_job. + +Stops node-agent and worker services, keeps clearing blockers for the release-index +DROP INDEX backend until it disappears, waits briefly for CREATE INDEX to appear, +then restores services. +""" + +from __future__ import annotations + +import os +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +DOMAINCHECK_ROOT = ROOT / "domainCheck" +if str(DOMAINCHECK_ROOT) not in sys.path: + sys.path.insert(0, str(DOMAINCHECK_ROOT)) + +from app.utils.database import Database # noqa: E402 + + +ENV_FILE = Path("/etc/default/domaincheck-worker") +INDEX_NAME = "idx_detect_job_items_release_node_job" +POLL_SECONDS = 5 +MAX_DROP_LOOPS = 72 # about 6 minutes +MAX_RESTORE_LOOPS = 24 + + +def load_env(path: Path) -> None: + if not path.exists(): + return + for raw in path.read_text(encoding="utf-8").splitlines(): + line = raw.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, value = line.split("=", 1) + os.environ[key] = value + + +def log(message: str) -> None: + stamp = time.strftime("%Y-%m-%d %H:%M:%S") + print(f"[{stamp}] {message}", flush=True) + + +def run(cmd: str) -> tuple[int, str, str]: + proc = subprocess.run( + cmd, + shell=True, + text=True, + capture_output=True, + check=False, + ) + return proc.returncode, proc.stdout.strip(), proc.stderr.strip() + + +def service_state(name: str) -> str: + _, out, _ = run(f"systemctl is-active {name} || true") + return out.strip() or "unknown" + + +def templated_running_count() -> int: + _, out, _ = run( + "systemctl list-units --type=service --state=running 'domaincheck-worker@*.service' --no-legend | wc -l" + ) + try: + return int((out or "0").strip()) + except Exception: + return -1 + + +def stop_services() -> None: + for cmd in [ + "systemctl stop domaincheck-node-agent.service", + "systemctl stop 'domaincheck-worker@*.service'", + "systemctl stop domaincheck-worker.service", + ]: + code, out, err = run(cmd) + log(f"stop cmd={cmd!r} code={code} out={out!r} err={err!r}") + + +def start_services() -> None: + for cmd in [ + "systemctl start domaincheck-node-agent.service", + "systemctl start domaincheck-worker.service", + "systemctl start --all 'domaincheck-worker@*.service'", + ]: + code, out, err = run(cmd) + log(f"start cmd={cmd!r} code={code} out={out!r} err={err!r}") + + +def wait_for_quiet() -> None: + for loop in range(MAX_RESTORE_LOOPS): + templated = templated_running_count() + base = service_state("domaincheck-worker.service") + node_agent = service_state("domaincheck-node-agent.service") + log( + f"quiet_check loop={loop} templated={templated} " + f"base={base} node_agent={node_agent}" + ) + if templated == 0 and base != "active" and node_agent != "active": + return + time.sleep(POLL_SECONDS) + + +def wait_for_restore() -> None: + for loop in range(MAX_RESTORE_LOOPS): + templated = templated_running_count() + base = service_state("domaincheck-worker.service") + node_agent = service_state("domaincheck-node-agent.service") + log( + f"restore_check loop={loop} templated={templated} " + f"base={base} node_agent={node_agent}" + ) + if templated >= 190 and base == "active" and node_agent == "active": + return + time.sleep(POLL_SECONDS) + + +def fetch_drop_pids(cur) -> list[int]: + cur.execute( + """ + SELECT pid + FROM pg_stat_activity + WHERE query LIKE %s + ORDER BY pid + """, + (f"DROP INDEX CONCURRENTLY IF EXISTS {INDEX_NAME}%",), + ) + return [int(row[0]) for row in cur.fetchall()] + + +def fetch_create_progress(cur) -> list[tuple]: + cur.execute( + """ + SELECT pid, phase, blocks_done, blocks_total, current_locker_pid + FROM pg_stat_progress_create_index + WHERE index_relid = (SELECT oid FROM pg_class WHERE relname = %s) + ORDER BY pid + """, + (INDEX_NAME,), + ) + return list(cur.fetchall()) + + +def fetch_index_state(cur) -> str: + cur.execute( + """ + SELECT + c.relname, + i.indisvalid, + i.indisready + FROM pg_class c + JOIN pg_index i ON i.indexrelid = c.oid + WHERE c.relname = %s + """, + (INDEX_NAME,), + ) + row = cur.fetchone() + if not row: + return "missing" + return f"{row[0]}|{int(bool(row[1]))}|{int(bool(row[2]))}" + + +def main() -> int: + load_env(ENV_FILE) + db = Database() + conn = db.get_connection() + if not conn: + log("db_connect_failed") + return 1 + conn.autocommit = True + cur = conn.cursor() + try: + log(f"quiet-window start index={INDEX_NAME}") + stop_services() + wait_for_quiet() + + for loop in range(MAX_DROP_LOOPS): + drop_pids = fetch_drop_pids(cur) + log(f"drop_loop={loop} drop_pids={drop_pids or ['none']}") + if not drop_pids: + break + for pid in drop_pids: + cur.execute("SELECT pg_blocking_pids(%s)", (pid,)) + row = cur.fetchone() + blockers = [int(v) for v in (row[0] or [])] if row and row[0] else [] + log(f"drop_pid={pid} blockers={blockers or ['none']}") + for blocker in blockers: + cur.execute("SELECT pg_terminate_backend(%s)", (blocker,)) + result = cur.fetchone() + log(f"terminate blocker={blocker} result={result!r}") + time.sleep(POLL_SECONDS) + + for loop in range(12): + progress = fetch_create_progress(cur) + state = fetch_index_state(cur) + log(f"create_wait loop={loop} index_state={state} progress={progress or ['none']}") + if progress: + break + time.sleep(POLL_SECONDS) + finally: + try: + start_services() + wait_for_restore() + finally: + cur.close() + db.close(conn=conn) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/rebuild_release_node_job_index.py b/tools/rebuild_release_node_job_index.py new file mode 100644 index 0000000..dee6e0e --- /dev/null +++ b/tools/rebuild_release_node_job_index.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +"""Drop and recreate idx_detect_job_items_release_node_job on mainland.""" + +from __future__ import annotations + +import os +import time +from pathlib import Path + +import psycopg2 +from psycopg2 import sql + + +ENV_FILE = Path("/etc/default/domaincheck-worker") +INDEX_NAME = "idx_detect_job_items_release_node_job" + + +def load_env(path: Path) -> None: + if not path.exists(): + return + for raw in path.read_text(encoding="utf-8").splitlines(): + line = raw.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, value = line.split("=", 1) + os.environ[key] = value + + +def log(message: str) -> None: + stamp = time.strftime("%Y-%m-%d %H:%M:%S") + print(f"[{stamp}] {message}", flush=True) + + +def fetch_index_state(cur: psycopg2.extensions.cursor, index_name: str) -> tuple[bool, bool, bool] | None: + cur.execute( + """ + SELECT i.indisvalid, i.indisready, i.indislive + FROM pg_index i + JOIN pg_class c ON c.oid = i.indexrelid + WHERE c.relname = %s + """, + (index_name,), + ) + row = cur.fetchone() + if not row: + return None + return bool(row[0]), bool(row[1]), bool(row[2]) + + +def fetch_conflicting_backends(cur: psycopg2.extensions.cursor) -> list[tuple[int, str, str]]: + cur.execute( + """ + SELECT pid, query, coalesce(wait_event_type, '') + FROM pg_stat_activity + WHERE state <> 'idle' + AND ( + query ILIKE %s OR + query ILIKE %s + ) + ORDER BY backend_start + """, + ( + "%idx_detect_job_items_release_node_job%", + "%idx_detect_job_items_stalled_job_activity%", + ), + ) + rows = [] + for pid, query, wait_event_type in cur.fetchall(): + snippet = " ".join((query or "").split())[:160] + rows.append((pid, wait_event_type, snippet)) + return rows + + +def main() -> int: + load_env(ENV_FILE) + conn = psycopg2.connect( + host="127.0.0.1", + port=5432, + user=os.getenv("DB_USER", "postgres"), + password=os.getenv("DB_PASSWORD"), + dbname="domain", + ) + conn.autocommit = True + cur = conn.cursor() + try: + conflicts = fetch_conflicting_backends(cur) + if conflicts: + log(f"preflight conflicting_backends={conflicts}") + + log("drop start") + cur.execute(sql.SQL("DROP INDEX CONCURRENTLY IF EXISTS {}").format(sql.Identifier(INDEX_NAME))) + log("drop done") + + residual_state = fetch_index_state(cur, INDEX_NAME) + if residual_state is not None: + raise RuntimeError( + f"index shell still present after drop: name={INDEX_NAME} " + f"state={residual_state}" + ) + + log("create start") + cur.execute( + sql.SQL( + """ + CREATE INDEX CONCURRENTLY {} + ON public.detect_job_items (claimed_by, job_id, status, id) + WHERE claimed_by <> '' + AND status IN ('claimed', 'running') + """ + ).format(sql.Identifier(INDEX_NAME)) + ) + log("create done") + log(f"final state={fetch_index_state(cur, INDEX_NAME)}") + finally: + cur.close() + conn.close() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/rebuild_stalled_job_activity_index.py b/tools/rebuild_stalled_job_activity_index.py new file mode 100644 index 0000000..f4d44fd --- /dev/null +++ b/tools/rebuild_stalled_job_activity_index.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +"""Drop and recreate idx_detect_job_items_stalled_job_activity on mainland.""" + +from __future__ import annotations + +import os +import time +from pathlib import Path + +import psycopg2 +from psycopg2 import sql + + +ENV_FILE = Path("/etc/default/domaincheck-worker") +INDEX_NAME = "idx_detect_job_items_stalled_job_activity" + + +def load_env(path: Path) -> None: + if not path.exists(): + return + for raw in path.read_text(encoding="utf-8").splitlines(): + line = raw.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, value = line.split("=", 1) + os.environ[key] = value + + +def log(message: str) -> None: + stamp = time.strftime("%Y-%m-%d %H:%M:%S") + print(f"[{stamp}] {message}", flush=True) + + +def fetch_index_state(cur: psycopg2.extensions.cursor, index_name: str) -> tuple[bool, bool, bool] | None: + cur.execute( + """ + SELECT i.indisvalid, i.indisready, i.indislive + FROM pg_index i + JOIN pg_class c ON c.oid = i.indexrelid + WHERE c.relname = %s + """, + (index_name,), + ) + row = cur.fetchone() + if not row: + return None + return bool(row[0]), bool(row[1]), bool(row[2]) + + +def fetch_conflicting_backends(cur: psycopg2.extensions.cursor) -> list[tuple[int, str, str]]: + cur.execute( + """ + SELECT pid, query, coalesce(wait_event_type, '') + FROM pg_stat_activity + WHERE state <> 'idle' + AND ( + query ILIKE %s OR + query ILIKE %s + ) + ORDER BY backend_start + """, + ( + "%idx_detect_job_items_stalled_job_activity%", + "%idx_detect_job_items_release_node_job%", + ), + ) + rows = [] + for pid, query, wait_event_type in cur.fetchall(): + snippet = " ".join((query or "").split())[:160] + rows.append((pid, wait_event_type, snippet)) + return rows + + +def main() -> int: + load_env(ENV_FILE) + conn = psycopg2.connect( + host="127.0.0.1", + port=5432, + user=os.getenv("DB_USER", "postgres"), + password=os.getenv("DB_PASSWORD"), + dbname="domain", + ) + conn.autocommit = True + cur = conn.cursor() + try: + conflicts = fetch_conflicting_backends(cur) + if conflicts: + log(f"preflight conflicting_backends={conflicts}") + + log("drop start") + cur.execute(sql.SQL("DROP INDEX CONCURRENTLY IF EXISTS {}").format(sql.Identifier(INDEX_NAME))) + log("drop done") + + residual_state = fetch_index_state(cur, INDEX_NAME) + if residual_state is not None: + raise RuntimeError( + f"index shell still present after drop: name={INDEX_NAME} " + f"state={residual_state}" + ) + + log("create start") + cur.execute( + sql.SQL( + """ + CREATE INDEX CONCURRENTLY {} + ON public.detect_job_items ( + job_id, + status, + (COALESCE(updated_at, started_at, create_time)), + id + ) + WHERE status IN ('claimed', 'running') + """ + ).format(sql.Identifier(INDEX_NAME)) + ) + log("create done") + log(f"final state={fetch_index_state(cur, INDEX_NAME)}") + finally: + cur.close() + conn.close() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/runtime_observer.py b/tools/runtime_observer.py new file mode 100755 index 0000000..66d8af2 --- /dev/null +++ b/tools/runtime_observer.py @@ -0,0 +1,401 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import os +import shutil +import sys +import time +import urllib.error +import urllib.request +from datetime import datetime + + +DEFAULT_API_BASE = "http://127.0.0.1:8100/api/v1" + + +def _now_text() -> str: + return datetime.now().strftime("%Y-%m-%d %H:%M:%S") + + +def _safe_int(value: object, default: int = 0) -> int: + try: + return int(value or 0) + except Exception: + return int(default) + + +def _safe_float(value: object, default: float = 0.0) -> float: + try: + return float(value or 0.0) + except Exception: + return float(default) + + +def _safe_text(value: object) -> str: + return str(value or "").strip() + + +def _fetch_api(api_base: str, path: str) -> dict: + url = f"{api_base.rstrip('/')}/{path.lstrip('/')}" + try: + request = urllib.request.Request(url, headers={"User-Agent": "domaincheck-runtime-observer/0.1"}) + with urllib.request.urlopen(request, timeout=8) as response: + payload = json.loads(response.read().decode("utf-8", errors="ignore")) + if isinstance(payload, dict): + data = payload.get("data") + if isinstance(data, dict): + return data + return payload + return {"_error": f"unexpected payload type: {type(payload).__name__}", "_url": url} + except urllib.error.HTTPError as exc: + body = exc.read().decode("utf-8", errors="ignore") + return {"_error": f"http {exc.code}", "_body": body[:1000], "_url": url} + except Exception as exc: + return {"_error": str(exc), "_url": url} + + +def _observe_payload(api_base: str) -> dict: + runtime_status = _fetch_api(api_base, "runtime/status") + direct_active_job = _fetch_api(api_base, "detect/job/active") + runtime_active_job = dict(((runtime_status.get("detect") or {}).get("active_job") or {})) + readiness = _fetch_api(api_base, "runtime/readiness") + sync_summary = _fetch_api(api_base, "runtime/sync-summary") + active_job = _merge_active_job_payloads(direct_active_job, runtime_active_job) + return { + "runtime_status": runtime_status, + "active_job": active_job, + "direct_active_job": direct_active_job, + "runtime_active_job": runtime_active_job, + "readiness": readiness, + "sync_summary": sync_summary, + } + + +def _real_node_count(active_job: dict) -> int: + count = 0 + for item in list((active_job or {}).get("node_stats") or []): + node_code = _safe_text((item or {}).get("node_code")) + if node_code and node_code != "unassigned": + count += 1 + return count + + +def _merge_active_job_payloads(direct_active_job: dict, runtime_active_job: dict) -> dict: + direct_active_job = dict(direct_active_job or {}) + runtime_active_job = dict(runtime_active_job or {}) + if not runtime_active_job: + return direct_active_job + if not direct_active_job: + return runtime_active_job + + merged = dict(direct_active_job) + same_job = ( + _safe_int(direct_active_job.get("job_id")) > 0 + and _safe_int(direct_active_job.get("job_id")) == _safe_int(runtime_active_job.get("job_id")) + ) or ( + _safe_text(direct_active_job.get("job_code")) + and _safe_text(direct_active_job.get("job_code")) == _safe_text(runtime_active_job.get("job_code")) + ) + if not same_job: + return direct_active_job + + preferred_runtime_keys = { + "node_stats", + "distributed_node_stats", + "display_items_claimed", + "display_items_running", + "display_current_load", + "display_active_threads", + "display_max_threads", + "display_items_completed", + "display_items_failed", + "display_active_node_codes", + "processed_recent", + "processed_per_minute", + "completed_recent", + "failed_recent", + "blacklisted_recent", + "runtime_snapshot_job_id", + "runtime_snapshot_job_code", + "runtime_snapshot_queue", + } + runtime_is_richer = _real_node_count(runtime_active_job) >= _real_node_count(direct_active_job) + for key in preferred_runtime_keys: + if key in runtime_active_job and runtime_is_richer: + merged[key] = runtime_active_job.get(key) + return merged + + +def _active_node_rows(active_job: dict) -> list[dict]: + rows: list[dict] = [] + for raw_item in list(active_job.get("node_stats") or []): + item = dict(raw_item or {}) + node_code = _safe_text(item.get("node_code")) + if not node_code or node_code == "unassigned": + continue + current_load = max( + _safe_int(item.get("current_load")), + _safe_int(item.get("active_threads")), + _safe_int(item.get("display_running")), + _safe_int(item.get("items_running")), + ) + processed_recent = _safe_int(item.get("processed_recent")) + items_claimed = _safe_int(item.get("items_claimed")) + status = _safe_text(item.get("status")) + if current_load <= 0 and processed_recent <= 0 and items_claimed <= 0 and status not in {"busy", "online"}: + continue + rows.append( + { + "node_code": node_code, + "status": status or "-", + "region": _safe_text(item.get("region")) or "-", + "role": _safe_text(item.get("role")) or "-", + "current_load": current_load, + "display_running": _safe_int(item.get("display_running")), + "active_threads": _safe_int(item.get("active_threads")), + "max_threads": _safe_int(item.get("max_threads")), + "items_claimed": items_claimed, + "items_running": _safe_int(item.get("items_running")), + "processed_recent": processed_recent, + "processed_per_minute": _safe_float(item.get("processed_per_minute")), + "completed_recent": _safe_int(item.get("completed_recent")), + "failed_recent": _safe_int(item.get("failed_recent")), + "blacklisted_recent": _safe_int(item.get("blacklisted_recent")), + "last_heartbeat_at": _safe_text(item.get("last_heartbeat_at")), + } + ) + rows.sort( + key=lambda item: ( + -int(item.get("current_load", 0)), + -int(item.get("processed_recent", 0)), + -int(item.get("items_claimed", 0)), + str(item.get("node_code") or ""), + ) + ) + return rows + + +def _step_rows(active_job: dict) -> list[dict]: + rows: list[dict] = [] + for raw_item in list(active_job.get("step_stats") or []): + item = dict(raw_item or {}) + rows.append( + { + "step_code": _safe_text(item.get("step_code")), + "step_name": _safe_text(item.get("step_name")), + "pending": _safe_int(item.get("items_pending")), + "running": _safe_int(item.get("items_running")), + "completed": _safe_int(item.get("items_completed")), + "failed": _safe_int(item.get("items_failed")), + "blacklisted": _safe_int(item.get("items_blacklisted")), + } + ) + rows.sort(key=lambda item: (-int(item["pending"]), -int(item["running"]), item["step_name"])) + return rows + + +def _summary_numbers(payload: dict) -> dict: + runtime_status = dict(payload.get("runtime_status") or {}) + active_job = dict(payload.get("active_job") or {}) + active_nodes = _active_node_rows(active_job) + active_processes = len(active_nodes) + active_threads = max( + _safe_int(active_job.get("display_active_threads")), + _safe_int(active_job.get("display_items_running")), + sum(max(_safe_int(item.get("current_load")), _safe_int(item.get("display_running"))) for item in active_nodes), + ) + max_threads = max( + _safe_int(active_job.get("display_max_threads")), + sum(_safe_int(item.get("max_threads")) for item in active_nodes if _safe_int(item.get("max_threads")) > 0), + ) + processed_recent = _safe_int(active_job.get("processed_recent")) + failed_recent = _safe_int(active_job.get("failed_recent")) + blacklisted_recent = _safe_int(active_job.get("blacklisted_recent")) + items_pending = _safe_int(active_job.get("items_pending")) + items_claimed = _safe_int(active_job.get("items_claimed")) + items_running = _safe_int(active_job.get("items_running")) + items_completed = _safe_int(active_job.get("items_completed")) + items_failed = _safe_int(active_job.get("items_failed")) + items_blacklisted = _safe_int(active_job.get("items_blacklisted")) + return { + "job_id": _safe_int(active_job.get("job_id")), + "job_code": _safe_text(active_job.get("job_code")), + "job_status": _safe_text(active_job.get("status")) or "-", + "progress_percent": _safe_float(active_job.get("progress_percent")), + "active_processes": active_processes, + "active_threads": active_threads, + "max_threads": max_threads, + "processed_recent": processed_recent, + "processed_per_minute": _safe_float(active_job.get("processed_per_minute")), + "completed_recent": _safe_int(active_job.get("completed_recent")), + "failed_recent": failed_recent, + "blacklisted_recent": blacklisted_recent, + "items_pending": items_pending, + "items_claimed": items_claimed, + "items_running": items_running, + "items_completed": items_completed, + "items_failed": items_failed, + "items_blacklisted": items_blacklisted, + "online_worker_nodes": _safe_int(((runtime_status.get("cluster_summary") or {}).get("online_worker_nodes"))), + "online_control_nodes": _safe_int(((runtime_status.get("cluster_summary") or {}).get("online_control_nodes"))), + } + + +def _diagnose_state(summary: dict) -> tuple[str, str]: + items_pending = int(summary["items_pending"]) + active_processes = int(summary["active_processes"]) + active_threads = int(summary["active_threads"]) + processed_recent = int(summary["processed_recent"]) + items_running = int(summary["items_running"]) + + if active_processes > 0 and active_threads > 0 and processed_recent > 0: + return "真跑中", "已经有真实执行面和最近吞吐，不是只剩显示残影。" + if active_processes > 0 and active_threads > 0 and items_running > 0: + return "在跑但偏慢", "有执行面，也有运行中任务，但最近吞吐还没完全拉起来。" + if items_pending > 0 and active_processes == 0 and active_threads == 0: + return "没跑起来", "队列还有积压，但当前没看到真实执行面在消化。" + if items_pending > 0 and processed_recent == 0: + return "疑似残影", "队列还有积压，但最近没有看到处理增量，需要继续查执行链。" + return "观察中", "当前口径没有明确异常，但还需要继续看最近吞吐是否持续变化。" + + +def _line(char: str = "-") -> str: + width = max(60, min(120, shutil.get_terminal_size((100, 20)).columns)) + return char * width + + +def _render_summary(payload: dict, *, top_nodes: int) -> str: + runtime_status = dict(payload.get("runtime_status") or {}) + active_job = dict(payload.get("active_job") or {}) + readiness = dict(payload.get("readiness") or {}) + sync_summary = dict(payload.get("sync_summary") or {}) + summary = _summary_numbers(payload) + state_label, state_reason = _diagnose_state(summary) + active_nodes = _active_node_rows(active_job) + step_rows = _step_rows(active_job) + lines: list[str] = [] + + lines.append(_line("=")) + lines.append(f"domainCheck 运行观察面板 {_now_text()}") + lines.append(_line("=")) + lines.append(f"状态判断: {state_label}") + lines.append(f"判断理由: {state_reason}") + lines.append( + f"当前任务: job_id={summary['job_id']} job_code={summary['job_code'] or '-'} status={summary['job_status']} 进度={summary['progress_percent']:.2f}%" + ) + lines.append( + f"就绪状态: { _safe_text(readiness.get('status')) or '-' } 摘要: { _safe_text(readiness.get('summary')) or '-' }" + ) + lines.append( + f"集群在线: mainland worker={summary['online_worker_nodes']} mainland controller={summary['online_control_nodes']}" + ) + + lines.append(_line()) + lines.append("一眼先看这 4 组数:") + lines.append( + f"任务积压: pending={summary['items_pending']} claimed={summary['items_claimed']} running={summary['items_running']}" + ) + lines.append( + f"结果产出: completed={summary['items_completed']} failed={summary['items_failed']} blacklisted={summary['items_blacklisted']}" + ) + lines.append( + f"执行面: active_processes={summary['active_processes']} active_threads={summary['active_threads']} max_threads={summary['max_threads']}" + ) + lines.append( + f"最近吞吐: processed_recent={summary['processed_recent']} per_minute={summary['processed_per_minute']:.2f} failed_recent={summary['failed_recent']} blacklisted_recent={summary['blacklisted_recent']}" + ) + + detect_runtime = dict(((runtime_status.get("detect") or {}).get("active_job") or {})) + if detect_runtime: + lines.append(_line()) + lines.append( + "运行口径提示: " + f"display_running={_safe_int(detect_runtime.get('display_items_running'))} " + f"display_threads={_safe_int(detect_runtime.get('display_active_threads'))} " + f"display_max={_safe_int(detect_runtime.get('display_max_threads'))}" + ) + + lines.append(_line()) + lines.append("当前真正有动作的节点:") + if not active_nodes: + lines.append("- 暂时没看到有真实负载或最近吞吐的节点。") + else: + for item in active_nodes[: max(1, int(top_nodes or 8))]: + lines.append( + "- " + f"{item['node_code']} " + f"load={item['current_load']} " + f"running={item['display_running']} " + f"max={item['max_threads']} " + f"processed_recent={item['processed_recent']} " + f"failed_recent={item['failed_recent']} " + f"blacklisted_recent={item['blacklisted_recent']} " + f"status={item['status']}" + ) + + lines.append(_line()) + lines.append("步骤分布:") + for item in step_rows: + lines.append( + "- " + f"{item['step_name'] or item['step_code']} " + f"pending={item['pending']} running={item['running']} " + f"completed={item['completed']} failed={item['failed']} blacklisted={item['blacklisted']}" + ) + + detect_batches = dict((sync_summary.get("detect_result_batches") or {})) + if sync_summary: + lines.append(_line()) + lines.append( + "结果回传: " + f"enabled={_safe_text(sync_summary.get('enabled')) or '-'} " + f"pending_batches={_safe_int(detect_batches.get('pending'))} " + f"pushing_batches={_safe_int(detect_batches.get('pushing'))} " + f"failed_batches={_safe_int(detect_batches.get('failed'))} " + f"synced_batches={_safe_int(detect_batches.get('synced'))}" + ) + + warnings = list(readiness.get("warnings") or []) + info = list(readiness.get("info") or []) + if warnings or info: + lines.append(_line()) + lines.append("当前提示:") + for text in warnings[:5]: + lines.append(f"- warning: {_safe_text(text)}") + for text in info[:3]: + lines.append(f"- info: {_safe_text(text)}") + + lines.append(_line("=")) + lines.append("建议:") + lines.append("- 先盯 `任务积压 / 执行面 / 最近吞吐`，这三组一起动，才算真跑。") + lines.append("- `日志` 只拿来辅助定位，不要拿日志多少判断是不是在跑。") + lines.append("- 如果 `pending` 很高，但 `active_threads` 和 `processed_recent` 都接近 0，就是没跑起来。") + lines.append("- 如果 `failed_recent` 很高而 `completed_recent`、`blacklisted_recent` 很低，说明更像外部步骤超时，不是黑名单在推进。") + return "\n".join(lines) + + +def main() -> int: + parser = argparse.ArgumentParser(description="Render a human-friendly runtime observation panel.") + parser.add_argument("--api-base", default=os.getenv("DOMAINCHECK_API_BASE", DEFAULT_API_BASE)) + parser.add_argument("--watch", type=int, default=0, help="refresh interval in seconds; 0 means run once") + parser.add_argument("--top-nodes", type=int, default=10) + parser.add_argument("--json", action="store_true", help="print raw normalized payload instead of panel") + args = parser.parse_args() + + while True: + payload = _observe_payload(args.api_base) + if args.json: + print(json.dumps(payload, ensure_ascii=False, indent=2, default=str)) + else: + if int(args.watch or 0) > 0: + print("\033[2J\033[H", end="") + print(_render_summary(payload, top_nodes=max(1, int(args.top_nodes or 10)))) + if int(args.watch or 0) <= 0: + return 0 + time.sleep(max(2, int(args.watch))) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/unstick_rebuild_release_index.py b/tools/unstick_rebuild_release_index.py new file mode 100644 index 0000000..6bdc1bf --- /dev/null +++ b/tools/unstick_rebuild_release_index.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +"""Repeatedly kill blockers holding DROP INDEX CONCURRENTLY for release index.""" + +from __future__ import annotations + +import os +import time +from pathlib import Path + +import psycopg2 + + +ENV_FILE = Path("/etc/default/domaincheck-worker") +INDEX_NAME = "idx_detect_job_items_release_node_job" +POLL_SECONDS = 3 +# Large enough to keep unblocking for roughly an hour without manual babysitting. +MAX_LOOPS = 1200 + + +def load_env(path: Path) -> None: + if not path.exists(): + return + for raw in path.read_text(encoding="utf-8").splitlines(): + line = raw.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, value = line.split("=", 1) + os.environ[key] = value + + +def log(message: str) -> None: + stamp = time.strftime("%Y-%m-%d %H:%M:%S") + print(f"[{stamp}] {message}", flush=True) + + +def main() -> int: + load_env(ENV_FILE) + conn = psycopg2.connect( + host="127.0.0.1", + port=5432, + user=os.getenv("DB_USER", "postgres"), + password=os.getenv("DB_PASSWORD"), + dbname="domain", + ) + conn.autocommit = True + cur = conn.cursor() + try: + for loop in range(MAX_LOOPS): + cur.execute( + """ + SELECT pid + FROM pg_stat_activity + WHERE pid <> pg_backend_pid() + AND state <> 'idle' + AND query ILIKE %s + ORDER BY pid + """, + (f"%DROP INDEX CONCURRENTLY IF EXISTS {INDEX_NAME}%",), + ) + drop_pids = [r[0] for r in cur.fetchall()] + if not drop_pids: + log("no_drop_pid") + break + + blockers: list[int] = [] + for pid in drop_pids: + cur.execute("SELECT pg_blocking_pids(%s)", (pid,)) + row = cur.fetchone() + if row and row[0]: + blockers.extend(int(v) for v in row[0]) + + blockers = sorted(set(blockers)) + log(f"loop={loop} drop_pids={drop_pids} blockers={blockers}") + if not blockers: + time.sleep(POLL_SECONDS) + continue + + for pid in blockers: + cur.execute("SELECT pg_cancel_backend(%s), pg_terminate_backend(%s)", (pid, pid)) + log(f"kill pid={pid} result={cur.fetchone()}") + time.sleep(POLL_SECONDS) + finally: + cur.close() + conn.close() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/unstick_rebuild_stalled_index.py b/tools/unstick_rebuild_stalled_index.py new file mode 100644 index 0000000..dbf0baf --- /dev/null +++ b/tools/unstick_rebuild_stalled_index.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +"""Repeatedly kill blockers holding DROP INDEX CONCURRENTLY for stalled index.""" + +from __future__ import annotations + +import os +import time +from pathlib import Path + +import psycopg2 + + +ENV_FILE = Path("/etc/default/domaincheck-worker") +INDEX_NAME = "idx_detect_job_items_stalled_job_activity" +POLL_SECONDS = 3 +MAX_LOOPS = 1200 + + +def load_env(path: Path) -> None: + if not path.exists(): + return + for raw in path.read_text(encoding="utf-8").splitlines(): + line = raw.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, value = line.split("=", 1) + os.environ[key] = value + + +def log(message: str) -> None: + stamp = time.strftime("%Y-%m-%d %H:%M:%S") + print(f"[{stamp}] {message}", flush=True) + + +def main() -> int: + load_env(ENV_FILE) + conn = psycopg2.connect( + host="127.0.0.1", + port=5432, + user=os.getenv("DB_USER", "postgres"), + password=os.getenv("DB_PASSWORD"), + dbname="domain", + ) + conn.autocommit = True + cur = conn.cursor() + try: + for loop in range(MAX_LOOPS): + cur.execute( + """ + SELECT pid + FROM pg_stat_activity + WHERE pid <> pg_backend_pid() + AND state <> 'idle' + AND query ILIKE %s + ORDER BY pid + """, + (f"%DROP INDEX CONCURRENTLY IF EXISTS {INDEX_NAME}%",), + ) + drop_pids = [r[0] for r in cur.fetchall()] + if not drop_pids: + log("no_drop_pid") + break + + blockers: list[int] = [] + for pid in drop_pids: + cur.execute("SELECT pg_blocking_pids(%s)", (pid,)) + row = cur.fetchone() + if row and row[0]: + blockers.extend(int(v) for v in row[0]) + + blockers = sorted(set(blockers)) + log(f"loop={loop} drop_pids={drop_pids} blockers={blockers}") + if not blockers: + time.sleep(POLL_SECONDS) + continue + + for pid in blockers: + cur.execute("SELECT pg_cancel_backend(%s), pg_terminate_backend(%s)", (pid, pid)) + log(f"kill pid={pid} result={cur.fetchone()}") + time.sleep(POLL_SECONDS) + finally: + cur.close() + conn.close() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/watch_job876_tail_release.py b/tools/watch_job876_tail_release.py new file mode 100644 index 0000000..9e920b4 --- /dev/null +++ b/tools/watch_job876_tail_release.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +"""Watch the release index build and reclaim known stuck job-876 tails.""" + +from __future__ import annotations + +import os +import sys +import time +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +DOMAINCHECK_ROOT = ROOT / "domainCheck" +if str(DOMAINCHECK_ROOT) not in sys.path: + sys.path.insert(0, str(DOMAINCHECK_ROOT)) + +from app.utils.database import Database # noqa: E402 + + +ENV_FILE = Path("/etc/default/domaincheck-worker") +INDEX_NAME = "idx_detect_job_items_release_node_job" +JOB_ID = 876 +TARGET_NODES = [ + "mainland-controller-01-ba", + "mainland-controller-01-bd", + "mainland-controller-01-aw", + "mainland-controller-01-ae", +] +POLL_SECONDS = 15 + + +def load_env(path: Path) -> None: + if not path.exists(): + return + for raw in path.read_text(encoding="utf-8").splitlines(): + line = raw.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, value = line.split("=", 1) + os.environ[key] = value + + +def log(message: str) -> None: + stamp = time.strftime("%Y-%m-%d %H:%M:%S") + print(f"[{stamp}] {message}", flush=True) + + +def fetch_index_state(db: Database) -> tuple[bool, bool, str | None, int | None, int | None]: + conn = db.get_connection() + try: + cur = conn.cursor() + try: + cur.execute( + """ + SELECT + i.indisvalid, + i.indisready, + p.phase, + p.blocks_done, + p.blocks_total + FROM pg_index i + JOIN pg_class c ON c.oid = i.indexrelid + LEFT JOIN pg_stat_progress_create_index p + ON p.index_relid = i.indexrelid + WHERE c.relname = %s + """, + (INDEX_NAME,), + ) + row = cur.fetchone() + finally: + cur.close() + finally: + db.close(conn=conn) + if not row: + return False, False, None, None, None + return bool(row[0]), bool(row[1]), row[2], row[3], row[4] + + +def main() -> int: + load_env(ENV_FILE) + db = Database() + log(f"watch start index={INDEX_NAME} job_id={JOB_ID} targets={','.join(TARGET_NODES)}") + while True: + try: + valid, ready, phase, blocks_done, blocks_total = fetch_index_state(db) + log( + "index_state " + f"valid={int(valid)} ready={int(ready)} " + f"phase={phase or 'none'} blocks_done={blocks_done} blocks_total={blocks_total}" + ) + if valid: + break + except Exception as exc: # pragma: no cover - operational script + log(f"index_state_error error={exc!r}") + db = Database() + time.sleep(POLL_SECONDS) + + released_total = 0 + for node in TARGET_NODES: + try: + released = db.release_detect_job_items_for_node_job(node, JOB_ID) + released_total += int(released or 0) + log(f"release node={node} released={released}") + except Exception as exc: # pragma: no cover - operational script + log(f"release_error node={node} error={exc!r}") + + try: + status = db.refresh_detect_job_status(JOB_ID) + log(f"refresh_status job_id={JOB_ID} status={status}") + except Exception as exc: # pragma: no cover - operational script + log(f"refresh_status_error job_id={JOB_ID} error={exc!r}") + + log(f"done released_total={released_total}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())