admin/getDomain
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

d

Browse Source
This commit is contained in:
Your Name
2026-04-22 14:13:21 +08:00
parent e0406b5d0e
commit 7cbde2aa78
145 changed files with 23086 additions and 2243 deletions
Download Patch File Download Diff File Expand all files Collapse all files

420
domain-api/app/services/juming_service.py
View File

@@ -1,6 +1,8 @@
from __future__ import annotations
import ast
import hashlib
from io import StringIO
import pickle
import re
import sys
@@ -35,6 +37,9 @@ LEGACY_JUMING_COOKIE_FILES = [
DELETE_LIST_SOURCE_TYPE = 2
FIXED_PRICE_SOURCE_TYPE = 1
JUMING_PREFERENCES_FILE = "juming_preferences.json"
JUMING_DELETE_IMPORT_STATE_FILE = "juming_delete_import_state.json"
IMPORT_BATCH_SIZE = 50000
IMPORT_PROGRESS_EVERY = 100000
class TaskStoppedError(RuntimeError):
@@ -125,6 +130,118 @@ def _persist_juming_cookie(cookie_jar: RequestsCookieJar) -> None:
pass
def _empty_import_stats() -> dict[str, int]:
return {
"total": 0,
"valid": 0,
"added": 0,
"exists": 0,
"invalid": 0,
"failed": 0,
}
def _merge_import_stats(base: dict[str, int], delta: dict[str, int]) -> dict[str, int]:
merged = dict(base or _empty_import_stats())
for key in ("total", "valid", "added", "exists", "invalid", "failed"):
merged[key] = int(merged.get(key, 0) or 0) + int((delta or {}).get(key, 0) or 0)
return merged
def _load_delete_import_state() -> dict[str, dict]:
payload = read_runtime_json(JUMING_DELETE_IMPORT_STATE_FILE, default={})
return payload if isinstance(payload, dict) else {}
def _save_delete_import_state(payload: dict[str, dict]) -> None:
write_runtime_json(JUMING_DELETE_IMPORT_STATE_FILE, payload)
def _compute_domains_signature(domains: list[str]) -> str:
digest = hashlib.sha1()
for domain in domains:
digest.update(str(domain).strip().encode("utf-8", errors="ignore"))
digest.update(b"\n")
return digest.hexdigest()
def _looks_like_login_redirect(location: str) -> bool:
normalized = str(location or "").strip().lower()
if not normalized:
return False
return any(
marker in normalized
for marker in (
"/login",
"user_zh",
"p_login",
"passport",
"sign",
)
)
def _looks_like_login_body(body: str) -> bool:
normalized = str(body or "").strip().lower()
if not normalized:
return False
return any(
marker in normalized
for marker in (
"账号登录",
"请先登录",
"登录后查看",
"登录聚名",
"user_zh",
"p_login",
)
)
def _validate_juming_cookie(
cookie_jar: RequestsCookieJar | None,
*,
probe_date: str | None = None,
) -> tuple[bool, str]:
if cookie_jar is None or not _cookie_jar_to_dict(cookie_jar):
return False, "未检测到有效 Cookie"
jm = JM()
jm.cookie = cookie_jar
probe_date = str(probe_date or date.today().isoformat())
url = f"{jm.base_url}/newcha/del_down?scsj={probe_date}"
try:
response = jm.session.get(
url,
headers=jm.headers,
cookies=jm.cookie,
allow_redirects=False,
timeout=10,
)
except Exception as exc:
return False, f"登录态校验失败: {exc}"
location = str(response.headers.get("Location") or "").strip()
if response.status_code in {301, 302, 303, 307, 308}:
if _looks_like_login_redirect(location):
return False, "聚名登录态已失效，请重新登录"
if location:
return True, f"删除列表下载链路校验通过: {probe_date}"
try:
body = response.text
except Exception:
body = ""
if _looks_like_login_body(body):
return False, "聚名登录态已失效，请重新登录"
if response.ok:
return True, f"聚名 Cookie 已通过远端校验: {probe_date}"
return False, f"聚名登录态校验失败，HTTP {response.status_code}"
def _jucha_cookie_status() -> dict:
if JUCHA_COOKIE_FILE.exists():
return {
@@ -186,11 +303,16 @@ def _load_juming_cookie() -> tuple[RequestsCookieJar | None, str]:
def get_juming_status() -> dict:
cookie_jar, storage = _load_juming_cookie()
cookie_valid, cookie_message = _validate_juming_cookie(cookie_jar)
cookie_count = len(_cookie_jar_to_dict(cookie_jar)) if cookie_jar is not None else 0
status = {
"cookie_ready": cookie_jar is not None,
"cookie_ready": bool(cookie_jar is not None and cookie_valid),
"cookie_present": cookie_jar is not None,
"cookie_valid": cookie_valid,
"cookie_message": cookie_message,
"cookie_storage": storage,
"cookie_file": str(JUMING_COOKIE_FILE),
"cookie_count": len(_cookie_jar_to_dict(cookie_jar)) if cookie_jar is not None else 0,
"cookie_count": cookie_count,
"jucha": _jucha_cookie_status(),
"supported_modes": [
{"label": "聚名一口价", "value": "fixed_price", "source_type": FIXED_PRICE_SOURCE_TYPE},
@@ -308,80 +430,146 @@ def _insert_domains(
source_type: int,
log: Callable[[str], None] | None = None,
should_stop: Callable[[], bool] | None = None,
*,
announce_total: bool = True,
progress_label: str = "",
) -> dict:
total = len(domains)
normalized_rows: list[tuple[str, str]] = []
invalid = 0
_emit_log(log, f"开始入库处理，共收到 {total} 个原始域名")
for value in domains:
_check_stop(should_stop)
normalized = normalize_domain(value)
if not normalized:
invalid += 1
continue
tld = normalized.rsplit(".", 1)[-1]
normalized_rows.append((normalized, tld))
existing_set: set[str] = set()
progress_prefix = f"{progress_label} " if str(progress_label or "").strip() else ""
if announce_total:
_emit_log(log, f"{progress_prefix}开始入库处理，共收到 {total} 个原始域名")
inserted = 0
existing = 0
processed = 0
valid = 0
last_progress_at = 0
pending_batch: list[tuple[str, str]] = []
pending_seen: set[str] = set()
stage_ready = False
def emit_progress(force: bool = False) -> None:
nonlocal last_progress_at
if not force and processed - last_progress_at < IMPORT_PROGRESS_EVERY:
return
last_progress_at = processed
_emit_log(
log,
(
f"{progress_prefix}入库进度：已处理 {processed}/{total}，"
f"有效 {valid}，新增 {inserted}，已存在 {existing}，无效 {invalid}"
),
)
def ensure_stage_table(cur) -> None:
nonlocal stage_ready
if stage_ready:
return
cur.execute(
"""
create temporary table if not exists juming_import_stage (
domain text primary key,
tld text not null
) on commit preserve rows
"""
)
stage_ready = True
def stage_rows(cur, rows: list[tuple[str, str]]) -> None:
buffer = StringIO()
for domain, tld in rows:
buffer.write(f"{domain}\t{tld}\n")
buffer.seek(0)
cur.copy_from(buffer, "juming_import_stage", columns=("domain", "tld"))
def flush_batch(cur, conn) -> None:
nonlocal inserted, existing
if not pending_batch:
return
ensure_stage_table(cur)
cur.execute("set local synchronous_commit = off")
cur.execute("truncate table juming_import_stage")
stage_rows(cur, pending_batch)
cur.execute(
"""
with existing_rows as (
select count(*)
from juming_import_stage stage
join domains existing on existing.domain = stage.domain
),
inserted as (
insert into domains (
domain, tld, source_type, use_status, detect_status, register_status,
has_beian, company_type, website_url, beian_year, snapshot_years,
expire_date, create_time, update_time, review_status, detect_time,
backlink_count, jucha_status, juziseo_status
)
select
stage.domain,
stage.tld,
%s,
0, 0, 0,
1, null, null, null, null,
null, now(), now(), 0, null,
0, 0, 0
from juming_import_stage stage
left join domains existing on existing.domain = stage.domain
where existing.id is null
returning id
),
task_insert as (
insert into detect_tasks (
domain_id, task_type, status, priority, retry_count, create_time, update_time
)
select id, 1, 1, 5, 0, now(), now()
from inserted
returning 1
)
select
(select count(*) from inserted),
(select count(*) from task_insert),
(select count(*) from existing_rows)
""",
(source_type,),
)
inserted_count, _task_count, existing_count = cur.fetchone()
inserted += int(inserted_count or 0)
existing += int(existing_count or 0)
conn.commit()
pending_batch.clear()
pending_seen.clear()
with get_db() as conn:
with conn.cursor() as cur:
normalized_domains = [row[0] for row in normalized_rows]
if normalized_domains:
cur.execute("select domain from domains where domain = any(%s)", (normalized_domains,))
existing_set = {row[0] for row in cur.fetchall()}
if existing_set:
_emit_log(log, f"检测到 {len(existing_set)} 个已存在域名，将自动跳过")
inserted_since_commit = 0
for domain, tld in normalized_rows:
for value in domains:
_check_stop(should_stop)
if domain in existing_set:
processed += 1
normalized = normalize_domain(value)
if not normalized:
invalid += 1
emit_progress()
continue
cur.execute(
"""
insert into domains (
domain, tld, source_type, use_status, detect_status, register_status,
has_beian, company_type, website_url, beian_year, snapshot_years,
expire_date, create_time, update_time, review_status, detect_time,
backlink_count, jucha_status, juziseo_status
) values (
%s, %s, %s, 0, 0, 0,
1, null, null, null, null,
null, now(), now(), 0, null,
0, 0, 0
)
returning id
""",
(domain, tld, source_type),
)
domain_id = cur.fetchone()[0]
cur.execute(
"""
insert into detect_tasks (domain_id, task_type, status, priority, retry_count, create_time, update_time)
values (%s, 1, 1, 5, 0, now(), now())
""",
(domain_id,),
)
inserted += 1
inserted_since_commit += 1
if inserted_since_commit >= 500:
conn.commit()
inserted_since_commit = 0
conn.commit()
valid += 1
if normalized in pending_seen:
existing += 1
emit_progress()
continue
pending_seen.add(normalized)
pending_batch.append((normalized, normalized.rsplit(".", 1)[-1]))
if len(pending_batch) >= IMPORT_BATCH_SIZE:
flush_batch(cur, conn)
emit_progress(force=True)
flush_batch(cur, conn)
emit_progress(force=True)
valid = len(normalized_rows)
exists = len(existing_set)
_emit_log(log, f"入库完成：有效 {valid}，新增 {inserted}，已存在 {exists}，无效 {invalid}")
_emit_log(log, f"{progress_prefix}入库完成：有效 {valid}，新增 {inserted}，已存在 {existing}，无效 {invalid}")
return {
"total": total,
"valid": valid,
"added": inserted,
"exists": exists,
"exists": existing,
"invalid": invalid,
"failed": max(valid - exists - inserted, 0),
"failed": max(valid - existing - inserted, 0),
}
@@ -450,6 +638,98 @@ def _crawl_delete_list(
return domains, dates
def _crawl_delete_list_and_import(
crawl_date: str,
auto_date: bool,
log: Callable[[str], None] | None = None,
should_stop: Callable[[], bool] | None = None,
) -> dict:
cookie_jar, _ = _load_juming_cookie()
jm = JM()
jm.cookie = cookie_jar or RequestsCookieJar()
start_date = datetime.strptime(crawl_date, "%Y-%m-%d").date()
end_date = date.today() + timedelta(days=4)
current_date = start_date
dates: list[dict[str, int]] = []
domains_found = 0
sample_domains: list[str] = []
stats = _empty_import_stats()
import_state = _load_delete_import_state()
_emit_log(log, f"开始采集删除列表：起始日期 {crawl_date}，自动追加日期 {'开启' if auto_date else '关闭'}")
while current_date <= end_date:
_check_stop(should_stop)
current_date_text = current_date.isoformat()
_emit_log(log, f"正在抓取 {current_date_text} 的删除列表")
domains_for_date = [item.strip() for item in jm.new_cha_del(current_date_text) if item.strip()]
domains_found += len(domains_for_date)
dates.append({"date": current_date_text, "count": len(domains_for_date)})
_emit_log(log, f"{current_date_text} 抓取到 {len(domains_for_date)} 个域名，累计 {domains_found} 个")
if domains_for_date:
if len(sample_domains) < 20:
sample_domains.extend(domains_for_date[: max(0, 20 - len(sample_domains))])
signature = _compute_domains_signature(domains_for_date)
cached = import_state.get(current_date_text) or {}
if (
cached.get("signature") == signature
and int(cached.get("total", 0) or 0) == len(domains_for_date)
):
cached_valid = int(cached.get("valid", 0) or 0)
cached_invalid = int(cached.get("invalid", 0) or 0)
date_stats = {
"total": len(domains_for_date),
"valid": cached_valid,
"added": 0,
"exists": cached_valid,
"invalid": cached_invalid,
"failed": 0,
}
_emit_log(
log,
(
f"{current_date_text} 删除列表内容未变化，跳过重复入库："
f"有效 {cached_valid}，视为已存在 {cached_valid}，无效 {cached_invalid}"
),
)
else:
date_stats = _insert_domains(
domains_for_date,
DELETE_LIST_SOURCE_TYPE,
log=log,
should_stop=should_stop,
announce_total=False,
progress_label=current_date_text,
)
import_state[current_date_text] = {
"signature": signature,
"total": int(date_stats.get("total", 0) or 0),
"valid": int(date_stats.get("valid", 0) or 0),
"invalid": int(date_stats.get("invalid", 0) or 0),
"updated_at": datetime.now().isoformat(sep=" ", timespec="seconds"),
}
_save_delete_import_state(import_state)
stats = _merge_import_stats(stats, date_stats)
if not auto_date:
break
current_date += timedelta(days=1)
_emit_log(
log,
(
f"删除列表采集+入库完成：抓取 {domains_found} 个域名，"
f"新增 {stats['added']}，已存在 {stats['exists']}，无效 {stats['invalid']}"
),
)
return {
"mode": "delete_list",
"dates": dates,
"domains_found": domains_found,
"stats": stats,
"sample_domains": sample_domains[:20],
}
def crawl_juming(
payload: dict,
log: Callable[[str], None] | None = None,
@@ -462,7 +742,10 @@ def crawl_juming(
cookie_jar, storage = _load_juming_cookie()
if cookie_jar is None:
raise ValueError("未找到聚名 Cookie，请先在桌面版系统设置完成聚名登录，或将 Cookie 同步到服务器")
_emit_log(log, f"检测到聚名登录态，来源：{storage}")
cookie_valid, cookie_message = _validate_juming_cookie(cookie_jar)
if not cookie_valid:
raise ValueError(cookie_message)
_emit_log(log, f"检测到聚名登录态，来源：{storage}，远端校验通过")
_check_stop(should_stop)
if mode == "fixed_price":
@@ -482,13 +765,6 @@ def crawl_juming(
crawl_date = str(payload.get("crawl_date") or date.today().isoformat())
auto_date = bool(payload.get("auto_date", True))
domains, dates = _crawl_delete_list(crawl_date, auto_date, log=log, should_stop=should_stop)
stats = _insert_domains(domains, DELETE_LIST_SOURCE_TYPE, log=log, should_stop=should_stop)
return {
"mode": mode,
"cookie_storage": storage,
"dates": dates,
"domains_found": len(domains),
"stats": stats,
"sample_domains": domains[:20],
}
result = _crawl_delete_list_and_import(crawl_date, auto_date, log=log, should_stop=should_stop)
result["cookie_storage"] = storage
return result