from __future__ import annotations import re from pathlib import Path from app.core.db import get_db from app.core.files import import_root DOMAIN_PATTERN = re.compile(r"^(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+(com|net)$", re.IGNORECASE) SOURCE_TYPE_LABELS = { 6: "手工录入", 7: "TXT 导入", 9: "其它", } def normalize_domain(value: str) -> str | None: candidate = value.strip().lower() candidate = re.sub(r"^https?://", "", candidate) candidate = candidate.split("/")[0].strip(".") if candidate.startswith("www."): candidate = candidate[4:] if not DOMAIN_PATTERN.match(candidate): return None return candidate def import_domains_from_path(file_path: Path, source_type: int = 7) -> dict: raw_lines = file_path.read_text(encoding="utf-8", errors="replace").splitlines() total = 0 normalized_rows: list[tuple[str, str]] = [] invalid = 0 for line in raw_lines: line = line.strip() if not line: continue total += 1 normalized = normalize_domain(line) if not normalized: invalid += 1 continue tld = normalized.rsplit(".", 1)[-1] normalized_rows.append((normalized, tld)) domains = [row[0] for row in normalized_rows] existing_set: set[str] = set() inserted = 0 exists = 0 seen_in_batch: set[str] = set() with get_db() as conn: with conn.cursor() as cur: if domains: cur.execute("select domain from domains where domain = any(%s)", (domains,)) existing_set = {row[0] for row in cur.fetchall()} for domain, tld in normalized_rows: if domain in seen_in_batch: exists += 1 continue seen_in_batch.add(domain) if domain in existing_set: exists += 1 continue cur.execute( """ insert into domains ( domain, tld, source_type, use_status, detect_status, register_status, has_beian, company_type, website_url, beian_year, snapshot_years, expire_date, create_time, update_time, review_status, detect_time, backlink_count, jucha_status, juziseo_status ) values ( %s, %s, %s, 0, 0, 0, 1, null, null, null, null, null, now(), now(), 0, null, 0, 0, 0 ) on conflict (domain) do nothing returning id """, (domain, tld, source_type), ) inserted_row = cur.fetchone() if not inserted_row: existing_set.add(domain) exists += 1 continue domain_id = inserted_row[0] cur.execute( """ insert into detect_tasks (domain_id, task_type, status, priority, retry_count, create_time, update_time) values (%s, 1, 1, 5, 0, now(), now()) """, (domain_id,), ) existing_set.add(domain) inserted += 1 conn.commit() valid = len(normalized_rows) stats = { "total": total, "valid": valid, "added": inserted, "exists": exists, "invalid": invalid, "failed": max(valid - exists - inserted, 0), } return { "filename": file_path.name, "source_type": source_type, "source_label": SOURCE_TYPE_LABELS.get(int(source_type or 7), "未知"), "stats": stats, } def import_domains_from_upload(content: bytes, filename: str, source_type: int = 7) -> dict: safe_name = Path(filename).name or "domains.txt" target = import_root() / safe_name target.write_bytes(content) return import_domains_from_path(target, source_type=source_type)