104 lines
3.4 KiB
Python
104 lines
3.4 KiB
Python
from __future__ import annotations
|
|
|
|
import re
|
|
from pathlib import Path
|
|
|
|
from app.core.db import get_db
|
|
from app.core.files import import_root
|
|
|
|
|
|
DOMAIN_PATTERN = re.compile(r"^(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+(com|net)$", re.IGNORECASE)
|
|
|
|
|
|
def normalize_domain(value: str) -> str | None:
|
|
candidate = value.strip().lower()
|
|
candidate = re.sub(r"^https?://", "", candidate)
|
|
candidate = candidate.split("/")[0].strip(".")
|
|
if candidate.startswith("www."):
|
|
candidate = candidate[4:]
|
|
if not DOMAIN_PATTERN.match(candidate):
|
|
return None
|
|
return candidate
|
|
|
|
|
|
def import_domains_from_path(file_path: Path, source_type: int = 7) -> dict:
|
|
raw_lines = file_path.read_text(encoding="utf-8", errors="replace").splitlines()
|
|
total = 0
|
|
normalized_rows: list[tuple[str, str]] = []
|
|
invalid = 0
|
|
|
|
for line in raw_lines:
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
total += 1
|
|
normalized = normalize_domain(line)
|
|
if not normalized:
|
|
invalid += 1
|
|
continue
|
|
tld = normalized.rsplit(".", 1)[-1]
|
|
normalized_rows.append((normalized, tld))
|
|
|
|
domains = [row[0] for row in normalized_rows]
|
|
existing_set: set[str] = set()
|
|
inserted = 0
|
|
|
|
with get_db() as conn:
|
|
with conn.cursor() as cur:
|
|
if domains:
|
|
cur.execute("select domain from domains where domain = any(%s)", (domains,))
|
|
existing_set = {row[0] for row in cur.fetchall()}
|
|
|
|
for domain, tld in normalized_rows:
|
|
if domain in existing_set:
|
|
continue
|
|
cur.execute(
|
|
"""
|
|
insert into domains (
|
|
domain, tld, source_type, use_status, detect_status, register_status,
|
|
has_beian, company_type, website_url, beian_year, snapshot_years,
|
|
expire_date, create_time, update_time, review_status, detect_time,
|
|
backlink_count, jucha_status, juziseo_status
|
|
) values (
|
|
%s, %s, %s, 0, 0, 0,
|
|
1, null, null, null, null,
|
|
null, now(), now(), 0, null,
|
|
0, 0, 0
|
|
)
|
|
returning id
|
|
""",
|
|
(domain, tld, source_type),
|
|
)
|
|
domain_id = cur.fetchone()[0]
|
|
cur.execute(
|
|
"""
|
|
insert into detect_tasks (domain_id, task_type, status, priority, retry_count, create_time, update_time)
|
|
values (%s, 1, 1, 5, 0, now(), now())
|
|
""",
|
|
(domain_id,),
|
|
)
|
|
inserted += 1
|
|
conn.commit()
|
|
|
|
exists = len(existing_set)
|
|
valid = len(normalized_rows)
|
|
stats = {
|
|
"total": total,
|
|
"valid": valid,
|
|
"added": inserted,
|
|
"exists": exists,
|
|
"invalid": invalid,
|
|
"failed": max(valid - exists - inserted, 0),
|
|
}
|
|
return {
|
|
"filename": file_path.name,
|
|
"stats": stats,
|
|
}
|
|
|
|
|
|
def import_domains_from_upload(content: bytes, filename: str, source_type: int = 7) -> dict:
|
|
safe_name = Path(filename).name or "domains.txt"
|
|
target = import_root() / safe_name
|
|
target.write_bytes(content)
|
|
return import_domains_from_path(target, source_type=source_type)
|