debug-grm

code/app/home/view/videoNuNu/sitemap/robots.txt

@@ -1,3 +1,38 @@
+# 允许的搜索引擎
+User-agent: Baiduspider
+Allow: /
+
+User-agent: 360Spider
+Allow: /
+
+User-agent: Sogou web spider
+Allow: /
+
+User-agent: Sogou inst spider
+Allow: /
+
+User-agent: YisouSpider
+Allow: /
+
+User-agent: Quarkbot
+Allow: /
+
+User-agent: Googlebot
+Allow: /
+
+# 禁止其他爬虫（AI 爬虫常见 UA）
+User-agent: GPTBot
+Disallow: /
+
+User-agent: ChatGPT-User
+Disallow: /
+
+User-agent: CCBot
+Disallow: /
+
+User-agent: ClaudeBot
+Disallow: /
+
 User-agent: *
-Disallow:
+Disallow: /
 Sitemap: https://{$DomainModel->d_domain}/sitemap_index.xml