admin/SEONexus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

baidu tuison

Browse Source
This commit is contained in:
make
2026-01-19 21:58:00 +08:00
parent 0323018c8c
commit 934ead3e11
10 changed files with 1075 additions and 466 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
code/app/task/crawler/CrawlerConfig.php
View File

@@ -21,6 +21,8 @@ class CrawlerConfig
'斯诺克',
'足球',
'篮球',
'网球',
'LPL',
];
}

174
code/app/task/logic/BaiduPushVideoUrlLogic.php Normal file
View File

@@ -0,0 +1,174 @@
<?php
declare(strict_types=1);
namespace app\task\logic;
use app\model\DomainModel;
class BaiduPushVideoUrlLogic
{
/**
* SiteMap 根目录
* @var string
*/
protected string $strSiteMapPath;
/**
* @var DomainModel[]
*/
protected array $arrDomainModel = [];
/**
* 每次推送条数
*/
protected int $limit = 10;
public function __construct()
{
$this->strSiteMapPath = root_path('storage/SiteMap');
}
/**
* 获取域名
*/
public function getDomain(): array
{
if (empty($this->arrDomainModel)) {
$this->arrDomainModel = DomainModel::getAllDomainOnCache();
}
return $this->arrDomainModel;
}
/**
* 推送百度 URL（任务入口）
*/
public function pushBaiduUrl(): void
{
foreach ($this->getDomain() as $DomainModel) {
$domain = $DomainModel->d_domain;
$strBaiduToken = $DomainModel->d_baidu_token ?? "";
$domainDir = $this->strSiteMapPath . DIRECTORY_SEPARATOR . $domain;
if (!is_dir($domainDir)) {
continue;
}
$this->pushDomainUrls($domain, $strBaiduToken, $domainDir);
}
}
/**
* 推送单个域名
*/
protected function pushDomainUrls(string $domain, string $strBaiduToken, string $domainDir): void
{
$stateFile = $domainDir . '/baidu_state.json';
if(empty($strBaiduToken)){
return ;
}
$strApi = 'http://data.zz.baidu.com/urls?site=https://' . $domain . '&token=' . $strBaiduToken;
// $strApi = 'http://data.zz.baidu.com/urls?site=https://www.aaggc.com&token=B4bzeDDBHhnoBnov';
// 1️⃣ 读取 txt 文件
$txtFiles = glob($domainDir . '/*.txt');
sort($txtFiles);
if (empty($txtFiles)) {
return;
}
// 2️⃣ 读取进度
if (file_exists($stateFile)) {
$state = json_decode(file_get_contents($stateFile), true);
} else {
$state = [
'file_index' => 0,
'line_index' => 0,
];
}
$fileIndex = (int)$state['file_index'];
$lineIndex = (int)$state['line_index'];
$arrPushUrls = [];
$newFileIndex = $fileIndex;
$newLineIndex = $lineIndex;
// 3️⃣ 按顺序取 URL
for ($i = $fileIndex; $i < count($txtFiles); $i++) {
$lines = file($txtFiles[$i], FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES);
$start = ($i === $fileIndex) ? $lineIndex : 0;
for ($j = $start; $j < count($lines); $j++) {
$url = trim($lines[$j]);
if ($url === '') {
continue;
}
$arrPushUrls[] = $url;
$newFileIndex = $i;
$newLineIndex = $j + 1;
if (count($arrPushUrls) >= $this->limit) {
break 2;
}
}
// 当前文件读完
$newFileIndex = $i + 1;
$newLineIndex = 0;
}
// 4️⃣ 没有可推送 URL
if (empty($arrPushUrls)) {
return;
}
// 5️⃣ curl 推送
$ch = curl_init();
curl_setopt_array($ch, [
CURLOPT_URL => $strApi,
CURLOPT_POST => true,
CURLOPT_RETURNTRANSFER => true,
CURLOPT_POSTFIELDS => implode("\n", $arrPushUrls),
CURLOPT_HTTPHEADER => [
'Content-Type: text/plain; charset=utf-8'
],
CURLOPT_TIMEOUT => 10,
CURLOPT_CONNECTTIMEOUT => 5,
CURLOPT_SSL_VERIFYPEER => false,
CURLOPT_SSL_VERIFYHOST => false,
]);
$result = curl_exec($ch);
if ($result === false) {
curl_close($ch);
return;
}
curl_close($ch);
// 6️⃣ 解析结果
$data = json_decode($result, true);
// 7️⃣ 成功才更新进度
if (isset($data['success']) && $data['success'] > 0) {
file_put_contents(
$stateFile,
json_encode([
'file_index' => $newFileIndex,
'line_index' => $newLineIndex,
'updated_at' => date('Y-m-d H:i:s'),
], JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE)
);
}
}
}

90
code/app/task/logic/VideoSiteMapLogic.php
View File

@@ -132,24 +132,6 @@ class VideoSiteMapLogic
*
* @return void
*/
// public function generateSiteMap()
// {
// $arrResult = $this->VideoModel->findPaginatedWithCache([], 1, 1);
// $this->intTotal = $arrResult['total'];
// $this->generateMapIndex();
// $this->generateMapMain();
// $intTotalPage = ceil($this->intTotal / $this->intPageSize);
// for ($intPage = 1; $intPage <= $intTotalPage; $intPage++) {
// $this->generateMapInfo($intPage);
// }
// echo date('Y-m-d H:i:s') . PHP_EOL;
// }
protected function generateMapInfo(int $intPage)
{
$intStartPage = $intPage;
@@ -208,6 +190,7 @@ EOF;
$arrArgs = [
'intVId' => $Video->v_id,
'strPinYin' => $Video->v_name_en,
'strPinyin' => $Video->v_name_en,
];
if ($arrUrlFamily && !empty($arrUrlFamily)) {
@@ -283,6 +266,10 @@ EOF;
foreach ($this->getDomain() as $DomainModel) {
// 用于 sitemap + txt 的 URL 去重池
$arrUrlPool = [];
$strDomainDir = $this->strSiteMapPath . $DomainModel->d_domain;
if (is_dir($strDomainDir) == false) {
mkdir($strDomainDir);
@@ -316,7 +303,15 @@ EOF;
<priority>0.8</priority>
</url>
EOF;
file_put_contents($strMapIndexFile, $strContent);
//file_put_contents($strMapIndexFile, $strContent);
$arrUrlPool["https://www.{$DomainModel->d_domain}/"] = [
'lastmod' => $this->strDate,
'changefreq' => 'daily',
'priority' => '1.0',
];
// 分类首页 '/vodtype/{strParentCategory}',
foreach ($this->arrVideoCategory as $strParentCategoryKey => $arrParentCategory) {
@@ -347,11 +342,15 @@ EOF;
<changefreq>weekly</changefreq>
<priority>0.8</priority>
</url>
EOF;
file_put_contents($strMapIndexFile, $strContent, FILE_APPEND);
EOF;
//file_put_contents($strMapIndexFile, $strContent, FILE_APPEND);
$arrUrlPool[$strUrl] = [
'lastmod' => $this->strDate,
'changefreq' => 'weekly',
'priority' => '0.8',
];
}
// 分类列表 '/vodtype-{strParentCategory}/{strCategory}-{strYear}-{strArea}-{strOrder}/{strLang}-page{intPage}',
foreach ($this->arrVideoCategory as $arrParentCategory) {
foreach ($arrParentCategory['children'] as $arrChildrenCategory) {
@@ -391,7 +390,12 @@ EOF;
<priority>0.8</priority>
</url>
EOF;
file_put_contents($strMapIndexFile, $strContent, FILE_APPEND);
//file_put_contents($strMapIndexFile, $strContent, FILE_APPEND);
$arrUrlPool[$strUrl] = [
'lastmod' => $this->strDate,
'changefreq' => 'weekly',
'priority' => '0.8',
];
// }
// }
// }
@@ -432,7 +436,12 @@ EOF;
<priority>0.8</priority>
</url>
EOF;
file_put_contents($strMapIndexFile, $strContent, FILE_APPEND);
//file_put_contents($strMapIndexFile, $strContent, FILE_APPEND);
$arrUrlPool[$strUrl] = [
'lastmod' => $this->strDate,
'changefreq' => 'weekly',
'priority' => '0.8',
];
}
}
}
@@ -442,7 +451,38 @@ EOF;
$strContent = <<<EOF
</urlset>
EOF;
file_put_contents($strMapIndexFile, $strContent, FILE_APPEND);
//file_put_contents($strMapIndexFile, $strContent, FILE_APPEND);
$strXml = <<<XML
<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
XML;
foreach ($arrUrlPool as $url => $meta) {
$strXml .= <<<XML
<url>
<loc>{$url}</loc>
<lastmod>{$meta['lastmod']}</lastmod>
<changefreq>{$meta['changefreq']}</changefreq>
<priority>{$meta['priority']}</priority>
</url>
XML;
}
$strXml .= "\n</urlset>";
file_put_contents($strMapIndexFile, $strXml);
$strTxtFile = $strDomainDir . '/sitemap-main.txt';
$strTxt = implode("\n", array_keys($arrUrlPool));
file_put_contents($strTxtFile, $strTxt);
}
}

22
code/app/task/module/PlanTask.php
View File

@@ -7,6 +7,7 @@ namespace app\task\module;
use app\model\PlanTaskModel;
use app\task\logic\SiteMapLogic;
use app\task\logic\VideoSiteMapLogic;
use app\task\logic\BaiduPushVideoUrlLogic;
use microserver\QueueManage;
use microserver\ProcessSanitizer;
@@ -90,6 +91,11 @@ class PlanTask
case 'GENERATE_VIDEO_SEO_WORDS':
self::generateVideoSEOWords();
break;
case 'PUSH_BAIDU_VIDEO_URL':
self::pushBaiduVideoUrl();
break;
}
}
}
@@ -303,6 +309,22 @@ class PlanTask
$QueueManage->set($arrTask);
}
/**
* Undocumented function
*
* @return void
*/
public static function pushBaiduVideoUrl()
{
$arrTask = [
'callback' => [BaiduPushVideoUrlLogic::class, 'pushBaiduUrl'],
'data' => []
];
$QueueManage = new QueueManage(2);
$QueueManage->set($arrTask);
}
/**
* Undocumented function
*