admin/SEONexus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

DEBUG

Browse Source
This commit is contained in:
NULL
2025-05-27 11:57:16 +08:00
parent 50d7df3869
commit b392a65948
15 changed files with 223 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
code/app/common.php
View File

@@ -452,7 +452,7 @@ namespace {
return $proxies[array_rand($proxies)];
}
function fetchContentWithCurl($url, $headers = [], $timeout = 10, $proxy = null, $proxyAuth = null)
function fetchContentWithCurl($url, $headers = [], $timeout = 10, $strProxy = null, $strProxyAuth = null)
{
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, $url);
@@ -469,10 +469,10 @@ namespace {
curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0 (Linux; Android 9; ASUS_X00TD; Flow) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/359.0.0.288 Mobile Safari/537.36');
// 如果需要代理
if ($proxy) {
curl_setopt($ch, CURLOPT_PROXY, $proxy);
if ($proxyAuth) {
curl_setopt($ch, CURLOPT_PROXYUSERPWD, $proxyAuth); // 代理用户名和密码
if ($strProxy) {
curl_setopt($ch, CURLOPT_PROXY, $strProxy);
if ($strProxyAuth) {
curl_setopt($ch, CURLOPT_PROXYUSERPWD, $strProxyAuth); // 代理用户名和密码
}
}

3
code/app/home/controller/Novel.php
View File

@@ -288,9 +288,6 @@ class Novel extends BaseController
$intCount = 20;
$arrNewsUpdateNovel = $NovelModel->getCommonNovelOnCache($strKey, $intLifeTime, $intCount, $intType, NULL, ['n_update_time' => -1]);
// echo "<pre>";
// var_dump($arrNewsUpdateNovel);exit;
// TODO n_description 改成 n_description
// 更新值
ConverterMovel::setVal([
'intPage' => 1,

1
code/app/model/ChapterModel.php
View File

@@ -129,7 +129,6 @@ class ChapterModel extends MongoModel
try {
applyToKeys($arrChapter, ['created_at', 'updated_at'], 'formatMongoDate');
} catch (\Exception $e) {
var_dump($arrChapter);
throw $e;
}

10
code/app/task/crawler/fengchao/page/Site.php
View File

@@ -102,14 +102,14 @@ class Site
break;
case 'QING':
$arrConfig['proxy'] = $strProxyHost;
$proxyAuth = base64_encode($strKey . ":" . $strSecret);
$arrConfig['headers']['Proxy-Authorization'] = "Basic " . $proxyAuth;
$strProxyAuth = base64_encode($strKey . ":" . $strSecret);
$arrConfig['headers']['Proxy-Authorization'] = "Basic " . $strProxyAuth;
break;
case 'JULIANG':
// 隧道代理信息
// $proxy = 'http://username:password@XXX.juliangip.com:23456';
$proxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
$arrConfig['proxy'] = $proxy;
// $strProxy = 'http://username:password@XXX.juliangip.com:23456';
$strProxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
$arrConfig['proxy'] = $strProxy;
break;
}
}

10
code/app/task/crawler/wuxian/page/Site.php
View File

@@ -102,14 +102,14 @@ class Site
break;
case 'QING':
$arrConfig['proxy'] = $strProxyHost;
$proxyAuth = base64_encode($strKey . ":" . $strSecret);
$arrConfig['headers']['Proxy-Authorization'] = "Basic " . $proxyAuth;
$strProxyAuth = base64_encode($strKey . ":" . $strSecret);
$arrConfig['headers']['Proxy-Authorization'] = "Basic " . $strProxyAuth;
break;
case 'JULIANG':
// 隧道代理信息
// $proxy = 'http://username:password@XXX.juliangip.com:23456';
$proxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
$arrConfig['proxy'] = $proxy;
// $strProxy = 'http://username:password@XXX.juliangip.com:23456';
$strProxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
$arrConfig['proxy'] = $strProxy;
break;
}
}

12
code/app/task/crawler/wxg/page/Site.php
View File

@@ -47,7 +47,7 @@ class Site
*
* @var boolean
*/
public $boolReInitClient = false;
public $boolReInitClient = true;
/**
* Undocumented function
@@ -110,14 +110,14 @@ class Site
break;
case 'QING':
$arrConfig['proxy'] = $strProxyHost;
$proxyAuth = base64_encode($strKey . ":" . $strSecret);
$arrConfig['headers']['Proxy-Authorization'] = "Basic " . $proxyAuth;
$strProxyAuth = base64_encode($strKey . ":" . $strSecret);
$arrConfig['headers']['Proxy-Authorization'] = "Basic " . $strProxyAuth;
break;
case 'JULIANG':
// 隧道代理信息
// $proxy = 'http://username:password@XXX.juliangip.com:23456';
$proxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
$arrConfig['proxy'] = $proxy;
// $strProxy = 'http://username:password@XXX.juliangip.com:23456';
$strProxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
$arrConfig['proxy'] = $strProxy;
break;
}
}

10
code/app/task/crawler/youzhi/page/Site.php
View File

@@ -102,14 +102,14 @@ class Site
break;
case 'QING':
$arrConfig['proxy'] = $strProxyHost;
$proxyAuth = base64_encode($strKey . ":" . $strSecret);
$arrConfig['headers']['Proxy-Authorization'] = "Basic " . $proxyAuth;
$strProxyAuth = base64_encode($strKey . ":" . $strSecret);
$arrConfig['headers']['Proxy-Authorization'] = "Basic " . $strProxyAuth;
break;
case 'JULIANG':
// 隧道代理信息
// $proxy = 'http://username:password@XXX.juliangip.com:23456';
$proxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
$arrConfig['proxy'] = $proxy;
// $strProxy = 'http://username:password@XXX.juliangip.com:23456';
$strProxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
$arrConfig['proxy'] = $strProxy;
break;
}
}

10
code/app/task/crawler/zw/page/Site.php
View File

@@ -110,14 +110,14 @@ class Site
break;
case 'QING':
$arrConfig['proxy'] = $strProxyHost;
$proxyAuth = base64_encode($strKey . ":" . $strSecret);
$arrConfig['headers']['Proxy-Authorization'] = "Basic " . $proxyAuth;
$strProxyAuth = base64_encode($strKey . ":" . $strSecret);
$arrConfig['headers']['Proxy-Authorization'] = "Basic " . $strProxyAuth;
break;
case 'JULIANG':
// 隧道代理信息
// $proxy = 'http://username:password@XXX.juliangip.com:23456';
$proxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
$arrConfig['proxy'] = $proxy;
// $strProxy = 'http://username:password@XXX.juliangip.com:23456';
$strProxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
$arrConfig['proxy'] = $strProxy;
break;
}
}

10
code/app/task/crawler/zw630/page/Site.php
View File

@@ -95,14 +95,14 @@ class Site extends ZwSite
break;
case 'QING':
$arrConfig['proxy'] = $strProxyHost;
$proxyAuth = base64_encode($strKey . ":" . $strSecret);
$arrConfig['headers']['Proxy-Authorization'] = "Basic " . $proxyAuth;
$strProxyAuth = base64_encode($strKey . ":" . $strSecret);
$arrConfig['headers']['Proxy-Authorization'] = "Basic " . $strProxyAuth;
break;
case 'JULIANG':
// 隧道代理信息
// $proxy = 'http://username:password@XXX.juliangip.com:23456';
$proxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
$arrConfig['proxy'] = $proxy;
// $strProxy = 'http://username:password@XXX.juliangip.com:23456';
$strProxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
$arrConfig['proxy'] = $strProxy;
break;
}
}

10
code/app/task/crawler/zw74/page/Site.php
View File

@@ -94,14 +94,14 @@ class Site extends ZwSite
break;
case 'QING':
$arrConfig['proxy'] = $strProxyHost;
$proxyAuth = base64_encode($strKey . ":" . $strSecret);
$arrConfig['headers']['Proxy-Authorization'] = "Basic " . $proxyAuth;
$strProxyAuth = base64_encode($strKey . ":" . $strSecret);
$arrConfig['headers']['Proxy-Authorization'] = "Basic " . $strProxyAuth;
break;
case 'JULIANG':
// 隧道代理信息
// $proxy = 'http://username:password@XXX.juliangip.com:23456';
$proxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
$arrConfig['proxy'] = $proxy;
// $strProxy = 'http://username:password@XXX.juliangip.com:23456';
$strProxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
$arrConfig['proxy'] = $strProxy;
break;
}

154
code/app/task/logic/VideoSeoLogic.php Normal file
View File

@@ -0,0 +1,154 @@
<?php
declare(strict_types=1);
namespace app\task\logic;
use app\model\VideoModel;
use GuzzleHttp\Client;
use GuzzleHttp\Promise;
use storage\StorageCore;
class VideoSeoLogic
{
/**
* VideoModel
*
* @var VideoModel
*/
public $VideoModel;
public $intPageSize = 40000;
public $strSiteMapPath = '';
public $intTotal = 0;
public $intBatchSize = 500;
public function __construct()
{
$this->VideoModel = VideoModel::getInstance();
VideoModel::$arrOptions['projection'] = [
'_id' => 0,
'v_id' => 1,
'v_name' => 1,
'v_seo_words' => 1,
];
}
public function generateVideoSEOWords()
{
$arrResult = $this->VideoModel->findPaginatedWithCache([], 1, 1);
$this->intTotal = $arrResult['total'];
$intTotalPage = ceil($this->intTotal / $this->intPageSize);
for ($intPage = 1; $intPage <= $intTotalPage; $intPage++) {
$intStartPage = $intPage;
$intStart = ($intStartPage -= 1) * $this->intPageSize;
foreach ($this->fetchVideosByBatch($intStart, $this->intPageSize) as $arrVideo) {
print_r($arrVideo);
if (!isset($arrVideo['v_seo_words']) || empty($arrVideo['v_seo_words'])) {
$arrWords = $this->getBaiduSEOWords($arrVideo['v_name']);
if (!empty($arrWords)) {
$this->VideoModel->updateOne(['v_id' => $arrVideo['v_id']], [
['$set' => ['v_seo_words' => $arrWords,]]
]);
}
}
}
}
}
private function getBaiduSEOWords(string $strVName): array
{
try {
$strUrl = 'https://suggestion.baidu.com/su?wd=%s&json=1&p=3&cb=callback';
$strUrl = sprintf($strUrl, $strVName);
$strContent = file_get_contents($strUrl);
$strContent = mb_convert_encoding($strContent, 'UTF-8', 'GBK');
$strContent = str_replace([
'callback(',
');',
], '', $strContent);
$arrWords = json_decode($strContent, true);
return $arrWords['s'] ?? [];
} catch (\Exception $e) {
echo $e->getMessage();
return [];
}
}
private function getOtherSEOWords(string $strVName): array
{
// 百度 API地址：http://suggestion.baidu.com/su?wd=关键词&p=3&cb=window.bdsug.sug
// suggestion.baidu.com/su?wd=关键词&p=100 后面的数字变化结果不同；
// 搜狗 API地址：http://w.sugg.sogou.com/sugg/ajaj_json.jsp?key=关键词&type=web
// 必应 API地址：http://sg1.api.bing.com/qsonhs.aspx?type=cb&cb=callback&q=关键词
// 必应搜索API接口地址：https://api.bing.com/qsonhs.aspx?type=cb&q=前端 返回的是 json 格式。
// 必应搜索API接口地址：https://api.bing.com/qsonhs.aspx?type=cb&q=前端&cb=window.bing.sug
// 谷歌API接口地址1：http://clients1.google.com/complete/search?hl=zh&output=toolbar&q=关键词
// 谷歌API接口地址2：http://suggestqueries.google.com/complete/search?output=toolbar&hl=zh&q=关键词
// 谷歌API接口地址2：http://suggestqueries.google.com/complete/search?client=youtube&q=关键词&jsonp=window.google.ac.h
// 360搜索API接口地址：https://sug.so.360.cn/suggest?callback=suggest_so&word=qianduan 测试只支持英文，不过可以支持拼音。
// 360搜索API接口地址：https://sug.so.360.cn/suggest?encodein=utf-8&encodeout=utf-8&format=json&word=前端&callback=window.so.sug 测试支持汉字。
// 淘宝搜索API接口地址：https://suggest.taobao.com/sug?code=utf-8&q=前端&callback=window.taobao.sug
// 一淘搜索API接口地址：https://suggest.taobao.com/sug?area=etao&code=utf-8&callback=KISSY.Suggest.callback&q=前端
return [];
}
/**
* 分批查询小说数据，并以生成器方式返回
*
* @param int $intStart 跳过的记录数（skip）
* @param int $intCount 需要查询的总记录数
* @return \Generator 逐条返回小说数据
* @throws \MongoDB\Exception\Exception
*/
public function fetchVideosByBatch(int $intStart, int $intCount): \Generator
{
$intRecordsFetched = 0;
$intRemainingRecords = min($intCount, $this->VideoModel->getCol()->countDocuments() - $intStart);
while ($intRecordsFetched < $intRemainingRecords) {
$intLimit = min($this->intBatchSize, $intRemainingRecords - $intRecordsFetched);
$Cursor = $this->VideoModel->getCol()->find(
[],
[
'skip' => $intStart + $intRecordsFetched,
'limit' => $intLimit,
'sort' => ['_id' => 1],
'projection' => VideoModel::$arrOptions['projection'],
'typeMap' => [
'root' => 'array',
'document' => 'array',
'array' => 'array',
],
]
);
foreach ($Cursor as $Video) {
yield $Video;
}
$intRecordsFetched += $intLimit;
unset($Cursor);
gc_collect_cycles();
}
}
}

3
code/app/task/logic/VideoSiteMapLogic.php
View File

@@ -189,8 +189,7 @@ EOF;
{
$intRecordsFetched = 0;
$intRemainingRecords = min($intCount, $this->VideoModel->getCol()->countDocuments() - $intStart);
var_dump('$intStart='.$intStart);
var_dump('$intRemainingRecords='.$intRemainingRecords);
while ($intRecordsFetched < $intRemainingRecords) {
$intLimit = min($this->intBatchSize, $intRemainingRecords - $intRecordsFetched);

22
code/app/task/module/PlanTask.php
View File

@@ -17,8 +17,7 @@ use app\task\crawler\fengchao\Scheduler as FengChaoScheduler;
use app\task\crawler\wuxian\Scheduler as WuXianScheduler;
use app\task\crawler\custom\Scheduler as CustomScheduler;
use app\task\logic\VideoSeoLogic;
class PlanTask
{
@@ -72,6 +71,9 @@ class PlanTask
case 'CUSTOM_CRAWLER':
self::customCrawler();
break;
case 'GENERATE_VIDEO_SEO_WORDS':
self::generateVideoSEOWords();
break;
}
}
}
@@ -236,4 +238,20 @@ class PlanTask
$QueueManage = new QueueManage(2);
$QueueManage->set($arrTask);
}
/**
* Undocumented function
*
* @return void
*/
public static function generateVideoSEOWords()
{
$arrTask = [
'callback' => [VideoSeoLogic::class, 'generateVideoSEOWords'],
'data' => []
];
$QueueManage = new QueueManage(2);
$QueueManage->set($arrTask);
}
}

6
code/database/seeders/PlanTaskSeeder.php
View File

@@ -85,6 +85,12 @@ class PlanTaskSeeder
'pt_enable' => 0,
'pt_limit' => 10,
],
[
'pt_name' => '生成视频SEO关键词',
'pt_code' => 'GENERATE_VIDEO_SEO_WORDS',
'pt_enable' => 0,
'pt_limit' => 7 * 24 * 3600,
],
];
foreach ($arrDataNovel as $arrPlanTask) {

5
code/extend/template/taglib/Video.php
View File

@@ -57,10 +57,7 @@ class Video extends TagLib
*/
public function tagRankList($tag, $content)
{
// if($tag['sort_type'] == 'weekly'){
// var_dump($tag);exit;
// }
$count = $tag['count'] ?? '10';
$sort_type = $tag['sort_type'] ?? "";
$v_status = $tag['v_status'] ?? "all";