diff --git a/code/app/common.php b/code/app/common.php index 3a19cb7..8a13cf0 100644 --- a/code/app/common.php +++ b/code/app/common.php @@ -452,7 +452,7 @@ namespace { return $proxies[array_rand($proxies)]; } - function fetchContentWithCurl($url, $headers = [], $timeout = 10, $proxy = null, $proxyAuth = null) + function fetchContentWithCurl($url, $headers = [], $timeout = 10, $strProxy = null, $strProxyAuth = null) { $ch = curl_init(); curl_setopt($ch, CURLOPT_URL, $url); @@ -469,10 +469,10 @@ namespace { curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0 (Linux; Android 9; ASUS_X00TD; Flow) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/359.0.0.288 Mobile Safari/537.36'); // 如果需要代理 - if ($proxy) { - curl_setopt($ch, CURLOPT_PROXY, $proxy); - if ($proxyAuth) { - curl_setopt($ch, CURLOPT_PROXYUSERPWD, $proxyAuth); // 代理用户名和密码 + if ($strProxy) { + curl_setopt($ch, CURLOPT_PROXY, $strProxy); + if ($strProxyAuth) { + curl_setopt($ch, CURLOPT_PROXYUSERPWD, $strProxyAuth); // 代理用户名和密码 } } diff --git a/code/app/home/controller/Novel.php b/code/app/home/controller/Novel.php index bef662d..3ede934 100644 --- a/code/app/home/controller/Novel.php +++ b/code/app/home/controller/Novel.php @@ -288,9 +288,6 @@ class Novel extends BaseController $intCount = 20; $arrNewsUpdateNovel = $NovelModel->getCommonNovelOnCache($strKey, $intLifeTime, $intCount, $intType, NULL, ['n_update_time' => -1]); - // echo "
";
- // var_dump($arrNewsUpdateNovel);exit;
- // TODO n_description 改成 n_description
// 更新值
ConverterMovel::setVal([
'intPage' => 1,
diff --git a/code/app/model/ChapterModel.php b/code/app/model/ChapterModel.php
index 2f73664..b7a41ef 100644
--- a/code/app/model/ChapterModel.php
+++ b/code/app/model/ChapterModel.php
@@ -129,7 +129,6 @@ class ChapterModel extends MongoModel
try {
applyToKeys($arrChapter, ['created_at', 'updated_at'], 'formatMongoDate');
} catch (\Exception $e) {
- var_dump($arrChapter);
throw $e;
}
diff --git a/code/app/task/crawler/fengchao/page/Site.php b/code/app/task/crawler/fengchao/page/Site.php
index 253e8b4..063c9a6 100644
--- a/code/app/task/crawler/fengchao/page/Site.php
+++ b/code/app/task/crawler/fengchao/page/Site.php
@@ -102,14 +102,14 @@ class Site
break;
case 'QING':
$arrConfig['proxy'] = $strProxyHost;
- $proxyAuth = base64_encode($strKey . ":" . $strSecret);
- $arrConfig['headers']['Proxy-Authorization'] = "Basic " . $proxyAuth;
+ $strProxyAuth = base64_encode($strKey . ":" . $strSecret);
+ $arrConfig['headers']['Proxy-Authorization'] = "Basic " . $strProxyAuth;
break;
case 'JULIANG':
// 隧道代理信息
- // $proxy = 'http://username:password@XXX.juliangip.com:23456';
- $proxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
- $arrConfig['proxy'] = $proxy;
+ // $strProxy = 'http://username:password@XXX.juliangip.com:23456';
+ $strProxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
+ $arrConfig['proxy'] = $strProxy;
break;
}
}
diff --git a/code/app/task/crawler/wuxian/page/Site.php b/code/app/task/crawler/wuxian/page/Site.php
index 4378c88..5e72b4b 100644
--- a/code/app/task/crawler/wuxian/page/Site.php
+++ b/code/app/task/crawler/wuxian/page/Site.php
@@ -102,14 +102,14 @@ class Site
break;
case 'QING':
$arrConfig['proxy'] = $strProxyHost;
- $proxyAuth = base64_encode($strKey . ":" . $strSecret);
- $arrConfig['headers']['Proxy-Authorization'] = "Basic " . $proxyAuth;
+ $strProxyAuth = base64_encode($strKey . ":" . $strSecret);
+ $arrConfig['headers']['Proxy-Authorization'] = "Basic " . $strProxyAuth;
break;
case 'JULIANG':
// 隧道代理信息
- // $proxy = 'http://username:password@XXX.juliangip.com:23456';
- $proxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
- $arrConfig['proxy'] = $proxy;
+ // $strProxy = 'http://username:password@XXX.juliangip.com:23456';
+ $strProxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
+ $arrConfig['proxy'] = $strProxy;
break;
}
}
diff --git a/code/app/task/crawler/wxg/page/Site.php b/code/app/task/crawler/wxg/page/Site.php
index fe9da3a..0e3deaf 100644
--- a/code/app/task/crawler/wxg/page/Site.php
+++ b/code/app/task/crawler/wxg/page/Site.php
@@ -47,7 +47,7 @@ class Site
*
* @var boolean
*/
- public $boolReInitClient = false;
+ public $boolReInitClient = true;
/**
* Undocumented function
@@ -110,14 +110,14 @@ class Site
break;
case 'QING':
$arrConfig['proxy'] = $strProxyHost;
- $proxyAuth = base64_encode($strKey . ":" . $strSecret);
- $arrConfig['headers']['Proxy-Authorization'] = "Basic " . $proxyAuth;
+ $strProxyAuth = base64_encode($strKey . ":" . $strSecret);
+ $arrConfig['headers']['Proxy-Authorization'] = "Basic " . $strProxyAuth;
break;
case 'JULIANG':
// 隧道代理信息
- // $proxy = 'http://username:password@XXX.juliangip.com:23456';
- $proxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
- $arrConfig['proxy'] = $proxy;
+ // $strProxy = 'http://username:password@XXX.juliangip.com:23456';
+ $strProxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
+ $arrConfig['proxy'] = $strProxy;
break;
}
}
diff --git a/code/app/task/crawler/youzhi/page/Site.php b/code/app/task/crawler/youzhi/page/Site.php
index 983cd87..51c6cc2 100644
--- a/code/app/task/crawler/youzhi/page/Site.php
+++ b/code/app/task/crawler/youzhi/page/Site.php
@@ -102,14 +102,14 @@ class Site
break;
case 'QING':
$arrConfig['proxy'] = $strProxyHost;
- $proxyAuth = base64_encode($strKey . ":" . $strSecret);
- $arrConfig['headers']['Proxy-Authorization'] = "Basic " . $proxyAuth;
+ $strProxyAuth = base64_encode($strKey . ":" . $strSecret);
+ $arrConfig['headers']['Proxy-Authorization'] = "Basic " . $strProxyAuth;
break;
case 'JULIANG':
// 隧道代理信息
- // $proxy = 'http://username:password@XXX.juliangip.com:23456';
- $proxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
- $arrConfig['proxy'] = $proxy;
+ // $strProxy = 'http://username:password@XXX.juliangip.com:23456';
+ $strProxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
+ $arrConfig['proxy'] = $strProxy;
break;
}
}
diff --git a/code/app/task/crawler/zw/page/Site.php b/code/app/task/crawler/zw/page/Site.php
index 130a688..db951c1 100644
--- a/code/app/task/crawler/zw/page/Site.php
+++ b/code/app/task/crawler/zw/page/Site.php
@@ -110,14 +110,14 @@ class Site
break;
case 'QING':
$arrConfig['proxy'] = $strProxyHost;
- $proxyAuth = base64_encode($strKey . ":" . $strSecret);
- $arrConfig['headers']['Proxy-Authorization'] = "Basic " . $proxyAuth;
+ $strProxyAuth = base64_encode($strKey . ":" . $strSecret);
+ $arrConfig['headers']['Proxy-Authorization'] = "Basic " . $strProxyAuth;
break;
case 'JULIANG':
// 隧道代理信息
- // $proxy = 'http://username:password@XXX.juliangip.com:23456';
- $proxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
- $arrConfig['proxy'] = $proxy;
+ // $strProxy = 'http://username:password@XXX.juliangip.com:23456';
+ $strProxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
+ $arrConfig['proxy'] = $strProxy;
break;
}
}
diff --git a/code/app/task/crawler/zw630/page/Site.php b/code/app/task/crawler/zw630/page/Site.php
index b457b52..26d5da3 100644
--- a/code/app/task/crawler/zw630/page/Site.php
+++ b/code/app/task/crawler/zw630/page/Site.php
@@ -95,14 +95,14 @@ class Site extends ZwSite
break;
case 'QING':
$arrConfig['proxy'] = $strProxyHost;
- $proxyAuth = base64_encode($strKey . ":" . $strSecret);
- $arrConfig['headers']['Proxy-Authorization'] = "Basic " . $proxyAuth;
+ $strProxyAuth = base64_encode($strKey . ":" . $strSecret);
+ $arrConfig['headers']['Proxy-Authorization'] = "Basic " . $strProxyAuth;
break;
case 'JULIANG':
// 隧道代理信息
- // $proxy = 'http://username:password@XXX.juliangip.com:23456';
- $proxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
- $arrConfig['proxy'] = $proxy;
+ // $strProxy = 'http://username:password@XXX.juliangip.com:23456';
+ $strProxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
+ $arrConfig['proxy'] = $strProxy;
break;
}
}
diff --git a/code/app/task/crawler/zw74/page/Site.php b/code/app/task/crawler/zw74/page/Site.php
index e61fb7f..489e9f3 100644
--- a/code/app/task/crawler/zw74/page/Site.php
+++ b/code/app/task/crawler/zw74/page/Site.php
@@ -94,14 +94,14 @@ class Site extends ZwSite
break;
case 'QING':
$arrConfig['proxy'] = $strProxyHost;
- $proxyAuth = base64_encode($strKey . ":" . $strSecret);
- $arrConfig['headers']['Proxy-Authorization'] = "Basic " . $proxyAuth;
+ $strProxyAuth = base64_encode($strKey . ":" . $strSecret);
+ $arrConfig['headers']['Proxy-Authorization'] = "Basic " . $strProxyAuth;
break;
case 'JULIANG':
// 隧道代理信息
- // $proxy = 'http://username:password@XXX.juliangip.com:23456';
- $proxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
- $arrConfig['proxy'] = $proxy;
+ // $strProxy = 'http://username:password@XXX.juliangip.com:23456';
+ $strProxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
+ $arrConfig['proxy'] = $strProxy;
break;
}
diff --git a/code/app/task/logic/VideoSeoLogic.php b/code/app/task/logic/VideoSeoLogic.php
new file mode 100644
index 0000000..55d0114
--- /dev/null
+++ b/code/app/task/logic/VideoSeoLogic.php
@@ -0,0 +1,154 @@
+VideoModel = VideoModel::getInstance();
+
+ VideoModel::$arrOptions['projection'] = [
+ '_id' => 0,
+ 'v_id' => 1,
+ 'v_name' => 1,
+ 'v_seo_words' => 1,
+ ];
+ }
+
+ public function generateVideoSEOWords()
+ {
+ $arrResult = $this->VideoModel->findPaginatedWithCache([], 1, 1);
+ $this->intTotal = $arrResult['total'];
+
+ $intTotalPage = ceil($this->intTotal / $this->intPageSize);
+ for ($intPage = 1; $intPage <= $intTotalPage; $intPage++) {
+
+ $intStartPage = $intPage;
+ $intStart = ($intStartPage -= 1) * $this->intPageSize;
+
+ foreach ($this->fetchVideosByBatch($intStart, $this->intPageSize) as $arrVideo) {
+
+ print_r($arrVideo);
+
+ if (!isset($arrVideo['v_seo_words']) || empty($arrVideo['v_seo_words'])) {
+ $arrWords = $this->getBaiduSEOWords($arrVideo['v_name']);
+ if (!empty($arrWords)) {
+ $this->VideoModel->updateOne(['v_id' => $arrVideo['v_id']], [
+ ['$set' => ['v_seo_words' => $arrWords,]]
+ ]);
+ }
+ }
+ }
+ }
+ }
+
+ private function getBaiduSEOWords(string $strVName): array
+ {
+ try {
+
+ $strUrl = 'https://suggestion.baidu.com/su?wd=%s&json=1&p=3&cb=callback';
+
+ $strUrl = sprintf($strUrl, $strVName);
+
+ $strContent = file_get_contents($strUrl);
+ $strContent = mb_convert_encoding($strContent, 'UTF-8', 'GBK');
+
+ $strContent = str_replace([
+ 'callback(',
+ ');',
+ ], '', $strContent);
+
+ $arrWords = json_decode($strContent, true);
+
+ return $arrWords['s'] ?? [];
+ } catch (\Exception $e) {
+ echo $e->getMessage();
+ return [];
+ }
+ }
+
+
+ private function getOtherSEOWords(string $strVName): array
+ {
+ // 百度 API地址:http://suggestion.baidu.com/su?wd=关键词&p=3&cb=window.bdsug.sug
+ // suggestion.baidu.com/su?wd=关键词&p=100 后面的数字变化结果不同;
+ // 搜狗 API地址:http://w.sugg.sogou.com/sugg/ajaj_json.jsp?key=关键词&type=web
+ // 必应 API地址:http://sg1.api.bing.com/qsonhs.aspx?type=cb&cb=callback&q=关键词
+ // 必应搜索API接口地址:https://api.bing.com/qsonhs.aspx?type=cb&q=前端 返回的是 json 格式。
+ // 必应搜索API接口地址:https://api.bing.com/qsonhs.aspx?type=cb&q=前端&cb=window.bing.sug
+ // 谷歌API接口地址1:http://clients1.google.com/complete/search?hl=zh&output=toolbar&q=关键词
+ // 谷歌API接口地址2:http://suggestqueries.google.com/complete/search?output=toolbar&hl=zh&q=关键词
+ // 谷歌API接口地址2:http://suggestqueries.google.com/complete/search?client=youtube&q=关键词&jsonp=window.google.ac.h
+ // 360搜索API接口地址:https://sug.so.360.cn/suggest?callback=suggest_so&word=qianduan 测试只支持英文,不过可以支持拼音。
+ // 360搜索API接口地址:https://sug.so.360.cn/suggest?encodein=utf-8&encodeout=utf-8&format=json&word=前端&callback=window.so.sug 测试支持汉字。
+ // 淘宝搜索API接口地址:https://suggest.taobao.com/sug?code=utf-8&q=前端&callback=window.taobao.sug
+ // 一淘搜索API接口地址:https://suggest.taobao.com/sug?area=etao&code=utf-8&callback=KISSY.Suggest.callback&q=前端
+
+ return [];
+ }
+
+ /**
+ * 分批查询小说数据,并以生成器方式返回
+ *
+ * @param int $intStart 跳过的记录数(skip)
+ * @param int $intCount 需要查询的总记录数
+ * @return \Generator 逐条返回小说数据
+ * @throws \MongoDB\Exception\Exception
+ */
+ public function fetchVideosByBatch(int $intStart, int $intCount): \Generator
+ {
+ $intRecordsFetched = 0;
+ $intRemainingRecords = min($intCount, $this->VideoModel->getCol()->countDocuments() - $intStart);
+
+
+ while ($intRecordsFetched < $intRemainingRecords) {
+ $intLimit = min($this->intBatchSize, $intRemainingRecords - $intRecordsFetched);
+ $Cursor = $this->VideoModel->getCol()->find(
+ [],
+ [
+ 'skip' => $intStart + $intRecordsFetched,
+ 'limit' => $intLimit,
+ 'sort' => ['_id' => 1],
+ 'projection' => VideoModel::$arrOptions['projection'],
+ 'typeMap' => [
+ 'root' => 'array',
+ 'document' => 'array',
+ 'array' => 'array',
+ ],
+ ]
+ );
+
+ foreach ($Cursor as $Video) {
+ yield $Video;
+ }
+
+ $intRecordsFetched += $intLimit;
+ unset($Cursor);
+ gc_collect_cycles();
+ }
+ }
+}
diff --git a/code/app/task/logic/VideoSiteMapLogic.php b/code/app/task/logic/VideoSiteMapLogic.php
index 271397c..9914444 100644
--- a/code/app/task/logic/VideoSiteMapLogic.php
+++ b/code/app/task/logic/VideoSiteMapLogic.php
@@ -189,8 +189,7 @@ EOF;
{
$intRecordsFetched = 0;
$intRemainingRecords = min($intCount, $this->VideoModel->getCol()->countDocuments() - $intStart);
- var_dump('$intStart='.$intStart);
- var_dump('$intRemainingRecords='.$intRemainingRecords);
+
while ($intRecordsFetched < $intRemainingRecords) {
$intLimit = min($this->intBatchSize, $intRemainingRecords - $intRecordsFetched);
diff --git a/code/app/task/module/PlanTask.php b/code/app/task/module/PlanTask.php
index 9e85aed..4b38752 100644
--- a/code/app/task/module/PlanTask.php
+++ b/code/app/task/module/PlanTask.php
@@ -17,8 +17,7 @@ use app\task\crawler\fengchao\Scheduler as FengChaoScheduler;
use app\task\crawler\wuxian\Scheduler as WuXianScheduler;
use app\task\crawler\custom\Scheduler as CustomScheduler;
-
-
+use app\task\logic\VideoSeoLogic;
class PlanTask
{
@@ -72,6 +71,9 @@ class PlanTask
case 'CUSTOM_CRAWLER':
self::customCrawler();
break;
+ case 'GENERATE_VIDEO_SEO_WORDS':
+ self::generateVideoSEOWords();
+ break;
}
}
}
@@ -236,4 +238,20 @@ class PlanTask
$QueueManage = new QueueManage(2);
$QueueManage->set($arrTask);
}
+
+ /**
+ * Undocumented function
+ *
+ * @return void
+ */
+ public static function generateVideoSEOWords()
+ {
+ $arrTask = [
+ 'callback' => [VideoSeoLogic::class, 'generateVideoSEOWords'],
+ 'data' => []
+ ];
+
+ $QueueManage = new QueueManage(2);
+ $QueueManage->set($arrTask);
+ }
}
diff --git a/code/database/seeders/PlanTaskSeeder.php b/code/database/seeders/PlanTaskSeeder.php
index 9e6b4ec..e145794 100644
--- a/code/database/seeders/PlanTaskSeeder.php
+++ b/code/database/seeders/PlanTaskSeeder.php
@@ -85,6 +85,12 @@ class PlanTaskSeeder
'pt_enable' => 0,
'pt_limit' => 10,
],
+ [
+ 'pt_name' => '生成视频SEO关键词',
+ 'pt_code' => 'GENERATE_VIDEO_SEO_WORDS',
+ 'pt_enable' => 0,
+ 'pt_limit' => 7 * 24 * 3600,
+ ],
];
foreach ($arrDataNovel as $arrPlanTask) {
diff --git a/code/extend/template/taglib/Video.php b/code/extend/template/taglib/Video.php
index 36d47dd..008513c 100644
--- a/code/extend/template/taglib/Video.php
+++ b/code/extend/template/taglib/Video.php
@@ -57,10 +57,7 @@ class Video extends TagLib
*/
public function tagRankList($tag, $content)
{
- // if($tag['sort_type'] == 'weekly'){
- // var_dump($tag);exit;
- // }
-
+
$count = $tag['count'] ?? '10';
$sort_type = $tag['sort_type'] ?? "";
$v_status = $tag['v_status'] ?? "all";