diff --git a/code/app/common.php b/code/app/common.php index 3a19cb7..8a13cf0 100644 --- a/code/app/common.php +++ b/code/app/common.php @@ -452,7 +452,7 @@ namespace { return $proxies[array_rand($proxies)]; } - function fetchContentWithCurl($url, $headers = [], $timeout = 10, $proxy = null, $proxyAuth = null) + function fetchContentWithCurl($url, $headers = [], $timeout = 10, $strProxy = null, $strProxyAuth = null) { $ch = curl_init(); curl_setopt($ch, CURLOPT_URL, $url); @@ -469,10 +469,10 @@ namespace { curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0 (Linux; Android 9; ASUS_X00TD; Flow) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/359.0.0.288 Mobile Safari/537.36'); // 如果需要代理 - if ($proxy) { - curl_setopt($ch, CURLOPT_PROXY, $proxy); - if ($proxyAuth) { - curl_setopt($ch, CURLOPT_PROXYUSERPWD, $proxyAuth); // 代理用户名和密码 + if ($strProxy) { + curl_setopt($ch, CURLOPT_PROXY, $strProxy); + if ($strProxyAuth) { + curl_setopt($ch, CURLOPT_PROXYUSERPWD, $strProxyAuth); // 代理用户名和密码 } } diff --git a/code/app/home/controller/Novel.php b/code/app/home/controller/Novel.php index bef662d..3ede934 100644 --- a/code/app/home/controller/Novel.php +++ b/code/app/home/controller/Novel.php @@ -288,9 +288,6 @@ class Novel extends BaseController $intCount = 20; $arrNewsUpdateNovel = $NovelModel->getCommonNovelOnCache($strKey, $intLifeTime, $intCount, $intType, NULL, ['n_update_time' => -1]); - // echo "
";
-        // var_dump($arrNewsUpdateNovel);exit;
-        // TODO   n_description  改成 n_description
         // 更新值
         ConverterMovel::setVal([
             'intPage' => 1,
diff --git a/code/app/model/ChapterModel.php b/code/app/model/ChapterModel.php
index 2f73664..b7a41ef 100644
--- a/code/app/model/ChapterModel.php
+++ b/code/app/model/ChapterModel.php
@@ -129,7 +129,6 @@ class ChapterModel extends MongoModel
             try {
                 applyToKeys($arrChapter, ['created_at', 'updated_at'], 'formatMongoDate');
             } catch (\Exception $e) {
-                var_dump($arrChapter);
                 throw $e;
             }
 
diff --git a/code/app/task/crawler/fengchao/page/Site.php b/code/app/task/crawler/fengchao/page/Site.php
index 253e8b4..063c9a6 100644
--- a/code/app/task/crawler/fengchao/page/Site.php
+++ b/code/app/task/crawler/fengchao/page/Site.php
@@ -102,14 +102,14 @@ class Site
                     break;
                 case 'QING':
                     $arrConfig['proxy'] = $strProxyHost;
-                    $proxyAuth = base64_encode($strKey . ":" . $strSecret);
-                    $arrConfig['headers']['Proxy-Authorization'] =  "Basic " . $proxyAuth;
+                    $strProxyAuth = base64_encode($strKey . ":" . $strSecret);
+                    $arrConfig['headers']['Proxy-Authorization'] =  "Basic " . $strProxyAuth;
                     break;
                 case 'JULIANG':
                     // 隧道代理信息
-                    // $proxy = 'http://username:password@XXX.juliangip.com:23456';
-                    $proxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
-                    $arrConfig['proxy'] = $proxy;
+                    // $strProxy = 'http://username:password@XXX.juliangip.com:23456';
+                    $strProxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
+                    $arrConfig['proxy'] = $strProxy;
                     break;
             }
         }
diff --git a/code/app/task/crawler/wuxian/page/Site.php b/code/app/task/crawler/wuxian/page/Site.php
index 4378c88..5e72b4b 100644
--- a/code/app/task/crawler/wuxian/page/Site.php
+++ b/code/app/task/crawler/wuxian/page/Site.php
@@ -102,14 +102,14 @@ class Site
                     break;
                 case 'QING':
                     $arrConfig['proxy'] = $strProxyHost;
-                    $proxyAuth = base64_encode($strKey . ":" . $strSecret);
-                    $arrConfig['headers']['Proxy-Authorization'] =  "Basic " . $proxyAuth;
+                    $strProxyAuth = base64_encode($strKey . ":" . $strSecret);
+                    $arrConfig['headers']['Proxy-Authorization'] =  "Basic " . $strProxyAuth;
                     break;
                 case 'JULIANG':
                     // 隧道代理信息
-                    // $proxy = 'http://username:password@XXX.juliangip.com:23456';
-                    $proxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
-                    $arrConfig['proxy'] = $proxy;
+                    // $strProxy = 'http://username:password@XXX.juliangip.com:23456';
+                    $strProxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
+                    $arrConfig['proxy'] = $strProxy;
                     break;
             }
         }
diff --git a/code/app/task/crawler/wxg/page/Site.php b/code/app/task/crawler/wxg/page/Site.php
index fe9da3a..0e3deaf 100644
--- a/code/app/task/crawler/wxg/page/Site.php
+++ b/code/app/task/crawler/wxg/page/Site.php
@@ -47,7 +47,7 @@ class Site
      *
      * @var boolean
      */
-    public $boolReInitClient = false;
+    public $boolReInitClient = true;
 
     /**
      * Undocumented function
@@ -110,14 +110,14 @@ class Site
                     break;
                 case 'QING':
                     $arrConfig['proxy'] = $strProxyHost;
-                    $proxyAuth = base64_encode($strKey . ":" . $strSecret);
-                    $arrConfig['headers']['Proxy-Authorization'] =  "Basic " . $proxyAuth;
+                    $strProxyAuth = base64_encode($strKey . ":" . $strSecret);
+                    $arrConfig['headers']['Proxy-Authorization'] =  "Basic " . $strProxyAuth;
                     break;
                 case 'JULIANG':
                     // 隧道代理信息
-                    // $proxy = 'http://username:password@XXX.juliangip.com:23456';
-                    $proxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
-                    $arrConfig['proxy'] = $proxy;
+                    // $strProxy = 'http://username:password@XXX.juliangip.com:23456';
+                    $strProxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
+                    $arrConfig['proxy'] = $strProxy;
                     break;
             }
         }
diff --git a/code/app/task/crawler/youzhi/page/Site.php b/code/app/task/crawler/youzhi/page/Site.php
index 983cd87..51c6cc2 100644
--- a/code/app/task/crawler/youzhi/page/Site.php
+++ b/code/app/task/crawler/youzhi/page/Site.php
@@ -102,14 +102,14 @@ class Site
                     break;
                 case 'QING':
                     $arrConfig['proxy'] = $strProxyHost;
-                    $proxyAuth = base64_encode($strKey . ":" . $strSecret);
-                    $arrConfig['headers']['Proxy-Authorization'] =  "Basic " . $proxyAuth;
+                    $strProxyAuth = base64_encode($strKey . ":" . $strSecret);
+                    $arrConfig['headers']['Proxy-Authorization'] =  "Basic " . $strProxyAuth;
                     break;
                 case 'JULIANG':
                     // 隧道代理信息
-                    // $proxy = 'http://username:password@XXX.juliangip.com:23456';
-                    $proxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
-                    $arrConfig['proxy'] = $proxy;
+                    // $strProxy = 'http://username:password@XXX.juliangip.com:23456';
+                    $strProxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
+                    $arrConfig['proxy'] = $strProxy;
                     break;
             }
         }
diff --git a/code/app/task/crawler/zw/page/Site.php b/code/app/task/crawler/zw/page/Site.php
index 130a688..db951c1 100644
--- a/code/app/task/crawler/zw/page/Site.php
+++ b/code/app/task/crawler/zw/page/Site.php
@@ -110,14 +110,14 @@ class Site
                     break;
                 case 'QING':
                     $arrConfig['proxy'] = $strProxyHost;
-                    $proxyAuth = base64_encode($strKey . ":" . $strSecret);
-                    $arrConfig['headers']['Proxy-Authorization'] =  "Basic " . $proxyAuth;
+                    $strProxyAuth = base64_encode($strKey . ":" . $strSecret);
+                    $arrConfig['headers']['Proxy-Authorization'] =  "Basic " . $strProxyAuth;
                     break;
                 case 'JULIANG':
                     // 隧道代理信息
-                    // $proxy = 'http://username:password@XXX.juliangip.com:23456';
-                    $proxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
-                    $arrConfig['proxy'] = $proxy;
+                    // $strProxy = 'http://username:password@XXX.juliangip.com:23456';
+                    $strProxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
+                    $arrConfig['proxy'] = $strProxy;
                     break;
             }
         }
diff --git a/code/app/task/crawler/zw630/page/Site.php b/code/app/task/crawler/zw630/page/Site.php
index b457b52..26d5da3 100644
--- a/code/app/task/crawler/zw630/page/Site.php
+++ b/code/app/task/crawler/zw630/page/Site.php
@@ -95,14 +95,14 @@ class Site extends  ZwSite
                     break;
                 case 'QING':
                     $arrConfig['proxy'] = $strProxyHost;
-                    $proxyAuth = base64_encode($strKey . ":" . $strSecret);
-                    $arrConfig['headers']['Proxy-Authorization'] =  "Basic " . $proxyAuth;
+                    $strProxyAuth = base64_encode($strKey . ":" . $strSecret);
+                    $arrConfig['headers']['Proxy-Authorization'] =  "Basic " . $strProxyAuth;
                     break;
                 case 'JULIANG':
                     // 隧道代理信息
-                    // $proxy = 'http://username:password@XXX.juliangip.com:23456';
-                    $proxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
-                    $arrConfig['proxy'] = $proxy;
+                    // $strProxy = 'http://username:password@XXX.juliangip.com:23456';
+                    $strProxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
+                    $arrConfig['proxy'] = $strProxy;
                     break;
             }
         }
diff --git a/code/app/task/crawler/zw74/page/Site.php b/code/app/task/crawler/zw74/page/Site.php
index e61fb7f..489e9f3 100644
--- a/code/app/task/crawler/zw74/page/Site.php
+++ b/code/app/task/crawler/zw74/page/Site.php
@@ -94,14 +94,14 @@ class Site extends  ZwSite
                     break;
                 case 'QING':
                     $arrConfig['proxy'] = $strProxyHost;
-                    $proxyAuth = base64_encode($strKey . ":" . $strSecret);
-                    $arrConfig['headers']['Proxy-Authorization'] =  "Basic " . $proxyAuth;
+                    $strProxyAuth = base64_encode($strKey . ":" . $strSecret);
+                    $arrConfig['headers']['Proxy-Authorization'] =  "Basic " . $strProxyAuth;
                     break;
                 case 'JULIANG':
                     // 隧道代理信息
-                    // $proxy = 'http://username:password@XXX.juliangip.com:23456';
-                    $proxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
-                    $arrConfig['proxy'] = $proxy;
+                    // $strProxy = 'http://username:password@XXX.juliangip.com:23456';
+                    $strProxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
+                    $arrConfig['proxy'] = $strProxy;
                     break;
                        
             }
diff --git a/code/app/task/logic/VideoSeoLogic.php b/code/app/task/logic/VideoSeoLogic.php
new file mode 100644
index 0000000..55d0114
--- /dev/null
+++ b/code/app/task/logic/VideoSeoLogic.php
@@ -0,0 +1,154 @@
+VideoModel = VideoModel::getInstance();
+
+        VideoModel::$arrOptions['projection'] = [
+            '_id' => 0,
+            'v_id' => 1,
+            'v_name' => 1,
+            'v_seo_words' => 1,
+        ];
+    }
+
+    public function generateVideoSEOWords()
+    {
+        $arrResult =  $this->VideoModel->findPaginatedWithCache([], 1, 1);
+        $this->intTotal = $arrResult['total'];
+
+        $intTotalPage = ceil($this->intTotal / $this->intPageSize);
+        for ($intPage = 1; $intPage <= $intTotalPage; $intPage++) {
+
+            $intStartPage = $intPage;
+            $intStart = ($intStartPage -= 1) * $this->intPageSize;
+
+            foreach ($this->fetchVideosByBatch($intStart, $this->intPageSize) as $arrVideo) {
+
+                print_r($arrVideo);
+
+                if (!isset($arrVideo['v_seo_words']) || empty($arrVideo['v_seo_words'])) {
+                    $arrWords =  $this->getBaiduSEOWords($arrVideo['v_name']);
+                    if (!empty($arrWords)) {
+                        $this->VideoModel->updateOne(['v_id' => $arrVideo['v_id']], [
+                            ['$set' => ['v_seo_words' =>   $arrWords,]]
+                        ]);
+                    }
+                }
+            }
+        }
+    }
+
+    private function getBaiduSEOWords(string $strVName): array
+    {
+        try {
+
+            $strUrl = 'https://suggestion.baidu.com/su?wd=%s&json=1&p=3&cb=callback';
+
+            $strUrl = sprintf($strUrl, $strVName);
+
+            $strContent = file_get_contents($strUrl);
+            $strContent = mb_convert_encoding($strContent, 'UTF-8', 'GBK');
+
+            $strContent = str_replace([
+                'callback(',
+                ');',
+            ], '', $strContent);
+
+            $arrWords = json_decode($strContent, true);
+
+            return $arrWords['s'] ?? [];
+        } catch (\Exception $e) {
+            echo $e->getMessage();
+            return [];
+        }
+    }
+
+
+    private function getOtherSEOWords(string $strVName): array
+    {
+        // 百度 API地址:http://suggestion.baidu.com/su?wd=关键词&p=3&cb=window.bdsug.sug
+        // suggestion.baidu.com/su?wd=关键词&p=100 后面的数字变化结果不同;
+        // 搜狗 API地址:http://w.sugg.sogou.com/sugg/ajaj_json.jsp?key=关键词&type=web
+        // 必应 API地址:http://sg1.api.bing.com/qsonhs.aspx?type=cb&cb=callback&q=关键词
+        // 必应搜索API接口地址:https://api.bing.com/qsonhs.aspx?type=cb&q=前端 返回的是 json 格式。
+        // 必应搜索API接口地址:https://api.bing.com/qsonhs.aspx?type=cb&q=前端&cb=window.bing.sug
+        // 谷歌API接口地址1:http://clients1.google.com/complete/search?hl=zh&output=toolbar&q=关键词
+        // 谷歌API接口地址2:http://suggestqueries.google.com/complete/search?output=toolbar&hl=zh&q=关键词
+        // 谷歌API接口地址2:http://suggestqueries.google.com/complete/search?client=youtube&q=关键词&jsonp=window.google.ac.h
+        // 360搜索API接口地址:https://sug.so.360.cn/suggest?callback=suggest_so&word=qianduan 测试只支持英文,不过可以支持拼音。
+        // 360搜索API接口地址:https://sug.so.360.cn/suggest?encodein=utf-8&encodeout=utf-8&format=json&word=前端&callback=window.so.sug 测试支持汉字。
+        // 淘宝搜索API接口地址:https://suggest.taobao.com/sug?code=utf-8&q=前端&callback=window.taobao.sug
+        // 一淘搜索API接口地址:https://suggest.taobao.com/sug?area=etao&code=utf-8&callback=KISSY.Suggest.callback&q=前端
+
+        return [];
+    }
+
+    /**
+     * 分批查询小说数据,并以生成器方式返回
+     *
+     * @param int $intStart  跳过的记录数(skip)
+     * @param int $intCount  需要查询的总记录数
+     * @return \Generator  逐条返回小说数据
+     * @throws \MongoDB\Exception\Exception
+     */
+    public function fetchVideosByBatch(int $intStart, int $intCount): \Generator
+    {
+        $intRecordsFetched = 0;
+        $intRemainingRecords = min($intCount, $this->VideoModel->getCol()->countDocuments() - $intStart);
+
+
+        while ($intRecordsFetched < $intRemainingRecords) {
+            $intLimit = min($this->intBatchSize, $intRemainingRecords - $intRecordsFetched);
+            $Cursor = $this->VideoModel->getCol()->find(
+                [],
+                [
+                    'skip' => $intStart + $intRecordsFetched,
+                    'limit' => $intLimit,
+                    'sort' => ['_id' => 1],
+                    'projection' =>  VideoModel::$arrOptions['projection'],
+                    'typeMap' => [
+                        'root' => 'array',
+                        'document' => 'array',
+                        'array' => 'array',
+                    ],
+                ]
+            );
+
+            foreach ($Cursor as $Video) {
+                yield $Video;
+            }
+
+            $intRecordsFetched += $intLimit;
+            unset($Cursor);
+            gc_collect_cycles();
+        }
+    }
+}
diff --git a/code/app/task/logic/VideoSiteMapLogic.php b/code/app/task/logic/VideoSiteMapLogic.php
index 271397c..9914444 100644
--- a/code/app/task/logic/VideoSiteMapLogic.php
+++ b/code/app/task/logic/VideoSiteMapLogic.php
@@ -189,8 +189,7 @@ EOF;
     {
         $intRecordsFetched = 0;
         $intRemainingRecords = min($intCount, $this->VideoModel->getCol()->countDocuments() - $intStart);
-        var_dump('$intStart='.$intStart);
-        var_dump('$intRemainingRecords='.$intRemainingRecords);
+
 
         while ($intRecordsFetched < $intRemainingRecords) {
             $intLimit = min($this->intBatchSize, $intRemainingRecords - $intRecordsFetched);
diff --git a/code/app/task/module/PlanTask.php b/code/app/task/module/PlanTask.php
index 9e85aed..4b38752 100644
--- a/code/app/task/module/PlanTask.php
+++ b/code/app/task/module/PlanTask.php
@@ -17,8 +17,7 @@ use app\task\crawler\fengchao\Scheduler as FengChaoScheduler;
 use app\task\crawler\wuxian\Scheduler as WuXianScheduler;
 
 use app\task\crawler\custom\Scheduler as CustomScheduler;
-
-
+use app\task\logic\VideoSeoLogic;
 
 class PlanTask
 {
@@ -72,6 +71,9 @@ class PlanTask
                 case 'CUSTOM_CRAWLER':
                     self::customCrawler();
                     break;
+                case 'GENERATE_VIDEO_SEO_WORDS':
+                    self::generateVideoSEOWords();
+                    break;
             }
         }
     }
@@ -236,4 +238,20 @@ class PlanTask
         $QueueManage = new QueueManage(2);
         $QueueManage->set($arrTask);
     }
+
+    /**
+     * Undocumented function
+     *
+     * @return void
+     */
+    public static function generateVideoSEOWords()
+    {
+        $arrTask = [
+            'callback' => [VideoSeoLogic::class, 'generateVideoSEOWords'],
+            'data' => []
+        ];
+
+        $QueueManage = new QueueManage(2);
+        $QueueManage->set($arrTask);
+    }
 }
diff --git a/code/database/seeders/PlanTaskSeeder.php b/code/database/seeders/PlanTaskSeeder.php
index 9e6b4ec..e145794 100644
--- a/code/database/seeders/PlanTaskSeeder.php
+++ b/code/database/seeders/PlanTaskSeeder.php
@@ -85,6 +85,12 @@ class PlanTaskSeeder
                 'pt_enable' => 0,
                 'pt_limit' => 10,
             ],
+            [
+                'pt_name' => '生成视频SEO关键词',
+                'pt_code' => 'GENERATE_VIDEO_SEO_WORDS',
+                'pt_enable' => 0,
+                'pt_limit' => 7 * 24 * 3600,
+            ],
         ];
 
         foreach ($arrDataNovel as $arrPlanTask) {
diff --git a/code/extend/template/taglib/Video.php b/code/extend/template/taglib/Video.php
index 36d47dd..008513c 100644
--- a/code/extend/template/taglib/Video.php
+++ b/code/extend/template/taglib/Video.php
@@ -57,10 +57,7 @@ class Video extends TagLib
          */
         public function tagRankList($tag, $content)
         {
-                // if($tag['sort_type'] == 'weekly'){
-                //         var_dump($tag);exit;
-                // }
-                
+  
                 $count = $tag['count'] ?? '10';
                 $sort_type = $tag['sort_type'] ?? "";
                 $v_status = $tag['v_status'] ?? "all";