DEBUG

2025-05-27 11:57:16 +08:00
parent 50d7df3869
commit b392a65948
15 changed files with 223 additions and 53 deletions
--- a/code/app/task/crawler/fengchao/page/Site.php
+++ b/code/app/task/crawler/fengchao/page/Site.php
@@ -102,14 +102,14 @@ class Site
                    break;
                case 'QING':
                    $arrConfig['proxy'] = $strProxyHost;
-                    $proxyAuth = base64_encode($strKey . ":" . $strSecret);
-                    $arrConfig['headers']['Proxy-Authorization'] =  "Basic " . $proxyAuth;
+                    $strProxyAuth = base64_encode($strKey . ":" . $strSecret);
+                    $arrConfig['headers']['Proxy-Authorization'] =  "Basic " . $strProxyAuth;
                    break;
                case 'JULIANG':
                    // 隧道代理信息
-                    // $proxy = 'http://username:password@XXX.juliangip.com:23456';
-                    $proxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
-                    $arrConfig['proxy'] = $proxy;
+                    // $strProxy = 'http://username:password@XXX.juliangip.com:23456';
+                    $strProxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
+                    $arrConfig['proxy'] = $strProxy;
                    break;
            }
        }
--- a/code/app/task/crawler/wuxian/page/Site.php
+++ b/code/app/task/crawler/wuxian/page/Site.php
@@ -102,14 +102,14 @@ class Site
                    break;
                case 'QING':
                    $arrConfig['proxy'] = $strProxyHost;
-                    $proxyAuth = base64_encode($strKey . ":" . $strSecret);
-                    $arrConfig['headers']['Proxy-Authorization'] =  "Basic " . $proxyAuth;
+                    $strProxyAuth = base64_encode($strKey . ":" . $strSecret);
+                    $arrConfig['headers']['Proxy-Authorization'] =  "Basic " . $strProxyAuth;
                    break;
                case 'JULIANG':
                    // 隧道代理信息
-                    // $proxy = 'http://username:password@XXX.juliangip.com:23456';
-                    $proxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
-                    $arrConfig['proxy'] = $proxy;
+                    // $strProxy = 'http://username:password@XXX.juliangip.com:23456';
+                    $strProxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
+                    $arrConfig['proxy'] = $strProxy;
                    break;
            }
        }
--- a/code/app/task/crawler/wxg/page/Site.php
+++ b/code/app/task/crawler/wxg/page/Site.php
@@ -47,7 +47,7 @@ class Site
     *
     * @var boolean
     */
-    public $boolReInitClient = false;
+    public $boolReInitClient = true;

    /**
     * Undocumented function
@@ -110,14 +110,14 @@ class Site
                    break;
                case 'QING':
                    $arrConfig['proxy'] = $strProxyHost;
-                    $proxyAuth = base64_encode($strKey . ":" . $strSecret);
-                    $arrConfig['headers']['Proxy-Authorization'] =  "Basic " . $proxyAuth;
+                    $strProxyAuth = base64_encode($strKey . ":" . $strSecret);
+                    $arrConfig['headers']['Proxy-Authorization'] =  "Basic " . $strProxyAuth;
                    break;
                case 'JULIANG':
                    // 隧道代理信息
-                    // $proxy = 'http://username:password@XXX.juliangip.com:23456';
-                    $proxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
-                    $arrConfig['proxy'] = $proxy;
+                    // $strProxy = 'http://username:password@XXX.juliangip.com:23456';
+                    $strProxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
+                    $arrConfig['proxy'] = $strProxy;
                    break;
            }
        }
--- a/code/app/task/crawler/youzhi/page/Site.php
+++ b/code/app/task/crawler/youzhi/page/Site.php
@@ -102,14 +102,14 @@ class Site
                    break;
                case 'QING':
                    $arrConfig['proxy'] = $strProxyHost;
-                    $proxyAuth = base64_encode($strKey . ":" . $strSecret);
-                    $arrConfig['headers']['Proxy-Authorization'] =  "Basic " . $proxyAuth;
+                    $strProxyAuth = base64_encode($strKey . ":" . $strSecret);
+                    $arrConfig['headers']['Proxy-Authorization'] =  "Basic " . $strProxyAuth;
                    break;
                case 'JULIANG':
                    // 隧道代理信息
-                    // $proxy = 'http://username:password@XXX.juliangip.com:23456';
-                    $proxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
-                    $arrConfig['proxy'] = $proxy;
+                    // $strProxy = 'http://username:password@XXX.juliangip.com:23456';
+                    $strProxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
+                    $arrConfig['proxy'] = $strProxy;
                    break;
            }
        }
--- a/code/app/task/crawler/zw/page/Site.php
+++ b/code/app/task/crawler/zw/page/Site.php
@@ -110,14 +110,14 @@ class Site
                    break;
                case 'QING':
                    $arrConfig['proxy'] = $strProxyHost;
-                    $proxyAuth = base64_encode($strKey . ":" . $strSecret);
-                    $arrConfig['headers']['Proxy-Authorization'] =  "Basic " . $proxyAuth;
+                    $strProxyAuth = base64_encode($strKey . ":" . $strSecret);
+                    $arrConfig['headers']['Proxy-Authorization'] =  "Basic " . $strProxyAuth;
                    break;
                case 'JULIANG':
                    // 隧道代理信息
-                    // $proxy = 'http://username:password@XXX.juliangip.com:23456';
-                    $proxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
-                    $arrConfig['proxy'] = $proxy;
+                    // $strProxy = 'http://username:password@XXX.juliangip.com:23456';
+                    $strProxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
+                    $arrConfig['proxy'] = $strProxy;
                    break;
            }
        }
--- a/code/app/task/crawler/zw630/page/Site.php
+++ b/code/app/task/crawler/zw630/page/Site.php
@@ -95,14 +95,14 @@ class Site extends  ZwSite
                    break;
                case 'QING':
                    $arrConfig['proxy'] = $strProxyHost;
-                    $proxyAuth = base64_encode($strKey . ":" . $strSecret);
-                    $arrConfig['headers']['Proxy-Authorization'] =  "Basic " . $proxyAuth;
+                    $strProxyAuth = base64_encode($strKey . ":" . $strSecret);
+                    $arrConfig['headers']['Proxy-Authorization'] =  "Basic " . $strProxyAuth;
                    break;
                case 'JULIANG':
                    // 隧道代理信息
-                    // $proxy = 'http://username:password@XXX.juliangip.com:23456';
-                    $proxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
-                    $arrConfig['proxy'] = $proxy;
+                    // $strProxy = 'http://username:password@XXX.juliangip.com:23456';
+                    $strProxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
+                    $arrConfig['proxy'] = $strProxy;
                    break;
            }
        }
--- a/code/app/task/crawler/zw74/page/Site.php
+++ b/code/app/task/crawler/zw74/page/Site.php
@@ -94,14 +94,14 @@ class Site extends  ZwSite
                    break;
                case 'QING':
                    $arrConfig['proxy'] = $strProxyHost;
-                    $proxyAuth = base64_encode($strKey . ":" . $strSecret);
-                    $arrConfig['headers']['Proxy-Authorization'] =  "Basic " . $proxyAuth;
+                    $strProxyAuth = base64_encode($strKey . ":" . $strSecret);
+                    $arrConfig['headers']['Proxy-Authorization'] =  "Basic " . $strProxyAuth;
                    break;
                case 'JULIANG':
                    // 隧道代理信息
-                    // $proxy = 'http://username:password@XXX.juliangip.com:23456';
-                    $proxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
-                    $arrConfig['proxy'] = $proxy;
+                    // $strProxy = 'http://username:password@XXX.juliangip.com:23456';
+                    $strProxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
+                    $arrConfig['proxy'] = $strProxy;
                    break;
                       
            }
--- a/code/app/task/logic/VideoSeoLogic.php
+++ b/code/app/task/logic/VideoSeoLogic.php
@@ -0,0 +1,154 @@
+<?php
+
+declare(strict_types=1);
+
+namespace app\task\logic;
+
+use app\model\VideoModel;
+
+use GuzzleHttp\Client;
+use GuzzleHttp\Promise;
+use storage\StorageCore;
+
+class VideoSeoLogic
+{
+    /**
+     * VideoModel
+     *
+     * @var VideoModel
+     */
+    public $VideoModel;
+
+    public $intPageSize = 40000;
+
+    public $strSiteMapPath = '';
+
+    public $intTotal = 0;
+
+    public $intBatchSize = 500;
+
+    public function __construct()
+    {
+        $this->VideoModel = VideoModel::getInstance();
+
+        VideoModel::$arrOptions['projection'] = [
+            '_id' => 0,
+            'v_id' => 1,
+            'v_name' => 1,
+            'v_seo_words' => 1,
+        ];
+    }
+
+    public function generateVideoSEOWords()
+    {
+        $arrResult =  $this->VideoModel->findPaginatedWithCache([], 1, 1);
+        $this->intTotal = $arrResult['total'];
+
+        $intTotalPage = ceil($this->intTotal / $this->intPageSize);
+        for ($intPage = 1; $intPage <= $intTotalPage; $intPage++) {
+
+            $intStartPage = $intPage;
+            $intStart = ($intStartPage -= 1) * $this->intPageSize;
+
+            foreach ($this->fetchVideosByBatch($intStart, $this->intPageSize) as $arrVideo) {
+
+                print_r($arrVideo);
+
+                if (!isset($arrVideo['v_seo_words']) || empty($arrVideo['v_seo_words'])) {
+                    $arrWords =  $this->getBaiduSEOWords($arrVideo['v_name']);
+                    if (!empty($arrWords)) {
+                        $this->VideoModel->updateOne(['v_id' => $arrVideo['v_id']], [
+                            ['$set' => ['v_seo_words' =>   $arrWords,]]
+                        ]);
+                    }
+                }
+            }
+        }
+    }
+
+    private function getBaiduSEOWords(string $strVName): array
+    {
+        try {
+
+            $strUrl = 'https://suggestion.baidu.com/su?wd=%s&json=1&p=3&cb=callback';
+
+            $strUrl = sprintf($strUrl, $strVName);
+
+            $strContent = file_get_contents($strUrl);
+            $strContent = mb_convert_encoding($strContent, 'UTF-8', 'GBK');
+
+            $strContent = str_replace([
+                'callback(',
+                ');',
+            ], '', $strContent);
+
+            $arrWords = json_decode($strContent, true);
+
+            return $arrWords['s'] ?? [];
+        } catch (\Exception $e) {
+            echo $e->getMessage();
+            return [];
+        }
+    }
+
+
+    private function getOtherSEOWords(string $strVName): array
+    {
+        // 百度 API地址：http://suggestion.baidu.com/su?wd=关键词&p=3&cb=window.bdsug.sug
+        // suggestion.baidu.com/su?wd=关键词&p=100 后面的数字变化结果不同；
+        // 搜狗 API地址：http://w.sugg.sogou.com/sugg/ajaj_json.jsp?key=关键词&type=web
+        // 必应 API地址：http://sg1.api.bing.com/qsonhs.aspx?type=cb&cb=callback&q=关键词
+        // 必应搜索API接口地址：https://api.bing.com/qsonhs.aspx?type=cb&q=前端 返回的是 json 格式。
+        // 必应搜索API接口地址：https://api.bing.com/qsonhs.aspx?type=cb&q=前端&cb=window.bing.sug
+        // 谷歌API接口地址1：http://clients1.google.com/complete/search?hl=zh&output=toolbar&q=关键词
+        // 谷歌API接口地址2：http://suggestqueries.google.com/complete/search?output=toolbar&hl=zh&q=关键词
+        // 谷歌API接口地址2：http://suggestqueries.google.com/complete/search?client=youtube&q=关键词&jsonp=window.google.ac.h
+        // 360搜索API接口地址：https://sug.so.360.cn/suggest?callback=suggest_so&word=qianduan 测试只支持英文，不过可以支持拼音。
+        // 360搜索API接口地址：https://sug.so.360.cn/suggest?encodein=utf-8&encodeout=utf-8&format=json&word=前端&callback=window.so.sug 测试支持汉字。
+        // 淘宝搜索API接口地址：https://suggest.taobao.com/sug?code=utf-8&q=前端&callback=window.taobao.sug
+        // 一淘搜索API接口地址：https://suggest.taobao.com/sug?area=etao&code=utf-8&callback=KISSY.Suggest.callback&q=前端
+
+        return [];
+    }
+
+    /**
+     * 分批查询小说数据，并以生成器方式返回
+     *
+     * @param int $intStart  跳过的记录数（skip）
+     * @param int $intCount  需要查询的总记录数
+     * @return \Generator  逐条返回小说数据
+     * @throws \MongoDB\Exception\Exception
+     */
+    public function fetchVideosByBatch(int $intStart, int $intCount): \Generator
+    {
+        $intRecordsFetched = 0;
+        $intRemainingRecords = min($intCount, $this->VideoModel->getCol()->countDocuments() - $intStart);
+
+
+        while ($intRecordsFetched < $intRemainingRecords) {
+            $intLimit = min($this->intBatchSize, $intRemainingRecords - $intRecordsFetched);
+            $Cursor = $this->VideoModel->getCol()->find(
+                [],
+                [
+                    'skip' => $intStart + $intRecordsFetched,
+                    'limit' => $intLimit,
+                    'sort' => ['_id' => 1],
+                    'projection' =>  VideoModel::$arrOptions['projection'],
+                    'typeMap' => [
+                        'root' => 'array',
+                        'document' => 'array',
+                        'array' => 'array',
+                    ],
+                ]
+            );
+
+            foreach ($Cursor as $Video) {
+                yield $Video;
+            }
+
+            $intRecordsFetched += $intLimit;
+            unset($Cursor);
+            gc_collect_cycles();
+        }
+    }
+}
--- a/code/app/task/logic/VideoSiteMapLogic.php
+++ b/code/app/task/logic/VideoSiteMapLogic.php
@@ -189,8 +189,7 @@ EOF;
    {
        $intRecordsFetched = 0;
        $intRemainingRecords = min($intCount, $this->VideoModel->getCol()->countDocuments() - $intStart);
-        var_dump('$intStart='.$intStart);
-        var_dump('$intRemainingRecords='.$intRemainingRecords);
+

        while ($intRecordsFetched < $intRemainingRecords) {
            $intLimit = min($this->intBatchSize, $intRemainingRecords - $intRecordsFetched);
--- a/code/app/task/module/PlanTask.php
+++ b/code/app/task/module/PlanTask.php
@@ -17,8 +17,7 @@ use app\task\crawler\fengchao\Scheduler as FengChaoScheduler;
 use app\task\crawler\wuxian\Scheduler as WuXianScheduler;

 use app\task\crawler\custom\Scheduler as CustomScheduler;
-
-
+use app\task\logic\VideoSeoLogic;

 class PlanTask
 {
@@ -72,6 +71,9 @@ class PlanTask
                case 'CUSTOM_CRAWLER':
                    self::customCrawler();
                    break;
+                case 'GENERATE_VIDEO_SEO_WORDS':
+                    self::generateVideoSEOWords();
+                    break;
            }
        }
    }
@@ -236,4 +238,20 @@ class PlanTask
        $QueueManage = new QueueManage(2);
        $QueueManage->set($arrTask);
    }
+
+    /**
+     * Undocumented function
+     *
+     * @return void
+     */
+    public static function generateVideoSEOWords()
+    {
+        $arrTask = [
+            'callback' => [VideoSeoLogic::class, 'generateVideoSEOWords'],
+            'data' => []
+        ];
+
+        $QueueManage = new QueueManage(2);
+        $QueueManage->set($arrTask);
+    }
 }