Merge branch 'dev' of ssh://git.bgbg.live:18150/root/SEONexus into dev

2025-05-05 16:26:56 +08:00
parent f827d86ee6 cac9177838
commit 6305233820
15 changed files with 297 additions and 156 deletions
--- a/code/app/api/controller/Statis.php
+++ b/code/app/api/controller/Statis.php
@@ -9,6 +9,9 @@ use app\model\CategoryModel;
 use app\model\NovelClicksModel;
 use app\model\NovelModel;
 use app\model\VideoClicksModel;
+use app\task\crawler\fengchao\page\VideoCategory;
+use app\task\crawler\wuxian\page\VideoCategory as PageVideoCategory;
+use app\task\crawler\youzhi\page\VideoCategory as YouzhiPageVideoCategory;
 use DateTime;
 use db\mongo\MongoBase;
 use ddl\DDLManage;
@@ -70,90 +73,23 @@ class Statis extends BaseController

    public function test0909()
    {
-        $a = [
-            'Hindi',
-            'Malay',
-            'Swahili',
-            'Tamil',
-            '上海',
-            '丹麦语',
-            '乌克兰',
-            '乌尔都语',
-            '他加',
-            '俄语',
-            '其它',
-            '匈牙利语',
-            '北印度语',
-            '印地语',
-            '印地语Hindi',
-            '印尼语',
-            '印度尼西亚语',
-            '印度语',
-            '印度语Hindi',
-            '因纽特语',
-            '国语',
-            '土库曼语',
-            '土耳',
-            '土耳其语',
-            '基库尤语',
-            '希腊',
-            '希腊语',
-            '库尔德语',
-            '德语',
-            '意大',
-            '意大利',
-            '意大利语',
-            '挪威语',
-            '捷克语',
-            '日语',
-            '普通话',
-            '林加拉语',
-            '毛利语',
-            '汉语',
-            '汉语普通',
-            '汉语普通话',
-            '汉语普通话Manda',
-            '河南方言',
-            '法语',
-            '波兰语',
-            '波斯语',
-            '泰卢固语',
-            '泰米尔',
-            '泰米尔语',
-            '泰米尔语Tamil',
-            '泰语',
-            '澳大利亚语',
-            '瑞典语',
-            '粤语',
-            '美国手语',
-            '英语',
-            '英语手语',
-            '荷兰语',
-            '菲律宾语',
-            '葡萄',
-            '葡萄牙语',
-            '藏语',
-            '西班',
-            '西班牙语',
-            '越南语',
-            '闽南语',
-            '阿拉',
-            '阿拉伯',
-            '阿拉伯语',
-            '韩语',
-            '马拉雅拉姆语',
-            '马来西亚语',
-            '马来语'
+        $a =  VideoCategory::$arrCategory;
+        $b = PageVideoCategory::$arrCategory;
+        $c = YouzhiPageVideoCategory::$arrCategory;

-        ];
-
-        $arrArea = [];
-        foreach ($a as $v) {
-            $arrArea[zhToPinYin($v)] = $v;
+        $x = [];
+        foreach ($a as $z) {
+            $x[$z['pinyin']] = $z['name'];
        }
-        echo '<pre>';

-        krsort($arrArea);
-        echo var_export($arrArea, true);
+        foreach ($b as $z) {
+            $x[$z['pinyin']] = $z['name'];
+        }
+
+        foreach ($c as $z) {
+            $x[$z['pinyin']] = $z['name'];
+        }
+
+        echo var_export($x, true);
    }
 }
--- a/code/app/model/CustomCrawlerModel.php
+++ b/code/app/model/CustomCrawlerModel.php
@@ -0,0 +1,17 @@
+<?php
+
+declare(strict_types=1);
+
+namespace app\model;
+
+use think\facade\Cache;
+use think\Model;
+
+/**
+ * @mixin think\Model
+ */
+class CustomCrawlerModel extends BaseModel
+{
+    protected $name = 'custom_crawler';
+    protected $pk = 'cc_id';
+}
--- a/code/app/model/VideoCategoryModel.php
+++ b/code/app/model/VideoCategoryModel.php
@@ -88,7 +88,7 @@ class VideoCategoryModel extends MongoModel
        ],
        [
            "v_category" => "电视剧",
-            "v_category_en" =>'dian-shi-ju',
+            "v_category_en" => 'dian-shi-ju',
            "children" => [
                [
                    "v_category_en" => "guo-chan-ju",
@@ -217,39 +217,57 @@ class VideoCategoryModel extends MongoModel
    }

    static $arrCategory = [
-        "da-lu-zong-yi" => "大陆综艺",
-        "ai-qing-pian" => "爱情片",
-        "ke-huan-pian" => "科幻片",
-        "ri-han-zong-yi" => "日韩综艺",
-        "ri-han-dong-man" => "日韩动漫",
-        "zhan-zheng-pian" => "战争片",
-        "ou-mei-ju" => "欧美剧",
-        "lun-li-pian" => "倫理片",
-        "han-guo-ju" => "韩国剧",
-        "yin-le" => "音乐",
-        "dong-hua-pian" => "动画片",
-        "guo-chan-dong-man" => "国产动漫",
-        "kong-bu-pian" => "恐怖片",
-        "tai-wan-ju" => "台湾剧",
-        "xiang-gang-ju" => "香港剧",
-        "xi-ju-pian" => "喜剧片",
-        "ying-shi" => "影视",
-        "qi-che" => "汽车",
-        "ou-mei-zong-yi" => "欧美综艺",
-        "ju-qing-pian" => "剧情片",
-        "tai-guo-ju" => "泰国剧",
-        "hai-wai-ju" => "海外剧",
-        "guo-chan-ju" => "国产剧",
-        "fu-li" => "福利",
-        "ou-mei-dong-man" => "欧美动漫",
-        "dong-zuo-pian" => "动作片",
-        "ri-ben-ju" => "日本剧",
-        "gang-tai-zong-yi" => "港台综艺",
-        "gang-tai-dong-man" => "港台动漫",
-        "ji-lu-pian" => "记录片",
-        "duan-ju-da-quan" => "短剧大全",
-        "gao-xiao" => "搞笑",
-        "hai-wai-dong-man" => "海外动漫",
+        'dian-ying' => '电影',
+        'dian-shi-ju' => '电视剧',
+        'zong-yi' => '综艺',
+        'dong-man' => '动漫',
+        'dong-zuo-pian' => '动作片',
+        'xi-ju-pian' => '喜剧片',
+        'ai-qing-pian' => '爱情片',
+        'ke-huan-pian' => '科幻片',
+        'kong-bu-pian' => '恐怖片',
+        'ju-qing-pian' => '剧情片',
+        'zhan-zheng-pian' => '战争片',
+        'guo-chan-ju' => '国产剧',
+        'ou-mei-ju' => '欧美剧',
+        'han-ju' => '韩剧',
+        'ri-ju' => '日剧',
+        'gang-ju' => '港剧',
+        'tai-ju' => '泰剧',
+        'ji-lu-pian' => '记录片',
+        'hai-wai-ju' => '海外剧',
+        'da-lu-zong-yi' => '大陆综艺',
+        'ri-han-zong-yi' => '日韩综艺',
+        'gang-tai-zong-yi' => '港台综艺',
+        'ou-mei-zong-yi' => '欧美综艺',
+        'guo-chan-dong-man' => '国产动漫',
+        'ri-han-dong-man' => '日韩动漫',
+        'ou-mei-dong-man' => '欧美动漫',
+        'dong-hua-pian' => '动画片',
+        'gang-tai-dong-man' => '港台动漫',
+        'hai-wai-dong-man' => '海外动漫',
+        'yan-chang-hui' => '演唱会',
+        'lun-li' => '伦理',
+        'shao-shi-dian-ying' => '邵氏电影',
+        'ti-yu-sai-shi' => '体育赛事',
+        'lan-qiu' => '篮球',
+        'zu-qiu' => '足球',
+        'si-nuo-ke' => '斯诺克',
+        'lian-xu-ju' => '连续剧',
+        'tai-wan-ju' => '台湾剧',
+        'han-guo-ju' => '韩国剧',
+        'xiang-gang-ju' => '香港剧',
+        'tai-guo-ju' => '泰国剧',
+        'ri-ben-ju' => '日本剧',
+        'fu-li' => '福利',
+        'lun-li-pian' => '伦理片',
+        'gao-xiao' => '搞笑',
+        'yin-le' => '音乐',
+        'ying-shi' => '影视',
+        'qi-che' => '汽车',
+        'duan-ju-da-quan' => '短剧大全',
+        'yu-gao-pian' => '预告片',
+        'ti-yu' => '体育',
    ];

    /**
--- a/code/app/task/crawler/custom/Scheduler.php
+++ b/code/app/task/crawler/custom/Scheduler.php
@@ -0,0 +1,75 @@
+<?php
+
+declare(strict_types=1);
+
+namespace app\task\crawler\custom;
+
+use app\model\ChapterModel;
+use app\model\CustomCrawlerModel;
+use app\task\crawler\custom\page\Site;
+use microserver\QueueManage;
+use storage\StorageCore;
+
+/**
+ * @mixin think\Model
+ */
+class Scheduler
+{
+    /**
+     * Undocumented variable
+     *
+     * @var \microserver\QueueManage
+     */
+    protected $QueueManage = NULL;
+
+    /**
+     * Undocumented function
+     */
+    public function __construct()
+    {
+        $this->QueueManage = new QueueManage();
+    }
+
+    /**
+     * create a site-wide craw task
+     *
+     * @return void
+     */
+    public function crawlFull()
+    {
+        $AllCustomCrawlerModel = CustomCrawlerModel::select();
+
+        foreach ($AllCustomCrawlerModel as $CustomCrawlerModel) {
+
+            if (
+                $CustomCrawlerModel->pt_last_exec != 0 &&
+                ($CustomCrawlerModel->pt_last_exec + $CustomCrawlerModel->pt_limit)  > time()
+            ) {
+                continue;
+            }
+
+            $CustomCrawlerModel->pt_last_exec = time();
+
+            $CustomCrawlerModel->save();
+
+            switch ($CustomCrawlerModel->cc_code) {
+                case 'PULL_WEN_XUE_GUAN':
+                    $arrTask = [
+                        'callback' => [self::class, 'createCrawlWenXueGuanTask'],
+                        'data' => [
+                            'cc_data' => $CustomCrawlerModel->cc_data
+                        ],
+                    ];
+                    break;
+                default:
+                    break;
+            }
+            $this->QueueManage->set($arrTask);
+        }
+    }
+
+    public function createCrawlWenXueGuanTask(array $arrData)
+    {
+        $arrCcData = json_decode($arrData['cc_data'], true);
+    }
+}
--- a/code/app/task/crawler/fengchao/Scheduler.php
+++ b/code/app/task/crawler/fengchao/Scheduler.php
@@ -66,6 +66,12 @@ class Scheduler
        return  $this->Site;
    }

+    public $arrExcludeCategory = [
+        '体育',
+        '福利',
+        '伦理',
+        '伦理片',
+    ];

    /**
     * create a site-wide craw task
@@ -106,6 +112,11 @@ class Scheduler
                continue;
            }

+            if (in_array($arrCategory['v_category_name'], $this->arrExcludeCategory)) {
+                continue;
+            }
+
+
            $intTotalPage = $arrCategory['total_page'];
            $arrFilter = [];
            for ($intStartPage = 1; $intStartPage <= $intTotalPage; $intStartPage++) {
--- a/code/app/task/crawler/wuxian/Scheduler.php
+++ b/code/app/task/crawler/wuxian/Scheduler.php
@@ -66,6 +66,12 @@ class Scheduler
        return  $this->Site;
    }

+    public $arrExcludeCategory = [
+        '体育',
+        '福利',
+        '伦理',
+        '伦理片',
+    ];

    /**
     * create a site-wide craw task
@@ -103,6 +109,10 @@ class Scheduler
                continue;
            }

+            if (in_array($arrCategory['v_category_name'], $this->arrExcludeCategory)) {
+                continue;
+            }
+
            $intTotalPage = $arrCategory['total_page'];
            $arrFilter = [];
            for ($intStartPage = 1; $intStartPage <= $intTotalPage; $intStartPage++) {
--- a/code/app/task/crawler/youzhi/Scheduler.php
+++ b/code/app/task/crawler/youzhi/Scheduler.php
@@ -66,6 +66,12 @@ class Scheduler
        return  $this->Site;
    }

+    public $arrExcludeCategory = [
+        '体育',
+        '福利',
+        '伦理',
+        '伦理片',
+    ];

    /**
     * create a site-wide craw task
@@ -104,6 +110,10 @@ class Scheduler
                continue;
            }

+            if (in_array($arrCategory['v_category_name'], $this->arrExcludeCategory)) {
+                continue;
+            }
+
            $intTotalPage = $arrCategory['total_page'];
            $arrFilter = [];
            for ($intStartPage = 1; $intStartPage <= $intTotalPage; $intStartPage++) {
--- a/code/app/task/crawler/youzhi/page/VideoCategory.php
+++ b/code/app/task/crawler/youzhi/page/VideoCategory.php
@@ -96,7 +96,7 @@ class VideoCategory
            'pinyin' => 'hai-wai-ju',
        ],
        61 => [
-            'name' => '?理片',
+            'name' => '伦理片',
            'pinyin' => 'lun-li-pian',
        ],
        62 => [
--- a/code/app/task/module/PlanTask.php
+++ b/code/app/task/module/PlanTask.php
@@ -15,7 +15,7 @@ use app\task\crawler\youzhi\Scheduler as YouZhiScheduler;
 use app\task\crawler\fengchao\Scheduler as FengChaoScheduler;
 use app\task\crawler\wuxian\Scheduler as WuXianScheduler;

-
+use app\task\crawler\custom\Scheduler as CustomScheduler;



@@ -29,49 +29,64 @@ class PlanTask

        foreach ($PlanTaskAll as $PlanTask) {

-            if ($PlanTask->pt_last_exec != 0 && ($PlanTask->pt_last_exec + $PlanTask->pt_limit)  > time()) continue;
+            if (
+                $PlanTask->pt_last_exec != 0 &&
+                ($PlanTask->pt_last_exec + $PlanTask->pt_limit)  > time()
+            ) {
+                continue;
+            }

            $PlanTask->pt_last_exec = time();

            $PlanTask->save();

            switch ($PlanTask->pt_code) {
-
+                case 'GENERATE_SITE_MAP':
+                    self::generateSiteMap();
+                    break;
                case 'PULL_630_ZW_NOVEL':
                    self::pull630ZwNovel();
                    break;
                case 'PULL_630_ZW_NOVEL_LATEST':
                    self::pull630ZwNovelLatest();
                    break;
-
                case 'PULL_74_ZW_NOVEL':
                    self::pull74ZwNovel();
                    break;
                case 'PULL_74_ZW_NOVEL_LATEST':
                    self::pull74ZwNovelLatest();
                    break;
-
                case 'PULL_YOU_ZHI_SHI_PIN':
                    self::pullYouZhiShiPin();
                    break;
-
                case 'PULL_WU_XIAN_SHI_PIN':
                    self::pullWuXianShiPin();
                    break;
-
-
                case 'PULL_FENG_CHAO_SHI_PIN':
                    self::pullFengChaoShiPin();
                    break;
-
-                case 'GENERATE_SITE_MAP':
-                    self::generateSiteMap();
+                case 'CUSTOM_CRAWLER':
+                    self::customCrawler();
                    break;
            }
        }
    }

-    
+    /**
+     * Undocumented function
+     *
+     * @return void
+     */
+    public static function customCrawler()
+    {
+        $arrTask = [
+            'callback' => [CustomScheduler::class, 'crawlFull'],
+            'data' => []
+        ];
+        $QueueManage =  QueueManage::getInstance();
+        $QueueManage->set($arrTask);
+    }
+
    /**
     * Undocumented function
     *
@@ -84,7 +99,7 @@ class PlanTask
            'data' => []
        ];

-        $QueueManage = new QueueManage();
+        $QueueManage =  QueueManage::getInstance();
        $QueueManage->set($arrTask);
    }

@@ -100,7 +115,7 @@ class PlanTask
            'data' => []
        ];

-        $QueueManage = new QueueManage();
+        $QueueManage = QueueManage::getInstance(2);
        $QueueManage->set($arrTask);
    }

@@ -116,7 +131,7 @@ class PlanTask
            'data' => []
        ];

-        $QueueManage = new QueueManage();
+        $QueueManage =  QueueManage::getInstance();
        $QueueManage->set($arrTask);
    }

@@ -133,7 +148,7 @@ class PlanTask
            'data' => []
        ];

-        $QueueManage = new QueueManage();
+        $QueueManage =  QueueManage::getInstance();
        $QueueManage->set($arrTask);
    }

@@ -149,7 +164,7 @@ class PlanTask
            'data' => []
        ];

-        $QueueManage = new QueueManage();
+        $QueueManage =  QueueManage::getInstance();
        $QueueManage->set($arrTask);
    }

@@ -166,7 +181,7 @@ class PlanTask
            'data' => []
        ];

-        $QueueManage = new QueueManage();
+        $QueueManage =  QueueManage::getInstance();
        $QueueManage->set($arrTask);
    }

@@ -182,7 +197,7 @@ class PlanTask
            'data' => []
        ];

-        $QueueManage = new QueueManage();
+        $QueueManage =  QueueManage::getInstance();
        $QueueManage->set($arrTask);
    }