fix: video model category

code/app/api/controller/Statis.php

@@ -9,6 +9,9 @@ use app\model\CategoryModel;
 use app\model\NovelClicksModel;
 use app\model\NovelModel;
 use app\model\VideoClicksModel;
+use app\task\crawler\fengchao\page\VideoCategory;
+use app\task\crawler\wuxian\page\VideoCategory as PageVideoCategory;
+use app\task\crawler\youzhi\page\VideoCategory as YouzhiPageVideoCategory;
 use DateTime;
 use db\mongo\MongoBase;
 use ddl\DDLManage;
@@ -70,90 +73,23 @@ class Statis extends BaseController
 
     public function test0909()
     {
-        $a = [
+        $a =  VideoCategory::$arrCategory;
-            'Hindi',
+        $b = PageVideoCategory::$arrCategory;
-            'Malay',
+        $c = YouzhiPageVideoCategory::$arrCategory;
-            'Swahili',
-            'Tamil',
-            '上海',
-            '丹麦语',
-            '乌克兰',
-            '乌尔都语',
-            '他加',
-            '俄语',
-            '其它',
-            '匈牙利语',
-            '北印度语',
-            '印地语',
-            '印地语Hindi',
-            '印尼语',
-            '印度尼西亚语',
-            '印度语',
-            '印度语Hindi',
-            '因纽特语',
-            '国语',
-            '土库曼语',
-            '土耳',
-            '土耳其语',
-            '基库尤语',
-            '希腊',
-            '希腊语',
-            '库尔德语',
-            '德语',
-            '意大',
-            '意大利',
-            '意大利语',
-            '挪威语',
-            '捷克语',
-            '日语',
-            '普通话',
-            '林加拉语',
-            '毛利语',
-            '汉语',
-            '汉语普通',
-            '汉语普通话',
-            '汉语普通话Manda',
-            '河南方言',
-            '法语',
-            '波兰语',
-            '波斯语',
-            '泰卢固语',
-            '泰米尔',
-            '泰米尔语',
-            '泰米尔语Tamil',
-            '泰语',
-            '澳大利亚语',
-            '瑞典语',
-            '粤语',
-            '美国手语',
-            '英语',
-            '英语手语',
-            '荷兰语',
-            '菲律宾语',
-            '葡萄',
-            '葡萄牙语',
-            '藏语',
-            '西班',
-            '西班牙语',
-            '越南语',
-            '闽南语',
-            '阿拉',
-            '阿拉伯',
-            '阿拉伯语',
-            '韩语',
-            '马拉雅拉姆语',
-            '马来西亚语',
-            '马来语'
 
-        ];
+        $x = [];
-
+        foreach ($a as $z) {
-        $arrArea = [];
+            $x[$z['pinyin']] = $z['name'];
-        foreach ($a as $v) {
-            $arrArea[zhToPinYin($v)] = $v;
         }
-        echo '<pre>';
 
-        krsort($arrArea);
+        foreach ($b as $z) {
-        echo var_export($arrArea, true);
+            $x[$z['pinyin']] = $z['name'];
+        }
+
+        foreach ($c as $z) {
+            $x[$z['pinyin']] = $z['name'];
+        }
+
+        echo var_export($x, true);
     }
 }

code/app/model/CustomCrawlerModel.php

@@ -0,0 +1,17 @@
+<?php
+
+declare(strict_types=1);
+
+namespace app\model;
+
+use think\facade\Cache;
+use think\Model;
+
+/**
+ * @mixin think\Model
+ */
+class CustomCrawlerModel extends BaseModel
+{
+    protected $name = 'custom_crawler';
+    protected $pk = 'cc_id';
+}

code/app/model/VideoCategoryModel.php

@@ -217,39 +217,57 @@ class VideoCategoryModel extends MongoModel
     }
 
     static $arrCategory = [
-        "da-lu-zong-yi" => "大陆综艺",
+        'dian-ying' => '电影',
-        "ai-qing-pian" => "爱情片",
+        'dian-shi-ju' => '电视剧',
-        "ke-huan-pian" => "科幻片",
+        'zong-yi' => '综艺',
-        "ri-han-zong-yi" => "日韩综艺",
+        'dong-man' => '动漫',
-        "ri-han-dong-man" => "日韩动漫",
+        'dong-zuo-pian' => '动作片',
-        "zhan-zheng-pian" => "战争片",
+        'xi-ju-pian' => '喜剧片',
-        "ou-mei-ju" => "欧美剧",
+        'ai-qing-pian' => '爱情片',
-        "lun-li-pian" => "倫理片",
+        'ke-huan-pian' => '科幻片',
-        "han-guo-ju" => "韩国剧",
+        'kong-bu-pian' => '恐怖片',
-        "yin-le" => "音乐",
+        'ju-qing-pian' => '剧情片',
-        "dong-hua-pian" => "动画片",
+        'zhan-zheng-pian' => '战争片',
-        "guo-chan-dong-man" => "国产动漫",
+        'guo-chan-ju' => '国产剧',
-        "kong-bu-pian" => "恐怖片",
+        'ou-mei-ju' => '欧美剧',
-        "tai-wan-ju" => "台湾剧",
+        'han-ju' => '韩剧',
-        "xiang-gang-ju" => "香港剧",
+        'ri-ju' => '日剧',
-        "xi-ju-pian" => "喜剧片",
+        'gang-ju' => '港剧',
-        "ying-shi" => "影视",
+        'tai-ju' => '泰剧',
-        "qi-che" => "汽车",
+        'ji-lu-pian' => '记录片',
-        "ou-mei-zong-yi" => "欧美综艺",
+        'hai-wai-ju' => '海外剧',
-        "ju-qing-pian" => "剧情片",
+        'da-lu-zong-yi' => '大陆综艺',
-        "tai-guo-ju" => "泰国剧",
+        'ri-han-zong-yi' => '日韩综艺',
-        "hai-wai-ju" => "海外剧",
+        'gang-tai-zong-yi' => '港台综艺',
-        "guo-chan-ju" => "国产剧",
+        'ou-mei-zong-yi' => '欧美综艺',
-        "fu-li" => "福利",
+        'guo-chan-dong-man' => '国产动漫',
-        "ou-mei-dong-man" => "欧美动漫",
+        'ri-han-dong-man' => '日韩动漫',
-        "dong-zuo-pian" => "动作片",
+        'ou-mei-dong-man' => '欧美动漫',
-        "ri-ben-ju" => "日本剧",
+        'dong-hua-pian' => '动画片',
-        "gang-tai-zong-yi" => "港台综艺",
+        'gang-tai-dong-man' => '港台动漫',
-        "gang-tai-dong-man" => "港台动漫",
+        'hai-wai-dong-man' => '海外动漫',
-        "ji-lu-pian" => "记录片",
+        'yan-chang-hui' => '演唱会',
-        "duan-ju-da-quan" => "短剧大全",
+        'lun-li' => '伦理',
-        "gao-xiao" => "搞笑",
+        'shao-shi-dian-ying' => '邵氏电影',
-        "hai-wai-dong-man" => "海外动漫",
+        'ti-yu-sai-shi' => '体育赛事',
+        'lan-qiu' => '篮球',
+        'zu-qiu' => '足球',
+        'si-nuo-ke' => '斯诺克',
+        'lian-xu-ju' => '连续剧',
+        'tai-wan-ju' => '台湾剧',
+        'han-guo-ju' => '韩国剧',
+        'xiang-gang-ju' => '香港剧',
+        'tai-guo-ju' => '泰国剧',
+        'ri-ben-ju' => '日本剧',
+        'fu-li' => '福利',
+        'lun-li-pian' => '伦理片',
+        'gao-xiao' => '搞笑',
+        'yin-le' => '音乐',
+        'ying-shi' => '影视',
+        'qi-che' => '汽车',
+        'duan-ju-da-quan' => '短剧大全',
+        'yu-gao-pian' => '预告片',
+        'ti-yu' => '体育',
     ];
 
     /**

code/app/task/crawler/custom/Scheduler.php

@@ -0,0 +1,75 @@
+<?php
+
+declare(strict_types=1);
+
+namespace app\task\crawler\custom;
+
+use app\model\ChapterModel;
+use app\model\CustomCrawlerModel;
+use app\task\crawler\custom\page\Site;
+use microserver\QueueManage;
+use storage\StorageCore;
+
+/**
+ * @mixin think\Model
+ */
+class Scheduler
+{
+    /**
+     * Undocumented variable
+     *
+     * @var \microserver\QueueManage
+     */
+    protected $QueueManage = NULL;
+
+    /**
+     * Undocumented function
+     */
+    public function __construct()
+    {
+        $this->QueueManage = new QueueManage();
+    }
+
+    /**
+     * create a site-wide craw task
+     *
+     * @return void
+     */
+    public function crawlFull()
+    {
+        $AllCustomCrawlerModel = CustomCrawlerModel::select();
+
+        foreach ($AllCustomCrawlerModel as $CustomCrawlerModel) {
+
+            if (
+                $CustomCrawlerModel->pt_last_exec != 0 &&
+                ($CustomCrawlerModel->pt_last_exec + $CustomCrawlerModel->pt_limit)  > time()
+            ) {
+                continue;
+            }
+
+            $CustomCrawlerModel->pt_last_exec = time();
+
+            $CustomCrawlerModel->save();
+
+            switch ($CustomCrawlerModel->cc_code) {
+                case 'PULL_WEN_XUE_GUAN':
+                    $arrTask = [
+                        'callback' => [self::class, 'createCrawlWenXueGuanTask'],
+                        'data' => [
+                            'cc_data' => $CustomCrawlerModel->cc_data
+                        ],
+                    ];
+                    break;
+                default:
+                    break;
+            }
+            $this->QueueManage->set($arrTask);
+        }
+    }
+
+    public function createCrawlWenXueGuanTask(array $arrData)
+    {
+        $arrCcData = json_decode($arrData['cc_data'], true);
+    }
+}

code/app/task/crawler/fengchao/Scheduler.php

@@ -66,6 +66,12 @@ class Scheduler
         return  $this->Site;
     }
 
+    public $arrExcludeCategory = [
+        '体育',
+        '福利',
+        '伦理',
+        '伦理片',
+    ];
 
     /**
      * create a site-wide craw task
@@ -106,6 +112,11 @@ class Scheduler
                 continue;
             }
 
+            if (in_array($arrCategory['v_category_name'], $this->arrExcludeCategory)) {
+                continue;
+            }
+
+
             $intTotalPage = $arrCategory['total_page'];
             $arrFilter = [];
             for ($intStartPage = 1; $intStartPage <= $intTotalPage; $intStartPage++) {

code/app/task/crawler/wuxian/Scheduler.php

@@ -66,6 +66,12 @@ class Scheduler
         return  $this->Site;
     }
 
+    public $arrExcludeCategory = [
+        '体育',
+        '福利',
+        '伦理',
+        '伦理片',
+    ];
 
     /**
      * create a site-wide craw task
@@ -103,6 +109,10 @@ class Scheduler
                 continue;
             }
 
+            if (in_array($arrCategory['v_category_name'], $this->arrExcludeCategory)) {
+                continue;
+            }
+
             $intTotalPage = $arrCategory['total_page'];
             $arrFilter = [];
             for ($intStartPage = 1; $intStartPage <= $intTotalPage; $intStartPage++) {

code/app/task/crawler/youzhi/Scheduler.php

@@ -66,6 +66,12 @@ class Scheduler
         return  $this->Site;
     }
 
+    public $arrExcludeCategory = [
+        '体育',
+        '福利',
+        '伦理',
+        '伦理片',
+    ];
 
     /**
      * create a site-wide craw task
@@ -104,6 +110,10 @@ class Scheduler
                 continue;
             }
 
+            if (in_array($arrCategory['v_category_name'], $this->arrExcludeCategory)) {
+                continue;
+            }
+
             $intTotalPage = $arrCategory['total_page'];
             $arrFilter = [];
             for ($intStartPage = 1; $intStartPage <= $intTotalPage; $intStartPage++) {

code/app/task/crawler/youzhi/page/VideoCategory.php

@@ -96,7 +96,7 @@ class VideoCategory
             'pinyin' => 'hai-wai-ju',
         ],
         61 => [
-            'name' => '?理片',
+            'name' => '伦理片',
             'pinyin' => 'lun-li-pian',
         ],
         62 => [

code/app/task/module/PlanTask.php

@@ -15,7 +15,7 @@ use app\task\crawler\youzhi\Scheduler as YouZhiScheduler;
 use app\task\crawler\fengchao\Scheduler as FengChaoScheduler;
 use app\task\crawler\wuxian\Scheduler as WuXianScheduler;
 
-
+use app\task\crawler\custom\Scheduler as CustomScheduler;
 
 
 
@@ -29,48 +29,63 @@ class PlanTask
 
         foreach ($PlanTaskAll as $PlanTask) {
 
-            if ($PlanTask->pt_last_exec != 0 && ($PlanTask->pt_last_exec + $PlanTask->pt_limit)  > time()) continue;
+            if (
+                $PlanTask->pt_last_exec != 0 &&
+                ($PlanTask->pt_last_exec + $PlanTask->pt_limit)  > time()
+            ) {
+                continue;
+            }
 
             $PlanTask->pt_last_exec = time();
 
             $PlanTask->save();
 
             switch ($PlanTask->pt_code) {
-
+                case 'GENERATE_SITE_MAP':
+                    self::generateSiteMap();
+                    break;
                 case 'PULL_630_ZW_NOVEL':
                     self::pull630ZwNovel();
                     break;
                 case 'PULL_630_ZW_NOVEL_LATEST':
                     self::pull630ZwNovelLatest();
                     break;
-
                 case 'PULL_74_ZW_NOVEL':
                     self::pull74ZwNovel();
                     break;
                 case 'PULL_74_ZW_NOVEL_LATEST':
                     self::pull74ZwNovelLatest();
                     break;
-
                 case 'PULL_YOU_ZHI_SHI_PIN':
                     self::pullYouZhiShiPin();
                     break;
-
                 case 'PULL_WU_XIAN_SHI_PIN':
                     self::pullWuXianShiPin();
                     break;
-
-
                 case 'PULL_FENG_CHAO_SHI_PIN':
                     self::pullFengChaoShiPin();
                     break;
-
+                case 'CUSTOM_CRAWLER':
-                case 'GENERATE_SITE_MAP':
+                    self::customCrawler();
-                    self::generateSiteMap();
                     break;
             }
         }
     }
 
+    /**
+     * Undocumented function
+     *
+     * @return void
+     */
+    public static function customCrawler()
+    {
+        $arrTask = [
+            'callback' => [CustomScheduler::class, 'crawlFull'],
+            'data' => []
+        ];
+        $QueueManage =  QueueManage::getInstance();
+        $QueueManage->set($arrTask);
+    }
 
     /**
      * Undocumented function
@@ -84,7 +99,7 @@ class PlanTask
             'data' => []
         ];
 
-        $QueueManage = new QueueManage();
+        $QueueManage =  QueueManage::getInstance();
         $QueueManage->set($arrTask);
     }
 
@@ -100,7 +115,7 @@ class PlanTask
             'data' => []
         ];
 
-        $QueueManage = new QueueManage();
+        $QueueManage = QueueManage::getInstance(2);
         $QueueManage->set($arrTask);
     }
 
@@ -116,7 +131,7 @@ class PlanTask
             'data' => []
         ];
 
-        $QueueManage = new QueueManage();
+        $QueueManage =  QueueManage::getInstance();
         $QueueManage->set($arrTask);
     }
 
@@ -133,7 +148,7 @@ class PlanTask
             'data' => []
         ];
 
-        $QueueManage = new QueueManage();
+        $QueueManage =  QueueManage::getInstance();
         $QueueManage->set($arrTask);
     }
 
@@ -149,7 +164,7 @@ class PlanTask
             'data' => []
         ];
 
-        $QueueManage = new QueueManage();
+        $QueueManage =  QueueManage::getInstance();
         $QueueManage->set($arrTask);
     }
 
@@ -166,7 +181,7 @@ class PlanTask
             'data' => []
         ];
 
-        $QueueManage = new QueueManage();
+        $QueueManage =  QueueManage::getInstance();
         $QueueManage->set($arrTask);
     }
 
@@ -182,7 +197,7 @@ class PlanTask
             'data' => []
         ];
 
-        $QueueManage = new QueueManage();
+        $QueueManage =  QueueManage::getInstance();
         $QueueManage->set($arrTask);
     }
 

code/database/migrations/20240717191901_add_gg_pai_xu_colum_on_guang_gao_table.php

@@ -1,20 +0,0 @@
-<?php
-
-use think\migration\Migrator;
-use think\migration\db\Column;
-
-class AddGgPaiXuColumOnGuangGaoTable extends Migrator
-{
-    public function up()
-    {
-        // $Table = $this->table('guang_gao');
-
-        // $Table->addColumn('gg_sort', 'integer', ['after' => 'gg_tiao_zhuan_di_zhi'])->setComment("排序")->save();
-    }
-
-    public function down()
-    {
-        // $Table = $this->table('guang_gao');
-        // $Table->removeColumn('gg_sort')->save();
-    }
-}

code/database/migrations/20250505062541_credate_custom_crawler_table.php

@@ -0,0 +1,26 @@
+<?php
+
+use think\migration\Migrator;
+use think\migration\db\Column;
+
+class CredateCustomCrawlerTable extends Migrator
+{
+    public function up()
+    {
+        $Table = $this->table('custom_crawler', ['id' => false]);
+        $Table
+            ->addColumn(Column::integer('cc_id')->setUnsigned()->setIdentity(true)->setComment('自增主键'))
+            ->addColumn(Column::string('cc_code', 50)->setNullable(false)->setComment('映射代码编码'))
+            ->addColumn(Column::text('cc_data')->setNullable(true)->setComment('数据内容'))
+            ->addColumn(Column::integer('cc_limit')->setDefault(0)->setComment('采集间隔（秒）'))
+            ->addColumn(Column::timestamp('cc_last_exec')->setNullable(true)->setComment('最后执行时间'))
+            ->addIndex(['cc_id'], ['type' => 'primary'])
+            ->setComment('自定义爬虫表')
+            ->create();
+    }
+
+    public function down()
+    {
+        $this->table('custom_crawler')->drop()->save();
+    }
+}

code/database/seeders/CustomCrawlerSeeder.php

@@ -0,0 +1,33 @@
+<?php
+
+namespace database\seeders;
+
+use app\model\AdminUserModel;
+use app\model\CustomCrawlerModel;
+
+class CustomCrawlerSeeder
+{
+    static public function handle()
+    {
+        self::addCustomCrawler();
+    }
+
+    static public function addCustomCrawler()
+    {
+        $arrData = [
+            [
+                'cc_id' => 1,
+                'cc_code' => 'PULL_WEN_XUE_GUAN',
+                'cc_data' => '[86114,90205]',
+                'cc_limit' => 24 * 3600,
+            ],
+        ];
+
+        foreach ($arrData as $arrCustomCrawler) {
+            $CustomCrawlerModel = CustomCrawlerModel::where('cc_code', $arrCustomCrawler['cc_code'])->find();
+            if (empty($CustomCrawlerModel)) {
+                CustomCrawlerModel::insert($arrCustomCrawler);
+            }
+        }
+    }
+}

code/database/seeders/PlanTaskSeeder.php

@@ -73,6 +73,12 @@ class PlanTaskSeeder
                 'pt_enable' => 0,
                 'pt_limit' => 1 * 24 * 3600,
             ],
+            [
+                'pt_name' => '自定义爬虫任务触发',
+                'pt_code' => 'CUSTOM_CRAWLER',
+                'pt_enable' => 0,
+                'pt_limit' => 10,
+            ],
         ];
 
         foreach ($arrDataNovel as $arrPlanTask) {

code/database/seeders/SeedManager.php

@@ -11,5 +11,6 @@ class SeedManager
         SystemConfigSeeder::handle();
         ZiYuanDuanDianSeeder::handle();
         TemplateSeeder::handle();
+        CustomCrawlerSeeder::handle();
     }
 }

doc/CustomCrawler.md

@@ -0,0 +1,3 @@
+# Custom crawler
+
+