diff --git a/code/app/api/controller/Statis.php b/code/app/api/controller/Statis.php index a37324f..76df3e6 100644 --- a/code/app/api/controller/Statis.php +++ b/code/app/api/controller/Statis.php @@ -9,6 +9,9 @@ use app\model\CategoryModel; use app\model\NovelClicksModel; use app\model\NovelModel; use app\model\VideoClicksModel; +use app\task\crawler\fengchao\page\VideoCategory; +use app\task\crawler\wuxian\page\VideoCategory as PageVideoCategory; +use app\task\crawler\youzhi\page\VideoCategory as YouzhiPageVideoCategory; use DateTime; use db\mongo\MongoBase; use ddl\DDLManage; @@ -70,90 +73,23 @@ class Statis extends BaseController public function test0909() { - $a = [ - 'Hindi', - 'Malay', - 'Swahili', - 'Tamil', - '上海', - '丹麦语', - '乌克兰', - '乌尔都语', - '他加', - '俄语', - '其它', - '匈牙利语', - '北印度语', - '印地语', - '印地语Hindi', - '印尼语', - '印度尼西亚语', - '印度语', - '印度语Hindi', - '因纽特语', - '国语', - '土库曼语', - '土耳', - '土耳其语', - '基库尤语', - '希腊', - '希腊语', - '库尔德语', - '德语', - '意大', - '意大利', - '意大利语', - '挪威语', - '捷克语', - '日语', - '普通话', - '林加拉语', - '毛利语', - '汉语', - '汉语普通', - '汉语普通话', - '汉语普通话Manda', - '河南方言', - '法语', - '波兰语', - '波斯语', - '泰卢固语', - '泰米尔', - '泰米尔语', - '泰米尔语Tamil', - '泰语', - '澳大利亚语', - '瑞典语', - '粤语', - '美国手语', - '英语', - '英语手语', - '荷兰语', - '菲律宾语', - '葡萄', - '葡萄牙语', - '藏语', - '西班', - '西班牙语', - '越南语', - '闽南语', - '阿拉', - '阿拉伯', - '阿拉伯语', - '韩语', - '马拉雅拉姆语', - '马来西亚语', - '马来语' + $a = VideoCategory::$arrCategory; + $b = PageVideoCategory::$arrCategory; + $c = YouzhiPageVideoCategory::$arrCategory; - ]; - - $arrArea = []; - foreach ($a as $v) { - $arrArea[zhToPinYin($v)] = $v; + $x = []; + foreach ($a as $z) { + $x[$z['pinyin']] = $z['name']; } - echo '
';
 
-        krsort($arrArea);
-        echo var_export($arrArea, true);
+        foreach ($b as $z) {
+            $x[$z['pinyin']] = $z['name'];
+        }
+
+        foreach ($c as $z) {
+            $x[$z['pinyin']] = $z['name'];
+        }
+
+        echo var_export($x, true);
     }
 }
diff --git a/code/app/model/CustomCrawlerModel.php b/code/app/model/CustomCrawlerModel.php
new file mode 100644
index 0000000..81e0023
--- /dev/null
+++ b/code/app/model/CustomCrawlerModel.php
@@ -0,0 +1,17 @@
+ "电视剧",
-            "v_category_en" =>'dian-shi-ju',
+            "v_category_en" => 'dian-shi-ju',
             "children" => [
                 [
                     "v_category_en" => "guo-chan-ju",
@@ -217,39 +217,57 @@ class VideoCategoryModel extends MongoModel
     }
 
     static $arrCategory = [
-        "da-lu-zong-yi" => "大陆综艺",
-        "ai-qing-pian" => "爱情片",
-        "ke-huan-pian" => "科幻片",
-        "ri-han-zong-yi" => "日韩综艺",
-        "ri-han-dong-man" => "日韩动漫",
-        "zhan-zheng-pian" => "战争片",
-        "ou-mei-ju" => "欧美剧",
-        "lun-li-pian" => "倫理片",
-        "han-guo-ju" => "韩国剧",
-        "yin-le" => "音乐",
-        "dong-hua-pian" => "动画片",
-        "guo-chan-dong-man" => "国产动漫",
-        "kong-bu-pian" => "恐怖片",
-        "tai-wan-ju" => "台湾剧",
-        "xiang-gang-ju" => "香港剧",
-        "xi-ju-pian" => "喜剧片",
-        "ying-shi" => "影视",
-        "qi-che" => "汽车",
-        "ou-mei-zong-yi" => "欧美综艺",
-        "ju-qing-pian" => "剧情片",
-        "tai-guo-ju" => "泰国剧",
-        "hai-wai-ju" => "海外剧",
-        "guo-chan-ju" => "国产剧",
-        "fu-li" => "福利",
-        "ou-mei-dong-man" => "欧美动漫",
-        "dong-zuo-pian" => "动作片",
-        "ri-ben-ju" => "日本剧",
-        "gang-tai-zong-yi" => "港台综艺",
-        "gang-tai-dong-man" => "港台动漫",
-        "ji-lu-pian" => "记录片",
-        "duan-ju-da-quan" => "短剧大全",
-        "gao-xiao" => "搞笑",
-        "hai-wai-dong-man" => "海外动漫",
+        'dian-ying' => '电影',
+        'dian-shi-ju' => '电视剧',
+        'zong-yi' => '综艺',
+        'dong-man' => '动漫',
+        'dong-zuo-pian' => '动作片',
+        'xi-ju-pian' => '喜剧片',
+        'ai-qing-pian' => '爱情片',
+        'ke-huan-pian' => '科幻片',
+        'kong-bu-pian' => '恐怖片',
+        'ju-qing-pian' => '剧情片',
+        'zhan-zheng-pian' => '战争片',
+        'guo-chan-ju' => '国产剧',
+        'ou-mei-ju' => '欧美剧',
+        'han-ju' => '韩剧',
+        'ri-ju' => '日剧',
+        'gang-ju' => '港剧',
+        'tai-ju' => '泰剧',
+        'ji-lu-pian' => '记录片',
+        'hai-wai-ju' => '海外剧',
+        'da-lu-zong-yi' => '大陆综艺',
+        'ri-han-zong-yi' => '日韩综艺',
+        'gang-tai-zong-yi' => '港台综艺',
+        'ou-mei-zong-yi' => '欧美综艺',
+        'guo-chan-dong-man' => '国产动漫',
+        'ri-han-dong-man' => '日韩动漫',
+        'ou-mei-dong-man' => '欧美动漫',
+        'dong-hua-pian' => '动画片',
+        'gang-tai-dong-man' => '港台动漫',
+        'hai-wai-dong-man' => '海外动漫',
+        'yan-chang-hui' => '演唱会',
+        'lun-li' => '伦理',
+        'shao-shi-dian-ying' => '邵氏电影',
+        'ti-yu-sai-shi' => '体育赛事',
+        'lan-qiu' => '篮球',
+        'zu-qiu' => '足球',
+        'si-nuo-ke' => '斯诺克',
+        'lian-xu-ju' => '连续剧',
+        'tai-wan-ju' => '台湾剧',
+        'han-guo-ju' => '韩国剧',
+        'xiang-gang-ju' => '香港剧',
+        'tai-guo-ju' => '泰国剧',
+        'ri-ben-ju' => '日本剧',
+        'fu-li' => '福利',
+        'lun-li-pian' => '伦理片',
+        'gao-xiao' => '搞笑',
+        'yin-le' => '音乐',
+        'ying-shi' => '影视',
+        'qi-che' => '汽车',
+        'duan-ju-da-quan' => '短剧大全',
+        'yu-gao-pian' => '预告片',
+        'ti-yu' => '体育',
     ];
 
     /**
diff --git a/code/app/task/crawler/custom/Scheduler.php b/code/app/task/crawler/custom/Scheduler.php
new file mode 100644
index 0000000..96604eb
--- /dev/null
+++ b/code/app/task/crawler/custom/Scheduler.php
@@ -0,0 +1,75 @@
+QueueManage = new QueueManage();
+    }
+
+    /**
+     * create a site-wide craw task
+     *
+     * @return void
+     */
+    public function crawlFull()
+    {
+        $AllCustomCrawlerModel = CustomCrawlerModel::select();
+
+        foreach ($AllCustomCrawlerModel as $CustomCrawlerModel) {
+
+            if (
+                $CustomCrawlerModel->pt_last_exec != 0 &&
+                ($CustomCrawlerModel->pt_last_exec + $CustomCrawlerModel->pt_limit)  > time()
+            ) {
+                continue;
+            }
+
+            $CustomCrawlerModel->pt_last_exec = time();
+
+            $CustomCrawlerModel->save();
+
+            switch ($CustomCrawlerModel->cc_code) {
+                case 'PULL_WEN_XUE_GUAN':
+                    $arrTask = [
+                        'callback' => [self::class, 'createCrawlWenXueGuanTask'],
+                        'data' => [
+                            'cc_data' => $CustomCrawlerModel->cc_data
+                        ],
+                    ];
+                    break;
+                default:
+                    break;
+            }
+            $this->QueueManage->set($arrTask);
+        }
+    }
+
+    public function createCrawlWenXueGuanTask(array $arrData)
+    {
+        $arrCcData = json_decode($arrData['cc_data'], true);
+    }
+}
diff --git a/code/app/task/crawler/fengchao/Scheduler.php b/code/app/task/crawler/fengchao/Scheduler.php
index 6d6f89c..3ba0d14 100644
--- a/code/app/task/crawler/fengchao/Scheduler.php
+++ b/code/app/task/crawler/fengchao/Scheduler.php
@@ -66,6 +66,12 @@ class Scheduler
         return  $this->Site;
     }
 
+    public $arrExcludeCategory = [
+        '体育',
+        '福利',
+        '伦理',
+        '伦理片',
+    ];
 
     /**
      * create a site-wide craw task
@@ -106,6 +112,11 @@ class Scheduler
                 continue;
             }
 
+            if (in_array($arrCategory['v_category_name'], $this->arrExcludeCategory)) {
+                continue;
+            }
+
+
             $intTotalPage = $arrCategory['total_page'];
             $arrFilter = [];
             for ($intStartPage = 1; $intStartPage <= $intTotalPage; $intStartPage++) {
diff --git a/code/app/task/crawler/wuxian/Scheduler.php b/code/app/task/crawler/wuxian/Scheduler.php
index 73fa5b3..05e08bd 100644
--- a/code/app/task/crawler/wuxian/Scheduler.php
+++ b/code/app/task/crawler/wuxian/Scheduler.php
@@ -66,6 +66,12 @@ class Scheduler
         return  $this->Site;
     }
 
+    public $arrExcludeCategory = [
+        '体育',
+        '福利',
+        '伦理',
+        '伦理片',
+    ];
 
     /**
      * create a site-wide craw task
@@ -103,6 +109,10 @@ class Scheduler
                 continue;
             }
 
+            if (in_array($arrCategory['v_category_name'], $this->arrExcludeCategory)) {
+                continue;
+            }
+
             $intTotalPage = $arrCategory['total_page'];
             $arrFilter = [];
             for ($intStartPage = 1; $intStartPage <= $intTotalPage; $intStartPage++) {
diff --git a/code/app/task/crawler/youzhi/Scheduler.php b/code/app/task/crawler/youzhi/Scheduler.php
index d41e45a..43b0195 100644
--- a/code/app/task/crawler/youzhi/Scheduler.php
+++ b/code/app/task/crawler/youzhi/Scheduler.php
@@ -66,6 +66,12 @@ class Scheduler
         return  $this->Site;
     }
 
+    public $arrExcludeCategory = [
+        '体育',
+        '福利',
+        '伦理',
+        '伦理片',
+    ];
 
     /**
      * create a site-wide craw task
@@ -104,6 +110,10 @@ class Scheduler
                 continue;
             }
 
+            if (in_array($arrCategory['v_category_name'], $this->arrExcludeCategory)) {
+                continue;
+            }
+
             $intTotalPage = $arrCategory['total_page'];
             $arrFilter = [];
             for ($intStartPage = 1; $intStartPage <= $intTotalPage; $intStartPage++) {
diff --git a/code/app/task/crawler/youzhi/page/VideoCategory.php b/code/app/task/crawler/youzhi/page/VideoCategory.php
index 4cc872f..03ecb18 100644
--- a/code/app/task/crawler/youzhi/page/VideoCategory.php
+++ b/code/app/task/crawler/youzhi/page/VideoCategory.php
@@ -96,7 +96,7 @@ class VideoCategory
             'pinyin' => 'hai-wai-ju',
         ],
         61 => [
-            'name' => '?理片',
+            'name' => '伦理片',
             'pinyin' => 'lun-li-pian',
         ],
         62 => [
diff --git a/code/app/task/module/PlanTask.php b/code/app/task/module/PlanTask.php
index 251dacb..b3a88ad 100644
--- a/code/app/task/module/PlanTask.php
+++ b/code/app/task/module/PlanTask.php
@@ -15,7 +15,7 @@ use app\task\crawler\youzhi\Scheduler as YouZhiScheduler;
 use app\task\crawler\fengchao\Scheduler as FengChaoScheduler;
 use app\task\crawler\wuxian\Scheduler as WuXianScheduler;
 
-
+use app\task\crawler\custom\Scheduler as CustomScheduler;
 
 
 
@@ -29,49 +29,64 @@ class PlanTask
 
         foreach ($PlanTaskAll as $PlanTask) {
 
-            if ($PlanTask->pt_last_exec != 0 && ($PlanTask->pt_last_exec + $PlanTask->pt_limit)  > time()) continue;
+            if (
+                $PlanTask->pt_last_exec != 0 &&
+                ($PlanTask->pt_last_exec + $PlanTask->pt_limit)  > time()
+            ) {
+                continue;
+            }
 
             $PlanTask->pt_last_exec = time();
 
             $PlanTask->save();
 
             switch ($PlanTask->pt_code) {
-
+                case 'GENERATE_SITE_MAP':
+                    self::generateSiteMap();
+                    break;
                 case 'PULL_630_ZW_NOVEL':
                     self::pull630ZwNovel();
                     break;
                 case 'PULL_630_ZW_NOVEL_LATEST':
                     self::pull630ZwNovelLatest();
                     break;
-
                 case 'PULL_74_ZW_NOVEL':
                     self::pull74ZwNovel();
                     break;
                 case 'PULL_74_ZW_NOVEL_LATEST':
                     self::pull74ZwNovelLatest();
                     break;
-
                 case 'PULL_YOU_ZHI_SHI_PIN':
                     self::pullYouZhiShiPin();
                     break;
-
                 case 'PULL_WU_XIAN_SHI_PIN':
                     self::pullWuXianShiPin();
                     break;
-
-
                 case 'PULL_FENG_CHAO_SHI_PIN':
                     self::pullFengChaoShiPin();
                     break;
-
-                case 'GENERATE_SITE_MAP':
-                    self::generateSiteMap();
+                case 'CUSTOM_CRAWLER':
+                    self::customCrawler();
                     break;
             }
         }
     }
 
-    
+    /**
+     * Undocumented function
+     *
+     * @return void
+     */
+    public static function customCrawler()
+    {
+        $arrTask = [
+            'callback' => [CustomScheduler::class, 'crawlFull'],
+            'data' => []
+        ];
+        $QueueManage =  QueueManage::getInstance();
+        $QueueManage->set($arrTask);
+    }
+
     /**
      * Undocumented function
      *
@@ -84,7 +99,7 @@ class PlanTask
             'data' => []
         ];
 
-        $QueueManage = new QueueManage();
+        $QueueManage =  QueueManage::getInstance();
         $QueueManage->set($arrTask);
     }
 
@@ -100,7 +115,7 @@ class PlanTask
             'data' => []
         ];
 
-        $QueueManage = new QueueManage();
+        $QueueManage = QueueManage::getInstance(2);
         $QueueManage->set($arrTask);
     }
 
@@ -116,7 +131,7 @@ class PlanTask
             'data' => []
         ];
 
-        $QueueManage = new QueueManage();
+        $QueueManage =  QueueManage::getInstance();
         $QueueManage->set($arrTask);
     }
 
@@ -133,7 +148,7 @@ class PlanTask
             'data' => []
         ];
 
-        $QueueManage = new QueueManage();
+        $QueueManage =  QueueManage::getInstance();
         $QueueManage->set($arrTask);
     }
 
@@ -149,7 +164,7 @@ class PlanTask
             'data' => []
         ];
 
-        $QueueManage = new QueueManage();
+        $QueueManage =  QueueManage::getInstance();
         $QueueManage->set($arrTask);
     }
 
@@ -166,7 +181,7 @@ class PlanTask
             'data' => []
         ];
 
-        $QueueManage = new QueueManage();
+        $QueueManage =  QueueManage::getInstance();
         $QueueManage->set($arrTask);
     }
 
@@ -182,7 +197,7 @@ class PlanTask
             'data' => []
         ];
 
-        $QueueManage = new QueueManage();
+        $QueueManage =  QueueManage::getInstance();
         $QueueManage->set($arrTask);
     }
 
diff --git a/code/database/migrations/20240717191901_add_gg_pai_xu_colum_on_guang_gao_table.php b/code/database/migrations/20240717191901_add_gg_pai_xu_colum_on_guang_gao_table.php
deleted file mode 100644
index 5c7b011..0000000
--- a/code/database/migrations/20240717191901_add_gg_pai_xu_colum_on_guang_gao_table.php
+++ /dev/null
@@ -1,20 +0,0 @@
-table('guang_gao');
-
-        // $Table->addColumn('gg_sort', 'integer', ['after' => 'gg_tiao_zhuan_di_zhi'])->setComment("排序")->save();
-    }
-
-    public function down()
-    {
-        // $Table = $this->table('guang_gao');
-        // $Table->removeColumn('gg_sort')->save();
-    }
-}
diff --git a/code/database/migrations/20250505062541_credate_custom_crawler_table.php b/code/database/migrations/20250505062541_credate_custom_crawler_table.php
new file mode 100644
index 0000000..2b334b5
--- /dev/null
+++ b/code/database/migrations/20250505062541_credate_custom_crawler_table.php
@@ -0,0 +1,26 @@
+table('custom_crawler', ['id' => false]);
+        $Table
+            ->addColumn(Column::integer('cc_id')->setUnsigned()->setIdentity(true)->setComment('自增主键'))
+            ->addColumn(Column::string('cc_code', 50)->setNullable(false)->setComment('映射代码编码'))
+            ->addColumn(Column::text('cc_data')->setNullable(true)->setComment('数据内容'))
+            ->addColumn(Column::integer('cc_limit')->setDefault(0)->setComment('采集间隔（秒）'))
+            ->addColumn(Column::timestamp('cc_last_exec')->setNullable(true)->setComment('最后执行时间'))
+            ->addIndex(['cc_id'], ['type' => 'primary'])
+            ->setComment('自定义爬虫表')
+            ->create();
+    }
+
+    public function down()
+    {
+        $this->table('custom_crawler')->drop()->save();
+    }
+}
diff --git a/code/database/seeders/CustomCrawlerSeeder.php b/code/database/seeders/CustomCrawlerSeeder.php
new file mode 100644
index 0000000..213ecc7
--- /dev/null
+++ b/code/database/seeders/CustomCrawlerSeeder.php
@@ -0,0 +1,33 @@
+ 1,
+                'cc_code' => 'PULL_WEN_XUE_GUAN',
+                'cc_data' => '[86114,90205]',
+                'cc_limit' => 24 * 3600,
+            ],
+        ];
+
+        foreach ($arrData as $arrCustomCrawler) {
+            $CustomCrawlerModel = CustomCrawlerModel::where('cc_code', $arrCustomCrawler['cc_code'])->find();
+            if (empty($CustomCrawlerModel)) {
+                CustomCrawlerModel::insert($arrCustomCrawler);
+            }
+        }
+    }
+}
diff --git a/code/database/seeders/PlanTaskSeeder.php b/code/database/seeders/PlanTaskSeeder.php
index a3f94f7..8edeceb 100644
--- a/code/database/seeders/PlanTaskSeeder.php
+++ b/code/database/seeders/PlanTaskSeeder.php
@@ -73,6 +73,12 @@ class PlanTaskSeeder
                 'pt_enable' => 0,
                 'pt_limit' => 1 * 24 * 3600,
             ],
+            [
+                'pt_name' => '自定义爬虫任务触发',
+                'pt_code' => 'CUSTOM_CRAWLER',
+                'pt_enable' => 0,
+                'pt_limit' => 10,
+            ],
         ];
 
         foreach ($arrDataNovel as $arrPlanTask) {
diff --git a/code/database/seeders/SeedManager.php b/code/database/seeders/SeedManager.php
index 71cb80d..953a2f3 100644
--- a/code/database/seeders/SeedManager.php
+++ b/code/database/seeders/SeedManager.php
@@ -11,5 +11,6 @@ class SeedManager
         SystemConfigSeeder::handle();
         ZiYuanDuanDianSeeder::handle();
         TemplateSeeder::handle();
+        CustomCrawlerSeeder::handle();
     }
 }
diff --git a/doc/CustomCrawler.md b/doc/CustomCrawler.md
new file mode 100644
index 0000000..60dfe55
--- /dev/null
+++ b/doc/CustomCrawler.md
@@ -0,0 +1,3 @@
+# Custom crawler
+
+