admin/SEONexus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'dev' of ssh://git.bgbg.live:18150/root/SEONexus into dev

Browse Source
This commit is contained in:
make
2025-05-05 16:26:56 +08:00
parent f827d86ee6 cac9177838
commit 6305233820
15 changed files with 297 additions and 156 deletions
Download Patch File Download Diff File Expand all files Collapse all files

100
code/app/api/controller/Statis.php
View File

@@ -9,6 +9,9 @@ use app\model\CategoryModel;
use app\model\NovelClicksModel;
use app\model\NovelModel;
use app\model\VideoClicksModel;
use app\task\crawler\fengchao\page\VideoCategory;
use app\task\crawler\wuxian\page\VideoCategory as PageVideoCategory;
use app\task\crawler\youzhi\page\VideoCategory as YouzhiPageVideoCategory;
use DateTime;
use db\mongo\MongoBase;
use ddl\DDLManage;
@@ -70,90 +73,23 @@ class Statis extends BaseController
public function test0909()
{
$a = [
'Hindi',
'Malay',
'Swahili',
'Tamil',
'上海',
'丹麦语',
'乌克兰',
'乌尔都语',
'他加',
'俄语',
'其它',
'匈牙利语',
'北印度语',
'印地语',
'印地语Hindi',
'印尼语',
'印度尼西亚语',
'印度语',
'印度语Hindi',
'因纽特语',
'国语',
'土库曼语',
'土耳',
'土耳其语',
'基库尤语',
'希腊',
'希腊语',
'库尔德语',
'德语',
'意大',
'意大利',
'意大利语',
'挪威语',
'捷克语',
'日语',
'普通话',
'林加拉语',
'毛利语',
'汉语',
'汉语普通',
'汉语普通话',
'汉语普通话Manda',
'河南方言',
'法语',
'波兰语',
'波斯语',
'泰卢固语',
'泰米尔',
'泰米尔语',
'泰米尔语Tamil',
'泰语',
'澳大利亚语',
'瑞典语',
'粤语',
'美国手语',
'英语',
'英语手语',
'荷兰语',
'菲律宾语',
'葡萄',
'葡萄牙语',
'藏语',
'西班',
'西班牙语',
'越南语',
'闽南语',
'阿拉',
'阿拉伯',
'阿拉伯语',
'韩语',
'马拉雅拉姆语',
'马来西亚语',
'马来语'
$a = VideoCategory::$arrCategory;
$b = PageVideoCategory::$arrCategory;
$c = YouzhiPageVideoCategory::$arrCategory;
];
$arrArea = [];
foreach ($a as $v) {
$arrArea[zhToPinYin($v)] = $v;
$x = [];
foreach ($a as $z) {
$x[$z['pinyin']] = $z['name'];
}
echo '<pre>';
krsort($arrArea);
echo var_export($arrArea, true);
foreach ($b as $z) {
$x[$z['pinyin']] = $z['name'];
}
foreach ($c as $z) {
$x[$z['pinyin']] = $z['name'];
}
echo var_export($x, true);
}
}

17
code/app/model/CustomCrawlerModel.php Normal file
View File

@@ -0,0 +1,17 @@
<?php
declare(strict_types=1);
namespace app\model;
use think\facade\Cache;
use think\Model;
/**
* @mixin think\Model
*/
class CustomCrawlerModel extends BaseModel
{
protected $name = 'custom_crawler';
protected $pk = 'cc_id';
}

86
code/app/model/VideoCategoryModel.php
View File

@@ -88,7 +88,7 @@ class VideoCategoryModel extends MongoModel
],
[
"v_category" => "电视剧",
"v_category_en" =>'dian-shi-ju',
"v_category_en" => 'dian-shi-ju',
"children" => [
[
"v_category_en" => "guo-chan-ju",
@@ -217,39 +217,57 @@ class VideoCategoryModel extends MongoModel
}
static $arrCategory = [
"da-lu-zong-yi" => "大陆综艺",
"ai-qing-pian" => "爱情片",
"ke-huan-pian" => "科幻片",
"ri-han-zong-yi" => "日韩综艺",
"ri-han-dong-man" => "日韩动漫",
"zhan-zheng-pian" => "战争片",
"ou-mei-ju" => "欧美剧",
"lun-li-pian" => "倫理片",
"han-guo-ju" => "韩国剧",
"yin-le" => "音乐",
"dong-hua-pian" => "动画片",
"guo-chan-dong-man" => "国产动漫",
"kong-bu-pian" => "恐怖片",
"tai-wan-ju" => "台湾剧",
"xiang-gang-ju" => "香港剧",
"xi-ju-pian" => "喜剧片",
"ying-shi" => "影视",
"qi-che" => "汽车",
"ou-mei-zong-yi" => "欧美综艺",
"ju-qing-pian" => "剧情片",
"tai-guo-ju" => "泰国剧",
"hai-wai-ju" => "海外剧",
"guo-chan-ju" => "国产剧",
"fu-li" => "福利",
"ou-mei-dong-man" => "欧美动漫",
"dong-zuo-pian" => "动作片",
"ri-ben-ju" => "日本剧",
"gang-tai-zong-yi" => "港台综艺",
"gang-tai-dong-man" => "港台动漫",
"ji-lu-pian" => "记录片",
"duan-ju-da-quan" => "短剧大全",
"gao-xiao" => "搞笑",
"hai-wai-dong-man" => "海外动漫",
'dian-ying' => '电影',
'dian-shi-ju' => '电视剧',
'zong-yi' => '综艺',
'dong-man' => '动漫',
'dong-zuo-pian' => '动作片',
'xi-ju-pian' => '喜剧片',
'ai-qing-pian' => '爱情片',
'ke-huan-pian' => '科幻片',
'kong-bu-pian' => '恐怖片',
'ju-qing-pian' => '剧情片',
'zhan-zheng-pian' => '战争片',
'guo-chan-ju' => '国产剧',
'ou-mei-ju' => '欧美剧',
'han-ju' => '韩剧',
'ri-ju' => '日剧',
'gang-ju' => '港剧',
'tai-ju' => '泰剧',
'ji-lu-pian' => '记录片',
'hai-wai-ju' => '海外剧',
'da-lu-zong-yi' => '大陆综艺',
'ri-han-zong-yi' => '日韩综艺',
'gang-tai-zong-yi' => '港台综艺',
'ou-mei-zong-yi' => '欧美综艺',
'guo-chan-dong-man' => '国产动漫',
'ri-han-dong-man' => '日韩动漫',
'ou-mei-dong-man' => '欧美动漫',
'dong-hua-pian' => '动画片',
'gang-tai-dong-man' => '港台动漫',
'hai-wai-dong-man' => '海外动漫',
'yan-chang-hui' => '演唱会',
'lun-li' => '伦理',
'shao-shi-dian-ying' => '邵氏电影',
'ti-yu-sai-shi' => '体育赛事',
'lan-qiu' => '篮球',
'zu-qiu' => '足球',
'si-nuo-ke' => '斯诺克',
'lian-xu-ju' => '连续剧',
'tai-wan-ju' => '台湾剧',
'han-guo-ju' => '韩国剧',
'xiang-gang-ju' => '香港剧',
'tai-guo-ju' => '泰国剧',
'ri-ben-ju' => '日本剧',
'fu-li' => '福利',
'lun-li-pian' => '伦理片',
'gao-xiao' => '搞笑',
'yin-le' => '音乐',
'ying-shi' => '影视',
'qi-che' => '汽车',
'duan-ju-da-quan' => '短剧大全',
'yu-gao-pian' => '预告片',
'ti-yu' => '体育',
];
/**

75
code/app/task/crawler/custom/Scheduler.php Normal file
View File

@@ -0,0 +1,75 @@
<?php
declare(strict_types=1);
namespace app\task\crawler\custom;
use app\model\ChapterModel;
use app\model\CustomCrawlerModel;
use app\task\crawler\custom\page\Site;
use microserver\QueueManage;
use storage\StorageCore;
/**
* @mixin think\Model
*/
class Scheduler
{
/**
* Undocumented variable
*
* @var \microserver\QueueManage
*/
protected $QueueManage = NULL;
/**
* Undocumented function
*/
public function __construct()
{
$this->QueueManage = new QueueManage();
}
/**
* create a site-wide craw task
*
* @return void
*/
public function crawlFull()
{
$AllCustomCrawlerModel = CustomCrawlerModel::select();
foreach ($AllCustomCrawlerModel as $CustomCrawlerModel) {
if (
$CustomCrawlerModel->pt_last_exec != 0 &&
($CustomCrawlerModel->pt_last_exec + $CustomCrawlerModel->pt_limit) > time()
) {
continue;
}
$CustomCrawlerModel->pt_last_exec = time();
$CustomCrawlerModel->save();
switch ($CustomCrawlerModel->cc_code) {
case 'PULL_WEN_XUE_GUAN':
$arrTask = [
'callback' => [self::class, 'createCrawlWenXueGuanTask'],
'data' => [
'cc_data' => $CustomCrawlerModel->cc_data
],
];
break;
default:
break;
}
$this->QueueManage->set($arrTask);
}
}
public function createCrawlWenXueGuanTask(array $arrData)
{
$arrCcData = json_decode($arrData['cc_data'], true);
}
}

11
code/app/task/crawler/fengchao/Scheduler.php
View File

@@ -66,6 +66,12 @@ class Scheduler
return $this->Site;
}
public $arrExcludeCategory = [
'体育',
'福利',
'伦理',
'伦理片',
];
/**
* create a site-wide craw task
@@ -106,6 +112,11 @@ class Scheduler
continue;
}
if (in_array($arrCategory['v_category_name'], $this->arrExcludeCategory)) {
continue;
}
$intTotalPage = $arrCategory['total_page'];
$arrFilter = [];
for ($intStartPage = 1; $intStartPage <= $intTotalPage; $intStartPage++) {

10
code/app/task/crawler/wuxian/Scheduler.php
View File

@@ -66,6 +66,12 @@ class Scheduler
return $this->Site;
}
public $arrExcludeCategory = [
'体育',
'福利',
'伦理',
'伦理片',
];
/**
* create a site-wide craw task
@@ -103,6 +109,10 @@ class Scheduler
continue;
}
if (in_array($arrCategory['v_category_name'], $this->arrExcludeCategory)) {
continue;
}
$intTotalPage = $arrCategory['total_page'];
$arrFilter = [];
for ($intStartPage = 1; $intStartPage <= $intTotalPage; $intStartPage++) {

10
code/app/task/crawler/youzhi/Scheduler.php
View File

@@ -66,6 +66,12 @@ class Scheduler
return $this->Site;
}
public $arrExcludeCategory = [
'体育',
'福利',
'伦理',
'伦理片',
];
/**
* create a site-wide craw task
@@ -104,6 +110,10 @@ class Scheduler
continue;
}
if (in_array($arrCategory['v_category_name'], $this->arrExcludeCategory)) {
continue;
}
$intTotalPage = $arrCategory['total_page'];
$arrFilter = [];
for ($intStartPage = 1; $intStartPage <= $intTotalPage; $intStartPage++) {

2
code/app/task/crawler/youzhi/page/VideoCategory.php
View File

@@ -96,7 +96,7 @@ class VideoCategory
'pinyin' => 'hai-wai-ju',
],
61 => [
'name' => '?理片',
'name' => '伦理片',
'pinyin' => 'lun-li-pian',
],
62 => [

51
code/app/task/module/PlanTask.php
View File

@@ -15,7 +15,7 @@ use app\task\crawler\youzhi\Scheduler as YouZhiScheduler;
use app\task\crawler\fengchao\Scheduler as FengChaoScheduler;
use app\task\crawler\wuxian\Scheduler as WuXianScheduler;
use app\task\crawler\custom\Scheduler as CustomScheduler;
@@ -29,48 +29,63 @@ class PlanTask
foreach ($PlanTaskAll as $PlanTask) {
if ($PlanTask->pt_last_exec != 0 && ($PlanTask->pt_last_exec + $PlanTask->pt_limit) > time()) continue;
if (
$PlanTask->pt_last_exec != 0 &&
($PlanTask->pt_last_exec + $PlanTask->pt_limit) > time()
) {
continue;
}
$PlanTask->pt_last_exec = time();
$PlanTask->save();
switch ($PlanTask->pt_code) {
case 'GENERATE_SITE_MAP':
self::generateSiteMap();
break;
case 'PULL_630_ZW_NOVEL':
self::pull630ZwNovel();
break;
case 'PULL_630_ZW_NOVEL_LATEST':
self::pull630ZwNovelLatest();
break;
case 'PULL_74_ZW_NOVEL':
self::pull74ZwNovel();
break;
case 'PULL_74_ZW_NOVEL_LATEST':
self::pull74ZwNovelLatest();
break;
case 'PULL_YOU_ZHI_SHI_PIN':
self::pullYouZhiShiPin();
break;
case 'PULL_WU_XIAN_SHI_PIN':
self::pullWuXianShiPin();
break;
case 'PULL_FENG_CHAO_SHI_PIN':
self::pullFengChaoShiPin();
break;
case 'GENERATE_SITE_MAP':
self::generateSiteMap();
case 'CUSTOM_CRAWLER':
self::customCrawler();
break;
}
}
}
/**
* Undocumented function
*
* @return void
*/
public static function customCrawler()
{
$arrTask = [
'callback' => [CustomScheduler::class, 'crawlFull'],
'data' => []
];
$QueueManage = QueueManage::getInstance();
$QueueManage->set($arrTask);
}
/**
* Undocumented function
@@ -84,7 +99,7 @@ class PlanTask
'data' => []
];
$QueueManage = new QueueManage();
$QueueManage = QueueManage::getInstance();
$QueueManage->set($arrTask);
}
@@ -100,7 +115,7 @@ class PlanTask
'data' => []
];
$QueueManage = new QueueManage();
$QueueManage = QueueManage::getInstance(2);
$QueueManage->set($arrTask);
}
@@ -116,7 +131,7 @@ class PlanTask
'data' => []
];
$QueueManage = new QueueManage();
$QueueManage = QueueManage::getInstance();
$QueueManage->set($arrTask);
}
@@ -133,7 +148,7 @@ class PlanTask
'data' => []
];
$QueueManage = new QueueManage();
$QueueManage = QueueManage::getInstance();
$QueueManage->set($arrTask);
}
@@ -149,7 +164,7 @@ class PlanTask
'data' => []
];
$QueueManage = new QueueManage();
$QueueManage = QueueManage::getInstance();
$QueueManage->set($arrTask);
}
@@ -166,7 +181,7 @@ class PlanTask
'data' => []
];
$QueueManage = new QueueManage();
$QueueManage = QueueManage::getInstance();
$QueueManage->set($arrTask);
}
@@ -182,7 +197,7 @@ class PlanTask
'data' => []
];
$QueueManage = new QueueManage();
$QueueManage = QueueManage::getInstance();
$QueueManage->set($arrTask);
}

20
code/database/migrations/20240717191901_add_gg_pai_xu_colum_on_guang_gao_table.php
View File

@@ -1,20 +0,0 @@
<?php
use think\migration\Migrator;
use think\migration\db\Column;
class AddGgPaiXuColumOnGuangGaoTable extends Migrator
{
public function up()
{
// $Table = $this->table('guang_gao');
// $Table->addColumn('gg_sort', 'integer', ['after' => 'gg_tiao_zhuan_di_zhi'])->setComment("排序")->save();
}
public function down()
{
// $Table = $this->table('guang_gao');
// $Table->removeColumn('gg_sort')->save();
}
}

26
code/database/migrations/20250505062541_credate_custom_crawler_table.php Normal file
View File

@@ -0,0 +1,26 @@
<?php
use think\migration\Migrator;
use think\migration\db\Column;
class CredateCustomCrawlerTable extends Migrator
{
public function up()
{
$Table = $this->table('custom_crawler', ['id' => false]);
$Table
->addColumn(Column::integer('cc_id')->setUnsigned()->setIdentity(true)->setComment('自增主键'))
->addColumn(Column::string('cc_code', 50)->setNullable(false)->setComment('映射代码编码'))
->addColumn(Column::text('cc_data')->setNullable(true)->setComment('数据内容'))
->addColumn(Column::integer('cc_limit')->setDefault(0)->setComment('采集间隔（秒）'))
->addColumn(Column::timestamp('cc_last_exec')->setNullable(true)->setComment('最后执行时间'))
->addIndex(['cc_id'], ['type' => 'primary'])
->setComment('自定义爬虫表')
->create();
}
public function down()
{
$this->table('custom_crawler')->drop()->save();
}
}

33
code/database/seeders/CustomCrawlerSeeder.php Normal file
View File

@@ -0,0 +1,33 @@
<?php
namespace database\seeders;
use app\model\AdminUserModel;
use app\model\CustomCrawlerModel;
class CustomCrawlerSeeder
{
static public function handle()
{
self::addCustomCrawler();
}
static public function addCustomCrawler()
{
$arrData = [
[
'cc_id' => 1,
'cc_code' => 'PULL_WEN_XUE_GUAN',
'cc_data' => '[86114,90205]',
'cc_limit' => 24 * 3600,
],
];
foreach ($arrData as $arrCustomCrawler) {
$CustomCrawlerModel = CustomCrawlerModel::where('cc_code', $arrCustomCrawler['cc_code'])->find();
if (empty($CustomCrawlerModel)) {
CustomCrawlerModel::insert($arrCustomCrawler);
}
}
}
}

6
code/database/seeders/PlanTaskSeeder.php
View File

@@ -73,6 +73,12 @@ class PlanTaskSeeder
'pt_enable' => 0,
'pt_limit' => 1 * 24 * 3600,
],
[
'pt_name' => '自定义爬虫任务触发',
'pt_code' => 'CUSTOM_CRAWLER',
'pt_enable' => 0,
'pt_limit' => 10,
],
];
foreach ($arrDataNovel as $arrPlanTask) {

1
code/database/seeders/SeedManager.php
View File

@@ -11,5 +11,6 @@ class SeedManager
SystemConfigSeeder::handle();
ZiYuanDuanDianSeeder::handle();
TemplateSeeder::handle();
CustomCrawlerSeeder::handle();
}
}

3
doc/CustomCrawler.md Normal file
View File

@@ -0,0 +1,3 @@
# Custom crawler