admin/SEONexus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: video model category

Browse Source
This commit is contained in:
Default
2025-05-05 16:25:13 +08:00
parent 017572b170
commit cac9177838
15 changed files with 297 additions and 156 deletions
Download Patch File Download Diff File Expand all files Collapse all files

75
code/app/task/crawler/custom/Scheduler.php Normal file
View File

@@ -0,0 +1,75 @@
<?php
declare(strict_types=1);
namespace app\task\crawler\custom;
use app\model\ChapterModel;
use app\model\CustomCrawlerModel;
use app\task\crawler\custom\page\Site;
use microserver\QueueManage;
use storage\StorageCore;
/**
* @mixin think\Model
*/
class Scheduler
{
/**
* Undocumented variable
*
* @var \microserver\QueueManage
*/
protected $QueueManage = NULL;
/**
* Undocumented function
*/
public function __construct()
{
$this->QueueManage = new QueueManage();
}
/**
* create a site-wide craw task
*
* @return void
*/
public function crawlFull()
{
$AllCustomCrawlerModel = CustomCrawlerModel::select();
foreach ($AllCustomCrawlerModel as $CustomCrawlerModel) {
if (
$CustomCrawlerModel->pt_last_exec != 0 &&
($CustomCrawlerModel->pt_last_exec + $CustomCrawlerModel->pt_limit) > time()
) {
continue;
}
$CustomCrawlerModel->pt_last_exec = time();
$CustomCrawlerModel->save();
switch ($CustomCrawlerModel->cc_code) {
case 'PULL_WEN_XUE_GUAN':
$arrTask = [
'callback' => [self::class, 'createCrawlWenXueGuanTask'],
'data' => [
'cc_data' => $CustomCrawlerModel->cc_data
],
];
break;
default:
break;
}
$this->QueueManage->set($arrTask);
}
}
public function createCrawlWenXueGuanTask(array $arrData)
{
$arrCcData = json_decode($arrData['cc_data'], true);
}
}

11
code/app/task/crawler/fengchao/Scheduler.php
View File

@@ -66,6 +66,12 @@ class Scheduler
return $this->Site;
}
public $arrExcludeCategory = [
'体育',
'福利',
'伦理',
'伦理片',
];
/**
* create a site-wide craw task
@@ -106,6 +112,11 @@ class Scheduler
continue;
}
if (in_array($arrCategory['v_category_name'], $this->arrExcludeCategory)) {
continue;
}
$intTotalPage = $arrCategory['total_page'];
$arrFilter = [];
for ($intStartPage = 1; $intStartPage <= $intTotalPage; $intStartPage++) {

10
code/app/task/crawler/wuxian/Scheduler.php
View File

@@ -66,6 +66,12 @@ class Scheduler
return $this->Site;
}
public $arrExcludeCategory = [
'体育',
'福利',
'伦理',
'伦理片',
];
/**
* create a site-wide craw task
@@ -103,6 +109,10 @@ class Scheduler
continue;
}
if (in_array($arrCategory['v_category_name'], $this->arrExcludeCategory)) {
continue;
}
$intTotalPage = $arrCategory['total_page'];
$arrFilter = [];
for ($intStartPage = 1; $intStartPage <= $intTotalPage; $intStartPage++) {

10
code/app/task/crawler/youzhi/Scheduler.php
View File

@@ -66,6 +66,12 @@ class Scheduler
return $this->Site;
}
public $arrExcludeCategory = [
'体育',
'福利',
'伦理',
'伦理片',
];
/**
* create a site-wide craw task
@@ -104,6 +110,10 @@ class Scheduler
continue;
}
if (in_array($arrCategory['v_category_name'], $this->arrExcludeCategory)) {
continue;
}
$intTotalPage = $arrCategory['total_page'];
$arrFilter = [];
for ($intStartPage = 1; $intStartPage <= $intTotalPage; $intStartPage++) {

2
code/app/task/crawler/youzhi/page/VideoCategory.php
View File

@@ -96,7 +96,7 @@ class VideoCategory
'pinyin' => 'hai-wai-ju',
],
61 => [
'name' => '?理片',
'name' => '伦理片',
'pinyin' => 'lun-li-pian',
],
62 => [

53
code/app/task/module/PlanTask.php
View File

@@ -15,7 +15,7 @@ use app\task\crawler\youzhi\Scheduler as YouZhiScheduler;
use app\task\crawler\fengchao\Scheduler as FengChaoScheduler;
use app\task\crawler\wuxian\Scheduler as WuXianScheduler;
use app\task\crawler\custom\Scheduler as CustomScheduler;
@@ -29,49 +29,64 @@ class PlanTask
foreach ($PlanTaskAll as $PlanTask) {
if ($PlanTask->pt_last_exec != 0 && ($PlanTask->pt_last_exec + $PlanTask->pt_limit) > time()) continue;
if (
$PlanTask->pt_last_exec != 0 &&
($PlanTask->pt_last_exec + $PlanTask->pt_limit) > time()
) {
continue;
}
$PlanTask->pt_last_exec = time();
$PlanTask->save();
switch ($PlanTask->pt_code) {
case 'GENERATE_SITE_MAP':
self::generateSiteMap();
break;
case 'PULL_630_ZW_NOVEL':
self::pull630ZwNovel();
break;
case 'PULL_630_ZW_NOVEL_LATEST':
self::pull630ZwNovelLatest();
break;
case 'PULL_74_ZW_NOVEL':
self::pull74ZwNovel();
break;
case 'PULL_74_ZW_NOVEL_LATEST':
self::pull74ZwNovelLatest();
break;
case 'PULL_YOU_ZHI_SHI_PIN':
self::pullYouZhiShiPin();
break;
case 'PULL_WU_XIAN_SHI_PIN':
self::pullWuXianShiPin();
break;
case 'PULL_FENG_CHAO_SHI_PIN':
self::pullFengChaoShiPin();
break;
case 'GENERATE_SITE_MAP':
self::generateSiteMap();
case 'CUSTOM_CRAWLER':
self::customCrawler();
break;
}
}
}
/**
* Undocumented function
*
* @return void
*/
public static function customCrawler()
{
$arrTask = [
'callback' => [CustomScheduler::class, 'crawlFull'],
'data' => []
];
$QueueManage = QueueManage::getInstance();
$QueueManage->set($arrTask);
}
/**
* Undocumented function
*
@@ -84,7 +99,7 @@ class PlanTask
'data' => []
];
$QueueManage = new QueueManage();
$QueueManage = QueueManage::getInstance();
$QueueManage->set($arrTask);
}
@@ -100,7 +115,7 @@ class PlanTask
'data' => []
];
$QueueManage = new QueueManage();
$QueueManage = QueueManage::getInstance(2);
$QueueManage->set($arrTask);
}
@@ -116,7 +131,7 @@ class PlanTask
'data' => []
];
$QueueManage = new QueueManage();
$QueueManage = QueueManage::getInstance();
$QueueManage->set($arrTask);
}
@@ -133,7 +148,7 @@ class PlanTask
'data' => []
];
$QueueManage = new QueueManage();
$QueueManage = QueueManage::getInstance();
$QueueManage->set($arrTask);
}
@@ -149,7 +164,7 @@ class PlanTask
'data' => []
];
$QueueManage = new QueueManage();
$QueueManage = QueueManage::getInstance();
$QueueManage->set($arrTask);
}
@@ -166,7 +181,7 @@ class PlanTask
'data' => []
];
$QueueManage = new QueueManage();
$QueueManage = QueueManage::getInstance();
$QueueManage->set($arrTask);
}
@@ -182,7 +197,7 @@ class PlanTask
'data' => []
];
$QueueManage = new QueueManage();
$QueueManage = QueueManage::getInstance();
$QueueManage->set($arrTask);
}