admin/SEONexus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

debug

Browse Source
This commit is contained in:
make
2025-06-03 23:02:57 +08:00
parent 806429f948
commit 499882f1a4
10 changed files with 121 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
code/app/common.php
View File

@@ -716,6 +716,24 @@ namespace {
return false;
}
/**
* 从 URL、URI 或路径中提取小说id（不带后缀）
*
* @param [type] $path /30/30345/
* @return void
*/
function extractNovelId($path)
{
if (!is_string($path) || empty($path)) {
return null;
}
$segments = explode('/', trim($path, '/'));
$last = end($segments);
return is_numeric($last) ? $last : null;
}
/**
* 从 URL、URI 或路径中提取文件名（不带后缀）

3
code/app/task/crawler/custom/Scheduler.php
View File

@@ -47,14 +47,15 @@ class Scheduler
*/
public function crawlFull()
{
var_dump('custom-crawlFull');
$AllCustomCrawlerModel = CustomCrawlerModel::select();
foreach ($AllCustomCrawlerModel as $CustomCrawlerModel) {
if (
$CustomCrawlerModel->pt_last_exec != 0 &&
($CustomCrawlerModel->pt_last_exec + $CustomCrawlerModel->pt_limit) > time()
) {
var_dump('continue');
continue;
}

1
code/app/task/crawler/custom/WenXueGuan.php
View File

@@ -46,6 +46,7 @@ class WenXueGuan
public function createCrawlWenXueGuanTask(array $arrData)
{
$arrCcData = json_decode($arrData['cc_data'], true);
// var_dump($arrCcData);
foreach ($arrCcData as $intNSourceId) {
$arrTask = [
'callback' => [WxgScheduler::class, 'getNovelInfo'],

1
code/app/task/crawler/wxg/Scheduler.php
View File

@@ -107,6 +107,7 @@ class Scheduler
// return;
$boolResult = $NovelPage->saveNovelInfo();
// var_dump($boolResult);
// return;
if ($boolResult == false) {

9
code/app/task/crawler/wxg/page/NovelPage.php
View File

@@ -124,10 +124,12 @@ class NovelPage extends BasePage
$this->arrNovelInfo[$strKey] = $Item->attr('content');
});
if (!empty($this->arrNovelInfo['og_novel_category']) && $this->arrNovelInfo['og_novel_category'] == '玄幻小说') {
$this->arrNovelInfo['og_novel_category'] = '玄幻修真';
}
if (!empty($this->arrNovelInfo['og_novel_category']) ) {
if($this->arrNovelInfo['og_novel_category'] == '玄幻小说' || $this->arrNovelInfo['og_novel_category'] == '武侠小说'){
$this->arrNovelInfo['og_novel_category'] = '玄幻修真';
}
}
$this->getReletedNovel();
}
@@ -242,6 +244,7 @@ class NovelPage extends BasePage
$arrUpdate = array_merge($arrUpdate, $arrNovelInfo);
$arrUpdate['n_related_novel'] = $this->arrRelatedNovel;
$arrUpdate['n_forge_novel'] = $this->arrForgeNovel;
$arrUpdate['n_category_pinyin'] = CategoryModel::getPinYin($this->arrNovelInfo['og_novel_category']);
$arrUpdate['n_category_sex_en'] = CategoryModel::getSexEn($this->arrNovelInfo['og_novel_category']);
$arrUpdate['n_category_sex_zh'] = CategoryModel::getSexZh($this->arrNovelInfo['og_novel_category']);

31
code/app/task/crawler/zw/Scheduler.php
View File

@@ -96,17 +96,20 @@ class Scheduler
*/
public function getNovelInfo(array $arrFilter)
{
// var_dump('getNovelInfo------------------------------------------------------------');
$Site = $this->getSite();
$arrNovelPageList = $Site->getNovelPageList($arrFilter);
// var_dump($arrNovelPageList);
// var_dump('$arrNovelPageList------------------------------------------------------------');
$arrCover = [];
foreach ($arrNovelPageList as $NovelPage) {
// $arrNovel = $NovelPage->getNovelInfo();
// print_r($arrNovel);
// return;
// var_dump('$NovelPage------------------------------------------------------------');
$boolResult = $NovelPage->saveNovelInfo();
// var_dump($boolResult);
// return;
if ($boolResult == false) {
@@ -130,6 +133,7 @@ class Scheduler
}
$arrChapterList = $NovelPage->getChapters();
// var_dump($arrChapterList);
$this->fetchChapter($NovelPage->intNId, $NovelPage->getData()['page'], $arrChapterList);
}
@@ -147,9 +151,13 @@ class Scheduler
{
$intSliceSize = $this->intSlice;
$arrUri = [];
$strUri = '/list/lastupdate_%s_0_0_%s.html';
// $strUri = '/list/lastupdate_%s_0_0_%s.html';
$strUri = '/sort/%s_%s/';
$intEndCategory = $intEndPage = 10;
$intEndCategory = 10;
$intEndPage = 10;
// $intEndCategory = 2;
// $intEndPage = 2;
for ($intCategoryKey = 0; $intCategoryKey < $intEndCategory; $intCategoryKey++) {
for ($intPage = 1; $intPage <= $intEndPage; $intPage++) {
@@ -159,6 +167,8 @@ class Scheduler
$arrUri = array_chunk($arrUri, $intSliceSize);
// var_dump($arrUri);
// return ;
$arrTask = [
'callback' => [static::class, 'getLatestChapter'],
'data' => [],
@@ -181,27 +191,34 @@ class Scheduler
$Site = $this->getSite();
$arrLatestListPageList = $Site->getLatestListPageList($arrFilter);
$arrNSourceId = [];
foreach ($arrLatestListPageList as $LatestListPage) {
$arrResult = $LatestListPage->getNovelFilter();
// var_dump($arrResult);
$arrNSourceId = array_merge($arrNSourceId, $arrResult);
}
// var_dump($arrNSourceId);
$arrNSourceId = array_unique($arrNSourceId);
$arrNSourceId = array_values($arrNSourceId);
$intSliceSize = $this->intSlice;
$arrFilter = [];
// var_dump($arrNSourceId);
// return ;
foreach ($arrNSourceId as $intKey => $intNSourceId) {
$arrFilter[] = [
'n_source_id' => $intNSourceId,
'page' => 0,
];
// var_dump($intKey);
// var_dump($intSliceSize);
// var_dump($intKey % $intSliceSize);
// var_dump('---------------------------------------------');
if ($intKey % $intSliceSize == 0) {
// var_dump('$intKey % $intSliceSize == 0');
$this->fetchNovel($arrFilter);
$arrFilter = [];
}

7
code/app/task/crawler/zw/page/LatestListPage.php
View File

@@ -49,13 +49,16 @@ class LatestListPage extends BasePage
if ($this->isMobile()) {
$arrFilter = $this->getQueryList()->find('div.wrap-box')->eq(0)->find('ul.sort_list li')->map(function ($li) {
$strUri = $li->find('a')->eq(0)->attr('href');
return supperExtractFilename($strUri);
$strNovelId = extractNovelId($strUri);
return (int)$strNovelId;
})->filter()->all();
} else {
$arrFilter = $this->getQueryList()->find('div.layout.layout2.layout-col2.fl ul.txt-list li')->map(function ($li) {
$strUri = $li->find('a')->eq(0)->attr('href');
return supperExtractFilename($strUri);
$strNovelId = extractNovelId($strUri);
return (int)$strNovelId;
return extractNovelId($strUri);
})->filter()->all();
}

7
code/app/task/crawler/zw/page/Site.php
View File

@@ -163,6 +163,7 @@ class Site
public function getNovelPageUri($intNSourceId, $intPage): string
{
$intNSourceId = int($intNSourceId);
$intUriFirst = intdiv($intNSourceId, 1000);
$strUri = '';
@@ -275,12 +276,16 @@ class Site
$arrPromises = [];
foreach ($arrFilter as $intKey => $strUri) {
$arrPromises[$intKey] = $this->getClient()->getAsync($strUri);
}
$arrResults = Promise\Utils::settle($arrPromises)->wait();
// var_dump($strUri);
// var_dump($arrResults);
$arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
// var_dump($arrResult);
return $arrResult['state'] === 'fulfilled';
});
@@ -296,7 +301,7 @@ class Site
$arrLatestListPage[] = $LatestListPage;
}
// var_dump($arrLatestListPage);
return $arrLatestListPage;
}

41
code/app/task/module/PlanTask.php
View File

@@ -12,6 +12,7 @@ use microserver\ProcessSanitizer;
use app\task\crawler\zw630\Scheduler as Zw630Scheduler;
use app\task\crawler\zw74\Scheduler as Zw74Scheduler;
use app\task\crawler\xbiqugu\Scheduler as XBiQuGeScheduler;
use app\task\crawler\youzhi\Scheduler as YouZhiScheduler;
use app\task\crawler\fengchao\Scheduler as FengChaoScheduler;
use app\task\crawler\wuxian\Scheduler as WuXianScheduler;
@@ -59,6 +60,13 @@ class PlanTask
case 'PULL_74_ZW_NOVEL_LATEST':
self::pull74ZwNovelLatest();
break;
case 'PULL_XBIQUGE_NOVEL':
self::pullXBiQuGeNovel();
break;
case 'PULL_XBIQUGE_NOVEL_LATEST':
self::pullXBiQuGeNovelLatest();
break;
case 'PULL_YOU_ZHI_SHI_PIN':
self::pullYouZhiShiPin();
break;
@@ -207,6 +215,39 @@ class PlanTask
$QueueManage->set($arrTask);
}
/**
* Undocumented function
*
* @return void
*/
public static function pullXBiQuGeNovel()
{
$arrTask = [
'callback' => [XBiQuGeScheduler::class, 'crawlFull'],
'data' => []
];
$QueueManage = QueueManage::getInstance();
$QueueManage->set($arrTask);
}
/**
* Undocumented function
*
* @return void
*/
public static function pullXBiQuGeNovelLatest()
{
$arrTask = [
'callback' => [XBiQuGeScheduler::class, 'crawLatest'],
'data' => []
];
$QueueManage = QueueManage::getInstance();
$QueueManage->set($arrTask);
}
/**
* Undocumented function
*

14
code/database/seeders/PlanTaskSeeder.php
View File

@@ -46,6 +46,20 @@ class PlanTaskSeeder
],
// [
// 'pt_name' => 'xbiqugu小说全量扫描',
// 'pt_code' => 'PULL_XBIQUGE_NOVEL',
// 'pt_enable' => 0,
// 'pt_limit' => 10 * 24 * 3600 * 3,
// ],
// [
// 'pt_name' => 'xbiqugu小说每日增量',
// 'pt_code' => 'PULL_XBIQUGE_NOVEL_LATEST',
// 'pt_enable' => 0,
// 'pt_limit' => 1 * 24 * 3600,
// ],
[
'pt_name' => '优质资源库全量采集',
'pt_code' => 'PULL_YOU_ZHI_SHI_PIN',