From 499882f1a4746650186ccc4d82295fb903fa09a6 Mon Sep 17 00:00:00 2001 From: make Date: Tue, 3 Jun 2025 23:02:57 +0800 Subject: [PATCH] debug --- code/app/common.php | 18 ++++++++ code/app/task/crawler/custom/Scheduler.php | 3 +- code/app/task/crawler/custom/WenXueGuan.php | 1 + code/app/task/crawler/wxg/Scheduler.php | 1 + code/app/task/crawler/wxg/page/NovelPage.php | 9 ++-- code/app/task/crawler/zw/Scheduler.php | 33 +++++++++++---- .../task/crawler/zw/page/LatestListPage.php | 9 ++-- code/app/task/crawler/zw/page/Site.php | 9 +++- code/app/task/module/PlanTask.php | 41 +++++++++++++++++++ code/database/seeders/PlanTaskSeeder.php | 14 +++++++ 10 files changed, 121 insertions(+), 17 deletions(-) diff --git a/code/app/common.php b/code/app/common.php index 8a13cf0..6b3c76a 100644 --- a/code/app/common.php +++ b/code/app/common.php @@ -716,6 +716,24 @@ namespace { return false; } + /** + * 从 URL、URI 或路径中提取小说id（不带后缀） + * + * @param [type] $path /30/30345/ + * @return void + */ + function extractNovelId($path) + { + if (!is_string($path) || empty($path)) { + return null; + } + + $segments = explode('/', trim($path, '/')); + $last = end($segments); + + return is_numeric($last) ? $last : null; + } + /** * 从 URL、URI 或路径中提取文件名（不带后缀） diff --git a/code/app/task/crawler/custom/Scheduler.php b/code/app/task/crawler/custom/Scheduler.php index bcff2cc..a9b8c94 100644 --- a/code/app/task/crawler/custom/Scheduler.php +++ b/code/app/task/crawler/custom/Scheduler.php @@ -47,14 +47,15 @@ class Scheduler */ public function crawlFull() { + var_dump('custom-crawlFull'); $AllCustomCrawlerModel = CustomCrawlerModel::select(); foreach ($AllCustomCrawlerModel as $CustomCrawlerModel) { - if ( $CustomCrawlerModel->pt_last_exec != 0 && ($CustomCrawlerModel->pt_last_exec + $CustomCrawlerModel->pt_limit) > time() ) { + var_dump('continue'); continue; } diff --git a/code/app/task/crawler/custom/WenXueGuan.php b/code/app/task/crawler/custom/WenXueGuan.php index 1c7bced..5599197 100644 --- a/code/app/task/crawler/custom/WenXueGuan.php +++ b/code/app/task/crawler/custom/WenXueGuan.php @@ -46,6 +46,7 @@ class WenXueGuan public function createCrawlWenXueGuanTask(array $arrData) { $arrCcData = json_decode($arrData['cc_data'], true); + // var_dump($arrCcData); foreach ($arrCcData as $intNSourceId) { $arrTask = [ 'callback' => [WxgScheduler::class, 'getNovelInfo'], diff --git a/code/app/task/crawler/wxg/Scheduler.php b/code/app/task/crawler/wxg/Scheduler.php index 538f877..803acd8 100644 --- a/code/app/task/crawler/wxg/Scheduler.php +++ b/code/app/task/crawler/wxg/Scheduler.php @@ -107,6 +107,7 @@ class Scheduler // return; $boolResult = $NovelPage->saveNovelInfo(); + // var_dump($boolResult); // return; if ($boolResult == false) { diff --git a/code/app/task/crawler/wxg/page/NovelPage.php b/code/app/task/crawler/wxg/page/NovelPage.php index d909975..83a9634 100644 --- a/code/app/task/crawler/wxg/page/NovelPage.php +++ b/code/app/task/crawler/wxg/page/NovelPage.php @@ -124,11 +124,13 @@ class NovelPage extends BasePage $this->arrNovelInfo[$strKey] = $Item->attr('content'); }); - if (!empty($this->arrNovelInfo['og_novel_category']) && $this->arrNovelInfo['og_novel_category'] == '玄幻小说') { - $this->arrNovelInfo['og_novel_category'] = '玄幻修真'; + if (!empty($this->arrNovelInfo['og_novel_category']) ) { + if($this->arrNovelInfo['og_novel_category'] == '玄幻小说' || $this->arrNovelInfo['og_novel_category'] == '武侠小说'){ + $this->arrNovelInfo['og_novel_category'] = '玄幻修真'; + } + } - $this->getReletedNovel(); } @@ -242,6 +244,7 @@ class NovelPage extends BasePage $arrUpdate = array_merge($arrUpdate, $arrNovelInfo); $arrUpdate['n_related_novel'] = $this->arrRelatedNovel; $arrUpdate['n_forge_novel'] = $this->arrForgeNovel; + $arrUpdate['n_category_pinyin'] = CategoryModel::getPinYin($this->arrNovelInfo['og_novel_category']); $arrUpdate['n_category_sex_en'] = CategoryModel::getSexEn($this->arrNovelInfo['og_novel_category']); $arrUpdate['n_category_sex_zh'] = CategoryModel::getSexZh($this->arrNovelInfo['og_novel_category']); diff --git a/code/app/task/crawler/zw/Scheduler.php b/code/app/task/crawler/zw/Scheduler.php index b031582..45d24cb 100644 --- a/code/app/task/crawler/zw/Scheduler.php +++ b/code/app/task/crawler/zw/Scheduler.php @@ -96,17 +96,20 @@ class Scheduler */ public function getNovelInfo(array $arrFilter) { + // var_dump('getNovelInfo------------------------------------------------------------'); $Site = $this->getSite(); $arrNovelPageList = $Site->getNovelPageList($arrFilter); - + // var_dump($arrNovelPageList); + // var_dump('$arrNovelPageList------------------------------------------------------------'); $arrCover = []; foreach ($arrNovelPageList as $NovelPage) { // $arrNovel = $NovelPage->getNovelInfo(); // print_r($arrNovel); // return; - + // var_dump('$NovelPage------------------------------------------------------------'); $boolResult = $NovelPage->saveNovelInfo(); + // var_dump($boolResult); // return; if ($boolResult == false) { @@ -130,6 +133,7 @@ class Scheduler } $arrChapterList = $NovelPage->getChapters(); + // var_dump($arrChapterList); $this->fetchChapter($NovelPage->intNId, $NovelPage->getData()['page'], $arrChapterList); } @@ -147,9 +151,13 @@ class Scheduler { $intSliceSize = $this->intSlice; $arrUri = []; - $strUri = '/list/lastupdate_%s_0_0_%s.html'; + // $strUri = '/list/lastupdate_%s_0_0_%s.html'; + $strUri = '/sort/%s_%s/'; - $intEndCategory = $intEndPage = 10; + $intEndCategory = 10; + $intEndPage = 10; + // $intEndCategory = 2; + // $intEndPage = 2; for ($intCategoryKey = 0; $intCategoryKey < $intEndCategory; $intCategoryKey++) { for ($intPage = 1; $intPage <= $intEndPage; $intPage++) { @@ -159,6 +167,8 @@ class Scheduler $arrUri = array_chunk($arrUri, $intSliceSize); + // var_dump($arrUri); + // return ; $arrTask = [ 'callback' => [static::class, 'getLatestChapter'], 'data' => [], @@ -179,29 +189,36 @@ class Scheduler public function getLatestChapter(array $arrFilter) { $Site = $this->getSite(); - + $arrLatestListPageList = $Site->getLatestListPageList($arrFilter); - $arrNSourceId = []; foreach ($arrLatestListPageList as $LatestListPage) { $arrResult = $LatestListPage->getNovelFilter(); + // var_dump($arrResult); $arrNSourceId = array_merge($arrNSourceId, $arrResult); } - + // var_dump($arrNSourceId); $arrNSourceId = array_unique($arrNSourceId); $arrNSourceId = array_values($arrNSourceId); $intSliceSize = $this->intSlice; $arrFilter = []; + // var_dump($arrNSourceId); + // return ; + foreach ($arrNSourceId as $intKey => $intNSourceId) { $arrFilter[] = [ 'n_source_id' => $intNSourceId, 'page' => 0, ]; - + // var_dump($intKey); + // var_dump($intSliceSize); + // var_dump($intKey % $intSliceSize); + // var_dump('---------------------------------------------'); if ($intKey % $intSliceSize == 0) { + // var_dump('$intKey % $intSliceSize == 0'); $this->fetchNovel($arrFilter); $arrFilter = []; } diff --git a/code/app/task/crawler/zw/page/LatestListPage.php b/code/app/task/crawler/zw/page/LatestListPage.php index 94565d1..5041299 100644 --- a/code/app/task/crawler/zw/page/LatestListPage.php +++ b/code/app/task/crawler/zw/page/LatestListPage.php @@ -49,13 +49,16 @@ class LatestListPage extends BasePage if ($this->isMobile()) { $arrFilter = $this->getQueryList()->find('div.wrap-box')->eq(0)->find('ul.sort_list li')->map(function ($li) { $strUri = $li->find('a')->eq(0)->attr('href'); - return supperExtractFilename($strUri); + $strNovelId = extractNovelId($strUri); + return (int)$strNovelId; })->filter()->all(); } else { - + $arrFilter = $this->getQueryList()->find('div.layout.layout2.layout-col2.fl ul.txt-list li')->map(function ($li) { $strUri = $li->find('a')->eq(0)->attr('href'); - return supperExtractFilename($strUri); + $strNovelId = extractNovelId($strUri); + return (int)$strNovelId; + return extractNovelId($strUri); })->filter()->all(); } diff --git a/code/app/task/crawler/zw/page/Site.php b/code/app/task/crawler/zw/page/Site.php index db951c1..50a1565 100644 --- a/code/app/task/crawler/zw/page/Site.php +++ b/code/app/task/crawler/zw/page/Site.php @@ -163,6 +163,7 @@ class Site public function getNovelPageUri($intNSourceId, $intPage): string { + $intNSourceId = int($intNSourceId); $intUriFirst = intdiv($intNSourceId, 1000); $strUri = ''; @@ -275,12 +276,16 @@ class Site $arrPromises = []; foreach ($arrFilter as $intKey => $strUri) { + $arrPromises[$intKey] = $this->getClient()->getAsync($strUri); } - + $arrResults = Promise\Utils::settle($arrPromises)->wait(); + // var_dump($strUri); + // var_dump($arrResults); $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) { + // var_dump($arrResult); return $arrResult['state'] === 'fulfilled'; }); @@ -296,7 +301,7 @@ class Site $arrLatestListPage[] = $LatestListPage; } - + // var_dump($arrLatestListPage); return $arrLatestListPage; } diff --git a/code/app/task/module/PlanTask.php b/code/app/task/module/PlanTask.php index 4b38752..daf0ce9 100644 --- a/code/app/task/module/PlanTask.php +++ b/code/app/task/module/PlanTask.php @@ -12,6 +12,7 @@ use microserver\ProcessSanitizer; use app\task\crawler\zw630\Scheduler as Zw630Scheduler; use app\task\crawler\zw74\Scheduler as Zw74Scheduler; +use app\task\crawler\xbiqugu\Scheduler as XBiQuGeScheduler; use app\task\crawler\youzhi\Scheduler as YouZhiScheduler; use app\task\crawler\fengchao\Scheduler as FengChaoScheduler; use app\task\crawler\wuxian\Scheduler as WuXianScheduler; @@ -59,6 +60,13 @@ class PlanTask case 'PULL_74_ZW_NOVEL_LATEST': self::pull74ZwNovelLatest(); break; + case 'PULL_XBIQUGE_NOVEL': + self::pullXBiQuGeNovel(); + break; + case 'PULL_XBIQUGE_NOVEL_LATEST': + self::pullXBiQuGeNovelLatest(); + break; + case 'PULL_YOU_ZHI_SHI_PIN': self::pullYouZhiShiPin(); break; @@ -207,6 +215,39 @@ class PlanTask $QueueManage->set($arrTask); } + + /** + * Undocumented function + * + * @return void + */ + public static function pullXBiQuGeNovel() + { + $arrTask = [ + 'callback' => [XBiQuGeScheduler::class, 'crawlFull'], + 'data' => [] + ]; + + $QueueManage = QueueManage::getInstance(); + $QueueManage->set($arrTask); + } + + /** + * Undocumented function + * + * @return void + */ + public static function pullXBiQuGeNovelLatest() + { + $arrTask = [ + 'callback' => [XBiQuGeScheduler::class, 'crawLatest'], + 'data' => [] + ]; + + $QueueManage = QueueManage::getInstance(); + $QueueManage->set($arrTask); + } + /** * Undocumented function * diff --git a/code/database/seeders/PlanTaskSeeder.php b/code/database/seeders/PlanTaskSeeder.php index e145794..7012825 100644 --- a/code/database/seeders/PlanTaskSeeder.php +++ b/code/database/seeders/PlanTaskSeeder.php @@ -46,6 +46,20 @@ class PlanTaskSeeder ], + // [ + // 'pt_name' => 'xbiqugu小说全量扫描', + // 'pt_code' => 'PULL_XBIQUGE_NOVEL', + // 'pt_enable' => 0, + // 'pt_limit' => 10 * 24 * 3600 * 3, + // ], + // [ + // 'pt_name' => 'xbiqugu小说每日增量', + // 'pt_code' => 'PULL_XBIQUGE_NOVEL_LATEST', + // 'pt_enable' => 0, + // 'pt_limit' => 1 * 24 * 3600, + // ], + + [ 'pt_name' => '优质资源库全量采集', 'pt_code' => 'PULL_YOU_ZHI_SHI_PIN',