diff --git a/code/app/task/crawler/custom/Scheduler.php b/code/app/task/crawler/custom/Scheduler.php index de76ec5..bcff2cc 100644 --- a/code/app/task/crawler/custom/Scheduler.php +++ b/code/app/task/crawler/custom/Scheduler.php @@ -65,7 +65,7 @@ class Scheduler switch ($CustomCrawlerModel->cc_code) { case 'PULL_WEN_XUE_GUAN': $arrTask = [ - 'callback' => [WenXueWang::class, 'createCrawlWenXueGuanTask'], + 'callback' => [WenXueGuan::class, 'createCrawlWenXueGuanTask'], 'data' => [ 'cc_data' => $CustomCrawlerModel->cc_data ], diff --git a/code/app/task/crawler/custom/WenXueWang.php b/code/app/task/crawler/custom/WenXueGuan.php similarity index 89% rename from code/app/task/crawler/custom/WenXueWang.php rename to code/app/task/crawler/custom/WenXueGuan.php index 1530d80..1c7bced 100644 --- a/code/app/task/crawler/custom/WenXueWang.php +++ b/code/app/task/crawler/custom/WenXueGuan.php @@ -8,7 +8,7 @@ use app\model\ChapterModel; use app\model\CustomCrawlerModel; use app\model\NovelModel; use app\task\crawler\custom\page\Site; -use app\task\crawler\wenxuewang\Scheduler as WxwScheduler; +use app\task\crawler\wxg\Scheduler as WxgScheduler; use app\task\crawler\zw\page\NovelPage; use GuzzleHttp\Client; use microserver\QueueManage; @@ -17,7 +17,7 @@ use storage\StorageCore; /** * @mixin think\Model */ -class WenXueWang +class WenXueGuan { /** * Undocumented variable @@ -48,7 +48,7 @@ class WenXueWang $arrCcData = json_decode($arrData['cc_data'], true); foreach ($arrCcData as $intNSourceId) { $arrTask = [ - 'callback' => [WxwScheduler::class, 'getNovelInfo'], + 'callback' => [WxgScheduler::class, 'getNovelInfo'], 'data' => [ [ 'n_source_id' => $intNSourceId, diff --git a/code/app/task/crawler/wenxuewang/Scheduler.php b/code/app/task/crawler/wxg/Scheduler.php similarity index 99% rename from code/app/task/crawler/wenxuewang/Scheduler.php rename to code/app/task/crawler/wxg/Scheduler.php index e2485d3..538f877 100644 --- a/code/app/task/crawler/wenxuewang/Scheduler.php +++ b/code/app/task/crawler/wxg/Scheduler.php @@ -2,10 +2,10 @@ declare(strict_types=1); -namespace app\task\crawler\wenxuewang; +namespace app\task\crawler\wxg; use app\model\ChapterModel; -use app\task\crawler\wenxuewang\page\Site; +use app\task\crawler\wxg\page\Site; use microserver\QueueManage; use storage\StorageCore; @@ -130,6 +130,7 @@ class Scheduler } $arrChapterList = $NovelPage->getChapters(); + $this->fetchChapter($NovelPage->intNId, $NovelPage->getData()['page'], $arrChapterList); } diff --git a/code/app/task/crawler/wenxuewang/page/BasePage.php b/code/app/task/crawler/wxg/page/BasePage.php similarity index 98% rename from code/app/task/crawler/wenxuewang/page/BasePage.php rename to code/app/task/crawler/wxg/page/BasePage.php index cc9a0d9..3c262e5 100644 --- a/code/app/task/crawler/wenxuewang/page/BasePage.php +++ b/code/app/task/crawler/wxg/page/BasePage.php @@ -2,7 +2,7 @@ declare(strict_types=1); -namespace app\task\crawler\wenxuewang\page; +namespace app\task\crawler\wxg\page; use db\mongo\MongoBase; use ddl\DDLManage; diff --git a/code/app/task/crawler/wenxuewang/page/ChapterPage.php b/code/app/task/crawler/wxg/page/ChapterPage.php similarity index 83% rename from code/app/task/crawler/wenxuewang/page/ChapterPage.php rename to code/app/task/crawler/wxg/page/ChapterPage.php index 8f0c231..a557c0f 100644 --- a/code/app/task/crawler/wenxuewang/page/ChapterPage.php +++ b/code/app/task/crawler/wxg/page/ChapterPage.php @@ -2,7 +2,7 @@ declare(strict_types=1); -namespace app\task\crawler\wenxuewang\page; +namespace app\task\crawler\wxg\page; use app\model\ChapterModel; use storage\StorageCore; @@ -55,15 +55,10 @@ class ChapterPage extends BasePage $this->arrChapterInfo = []; - $strPattern = "/qsbs\.bb\('([^']*)'\)/"; - preg_match_all($strPattern, $this->getContent(), $arrMatches); + $strContent = $this->getQueryList()->find('.showtxt')->eq(0)->html(); - $strContent = ''; - - foreach ($arrMatches[1] as $str) { - $strContent .= base64_decode($str); - } + $strContent = strip_tags($strContent, ['']); $this->strChapterContent = $strContent; @@ -71,21 +66,8 @@ class ChapterPage extends BasePage $this->arrChapterInfo['c_page_keywords'] = $this->getQueryList()->find('meta[name="keywords"]')->attr('content'); $this->arrChapterInfo['c_page_description'] = $this->getQueryList()->find('meta[name="description"]')->attr('content'); - $strPattern = "/lastread\.set\((.*?)\);/"; - - if (preg_match($strPattern, $this->getContent(), $arrMatches)) { - $strMatch = $arrMatches[1]; // 提取括号内的内容 - - $arrData = array_map('trim', explode(",", $strMatch)); - - $arrData = array_map(function ($item) { - return trim($item, "'"); - }, $arrData); - - $this->arrChapterInfo['c_page_data'] = $arrData; - } else { - $this->arrChapterInfo['c_page_data'] = NULL; - } + $this->arrChapterInfo['c_name'] = $this->getQueryList()->find('.content')->eq(0)->find('h1')->eq(0)->text(); + $this->arrChapterInfo['c_name'] = trim($this->arrChapterInfo['c_name']); } return $this->arrChapterInfo; @@ -112,8 +94,7 @@ class ChapterPage extends BasePage if ( empty($arrPageChapter) || empty($arrPageChapter['c_page_title']) || - empty($arrPageChapter['c_page_data']) || - count($arrPageChapter['c_page_data']) < 5 + empty($arrPageChapter['c_name']) ) { return false; } @@ -122,7 +103,7 @@ class ChapterPage extends BasePage $arrChapter = [ 'n_id' => $intNId, - 'c_name' => $arrPageChapter['c_page_data'][3], + 'c_name' => $arrPageChapter['c_name'], 'c_sort_num' => $this->arrData['c_sort_num'], 'c_page' => $this->arrData['c_page'], 'c_source_id' => $this->arrData['c_source_id'], @@ -130,6 +111,7 @@ class ChapterPage extends BasePage 'c_source_uuid' => $strUUId, ]; + $arrExistsChapter = $this->getChapterModel()->findOne(['c_source_uuid' => $strUUId], $this->arrOptions); if ($arrExistsChapter) { diff --git a/code/app/task/crawler/wenxuewang/page/LatestListPage.php b/code/app/task/crawler/wxg/page/LatestListPage.php similarity index 97% rename from code/app/task/crawler/wenxuewang/page/LatestListPage.php rename to code/app/task/crawler/wxg/page/LatestListPage.php index c5a5665..328a778 100644 --- a/code/app/task/crawler/wenxuewang/page/LatestListPage.php +++ b/code/app/task/crawler/wxg/page/LatestListPage.php @@ -2,7 +2,7 @@ declare(strict_types=1); -namespace app\task\crawler\wenxuewang\page; +namespace app\task\crawler\wxg\page; use app\model\NovelModel; diff --git a/code/app/task/crawler/wenxuewang/page/NovelPage.php b/code/app/task/crawler/wxg/page/NovelPage.php similarity index 95% rename from code/app/task/crawler/wenxuewang/page/NovelPage.php rename to code/app/task/crawler/wxg/page/NovelPage.php index 0651ffe..d909975 100644 --- a/code/app/task/crawler/wenxuewang/page/NovelPage.php +++ b/code/app/task/crawler/wxg/page/NovelPage.php @@ -2,7 +2,7 @@ declare(strict_types=1); -namespace app\task\crawler\wenxuewang\page; +namespace app\task\crawler\wxg\page; use app\model\CategoryModel; use app\model\CountersModel; @@ -360,22 +360,21 @@ class NovelPage extends BasePage { if ($this->arrChapters === NULL) { $this->arrChapters = []; - if ($this->isMobile()) { - $strClass = '.chapter-list'; - } else { - $strClass = '.section-list'; - } - $intDivIndex = 0; - if ($this->arrData['page'] == 0) { - $intDivIndex = 1; - } - - $this->getQueryList()->find($strClass)->eq($intDivIndex)->find("a")->map(function ($A) { - $this->arrChapters[] = [ - 'name' => $A->text(), - 'uri' => $A->attr('href'), - ]; + $this->getQueryList()->find('.listmain dl')->children('dt, dd')->each(function ($item) { + if ($item->is('dt')) { + if ($item->find('a')->count() === 0) { + $this->arrChapters = []; + } + } elseif ($item->is('dd')) { + $a = $item->find('a'); + if ($a->count() > 0) { + $this->arrChapters[] = [ + 'uri' => $a->attr('href'), + 'name' => $a->text() + ]; + } + } }); } return $this->arrChapters; diff --git a/code/app/task/crawler/wenxuewang/page/Site.php b/code/app/task/crawler/wxg/page/Site.php similarity index 99% rename from code/app/task/crawler/wenxuewang/page/Site.php rename to code/app/task/crawler/wxg/page/Site.php index 247088d..b87b103 100644 --- a/code/app/task/crawler/wenxuewang/page/Site.php +++ b/code/app/task/crawler/wxg/page/Site.php @@ -2,7 +2,7 @@ declare(strict_types=1); -namespace app\task\crawler\wenxuewang\page; +namespace app\task\crawler\wxg\page; use GuzzleHttp\Client; use GuzzleHttp\Promise; @@ -32,7 +32,7 @@ class Site * * @var string */ - protected $strSiteCode = 'wenxuewang'; + protected $strSiteCode = 'wxg'; /** * Undocumented function