admin/SEONexus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feature: wen xue guan crawler

Browse Source
This commit is contained in:
Default
2025-05-05 18:04:33 +08:00
parent d9bb8b727f
commit 68e0a40329
8 changed files with 34 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
code/app/task/crawler/custom/Scheduler.php
View File

@@ -65,7 +65,7 @@ class Scheduler
switch ($CustomCrawlerModel->cc_code) {
case 'PULL_WEN_XUE_GUAN':
$arrTask = [
'callback' => [WenXueWang::class, 'createCrawlWenXueGuanTask'],
'callback' => [WenXueGuan::class, 'createCrawlWenXueGuanTask'],
'data' => [
'cc_data' => $CustomCrawlerModel->cc_data
],

6
code/app/task/crawler/custom/WenXueWang.php → code/app/task/crawler/custom/WenXueGuan.php
View File

@@ -8,7 +8,7 @@ use app\model\ChapterModel;
use app\model\CustomCrawlerModel;
use app\model\NovelModel;
use app\task\crawler\custom\page\Site;
use app\task\crawler\wenxuewang\Scheduler as WxwScheduler;
use app\task\crawler\wxg\Scheduler as WxgScheduler;
use app\task\crawler\zw\page\NovelPage;
use GuzzleHttp\Client;
use microserver\QueueManage;
@@ -17,7 +17,7 @@ use storage\StorageCore;
/**
* @mixin think\Model
*/
class WenXueWang
class WenXueGuan
{
/**
* Undocumented variable
@@ -48,7 +48,7 @@ class WenXueWang
$arrCcData = json_decode($arrData['cc_data'], true);
foreach ($arrCcData as $intNSourceId) {
$arrTask = [
'callback' => [WxwScheduler::class, 'getNovelInfo'],
'callback' => [WxgScheduler::class, 'getNovelInfo'],
'data' => [
[
'n_source_id' => $intNSourceId,

5
code/app/task/crawler/wenxuewang/Scheduler.php → code/app/task/crawler/wxg/Scheduler.php
View File

@@ -2,10 +2,10 @@
declare(strict_types=1);
namespace app\task\crawler\wenxuewang;
namespace app\task\crawler\wxg;
use app\model\ChapterModel;
use app\task\crawler\wenxuewang\page\Site;
use app\task\crawler\wxg\page\Site;
use microserver\QueueManage;
use storage\StorageCore;
@@ -130,6 +130,7 @@ class Scheduler
}
$arrChapterList = $NovelPage->getChapters();
$this->fetchChapter($NovelPage->intNId, $NovelPage->getData()['page'], $arrChapterList);
}

2
code/app/task/crawler/wenxuewang/page/BasePage.php → code/app/task/crawler/wxg/page/BasePage.php
View File

@@ -2,7 +2,7 @@
declare(strict_types=1);
namespace app\task\crawler\wenxuewang\page;
namespace app\task\crawler\wxg\page;
use db\mongo\MongoBase;
use ddl\DDLManage;

34
code/app/task/crawler/wenxuewang/page/ChapterPage.php → code/app/task/crawler/wxg/page/ChapterPage.php
View File

@@ -2,7 +2,7 @@
declare(strict_types=1);
namespace app\task\crawler\wenxuewang\page;
namespace app\task\crawler\wxg\page;
use app\model\ChapterModel;
use storage\StorageCore;
@@ -55,15 +55,10 @@ class ChapterPage extends BasePage
$this->arrChapterInfo = [];
$strPattern = "/qsbs\.bb\('([^']*)'\)/";
preg_match_all($strPattern, $this->getContent(), $arrMatches);
$strContent = $this->getQueryList()->find('.showtxt')->eq(0)->html();
$strContent = '';
foreach ($arrMatches[1] as $str) {
$strContent .= base64_decode($str);
}
$strContent = strip_tags($strContent, ['<script>', '</script>']);
$this->strChapterContent = $strContent;
@@ -71,21 +66,8 @@ class ChapterPage extends BasePage
$this->arrChapterInfo['c_page_keywords'] = $this->getQueryList()->find('meta[name="keywords"]')->attr('content');
$this->arrChapterInfo['c_page_description'] = $this->getQueryList()->find('meta[name="description"]')->attr('content');
$strPattern = "/lastread\.set\((.*?)\);/";
if (preg_match($strPattern, $this->getContent(), $arrMatches)) {
$strMatch = $arrMatches[1]; // 提取括号内的内容
$arrData = array_map('trim', explode(",", $strMatch));
$arrData = array_map(function ($item) {
return trim($item, "'");
}, $arrData);
$this->arrChapterInfo['c_page_data'] = $arrData;
} else {
$this->arrChapterInfo['c_page_data'] = NULL;
}
$this->arrChapterInfo['c_name'] = $this->getQueryList()->find('.content')->eq(0)->find('h1')->eq(0)->text();
$this->arrChapterInfo['c_name'] = trim($this->arrChapterInfo['c_name']);
}
return $this->arrChapterInfo;
@@ -112,8 +94,7 @@ class ChapterPage extends BasePage
if (
empty($arrPageChapter) ||
empty($arrPageChapter['c_page_title']) ||
empty($arrPageChapter['c_page_data']) ||
count($arrPageChapter['c_page_data']) < 5
empty($arrPageChapter['c_name'])
) {
return false;
}
@@ -122,7 +103,7 @@ class ChapterPage extends BasePage
$arrChapter = [
'n_id' => $intNId,
'c_name' => $arrPageChapter['c_page_data'][3],
'c_name' => $arrPageChapter['c_name'],
'c_sort_num' => $this->arrData['c_sort_num'],
'c_page' => $this->arrData['c_page'],
'c_source_id' => $this->arrData['c_source_id'],
@@ -130,6 +111,7 @@ class ChapterPage extends BasePage
'c_source_uuid' => $strUUId,
];
$arrExistsChapter = $this->getChapterModel()->findOne(['c_source_uuid' => $strUUId], $this->arrOptions);
if ($arrExistsChapter) {

2
code/app/task/crawler/wenxuewang/page/LatestListPage.php → code/app/task/crawler/wxg/page/LatestListPage.php
View File

@@ -2,7 +2,7 @@
declare(strict_types=1);
namespace app\task\crawler\wenxuewang\page;
namespace app\task\crawler\wxg\page;
use app\model\NovelModel;

25
code/app/task/crawler/wenxuewang/page/NovelPage.php → code/app/task/crawler/wxg/page/NovelPage.php
View File

@@ -2,7 +2,7 @@
declare(strict_types=1);
namespace app\task\crawler\wenxuewang\page;
namespace app\task\crawler\wxg\page;
use app\model\CategoryModel;
use app\model\CountersModel;
@@ -360,22 +360,21 @@ class NovelPage extends BasePage
{
if ($this->arrChapters === NULL) {
$this->arrChapters = [];
if ($this->isMobile()) {
$strClass = '.chapter-list';
} else {
$strClass = '.section-list';
}
$intDivIndex = 0;
if ($this->arrData['page'] == 0) {
$intDivIndex = 1;
$this->getQueryList()->find('.listmain dl')->children('dt, dd')->each(function ($item) {
if ($item->is('dt')) {
if ($item->find('a')->count() === 0) {
$this->arrChapters = [];
}
$this->getQueryList()->find($strClass)->eq($intDivIndex)->find("a")->map(function ($A) {
} elseif ($item->is('dd')) {
$a = $item->find('a');
if ($a->count() > 0) {
$this->arrChapters[] = [
'name' => $A->text(),
'uri' => $A->attr('href'),
'uri' => $a->attr('href'),
'name' => $a->text()
];
}
}
});
}
return $this->arrChapters;

4
code/app/task/crawler/wenxuewang/page/Site.php → code/app/task/crawler/wxg/page/Site.php
View File

@@ -2,7 +2,7 @@
declare(strict_types=1);
namespace app\task\crawler\wenxuewang\page;
namespace app\task\crawler\wxg\page;
use GuzzleHttp\Client;
use GuzzleHttp\Promise;
@@ -32,7 +32,7 @@ class Site
*
* @var string
*/
protected $strSiteCode = 'wenxuewang';
protected $strSiteCode = 'wxg';
/**
* Undocumented function