From d9bb8b727f9b10a216dc5a16255daf47635daf1e Mon Sep 17 00:00:00 2001 From: Default Date: Mon, 5 May 2025 17:28:05 +0800 Subject: [PATCH] debug --- code/app/model/VideoClicksModel.php | 2 +- code/app/task/crawler/custom/Scheduler.php | 20 +- code/app/task/crawler/custom/WenXueWang.php | 62 +++ .../app/task/crawler/wenxuewang/Scheduler.php | 343 ++++++++++++++ .../task/crawler/wenxuewang/page/BasePage.php | 251 +++++++++++ .../crawler/wenxuewang/page/ChapterPage.php | 226 ++++++++++ .../wenxuewang/page/LatestListPage.php | 64 +++ .../crawler/wenxuewang/page/NovelPage.php | 423 ++++++++++++++++++ .../app/task/crawler/wenxuewang/page/Site.php | 352 +++++++++++++++ code/app/task/crawler/zw/Scheduler.php | 3 + code/app/task/module/PlanTask.php | 2 +- 11 files changed, 1738 insertions(+), 10 deletions(-) create mode 100644 code/app/task/crawler/custom/WenXueWang.php create mode 100644 code/app/task/crawler/wenxuewang/Scheduler.php create mode 100644 code/app/task/crawler/wenxuewang/page/BasePage.php create mode 100644 code/app/task/crawler/wenxuewang/page/ChapterPage.php create mode 100644 code/app/task/crawler/wenxuewang/page/LatestListPage.php create mode 100644 code/app/task/crawler/wenxuewang/page/NovelPage.php create mode 100644 code/app/task/crawler/wenxuewang/page/Site.php diff --git a/code/app/model/VideoClicksModel.php b/code/app/model/VideoClicksModel.php index 9a00672..1da836c 100644 --- a/code/app/model/VideoClicksModel.php +++ b/code/app/model/VideoClicksModel.php @@ -74,7 +74,7 @@ class VideoClicksModel extends MongoModel 'v_category' => $Video->v_category, 'v_parent_category' => $Video->v_parent_category, 'v_category_en' => $Video->v_category_en, - 'v_parent_categoryen' => $Video->v_parent_category_en, + 'v_parent_category_en' => $Video->v_parent_category_en, ], ], [ diff --git a/code/app/task/crawler/custom/Scheduler.php b/code/app/task/crawler/custom/Scheduler.php index ff84174..de76ec5 100644 --- a/code/app/task/crawler/custom/Scheduler.php +++ b/code/app/task/crawler/custom/Scheduler.php @@ -6,7 +6,9 @@ namespace app\task\crawler\custom; use app\model\ChapterModel; use app\model\CustomCrawlerModel; +use app\model\NovelModel; use app\task\crawler\custom\page\Site; +use GuzzleHttp\Client; use microserver\QueueManage; use storage\StorageCore; @@ -22,12 +24,20 @@ class Scheduler */ protected $QueueManage = NULL; + /** + * Undocumented variable + * + * @var NovelModel + */ + protected $NovelModel = NULL; + /** * Undocumented function */ public function __construct() { - $this->QueueManage = new QueueManage(); + $this->QueueManage = QueueManage::getInstance(); + $this->NovelModel = NovelModel::getInstance(); } /** @@ -55,7 +65,7 @@ class Scheduler switch ($CustomCrawlerModel->cc_code) { case 'PULL_WEN_XUE_GUAN': $arrTask = [ - 'callback' => [self::class, 'createCrawlWenXueGuanTask'], + 'callback' => [WenXueWang::class, 'createCrawlWenXueGuanTask'], 'data' => [ 'cc_data' => $CustomCrawlerModel->cc_data ], @@ -67,10 +77,4 @@ class Scheduler $this->QueueManage->set($arrTask); } } - - public function createCrawlWenXueGuanTask(array $arrData) - { - $arrCcData = json_decode($arrData['cc_data'], true); - print_r($arrCcData); - } } diff --git a/code/app/task/crawler/custom/WenXueWang.php b/code/app/task/crawler/custom/WenXueWang.php new file mode 100644 index 0000000..1530d80 --- /dev/null +++ b/code/app/task/crawler/custom/WenXueWang.php @@ -0,0 +1,62 @@ +QueueManage = QueueManage::getInstance(); + $this->NovelModel = NovelModel::getInstance(); + } + + + public function createCrawlWenXueGuanTask(array $arrData) + { + $arrCcData = json_decode($arrData['cc_data'], true); + foreach ($arrCcData as $intNSourceId) { + $arrTask = [ + 'callback' => [WxwScheduler::class, 'getNovelInfo'], + 'data' => [ + [ + 'n_source_id' => $intNSourceId, + 'page' => 0, + ] + ], + ]; + $this->QueueManage->set($arrTask); + } + } +} diff --git a/code/app/task/crawler/wenxuewang/Scheduler.php b/code/app/task/crawler/wenxuewang/Scheduler.php new file mode 100644 index 0000000..e2485d3 --- /dev/null +++ b/code/app/task/crawler/wenxuewang/Scheduler.php @@ -0,0 +1,343 @@ +QueueManage = new QueueManage(); + $this->init(); + } + + /** + * Undocumented function + * + * @return void + */ + public function init() + { + $this->intSlice = 2; + } + + /** + * create a site-wide craw task + * + * @return void + */ + public function crawlFull() + { + $intSliceSize = $this->intSlice; + + $arrFilter = []; + for ($intNId = $this->intStartNSourceId; $intNId <= $this->intEndNSourceId; $intNId++) { + $arrFilter[] = [ + 'n_source_id' => $intNId, + 'page' => 0, + ]; + + if ($intNId % $intSliceSize == 0) { + $this->fetchNovel($arrFilter); + $arrFilter = []; + } + } + if (!empty($arrFilter)) { + $this->fetchNovel($arrFilter); + } + } + + /** + * Undocumented function + * + * @param array $arrFilter + * @return void + */ + public function fetchNovel(array $arrFilter) + { + $arrTask = [ + 'callback' => [static::class, 'getNovelInfo'], + 'data' => $arrFilter, + ]; + + $this->QueueManage->set($arrTask); + } + + /** + * Undocumented function + * + * @param array $arrFilter + * @return void + */ + public function getNovelInfo(array $arrFilter) + { + $Site = $this->getSite(); + + $arrNovelPageList = $Site->getNovelPageList($arrFilter); + + $arrCover = []; + foreach ($arrNovelPageList as $NovelPage) { + // $arrNovel = $NovelPage->getNovelInfo(); + // print_r($arrNovel); + // return; + + $boolResult = $NovelPage->saveNovelInfo(); + // return; + + if ($boolResult == false) { + continue; + } + + // $arrNextPageData = $NovelPage->getNextPageArgs(); + // if (!empty($arrNextPageData)) { + // $this->fetchNovel([ + // [ + // 'n_source_id' => $arrNextPageData['n_source_id'], + // 'page' => $arrNextPageData['page'], + // ] + // ]); + // } + + $arrCoverItem = $NovelPage->getDownCoverInfo(); + + if (!StorageCore::getInstance()->has($arrCoverItem['local_path'])) { + $arrCover[] = $arrCoverItem; + } + + $arrChapterList = $NovelPage->getChapters(); + $this->fetchChapter($NovelPage->intNId, $NovelPage->getData()['page'], $arrChapterList); + } + + if ($arrCover) { + $Site->downRemoteImgToLocal($arrCover); + } + } + + /** + * create a task to craw the latest chapter + * + * @return void + */ + public function crawLatest() + { + $intSliceSize = $this->intSlice; + $arrUri = []; + $strUri = '/list/lastupdate_%s_0_0_%s.html'; + + $intEndCategory = $intEndPage = 10; + + for ($intCategoryKey = 0; $intCategoryKey < $intEndCategory; $intCategoryKey++) { + for ($intPage = 1; $intPage <= $intEndPage; $intPage++) { + $arrUri[] = sprintf($strUri, $intCategoryKey, $intPage); + } + } + + $arrUri = array_chunk($arrUri, $intSliceSize); + + $arrTask = [ + 'callback' => [static::class, 'getLatestChapter'], + 'data' => [], + ]; + + foreach ($arrUri as $arrUriSlice) { + $arrTask['data'] = $arrUriSlice; + $this->QueueManage->set($arrTask); + } + } + + /** + * Undocumented function + * + * @param array $arrData + * @return void + */ + public function getLatestChapter(array $arrFilter) + { + $Site = $this->getSite(); + + $arrLatestListPageList = $Site->getLatestListPageList($arrFilter); + + $arrNSourceId = []; + + foreach ($arrLatestListPageList as $LatestListPage) { + $arrResult = $LatestListPage->getNovelFilter(); + $arrNSourceId = array_merge($arrNSourceId, $arrResult); + } + + $arrNSourceId = array_unique($arrNSourceId); + $arrNSourceId = array_values($arrNSourceId); + + $intSliceSize = $this->intSlice; + $arrFilter = []; + + foreach ($arrNSourceId as $intKey => $intNSourceId) { + $arrFilter[] = [ + 'n_source_id' => $intNSourceId, + 'page' => 0, + ]; + + if ($intKey % $intSliceSize == 0) { + $this->fetchNovel($arrFilter); + $arrFilter = []; + } + } + + if (!empty($arrFilter)) { + $this->fetchNovel($arrFilter); + } + } + + /** + * Undocumented function + * + * @param integer $intNId + * @param integer $intPage + * @param array $arrChapterList + * @return void + */ + public function fetchChapter(int $intNId, int $intPage, array $arrChapterList = []) + { + $intSliceSize = $this->intSlice; + $intPageSize = 100; + + $arrTask = [ + 'callback' => [static::class, 'getChapterInfo'], + 'data' => [ + 'n_id' => $intNId, + ] + ]; + $arrFilter = []; + foreach ($arrChapterList as $intKey => $arrChapter) { + $arrChapter['c_sort_num'] = $intPage * $intPageSize + $intKey + 1; + $arrChapter['c_source_id'] = supperExtractFilename($arrChapter['uri']); + $arrChapter['c_page'] = 0; + + $arrFilter[] = $arrChapter; + if ($intKey % $intSliceSize == 0) { + $arrTask['data']['filter'] = $arrFilter; + $this->QueueManage->set($arrTask); + $arrFilter = []; + } + } + + if (!empty($arrFilter)) { + $arrTask['data']['filter'] = $arrFilter; + $this->QueueManage->set($arrTask); + } + } + + /** + * Undocumented variable + * + * @var Site + */ + public $Site; + + /** + * Undocumented function + * + * @return Site + */ + public function getSite() + { + if ($this->Site == NULL) { + $this->Site = new Site; + } + return $this->Site; + } + + /** + * Undocumented function + * + * @param array $arrData + * @return void + */ + public function getChapterInfo(array $arrData) + { + + $Site = $this->getSite(); + + $ChapterModel = ChapterModel::getInstance(); + + #FIXED 这里增加临时代码，将章节任务拆分为一个章节用一个进程请求，防止代理同时请求多个异常问题。 + $arrTask = [ + 'callback' => [static::class, 'getChapterInfo'], + 'data' => [ + 'n_id' => $arrData['n_id'], + 'filter' => [], + ] + ]; + + foreach ($arrData['filter'] as $intKey => $arrChapterFilter) { + $strUUId = sprintf('%s-chapter-%s-%s', $Site->getSiteCode(), $arrChapterFilter['c_source_id'], $arrChapterFilter['c_page']); + $arrChapter = $ChapterModel->findOne(['c_source_uuid' => $strUUId]); + + if (!empty($arrChapter) && !empty($arrChapter['content'])) { + unset($arrData['filter'][$intKey]); + } else { + #FIXED 这里增加临时代码，将章节任务拆分为一个章节用一个进程请求，防止代理同时请求多个异常问题。 + + $arrFilter = [$arrChapterFilter]; + $arrTask['data']['filter'] = $arrFilter; + $this->QueueManage->set($arrTask); + } + } + if (count($arrData['filter']) > 1 || count($arrData['filter']) <= 0) { + #FIXED 这里增加临时代码，将章节任务拆分为一个章节用一个进程请求，防止代理同时请求多个异常问题。 + return false; + } + + + $arrData['filter'] = array_values($arrData['filter']); + + $arrChapterPageList = $Site->getChapterPageList($arrData['filter']); + + $arrNextPageFilterList = []; + foreach ($arrChapterPageList as $ChapterPage) { + + $ChapterPage->saveChapter($arrData['n_id']); + + $arrNextPageFilter = $ChapterPage->getNextPage(); + + if (!empty($arrNextPageFilter)) { + $arrNextPageFilterList[] = $arrNextPageFilter; + } + } + + if (!empty($arrNextPageFilterList)) { + + $arrTask = $arrData; + $arrTask['filter'] = $arrNextPageFilterList; + $arrTask = [ + 'callback' => [static::class, 'getChapterInfo'], + 'data' => $arrTask + ]; + $this->QueueManage->set($arrTask); + } + } +} diff --git a/code/app/task/crawler/wenxuewang/page/BasePage.php b/code/app/task/crawler/wenxuewang/page/BasePage.php new file mode 100644 index 0000000..cc9a0d9 --- /dev/null +++ b/code/app/task/crawler/wenxuewang/page/BasePage.php @@ -0,0 +1,251 @@ +setSite($Site); + + $this->strUri = $strUri; + } + + /** + * Undocumented function + * + * @return string + */ + public function getUri(): string + { + return $this->strUri; + } + + /** + * Undocumented function + * + * @param array $arrData + * @return void + */ + public function setData(array $arrData) + { + $this->arrData = $arrData; + } + + /** + * Undocumented function + * + * @return array + */ + public function getData(): array + { + return $this->arrData; + } + + /** + * Undocumented function + * + * @return QueueManage + */ + public function getQueueManage(): QueueManage + { + if ($this->QueueManage == NULL) { + $this->QueueManage = QueueManage::getInstance(); + } + + return $this->QueueManage; + } + + /** + * Undocumented function + * + * @return MongoBase + */ + public function getMongoBase(): MongoBase + { + if ($this->MongoBase == NULL) { + DDLManage::load('MongoBase'); + $this->MongoBase = MongoBase::getInstance(); + } + + return $this->MongoBase; + } + + public function getQueryList(): QueryList + { + if ($this->QueryList == NULL) { + $this->QueryList = QueryList::html($this->getContent()); + } + + return $this->QueryList; + } + + + /** + * Undocumented function + * + * @return string + */ + public function getContent(): string + { + if ($this->strContent == NULL) { + $this->Response = $this->getClient()->get($this->strUri); + $this->strContent = $this->Response->getBody()->getContents(); + } + + return $this->strContent; + } + + /** + * force set page content + * + * @param string $strContent + * @return void + */ + public function setContent($strContent) + { + $this->strContent = $strContent; + } + + + /** + * Undocumented function + * + * @return boolean + */ + public function isMobile($strMark = 'content="mobile"'): bool + { + if ($this->boolIsMobile === NULL) { + $this->boolIsMobile = is_numeric(strpos($this->getContent(), $strMark)); + } + return $this->boolIsMobile; + } + + + /** + * Undocumented function + * + * @return Client + */ + protected function getClient(): Client + { + if ($this->Client == NULL) { + $this->Client = $this->getSite()->getClient(); + } + return $this->Client; + } + + + /** + * Undocumented variable + * + * @var Site + */ + protected $Site; + + public function setSite(Site $Site) + { + $this->Site = $Site; + } + + public function getSite() + { + return $this->Site; + } +} diff --git a/code/app/task/crawler/wenxuewang/page/ChapterPage.php b/code/app/task/crawler/wenxuewang/page/ChapterPage.php new file mode 100644 index 0000000..8f0c231 --- /dev/null +++ b/code/app/task/crawler/wenxuewang/page/ChapterPage.php @@ -0,0 +1,226 @@ +ChapterModel == NULL) { + $this->ChapterModel = ChapterModel::getInstance(); + } + return $this->ChapterModel; + } + + public $arrOptions = [ + 'typeMap' => [ + 'root' => 'array', + 'document' => 'array', + 'array' => 'array', + ], + ]; + + /** + * Undocumented function + * + * @return array + */ + public function getChapterInfo(): array + { + if ($this->arrChapterInfo === NULL) { + + $this->arrChapterInfo = []; + + $strPattern = "/qsbs\.bb\('([^']*)'\)/"; + + preg_match_all($strPattern, $this->getContent(), $arrMatches); + + $strContent = ''; + + foreach ($arrMatches[1] as $str) { + $strContent .= base64_decode($str); + } + + $this->strChapterContent = $strContent; + + $this->arrChapterInfo['c_page_title'] = $this->getQueryList()->find('title')->text(); + $this->arrChapterInfo['c_page_keywords'] = $this->getQueryList()->find('meta[name="keywords"]')->attr('content'); + $this->arrChapterInfo['c_page_description'] = $this->getQueryList()->find('meta[name="description"]')->attr('content'); + + $strPattern = "/lastread\.set\((.*?)\);/"; + + if (preg_match($strPattern, $this->getContent(), $arrMatches)) { + $strMatch = $arrMatches[1]; // 提取括号内的内容 + + $arrData = array_map('trim', explode(",", $strMatch)); + + $arrData = array_map(function ($item) { + return trim($item, "'"); + }, $arrData); + + $this->arrChapterInfo['c_page_data'] = $arrData; + } else { + $this->arrChapterInfo['c_page_data'] = NULL; + } + } + + return $this->arrChapterInfo; + } + + /** + * Undocumented variable + * + * @var array + */ + protected $arrChapterInfo; + + + /** + * Undocumented function + * + * @param integer $intNId + * @return boolean + */ + public function saveChapter(int $intNId): bool + { + $arrPageChapter = $this->getChapterInfo(); + + if ( + empty($arrPageChapter) || + empty($arrPageChapter['c_page_title']) || + empty($arrPageChapter['c_page_data']) || + count($arrPageChapter['c_page_data']) < 5 + ) { + return false; + } + + $strUUId = $this->getSite()->getSiteUUId('chapter-' . $this->arrData['c_source_id'] . '-' . $this->arrData['c_page']); + + $arrChapter = [ + 'n_id' => $intNId, + 'c_name' => $arrPageChapter['c_page_data'][3], + 'c_sort_num' => $this->arrData['c_sort_num'], + 'c_page' => $this->arrData['c_page'], + 'c_source_id' => $this->arrData['c_source_id'], + 'c_site_code' => $this->getSite()->getSiteCode(), + 'c_source_uuid' => $strUUId, + ]; + + $arrExistsChapter = $this->getChapterModel()->findOne(['c_source_uuid' => $strUUId], $this->arrOptions); + + if ($arrExistsChapter) { + + $arrExistsChapter = (array) $arrExistsChapter; + + $arrUpdate = [ + 'updated_at' => new \MongoDB\BSON\UTCDateTime(), + ]; + + $this->getChapterModel()->updateOne( + ['c_source_uuid' => $strUUId], + ['$set' => $arrUpdate], + ); + $this->strCContentPath = $arrExistsChapter['c_content_path']; + } else { + + $arrInsertNovel = $arrChapter; + + $arrInsertNovel['c_content_path'] = '/novel/' . ($intNId % 100) . '/' . $intNId . '/' . $arrChapter['c_sort_num'] . '_' . $arrChapter['c_page'] . '.txt'; + $arrInsertNovel['created_at'] = new \MongoDB\BSON\UTCDateTime(); + + try { + $this->getChapterModel()->insert($arrInsertNovel); + $this->updateNovelHaveChapter($intNId); + $this->strCContentPath = $arrInsertNovel['c_content_path']; + } catch (\Exception $e) { + if ($e->getCode() === 11000) { + return $this->saveChapter($intNId); + } else { + throw $e; + } + } + } + + StorageCore::getInstance()->put($this->strCContentPath, $this->strChapterContent); + + return true; + } + + public $strCContentPath; + /** + * Undocumented function + * + * @param integer $intNId + * @return void + */ + protected function updateNovelHaveChapter(int $intNId) + { + $arrFilter = [ + 'n_id' => $intNId, + ]; + + $ChapterModel = ChapterModel::getInstance(); + $arrLasteChapter = $ChapterModel->getLatestChapterByNId($intNId, false); + + if (!empty($arrLasteChapter)) { + $arrUpdate['$set']['n_have_chapter'] = 1; + $arrUpdate['$set']['n_latest_chapter_name'] = $arrLasteChapter['c_name']; + $this->getMongoBase()->updateOne('novel', $arrFilter, $arrUpdate); + } + } + + protected $arrNextPage; + + /** + * Undocumented function + * + * @return array + */ + public function getNextPage(): array + { + if ($this->arrNextPage === NULL) { + $this->arrNextPage = []; + + if ($this->isMobile()) { + $strPattern = "/var\s+hhekgsv\s*=\s*['\"](.*?)['\"]/"; + preg_match_all($strPattern, $this->getContent(), $arrMatches); + $strNextPageUri = end($arrMatches[1]) ?: ''; + } else { + $strHref = $this->getQueryList()->find('.read_btn a:contains("下一章")')->attr('href'); + $strNextPageUri = $strHref ?: ''; + } + + if (is_numeric(strpos($strNextPageUri, '_'))) { + $this->arrNextPage = $this->arrData; + $this->arrNextPage['c_page'] += 1; + $this->arrNextPage['uri'] = $strNextPageUri; + } + } + + return $this->arrNextPage; + } +} diff --git a/code/app/task/crawler/wenxuewang/page/LatestListPage.php b/code/app/task/crawler/wenxuewang/page/LatestListPage.php new file mode 100644 index 0000000..c5a5665 --- /dev/null +++ b/code/app/task/crawler/wenxuewang/page/LatestListPage.php @@ -0,0 +1,64 @@ +NovelModel == NULL) { + $this->NovelModel = NovelModel::getInstance(); + } + return $this->NovelModel; + } + + static public $arrOptions = [ + 'typeMap' => [ + 'root' => 'array', + 'document' => 'array', + 'array' => 'array', + ], + ]; + + /** + * Undocumented function + * + * @return array + */ + public function getNovelFilter(): array + { + $arrFilter = []; + if ($this->isMobile()) { + $arrFilter = $this->getQueryList()->find('div.wrap-box')->eq(0)->find('ul.sort_list li')->map(function ($li) { + $strUri = $li->find('a')->eq(0)->attr('href'); + return supperExtractFilename($strUri); + })->filter()->all(); + } else { + + $arrFilter = $this->getQueryList()->find('div.layout.layout2.layout-col2.fl ul.txt-list li')->map(function ($li) { + $strUri = $li->find('a')->eq(0)->attr('href'); + return supperExtractFilename($strUri); + })->filter()->all(); + } + + return $arrFilter; + } +} diff --git a/code/app/task/crawler/wenxuewang/page/NovelPage.php b/code/app/task/crawler/wenxuewang/page/NovelPage.php new file mode 100644 index 0000000..0651ffe --- /dev/null +++ b/code/app/task/crawler/wenxuewang/page/NovelPage.php @@ -0,0 +1,423 @@ +NovelModel == NULL) { + $this->NovelModel = NovelModel::getInstance(); + } + return $this->NovelModel; + } + + public $arrOptions = [ + 'typeMap' => [ + 'root' => 'array', + 'document' => 'array', + 'array' => 'array', + ], + ]; + + /** + * 查询看的小说 + * + * @param integer $intNSourceId + * @param string $strName + * @return array + */ + protected function findOrCreateNovel(int $intNSourceId, string $strName): array + { + $strUUId = $this->getSite()->getSiteUUId($intNSourceId); + + $arrExistsNovel = $this->getNovelModel()->findOne(['n_source_uuid' => $strUUId], $this->arrOptions); + + if ($arrExistsNovel) { + $arrNovelInfo = (array) $arrExistsNovel; + } else { + $arrNovelInfo = [ + 'n_id' => CountersModel::getInstance()->getNextSequence('novel'), + 'n_name' => $strName, + 'n_name_pinyin' => zhToPinYin($strName), + 'n_site_code' => $this->getSite()->getSiteCode(), + 'n_source_uuid' => $strUUId, + 'n_source_id' => $intNSourceId, + 'n_level' => 1, + 'n_have_chapter' => 0, + 'created_at' => new \MongoDB\BSON\UTCDateTime(), + ]; + try { + $this->getNovelModel()->insert($arrNovelInfo); + } catch (\Exception $e) { + if ($e->getCode() === 11000) { + return $this->findOrCreateNovel($intNSourceId, $strName); + } else { + throw $e; + } + } + } + return $arrNovelInfo; + } + + + /** + * 分析关联数据 + * + * @return void + */ + public function getReletedNovel() {} + + + /** + * Undocumented function + * + * @return array + */ + public function getNovelInfo(): array + { + if (empty($this->arrNovelInfo)) { + + # 获取head字段 + $this->getQueryList()->find('meta[property]')->each(function ($Item) { + $strKey = str_replace(':', '_', $Item->attr('property')); + $this->arrNovelInfo[$strKey] = $Item->attr('content'); + }); + + if (!empty($this->arrNovelInfo['og_novel_category']) && $this->arrNovelInfo['og_novel_category'] == '玄幻小说') { + $this->arrNovelInfo['og_novel_category'] = '玄幻修真'; + } + + + $this->getReletedNovel(); + } + + return $this->arrNovelInfo; + } + + /** + * Undocumented function + * + * @param string $strName + * @return string + */ + public function generateCoverPath($strName): string + { + $intTime = time() - mt_rand(1, 2 * 24 * 3600 * 360); + + return sprintf('/img/%s/%s.jpg', date('Y/m/d/H', $intTime), $strName); + } + + public function filterOtherFields(&$arrNovel) + { + $arrFields = [ + 'n_id', + 'n_name', + 'n_name_pinyin', + 'n_status', + 'n_author', + 'n_description', + 'n_category', + 'n_category_pinyin', + 'n_category_sex_en', + 'n_category_sex_zh', + 'n_cover_path', + 'n_latest_chapter_name', + 'n_update_time', + 'n_forge_novel', + 'n_related_novel', + 'n_site_code', + 'n_source_uuid', + 'n_source_id', + 'n_have_chapter', + 'created_at', + 'updated_at', + ]; + + foreach ($arrNovel as $strKey => $strVal) { + if (!in_array($strKey, $arrFields)) { + unset($arrNovel[$strKey]); + } + } + } + + /** + * Undocumented function + * + * @return bool + */ + public function saveNovelInfo(): bool + { + $arrPageInfo = $this->getNovelInfo(); + if ( + empty($arrPageInfo) || + empty($arrPageInfo['og_title']) || + empty($arrPageInfo['og_image']) || + empty($arrPageInfo['og_novel_category']) + ) { + return false; + } + + $arrNovelInfo = [ + // 'n_id'=> $arrPageInfo['xxxxx'], + 'n_name' => $arrPageInfo['og_title'], + 'n_name_pinyin' => zhToPinYin($arrPageInfo['og_title']), + 'n_status' => $arrPageInfo['og_novel_status'], + 'n_author' => $arrPageInfo['og_novel_author'], + 'n_description' => $arrPageInfo['og_description'], + 'n_category' => $arrPageInfo['og_novel_category'], + // 'n_category_pinyin' => $arrPageInfo['xxxxx'], + // 'n_category_sex_en' => $arrPageInfo['xxxxx'], + // 'n_category_sex_zh' => $arrPageInfo['xxxxx'], + // 'n_cover_path' => $arrPageInfo['xxxxx'], + 'n_latest_chapter_name' => $arrPageInfo['og_novel_latest_chapter_name'], + 'n_update_time' => $arrPageInfo['og_novel_update_time'], + 'n_forge_novel' => $this->arrForgeNovel, + 'n_related_novel' => $this->arrRelatedNovel, + + 'n_site_code' => $this->getSite()->getSiteCode(), + 'n_source_id' => $this->arrData['n_source_id'], + 'n_source_uuid' => $this->getSite()->getSiteUUId($this->arrData['n_source_id']), + + // 'n_have_chapter' => $arrPageInfo['xxxxx'], + // 'created_at' => $arrPageInfo['xxxxx'], + // 'updated_at' => $arrPageInfo['xxxxx'], + + ]; + + $arrExistsNovel = $this->getNovelModel()->findOne(['n_source_uuid' => $arrNovelInfo['n_source_uuid']], $this->arrOptions); + + if ($arrExistsNovel) { + + $arrExistsNovel = (array) $arrExistsNovel; + + $arrUpdate = [ + 'n_latest_chapter_name' => $arrNovelInfo['n_latest_chapter_name'], + 'n_update_time' => $arrNovelInfo['n_update_time'], + 'updated_at' => new \MongoDB\BSON\UTCDateTime(), + ]; + + # 如果没有分类，说明这条数据是最初从关联shu那里采集的，需要补全全部信息 + if (empty($arrExistsNovel['n_category'])) { + $arrUpdate = array_merge($arrUpdate, $arrNovelInfo); + $arrUpdate['n_related_novel'] = $this->arrRelatedNovel; + $arrUpdate['n_forge_novel'] = $this->arrForgeNovel; + $arrUpdate['n_category_pinyin'] = CategoryModel::getPinYin($this->arrNovelInfo['og_novel_category']); + $arrUpdate['n_category_sex_en'] = CategoryModel::getSexEn($this->arrNovelInfo['og_novel_category']); + $arrUpdate['n_category_sex_zh'] = CategoryModel::getSexZh($this->arrNovelInfo['og_novel_category']); + $arrUpdate['n_related_novel'] = $this->arrRelatedNovel; + $arrUpdate['n_forge_novel'] = $this->arrForgeNovel; + } + + # 图片不存在的时候，需要重新生成 + if ( + !isset($arrExistsNovel['n_cover_path']) || + empty($arrExistsNovel['n_cover_path']) || + !StorageCore::getInstance()->has($arrExistsNovel['n_cover_path']) || + (basename($arrExistsNovel['n_cover_path']) == '.jpg') + ) { + $arrUpdate['n_cover_path'] = $this->generateCoverPath($arrNovelInfo['n_name_pinyin']); + $this->strCoverLocalPath = $arrUpdate['n_cover_path']; + } else { + $this->strCoverLocalPath = $arrExistsNovel['n_cover_path']; + } + + $this->getNovelModel()->updateOne( + ['n_source_uuid' => $arrNovelInfo['n_source_uuid']], + ['$set' => $arrUpdate], + ); + $this->intNId = $arrExistsNovel['n_id']; + } else { + + $arrNovelInfo['n_category_pinyin'] = CategoryModel::getPinYin($this->arrNovelInfo['og_novel_category']); + $arrNovelInfo['n_category_sex_en'] = CategoryModel::getSexEn($this->arrNovelInfo['og_novel_category']); + $arrNovelInfo['n_category_sex_zh'] = CategoryModel::getSexZh($this->arrNovelInfo['og_novel_category']); + $arrNovelInfo['n_id'] = CountersModel::getInstance()->getNextSequence('novel'); + $arrNovelInfo['n_level'] = 1; + $arrNovelInfo['n_have_chapter'] = 0; + $arrNovelInfo['n_cover_path'] = $this->generateCoverPath($arrNovelInfo['n_name_pinyin']); + $arrNovelInfo['n_related_novel'] = $this->arrRelatedNovel; + $arrNovelInfo['n_forge_novel'] = $this->arrForgeNovel; + $arrNovelInfo['created_at'] = new \MongoDB\BSON\UTCDateTime(); + + try { + $this->getNovelModel()->insert($arrNovelInfo); + $this->intNId = $arrNovelInfo['n_id']; + $this->strCoverLocalPath = $arrNovelInfo['n_cover_path']; + } catch (\Exception $e) { + if ($e->getCode() === 11000) { + return $this->saveNovelInfo(); + } else { + throw $e; + } + } + } + + return true; + } + + + public $strCoverLocalPath = ''; + + public $intNId; + + /** + * Undocumented function + * + * @return array + */ + public function getDownCoverInfo(): array + { + return [ + 'uri' => $this->arrNovelInfo['og_image'], + 'local_path' => $this->strCoverLocalPath, + ]; + } + + /** + * Undocumented variable + * + * @var array + */ + protected $arrLatestChapters = NULL; + + /** + * Undocumented variable + * + * @var array + */ + protected $arrChapters = NULL; + + public function getLatestChapterByNIdsList(): array + { + if ($this->arrLatestChapters === NULL) { + $this->arrLatestChapters = []; + + if ($this->isMobile()) { + $strClass = '.chapter-list'; + } else { + $strClass = '.section-list'; + } + + if ($this->arrData['page'] == 0) { + $this->getQueryList()->find($strClass)->eq(0)->find("a")->map(function ($A) { + $this->arrLatestChapters[] = [ + 'name' => $A->text(), + 'uri' => $A->attr('href'), + ]; + }); + } + } + return $this->arrLatestChapters; + } + + /** + * Undocumented function + * + * @return array + */ + public function getChapters(): array + { + if ($this->arrChapters === NULL) { + $this->arrChapters = []; + if ($this->isMobile()) { + $strClass = '.chapter-list'; + } else { + $strClass = '.section-list'; + } + + $intDivIndex = 0; + if ($this->arrData['page'] == 0) { + $intDivIndex = 1; + } + + $this->getQueryList()->find($strClass)->eq($intDivIndex)->find("a")->map(function ($A) { + $this->arrChapters[] = [ + 'name' => $A->text(), + 'uri' => $A->attr('href'), + ]; + }); + } + return $this->arrChapters; + } + + + protected $arrNextPageArgs = NULL; + + /** + * Undocumented function + * + * @return array + */ + public function getNextPageArgs(): array + { + $this->arrNextPageArgs = []; + if ($this->arrData['page'] == 0) { + $this->getLatestChapterByNIdsList(); + $this->getChapters(); + + $arrLatestChapterFirstBlock = count($this->arrLatestChapters) > 0 ? $this->arrLatestChapters[0] : null; + $arrLatestChapterSecondBlock = count($this->arrChapters) > 0 ? $this->arrChapters[count($this->arrChapters) - 1] : null; + + if ( + !empty($arrLatestChapterFirstBlock) && !empty($arrLatestChapterSecondBlock) + && ($arrLatestChapterFirstBlock['uri'] != $arrLatestChapterSecondBlock['uri']) + + ) { + $this->arrNextPageArgs = [ + 'n_source_id' => $this->arrData['n_source_id'], + 'page' => $this->arrData['page'] + 1, + ]; + } + } else { + $strLastUri = $this->getQueryList()->find('select:first option:last')->attr('value'); + if ($strLastUri != '' && $strLastUri != $this->strUri) { + $this->arrNextPageArgs = [ + 'n_source_id' => $this->arrData['n_source_id'], + 'page' => $this->arrData['page'] + 1, + ]; + } + } + return $this->arrNextPageArgs; + } +} diff --git a/code/app/task/crawler/wenxuewang/page/Site.php b/code/app/task/crawler/wenxuewang/page/Site.php new file mode 100644 index 0000000..247088d --- /dev/null +++ b/code/app/task/crawler/wenxuewang/page/Site.php @@ -0,0 +1,352 @@ +initClient(); + } + + /** + * Undocumented variable + * + * @var boolean + */ + public $boolReInitClient = false; + + /** + * Undocumented function + */ + public function initClient() + { + $arrConfig = [ + 'http_version' => '1.1', // 强制用 HTTP/1.1，已有，保持 + 'base_uri' => $this->strDomain, // 基础域名，已有，保持 + 'http_errors' => false, // 不抛 HTTP 错误，已有，保持 + 'timeout' => 60, // 从 160 秒改为 60 秒 + 'connect_timeout' => 10, // 从 30 秒改为 10 秒 + 'curl' => [ + CURLOPT_TCP_KEEPALIVE => 1, // 启用 TCP Keep-Alive，已有，保持 + CURLOPT_TCP_KEEPIDLE => 120, // Keep-Alive 空闲时间 + CURLOPT_TCP_KEEPINTVL => 60, // Keep-Alive 探测间隔 + // CURLOPT_SSL_VERIFYPEER => true, // 验证 SSL，已有，保持 + ], + 'headers' => [ + 'User-Agent' => 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36', + 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7', + 'Accept-Language' => 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6', + 'Referer' => $this->strDomain, + 'Connection' => 'keep-alive', + 'sec-ch-ua' => '"Not/A)Brand";v="8", "Chromium";v="130", "Google Chrome";v="130"', + 'sec-ch-ua-mobile' => '?1', + 'sec-ch-ua-platform' => '"Android"', + 'sec-fetch-dest' => 'document', + 'sec-fetch-mode' => 'navigate', + 'sec-fetch-site' => 'same-origin', + 'sec-fetch-user' => '?1', + 'upgrade-insecure-requests' => '1', + ], + 'pool_size' => 10, + 'verify' => false, + ]; + + $arrProxy = config('proxy'); + if ($arrProxy['proxy_status']) { + + $strCode = $arrProxy['proxy_code']; + $strProxyHost = $arrProxy['proxy_host']; + $strKey = $arrProxy['proxy_key']; + $strSecret = $arrProxy['proxy_secret']; + + + switch ($strCode) { + case 'XIONGMAO': + $intTime = time(); + $strTxt = "orderno=" . $strKey . ",secret=" . $strSecret . ",timestamp=" . $intTime; + $strSign = strtoupper(md5($strTxt)); + $strAuth = 'sign=' . $strSign . '&orderno=' . $strKey . '×tamp=' . $intTime; + $arrConfig['proxy'] = $strProxyHost; + $arrConfig['headers']['Proxy-Authorization'] = $strAuth; + break; + case 'KUAI': + $arrConfig['proxy'] = $strProxyHost; + $arrConfig['curl'][CURLOPT_PROXYAUTH] = CURLAUTH_BASIC; + $arrConfig['curl'][CURLOPT_PROXYUSERPWD] = "$strKey:$strSecret"; + break; + } + } + + + $this->Client = new Client($arrConfig); + } + + /** + * Undocumented function + * + * @return string + */ + public function getSiteCode(): string + { + return $this->strSiteCode; + } + + /** + * Undocumented function + * + * @param integer|string $strSign + * @return string + */ + public function getSiteUUId(int|string $strSign): string + { + return $this->getSiteCode() . '-' . $strSign; + } + + /** + * Undocumented function + * + * @return Client + */ + public function getClient(): Client + { + if ($this->boolReInitClient) { + $this->initClient(); + } + + return $this->Client; + } + + public function getNovelPageUri($intNSourceId, $intPage): string + { + $intUriFirst = intdiv($intNSourceId, 1000); + + $strUri = sprintf('/wenzhang/%s/%s/', $intUriFirst, $intNSourceId); + + return $strUri; + } + + /** + * retrieve multiple novel page model + * + * @param string $strUri + * @return NovelPage[] + */ + public function getNovelPageList(array $arrFilter): array + { + + /** @var NovelPage[] */ + $arrNovelPage = []; + + foreach ($arrFilter as $intKey => $arrItem) { + + $arrFilter[$intKey]['uri'] = $this->getNovelPageUri($arrItem['n_source_id'], $arrItem['page']); + + $arrPromises[$intKey] = $this->getClient()->getAsync($arrFilter[$intKey]['uri']); + } + + $arrResults = Promise\Utils::settle($arrPromises)->wait(); + + + $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) { + return $arrResult['state'] === 'fulfilled'; + }); + + foreach ($arrSuccessfulResults as $intKey => $arrResult) { + $Response = $arrResult['value']; + + $strContent = $Response->getBody()->getContents(); + if ($strContent == '索引文件不存在！') { + continue; + } + + $NovelPage = $this->getNovelPage($this, $arrFilter[$intKey]['uri']); + + $NovelPage->setContent($strContent); + $NovelPage->setData($arrFilter[$intKey]); + + $arrNovelPage[] = $NovelPage; + } + + return $arrNovelPage; + } + + /** + * retrieve multiple chapter page model + * + * @param string $strUri + * @return ChapterPage[] + */ + public function getChapterPageList(array $arrFilter): array + { + + /** @var ChapterPage[] */ + $arrChapterPage = []; + + $arrPromises = []; + foreach ($arrFilter as $intKey => $arrItem) { + $arrPromises[$intKey] = $this->getClient()->getAsync($arrItem['uri']); + } + + $arrResults = Promise\Utils::settle($arrPromises)->wait(); + + $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) { + return $arrResult['state'] === 'fulfilled'; + }); + + foreach ($arrSuccessfulResults as $intKey => $arrResult) { + $Response = $arrResult['value']; + + $strContent = $Response->getBody()->getContents();; + + $ChapterPage = $this->getChapterPage($this, $arrFilter[$intKey]['uri']); + + $ChapterPage->setContent($strContent); + $ChapterPage->setData($arrFilter[$intKey]); + + $arrChapterPage[] = $ChapterPage; + } + + return $arrChapterPage; + } + + + /** + * retrieve multiple chapter page model + * + * @param string $strUri + * @return LatestListPage[] + */ + public function getLatestListPageList(array $arrFilter): array + { + + /** @var LatestListPage[] */ + $arrLatestListPage = []; + + $arrPromises = []; + foreach ($arrFilter as $intKey => $strUri) { + $arrPromises[$intKey] = $this->getClient()->getAsync($strUri); + } + + $arrResults = Promise\Utils::settle($arrPromises)->wait(); + + $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) { + return $arrResult['state'] === 'fulfilled'; + }); + + foreach ($arrSuccessfulResults as $intKey => $arrResult) { + $Response = $arrResult['value']; + + $strContent = $Response->getBody()->getContents();; + + $LatestListPage = $this->getLatestListPage($this, $arrFilter[$intKey]); + + $LatestListPage->setContent($strContent); + + $arrLatestListPage[] = $LatestListPage; + } + + + return $arrLatestListPage; + } + + /** + * download the image to the local system + * + * @param array $arrFilter + * @return void + */ + public function downRemoteImgToLocal(array $arrFilter) + { + $StorageCore = StorageCore::getInstance(); + + $arrPromises = []; + + foreach ($arrFilter as $intKey => $arrItem) { + if (!$StorageCore->has($arrItem['local_path'])) { + $arrPromises[$intKey] = $this->getClient()->getAsync($arrItem['uri']); + } + } + + $arrResults = Promise\Utils::settle($arrPromises)->wait(); + + $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) { + return $arrResult['state'] === 'fulfilled'; + }); + + foreach ($arrSuccessfulResults as $intKey => $arrResult) { + $Response = $arrResult['value']; + + $StorageCore->writeStream($arrFilter[$intKey]['local_path'], $Response->getBody()->detach());; + } + } + + /** + * Undocumented function + * + * @param object $Site + * @param string $strUri + * @return NovelPage + */ + public function getNovelPage($Site, $strUri) + { + return new NovelPage($Site, $strUri); + } + + /** + * Undocumented function + * + * @param object $Site + * @param string $strUri + * @return ChapterPage + */ + public function getChapterPage($Site, $strUri) + { + return new ChapterPage($Site, $strUri); + } + + /** + * Undocumented function + * + * @param object $Site + * @param string $strUri + * @return LatestListPage + */ + public function getLatestListPage($Site, $strUri) + { + return new LatestListPage($Site, $strUri); + } +} diff --git a/code/app/task/crawler/zw/Scheduler.php b/code/app/task/crawler/zw/Scheduler.php index be22724..b031582 100644 --- a/code/app/task/crawler/zw/Scheduler.php +++ b/code/app/task/crawler/zw/Scheduler.php @@ -103,6 +103,9 @@ class Scheduler $arrCover = []; foreach ($arrNovelPageList as $NovelPage) { // $arrNovel = $NovelPage->getNovelInfo(); + // print_r($arrNovel); + // return; + $boolResult = $NovelPage->saveNovelInfo(); // return; diff --git a/code/app/task/module/PlanTask.php b/code/app/task/module/PlanTask.php index b3a88ad..88d4f04 100644 --- a/code/app/task/module/PlanTask.php +++ b/code/app/task/module/PlanTask.php @@ -115,7 +115,7 @@ class PlanTask 'data' => [] ]; - $QueueManage = QueueManage::getInstance(2); + $QueueManage = QueueManage::getInstance(); $QueueManage->set($arrTask); }