From 5893a411660ab526c46f9b56bf571374929284b7 Mon Sep 17 00:00:00 2001 From: Default Date: Tue, 15 Apr 2025 16:33:25 +0800 Subject: [PATCH 1/3] debug --- code/.example.env | 17 +- code/app/common.php | 7 +- code/app/task/crawler/zw/Scheduler.php | 321 +++++++++++++++ .../crawler/{zw630 => zw}/page/BasePage.php | 4 +- code/app/task/crawler/zw/page/ChapterPage.php | 217 ++++++++++ .../task/crawler/zw/page/LatestListPage.php | 64 +++ code/app/task/crawler/zw/page/NovelPage.php | 370 ++++++++++++++++++ code/app/task/crawler/zw/page/Site.php | 283 ++++++++++++++ code/app/task/crawler/zw630/Scheduler.php | 264 +------------ .../task/crawler/zw630/page/ChapterPage.php | 212 +--------- .../crawler/zw630/page/LatestListPage.php | 61 +-- .../app/task/crawler/zw630/page/NovelPage.php | 343 +--------------- code/app/task/crawler/zw630/page/Site.php | 262 +++---------- code/app/task/crawler/zw74/Scheduler.php | 37 ++ .../task/crawler/zw74/page/ChapterPage.php | 9 + .../task/crawler/zw74/page/LatestListPage.php | 10 + code/app/task/crawler/zw74/page/NovelPage.php | 48 +++ code/app/task/crawler/zw74/page/Site.php | 152 +++++++ code/app/task/module/PlanTask.php | 41 ++ code/config/proxy.php | 3 +- code/database/mongo/migrations/novel.php | 10 +- code/database/seeders/PlanTaskSeeder.php | 19 +- code/extend/storage/StorageCore.php | 12 +- 23 files changed, 1686 insertions(+), 1080 deletions(-) create mode 100644 code/app/task/crawler/zw/Scheduler.php rename code/app/task/crawler/{zw630 => zw}/page/BasePage.php (97%) create mode 100644 code/app/task/crawler/zw/page/ChapterPage.php create mode 100644 code/app/task/crawler/zw/page/LatestListPage.php create mode 100644 code/app/task/crawler/zw/page/NovelPage.php create mode 100644 code/app/task/crawler/zw/page/Site.php create mode 100644 code/app/task/crawler/zw74/Scheduler.php create mode 100644 code/app/task/crawler/zw74/page/ChapterPage.php create mode 100644 code/app/task/crawler/zw74/page/LatestListPage.php create mode 100644 code/app/task/crawler/zw74/page/NovelPage.php create mode 100644 code/app/task/crawler/zw74/page/Site.php diff --git a/code/.example.env b/code/.example.env index 05ebaf5..acda0d5 100644 --- a/code/.example.env +++ b/code/.example.env @@ -6,7 +6,7 @@ DEFAULT_LANG = zh-cn # 数据库 DB_TYPE = mysql DB_HOST = 127.0.0.1 -DB_NAME = seo +DB_NAME = novel DB_USER = root DB_PASS = qq123123 DB_PORT = 3306 @@ -51,7 +51,14 @@ VIEW_TPL_CACHE = false VIEW_DATA_CACHE = false # PROXY -PROXY_STATUS = false -PROXY_CODE = XIONGMAO -PROXY_ORDER = DT20250413204710pFcgjMR9 -PROXY_SECRET = 583b141edcf65ce94f83591c5cb592ed \ No newline at end of file +PROXY_STATUS = true +# PROXY_CODE = XIONGMAO +# PROXY_HOST = dtbf.xiongmaodaili.com:8089 +# PROXY_KEY = DT20250413204710pFcgjMR9 +# PROXY_SECRET = 583b141edcf65ce94f83591c5cb592ed + +PROXY_CODE = KUAI +#PROXY_HOST = w739.kdltps.com +PROXY_HOST = w740.kdltps.com +PROXY_KEY = t14468368981914 +PROXY_SECRET = 03xvfdc0 diff --git a/code/app/common.php b/code/app/common.php index 8c4f07a..9ab0186 100644 --- a/code/app/common.php +++ b/code/app/common.php @@ -1,4 +1,5 @@ QueueManage = new QueueManage(); + $this->init(); + } + + /** + * Undocumented function + * + * @return void + */ + public function init() + { + $this->intSlice = 8; + } + + /** + * create a site-wide craw task + * + * @return void + */ + public function crawlFull() + { + $intSliceSize = $this->intSlice; + + $arrFilter = []; + for ($intNId = $this->intStartNSourceId; $intNId <= $this->intEndNSourceId; $intNId++) { + $arrFilter[] = [ + 'n_source_id' => $intNId, + 'page' => 0, + ]; + + if ($intNId % $intSliceSize == 0) { + $this->fetchNovel($arrFilter); + $arrFilter = []; + } + } + if (!empty($arrFilter)) { + $this->fetchNovel($arrFilter); + } + } + + /** + * Undocumented function + * + * @param array $arrFilter + * @return void + */ + public function fetchNovel(array $arrFilter) + { + $arrTask = [ + 'callback' => [static::class, 'getNovelInfo'], + 'data' => $arrFilter, + ]; + + $this->QueueManage->set($arrTask); + } + + /** + * Undocumented function + * + * @param array $arrFilter + * @return void + */ + public function getNovelInfo(array $arrFilter) + { + $Site = $this->getSite(); + + $arrNovelPageList = $Site->getNovelPageList($arrFilter); + + $arrCover = []; + foreach ($arrNovelPageList as $NovelPage) { + $arrNovel = $NovelPage->getNovelInfo(); + // $boolResult = $NovelPage->saveNovelInfo(); + + + return; + + if ($boolResult == false) { + continue; + } + + $arrNextPageData = $NovelPage->getNextPageArgs(); + if (!empty($arrNextPageData)) { + $this->fetchNovel([ + [ + 'n_source_id' => $arrNextPageData['n_source_id'], + 'page' => $arrNextPageData['page'], + ] + ]); + } + + $arrCoverItem = $NovelPage->getDownCoverInfo(); + + if (!StorageCore::getInstance()->has($arrCoverItem['local_path'])) { + $arrCover[] = $arrCoverItem; + } + + $arrChapterList = $NovelPage->getChapters(); + $this->fetchChapter($arrNovel['n_id'], $NovelPage->getData()['page'], $arrChapterList); + } + + if ($arrCover) { + $Site->downRemoteImgToLocal($arrCover); + } + } + + /** + * create a task to craw the latest chapter + * + * @return void + */ + public function crawLatest() + { + $intSliceSize = $this->intSlice; + $arrUri = []; + $strUri = '/list/lastupdate_%s_0_0_%s.html'; + + $intEndCategory = $intEndPage = 10; + + for ($intCategoryKey = 0; $intCategoryKey < $intEndCategory; $intCategoryKey++) { + for ($intPage = 1; $intPage <= $intEndPage; $intPage++) { + $arrUri[] = sprintf($strUri, $intCategoryKey, $intPage); + } + } + + $arrUri = array_chunk($arrUri, $intSliceSize); + + $arrTask = [ + 'callback' => [static::class, 'getLatestChapter'], + 'data' => [], + ]; + + foreach ($arrUri as $arrUriSlice) { + $arrTask['data'] = $arrUriSlice; + $this->QueueManage->set($arrTask); + } + } + + /** + * Undocumented function + * + * @param array $arrData + * @return void + */ + public function getLatestChapter(array $arrFilter) + { + $Site = $this->getSite(); + + $arrLatestListPageList = $Site->getLatestListPageList($arrFilter); + + $arrNSourceId = []; + + foreach ($arrLatestListPageList as $LatestListPage) { + $arrResult = $LatestListPage->getNovelFilter(); + $arrNSourceId = array_merge($arrNSourceId, $arrResult); + } + + $arrNSourceId = array_unique($arrNSourceId); + $arrNSourceId = array_values($arrNSourceId); + + $intSliceSize = $this->intSlice; + $arrFilter = []; + + foreach ($arrNSourceId as $intKey => $intNSourceId) { + $arrFilter[] = [ + 'n_source_id' => $intNSourceId, + 'page' => 0, + ]; + + if ($intKey % $intSliceSize == 0) { + $this->fetchNovel($arrFilter); + $arrFilter = []; + } + } + + if (!empty($arrFilter)) { + $this->fetchNovel($arrFilter); + } + } + + /** + * Undocumented function + * + * @param integer $intNId + * @param integer $intPage + * @param array $arrChapterList + * @return void + */ + public function fetchChapter(int $intNId, int $intPage, array $arrChapterList = []) + { + $intSliceSize = $this->intSlice; + $intPageSize = 100; + + $arrTask = [ + 'callback' => [static::class, 'getChapterInfo'], + 'data' => [ + 'n_id' => $intNId, + ] + ]; + $arrFilter = []; + foreach ($arrChapterList as $intKey => $arrChapter) { + $arrChapter['c_sort_num'] = $intPage * $intPageSize + $intKey + 1; + $arrChapter['c_source_id'] = supperExtractFilename($arrChapter['uri']); + $arrChapter['c_page'] = 0; + + $arrFilter[] = $arrChapter; + if ($intKey % $intSliceSize == 0) { + $arrTask['data']['filter'] = $arrFilter; + $this->QueueManage->set($arrTask); + $arrFilter = []; + } + } + + if (!empty($arrFilter)) { + $arrTask['data']['filter'] = $arrFilter; + $this->QueueManage->set($arrTask); + } + } + + /** + * Undocumented variable + * + * @var Site + */ + public $Site; + + /** + * Undocumented function + * + * @return Site + */ + public function getSite() + { + if ($this->Site == NULL) { + $this->Site = new Site; + } + return $this->Site; + } + + /** + * Undocumented function + * + * @param array $arrData + * @return void + */ + public function getChapterInfo(array $arrData) + { + $Site = $this->getSite(); + + $ChapterModel = ChapterModel::getInstance(); + + foreach ($arrData['filter'] as $intKey => $arrChapterFilter) { + $strUUId = sprintf('%s-chapter-%s-%s', $Site->getSiteCode(), $arrChapterFilter['c_source_id'], $arrChapterFilter['c_page']); + $arrChapter = $ChapterModel->findOne(['c_site_uuid' => $strUUId]); + + if (!empty($arrChapter) && !empty($arrChapter['content'])) { + unset($arrData['filter'][$intKey]); + } + } + + $arrData['filter'] = array_values($arrData['filter']); + + $arrChapterPageList = $Site->getChapterPageList($arrData['filter']); + + $arrNextPageFilterList = []; + foreach ($arrChapterPageList as $ChapterPage) { + + $ChapterPage->saveChapter($arrData['n_id']); + + $arrNextPageFilter = $ChapterPage->getNextPage(); + + if (!empty($arrNextPageFilter)) { + $arrNextPageFilterList[] = $arrNextPageFilter; + } + } + + if (!empty($arrNextPageFilterList)) { + + $arrTask = $arrData; + $arrTask['filter'] = $arrNextPageFilterList; + $arrTask = [ + 'callback' => [static::class, 'getChapterInfo'], + 'data' => $arrTask + ]; + $this->QueueManage->set($arrTask); + } + } +} diff --git a/code/app/task/crawler/zw630/page/BasePage.php b/code/app/task/crawler/zw/page/BasePage.php similarity index 97% rename from code/app/task/crawler/zw630/page/BasePage.php rename to code/app/task/crawler/zw/page/BasePage.php index bf00316..e0c8c99 100644 --- a/code/app/task/crawler/zw630/page/BasePage.php +++ b/code/app/task/crawler/zw/page/BasePage.php @@ -2,7 +2,7 @@ declare(strict_types=1); -namespace app\task\crawler\zw630\page; +namespace app\task\crawler\zw\page; use db\mongo\MongoBase; use ddl\DDLManage; @@ -100,7 +100,7 @@ abstract class BasePage * * @param string $strUri */ - public function __construct(Site $Site, string $strUri) + public function __construct($Site, string $strUri) { $this->setSite($Site); diff --git a/code/app/task/crawler/zw/page/ChapterPage.php b/code/app/task/crawler/zw/page/ChapterPage.php new file mode 100644 index 0000000..96e7255 --- /dev/null +++ b/code/app/task/crawler/zw/page/ChapterPage.php @@ -0,0 +1,217 @@ +ChapterModel == NULL) { + $this->ChapterModel = ChapterModel::getInstance(); + } + return $this->ChapterModel; + } + + public $arrOptions = [ + 'typeMap' => [ + 'root' => 'array', + 'document' => 'array', + 'array' => 'array', + ], + ]; + + /** + * Undocumented function + * + * @return array + */ + public function getChapterInfo(): array + { + if ($this->arrChapterInfo === NULL) { + + $this->arrChapterInfo = []; + + $strPattern = "/qsbs\.bb\('([^']*)'\)/"; + + preg_match_all($strPattern, $this->getContent(), $arrMatches); + + $strContent = ''; + + foreach ($arrMatches[1] as $str) { + $strContent .= base64_decode($str); + } + + $this->strChapterContent = $strContent; + + $this->arrChapterInfo['c_page_title'] = $this->getQueryList()->find('title')->text(); + $this->arrChapterInfo['c_page_keywords'] = $this->getQueryList()->find('meta[name="keywords"]')->attr('content'); + $this->arrChapterInfo['c_page_description'] = $this->getQueryList()->find('meta[name="description"]')->attr('content'); + } + + return $this->arrChapterInfo; + } + + /** + * Undocumented variable + * + * @var array + */ + protected $arrChapterInfo; + + + public function filterOtherFields(&$arrChapter) + { + $arrFields = [ + 'n_id', + 'name', + 'c_sort_num', + 'c_page', + 'c_source_id', + 'c_site_code', + 'c_site_uuid', + 'c_content_path', + 'created_at', + 'updated_at', + + ]; + + foreach ($arrChapter as $strKey => $strVal) { + if (!in_array($strKey, $arrFields)) { + unset($arrChapter[$strKey]); + } + } + } + + public function saveChapter(int $intNId): array + { + + try { + $this->getChapterInfo(); + + $strUUId = $this->getSite()->getSiteUUId('chapter-' . $this->arrData['c_source_id'] . '-' . $this->arrData['c_page']); + $arrExisting = $this->getChapterModel()->findOne(['c_site_uuid' => $strUUId], $this->arrOptions); + + if ($arrExisting) { + + $arrExisting = (array) $arrExisting; + $this->arrChapterInfo = array_merge($arrExisting, $this->getChapterInfo()); + + $this->arrChapterInfo['updated_at'] = new \MongoDB\BSON\UTCDateTime(); + + $arrUpdate = $this->arrChapterInfo; + + $this->filterOtherFields($arrUpdate); + + $this->getChapterModel()->updateOne( + ['c_site_uuid' => $strUUId], + ['$set' => $arrUpdate], + ); + } else { + + $this->arrChapterInfo['n_id'] = $intNId; + $this->arrChapterInfo['name'] = $this->arrData['name']; + $this->arrChapterInfo['c_sort_num'] = $this->arrData['c_sort_num']; + $this->arrChapterInfo['c_page'] = $this->arrData['c_page']; + $this->arrChapterInfo['c_source_id'] = $this->arrData['c_source_id']; + $this->arrChapterInfo['c_site_code'] = $this->getSite()->getSiteCode(); + $this->arrChapterInfo['c_site_uuid'] = $strUUId; + $this->arrChapterInfo['c_content_path'] = '/novel/' . ($intNId % 100) . '/' . $intNId . '/' . $this->arrChapterInfo['c_sort_num'] . '_' . $this->arrChapterInfo['c_page'] . '.txt'; + $this->arrChapterInfo['created_at'] = new \MongoDB\BSON\UTCDateTime(); + $arrInsert = $this->arrChapterInfo; + + $this->filterOtherFields($arrInsert); + + $this->getChapterModel()->insert($arrInsert); + + $this->updateNovelHaveChapter($intNId); + } + + if (mb_strlen($this->strChapterContent) > 20) { + if (!StorageCore::getInstance()->has($this->arrChapterInfo['c_content_path'])) { + StorageCore::getInstance()->set($this->arrChapterInfo['c_content_path'], $this->strChapterContent); + } + } + return $this->arrChapterInfo; + } catch (\Exception $e) { + throw new \Exception("Upsert failed: " . $e->getMessage()); + } + } + + + /** + * Undocumented function + * + * @param integer $intNId + * @return void + */ + protected function updateNovelHaveChapter(int $intNId) + { + $arrFilter = [ + 'n_id' => $intNId, + ]; + + $ChapterModel = ChapterModel::getInstance(); + $arrLasteChapter = $ChapterModel->getLatestChapterByNId($intNId, false); + + if (!empty($arrLasteChapter)) { + $arrUpdate['$set']['n_have_chapter'] = 1; + $arrUpdate['$set']['og_novel_latest_chapter_name'] = $arrLasteChapter['name']; + $this->getMongoBase()->updateOne('novel', $arrFilter, $arrUpdate); + } + } + + protected $arrNextPage; + + /** + * Undocumented function + * + * @return array + */ + public function getNextPage(): array + { + if ($this->arrNextPage === NULL) { + $this->arrNextPage = []; + + if ($this->isMobile()) { + $strPattern = "/var\s+hhekgsv\s*=\s*['\"](.*?)['\"]/"; + preg_match_all($strPattern, $this->getContent(), $arrMatches); + $strNextPageUri = end($arrMatches[1]) ?: ''; + } else { + $strHref = $this->getQueryList()->find('.read_btn a:contains("下一章")')->attr('href'); + $strNextPageUri = $strHref ?: ''; + } + + if (is_numeric(strpos($strNextPageUri, '_'))) { + $this->arrNextPage = $this->arrData; + $this->arrNextPage['c_page'] += 1; + $this->arrNextPage['uri'] = $strNextPageUri; + } + } + + return $this->arrNextPage; + } +} diff --git a/code/app/task/crawler/zw/page/LatestListPage.php b/code/app/task/crawler/zw/page/LatestListPage.php new file mode 100644 index 0000000..94565d1 --- /dev/null +++ b/code/app/task/crawler/zw/page/LatestListPage.php @@ -0,0 +1,64 @@ +NovelModel == NULL) { + $this->NovelModel = NovelModel::getInstance(); + } + return $this->NovelModel; + } + + static public $arrOptions = [ + 'typeMap' => [ + 'root' => 'array', + 'document' => 'array', + 'array' => 'array', + ], + ]; + + /** + * Undocumented function + * + * @return array + */ + public function getNovelFilter(): array + { + $arrFilter = []; + if ($this->isMobile()) { + $arrFilter = $this->getQueryList()->find('div.wrap-box')->eq(0)->find('ul.sort_list li')->map(function ($li) { + $strUri = $li->find('a')->eq(0)->attr('href'); + return supperExtractFilename($strUri); + })->filter()->all(); + } else { + + $arrFilter = $this->getQueryList()->find('div.layout.layout2.layout-col2.fl ul.txt-list li')->map(function ($li) { + $strUri = $li->find('a')->eq(0)->attr('href'); + return supperExtractFilename($strUri); + })->filter()->all(); + } + + return $arrFilter; + } +} diff --git a/code/app/task/crawler/zw/page/NovelPage.php b/code/app/task/crawler/zw/page/NovelPage.php new file mode 100644 index 0000000..7be5f5c --- /dev/null +++ b/code/app/task/crawler/zw/page/NovelPage.php @@ -0,0 +1,370 @@ +NovelModel == NULL) { + $this->NovelModel = NovelModel::getInstance(); + } + return $this->NovelModel; + } + + public $arrOptions = [ + 'typeMap' => [ + 'root' => 'array', + 'document' => 'array', + 'array' => 'array', + ], + ]; + + /** + * 查询看的小说 + * + * @param integer $intNSourceId + * @param string $strName + * @return array + */ + protected function findOrCreateNovel(int $intNSourceId, string $strName): array + { + $strUUId = $this->getSite()->getSiteUUId($intNSourceId); + + $arrExisting = $this->getNovelModel()->findOne(['n_site_uuid' => $strUUId], $this->arrOptions); + + if ($arrExisting) { + $arrNovelInfo = (array) $arrExisting; + } else { + $arrNovelInfo = [ + 'og_novel_book_name' => $strName, + 'og_novel_pin_yin' => zhToPinYin($strName), + 'n_site_code' => $this->getSite()->getSiteCode(), + 'n_site_uuid' => $strUUId, + 'n_source_id' => $intNSourceId, + 'n_level' => 1, + 'n_have_chapter' => 0, + 'n_id' => CountersModel::getInstance()->getNextSequence('novel'), + ]; + $this->getNovelModel()->insert($arrNovelInfo); + } + return $arrNovelInfo; + } + + + /** + * 分析关联数据 + * + * @return void + */ + public function getReletedNovel() {} + + + /** + * Undocumented function + * + * @return array + */ + public function getNovelInfo(): array + { + if (empty($this->arrNovelInfo)) { + + # 获取head字段 + $this->getQueryList()->find('meta[property]')->each(function ($Item) { + $strKey = str_replace(':', '_', $Item->attr('property')); + $this->arrNovelInfo[$strKey] = $Item->attr('content'); + }); + + $this->getReletedNovel(); + + $this->arrNovelInfo['n_related_novel'] = $this->arrRelatedNovel; + $this->arrNovelInfo['n_forge_novel'] = $this->arrForgeNovel; + + // $this->arrNovelInfo['og_novel_pin_yin'] = zhToPinYin($this->arrNovelInfo['og_novel_book_name'] ?? ''); + + // if (!empty($this->arrNovelInfo['og_novel_category'])) { + // $this->arrNovelInfo['category_pinyin'] = CategoryModel::getPinYinByZh($this->arrNovelInfo['og_novel_category']); + // $this->arrNovelInfo['category_sex'] = CategoryModel::checkSex($this->arrNovelInfo['og_novel_category']); + // $this->arrNovelInfo['category_sex_zh'] = CategoryModel::checkSexZh($this->arrNovelInfo['og_novel_category']); + // } + + // $this->arrNovelInfo['n_site_code'] = $this->getSite()->getSiteCode(); + // $this->arrNovelInfo['n_source_id'] = $this->arrData['n_source_id']; + } + + print_r($this->arrNovelInfo); + + return $this->arrNovelInfo; + } + + /** + * Undocumented function + * + * @param string $strName + * @return string + */ + public function generateCoverPath($strName): string + { + return sprintf('/img/%s/%s.jpg', date('Y/m/d', time() - 24 * 3600 * 14), $strName); + } + + public function filterOtherFields(&$arrNovel) + { + $arrFields = [ + 'n_id', + 'og_novel_book_name', + 'og_novel_pin_yin', + 'og_novel_status', + 'og_novel_author', + 'og_description', + 'og_novel_category', + 'category_pinyin', + 'category_sex', + 'category_sex_zh', + 'n_cover_path', + 'og_novel_latest_chapter_name', + 'og_novel_update_time', + 'n_forge_novel', + 'n_related_novel', + 'og_site_code', + 'n_site_uuid', + 'n_source_id', + 'n_have_chapter', + 'created_at', + 'updated_at', + ]; + + foreach ($arrNovel as $strKey => $strVal) { + if (!in_array($strKey, $arrFields)) { + unset($arrNovel[$strKey]); + } + } + } + + /** + * Undocumented function + * + * @return void + */ + public function saveNovelInfo() + { + try { + + $this->getNovelInfo(); + + if (!isset($this->arrNovelInfo['og_novel_book_name'])) { + return false; + } + + + $strUUId = $this->getSite()->getSiteUUId($this->arrData['n_source_id']); + + $arrExisting = $this->getNovelModel()->findOne(['n_site_uuid' => $strUUId], $this->arrOptions); + + if ($arrExisting) { + + $arrExisting = (array) $arrExisting; + $this->arrNovelInfo = array_merge($arrExisting, $this->arrNovelInfo); + + if ( + !isset($this->arrNovelInfo['n_cover_path']) || + empty($this->arrNovelInfo['n_cover_path']) || + !file_exists($this->arrNovelInfo['n_cover_path']) || + + (basename($this->arrNovelInfo['n_cover_path']) == '.jpg') + ) { + $this->arrNovelInfo['n_cover_path'] = $this->generateCoverPath($this->arrNovelInfo['og_novel_pin_yin']); + } + + $this->arrNovelInfo['updated_at'] = new \MongoDB\BSON\UTCDateTime(); + + $arrUpdate = $this->arrNovelInfo; + $this->filterOtherFields($arrUpdate); + + $this->getNovelModel()->updateOne( + ['n_site_uuid' => $strUUId], + ['$set' => $arrUpdate], + ); + } else { + $this->arrNovelInfo['n_site_uuid'] = $strUUId; + $this->arrNovelInfo['n_id'] = CountersModel::getInstance()->getNextSequence('novel'); + $this->arrNovelInfo['n_level'] = 1; + $this->arrNovelInfo['n_have_chapter'] = 0; + $this->arrNovelInfo['n_cover_path'] = $this->generateCoverPath($this->arrNovelInfo['og_novel_pin_yin']); + + $arrInsert = $this->arrNovelInfo; + $this->filterOtherFields($arrInsert); + + $this->getNovelModel()->insert($arrInsert); + } + return $this->arrNovelInfo; + } catch (\Exception $e) { + throw new \Exception("Upsert failed: " . $e->getMessage()); + } + } + + /** + * Undocumented function + * + * @return array + */ + public function getDownCoverInfo(): array + { + if (!key_exists('og_image', $this->arrNovelInfo)) { + print_r($this->arrNovelInfo); + echo $this->getContent(); + } + + return [ + 'uri' => $this->arrNovelInfo['og_image'], + 'local_path' => $this->arrNovelInfo['n_cover_path'], + ]; + } + + /** + * Undocumented variable + * + * @var array + */ + protected $arrLatestChapters = NULL; + + /** + * Undocumented variable + * + * @var array + */ + protected $arrChapters = NULL; + + public function getLatestChapterByNIdsList(): array + { + if ($this->arrLatestChapters === NULL) { + $this->arrLatestChapters = []; + + if ($this->isMobile()) { + $strClass = '.chapter-list'; + } else { + $strClass = '.section-list'; + } + + if ($this->arrData['page'] == 0) { + $this->getQueryList()->find($strClass)->eq(0)->find("a")->map(function ($A) { + $this->arrLatestChapters[] = [ + 'name' => $A->text(), + 'uri' => $A->attr('href'), + ]; + }); + } + } + return $this->arrLatestChapters; + } + + /** + * Undocumented function + * + * @return array + */ + public function getChapters(): array + { + if ($this->arrChapters === NULL) { + $this->arrChapters = []; + if ($this->isMobile()) { + $strClass = '.chapter-list'; + } else { + $strClass = '.section-list'; + } + + $intDivIndex = 0; + if ($this->arrData['page'] == 0) { + $intDivIndex = 1; + } + + $this->getQueryList()->find($strClass)->eq($intDivIndex)->find("a")->map(function ($A) { + $this->arrChapters[] = [ + 'name' => $A->text(), + 'uri' => $A->attr('href'), + ]; + }); + } + return $this->arrChapters; + } + + + protected $arrNextPageArgs = NULL; + + /** + * Undocumented function + * + * @return array + */ + public function getNextPageArgs(): array + { + $this->arrNextPageArgs = []; + if ($this->arrData['page'] == 0) { + $this->getLatestChapterByNIdsList(); + $this->getChapters(); + + $arrLatestChapterFirstBlock = count($this->arrLatestChapters) > 0 ? $this->arrLatestChapters[0] : null; + $arrLatestChapterSecondBlock = count($this->arrChapters) > 0 ? $this->arrChapters[count($this->arrChapters) - 1] : null; + + if ( + !empty($arrLatestChapterFirstBlock) && !empty($arrLatestChapterSecondBlock) + && ($arrLatestChapterFirstBlock['uri'] != $arrLatestChapterSecondBlock['uri']) + + ) { + $this->arrNextPageArgs = [ + 'n_source_id' => $this->arrData['n_source_id'], + 'page' => $this->arrData['page'] + 1, + ]; + } + } else { + $strLastUri = $this->getQueryList()->find('select:first option:last')->attr('value'); + if ($strLastUri != '' && $strLastUri != $this->strUri) { + $this->arrNextPageArgs = [ + 'n_source_id' => $this->arrData['n_source_id'], + 'page' => $this->arrData['page'] + 1, + ]; + } + } + return $this->arrNextPageArgs; + } +} diff --git a/code/app/task/crawler/zw/page/Site.php b/code/app/task/crawler/zw/page/Site.php new file mode 100644 index 0000000..5d5c768 --- /dev/null +++ b/code/app/task/crawler/zw/page/Site.php @@ -0,0 +1,283 @@ +initClient(); + } + + + + /** + * Undocumented function + * + * @return string + */ + public function getSiteCode(): string + { + return $this->strSiteCode; + } + + /** + * Undocumented function + * + * @param integer|string $strSign + * @return string + */ + public function getSiteUUId(int|string $strSign): string + { + return $this->getSiteCode() . '-' . $strSign; + } + + /** + * Undocumented function + * + * @return Client + */ + public function getClient(): Client + { + return $this->Client; + } + + public function getNovelPageUri($intNSourceId, $intPage): string + { + $intUriFirst = intdiv($intNSourceId, 1000); + + $strUri = ''; + + if ($intPage == 0) { + $strUri = sprintf('/%s/%s/', $intUriFirst, $intNSourceId); + } else { + $strUri = sprintf('/book/%s/list%s.html', $intNSourceId, $intPage); + } + + return $strUri; + } + + /** + * retrieve multiple novel page model + * + * @param string $strUri + * @return NovelPage[] + */ + public function getNovelPageList(array $arrFilter): array + { + + /** @var NovelPage[] */ + $arrNovelPage = []; + + foreach ($arrFilter as $intKey => $arrItem) { + + $arrFilter[$intKey]['uri'] = $this->getNovelPageUri($arrItem['n_source_id'], $arrItem['page']); + + $arrPromises[$intKey] = $this->getClient()->getAsync($arrFilter[$intKey]['uri']); + } + + $arrResults = Promise\Utils::settle($arrPromises)->wait(); + + + $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) { + return $arrResult['state'] === 'fulfilled'; + }); + + foreach ($arrSuccessfulResults as $intKey => $arrResult) { + $Response = $arrResult['value']; + + $strContent = $Response->getBody()->getContents(); + if ($strContent == '索引文件不存在！') { + continue; + } + + $NovelPage = $this->getNovelPage($this, $arrFilter[$intKey]['uri']); + + $NovelPage->setContent($strContent); + $NovelPage->setData($arrFilter[$intKey]); + + $arrNovelPage[] = $NovelPage; + } + + return $arrNovelPage; + } + + /** + * retrieve multiple chapter page model + * + * @param string $strUri + * @return ChapterPage[] + */ + public function getChapterPageList(array $arrFilter): array + { + + /** @var ChapterPage[] */ + $arrChapterPage = []; + + $arrPromises = []; + foreach ($arrFilter as $intKey => $arrItem) { + $arrPromises[$intKey] = $this->getClient()->getAsync($arrItem['uri']); + } + + $arrResults = Promise\Utils::settle($arrPromises)->wait(); + + $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) { + return $arrResult['state'] === 'fulfilled'; + }); + + foreach ($arrSuccessfulResults as $intKey => $arrResult) { + $Response = $arrResult['value']; + + $strContent = $Response->getBody()->getContents();; + + $ChapterPage = $this->getChapterPage($this, $arrFilter[$intKey]['uri']); + + $ChapterPage->setContent($strContent); + $ChapterPage->setData($arrFilter[$intKey]); + + $arrChapterPage[] = $ChapterPage; + } + + return $arrChapterPage; + } + + + /** + * retrieve multiple chapter page model + * + * @param string $strUri + * @return LatestListPage[] + */ + public function getLatestListPageList(array $arrFilter): array + { + + /** @var LatestListPage[] */ + $arrLatestListPage = []; + + $arrPromises = []; + foreach ($arrFilter as $intKey => $strUri) { + $arrPromises[$intKey] = $this->getClient()->getAsync($strUri); + } + + $arrResults = Promise\Utils::settle($arrPromises)->wait(); + + $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) { + return $arrResult['state'] === 'fulfilled'; + }); + + foreach ($arrSuccessfulResults as $intKey => $arrResult) { + $Response = $arrResult['value']; + + $strContent = $Response->getBody()->getContents();; + + $LatestListPage = $this->getLatestListPage($this, $arrFilter[$intKey]); + + $LatestListPage->setContent($strContent); + + $arrLatestListPage[] = $LatestListPage; + } + + + return $arrLatestListPage; + } + + /** + * download the image to the local system + * + * @param array $arrFilter + * @return void + */ + public function downRemoteImgToLocal(array $arrFilter) + { + $StorageCore = StorageCore::getInstance(); + + $arrPromises = []; + + foreach ($arrFilter as $intKey => $arrItem) { + if (!$StorageCore->has($arrItem['local_path'])) { + $arrPromises[$intKey] = $this->getClient()->getAsync($arrItem['uri']); + } + } + + $arrResults = Promise\Utils::settle($arrPromises)->wait(); + + $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) { + return $arrResult['state'] === 'fulfilled'; + }); + + foreach ($arrSuccessfulResults as $intKey => $arrResult) { + $Response = $arrResult['value']; + + $StorageCore->writeStream($arrFilter[$intKey]['local_path'], $Response->getBody()->detach());; + } + } + + /** + * Undocumented function + * + * @param object $Site + * @param string $strUri + * @return NovelPage + */ + public function getNovelPage($Site, $strUri) + { + return new NovelPage($Site, $strUri); + } + + /** + * Undocumented function + * + * @param object $Site + * @param string $strUri + * @return ChapterPage + */ + public function getChapterPage($Site, $strUri) + { + return new ChapterPage($Site, $strUri); + } + + /** + * Undocumented function + * + * @param object $Site + * @param string $strUri + * @return LatestListPage + */ + public function getLatestListPage($Site, $strUri) + { + return new LatestListPage($Site, $strUri); + } +} diff --git a/code/app/task/crawler/zw630/Scheduler.php b/code/app/task/crawler/zw630/Scheduler.php index 2ef9d0b..17d7427 100644 --- a/code/app/task/crawler/zw630/Scheduler.php +++ b/code/app/task/crawler/zw630/Scheduler.php @@ -4,278 +4,34 @@ declare(strict_types=1); namespace app\task\crawler\zw630; -use app\model\ChapterModel; -use app\task\crawler\zw630\page\NovelPage; use app\task\crawler\zw630\page\Site; -use microserver\QueueManage; -use storage\StorageCore; +use app\task\crawler\zw\Scheduler as ZwScheduler; /** * @mixin think\Model */ -class Scheduler +class Scheduler extends ZwScheduler { - - static $intSlice = 3; - /** - * Undocumented variable - * - * @var \microserver\QueueManage - */ - protected $QueueManage = NULL; - - public function __construct() - { - $this->QueueManage = new QueueManage(); - } - - /** - * create a site-wide craw task + * Undocumented function * * @return void */ - public function crawlFull() + public function init() { - $intSliceSize = self::$intSlice; - - $arrFilter = []; - for ($intNId = 1; $intNId <= 531208; $intNId++) { - // for ($intNId = 1; $intNId <= 10000; $intNId++) { - // for ($intNId = 1; $intNId <= 1; $intNId++) { - $arrFilter[] = [ - 'n_source_id' => $intNId, - 'page' => 0, - ]; - - if ($intNId % $intSliceSize == 0) { - $this->fetchNovel($arrFilter); - $arrFilter = []; - } - } - if (!empty($arrFilter)) { - $this->fetchNovel($arrFilter); - } - } - - /** - * create a task to craw the latest chapter - * - * @return void - */ - public function crawLatest() - { - $intSliceSize = self::$intSlice; - $arrUri = []; - $strUri = '/list/lastupdate_%s_0_0_%s.html'; - - $intEndCategory = $intEndPage = 10; - - for ($intCategoryKey = 0; $intCategoryKey < $intEndCategory; $intCategoryKey++) { - for ($intPage = 1; $intPage <= $intEndPage; $intPage++) { - $arrUri[] = sprintf($strUri, $intCategoryKey, $intPage); - } - } - - $arrUri = array_chunk($arrUri, $intSliceSize); - - $arrTask = [ - 'callback' => [self::class, 'getLatestChapter'], - 'data' => [], - ]; - - foreach ($arrUri as $arrUriSlice) { - $arrTask['data'] = $arrUriSlice; - $this->QueueManage->set($arrTask); - } + $this->intSlice = 8; } /** * Undocumented function * - * @param array $arrData - * @return void + * @return Site */ - public function getLatestChapter(array $arrFilter) + public function getSite() { - $Site = new Site; - - $arrLatestListPageList = $Site->getLatestListPageList($arrFilter); - - $arrNSourceId = []; - - foreach ($arrLatestListPageList as $LatestListPage) { - $arrResult = $LatestListPage->getNovelFilter(); - $arrNSourceId = array_merge($arrNSourceId, $arrResult); - } - - $arrNSourceId = array_unique($arrNSourceId); - $arrNSourceId = array_values($arrNSourceId); - - $intSliceSize = self::$intSlice; - $arrFilter = []; - - foreach ($arrNSourceId as $intKey => $intNSourceId) { - $arrFilter[] = [ - 'n_source_id' => $intNSourceId, - 'page' => 0, - ]; - - if ($intKey % $intSliceSize == 0) { - $this->fetchNovel($arrFilter); - $arrFilter = []; - } - } - - if (!empty($arrFilter)) { - $this->fetchNovel($arrFilter); - } - } - - /** - * Undocumented function - * - * @param array $arrFilter - * @return void - */ - public function fetchNovel(array $arrFilter) - { - $arrTask = [ - 'callback' => [self::class, 'getNovelInfo'], - 'data' => $arrFilter, - ]; - $this->QueueManage->set($arrTask); - } - - /** - * Undocumented function - * - * @param integer $intNId - * @param integer $intPage - * @param array $arrChapterList - * @return void - */ - public function fetchChapter(int $intNId, int $intPage, array $arrChapterList = []) - { - $intSliceSize = self::$intSlice; - $intPageSize = 100; - - $arrTask = [ - 'callback' => [self::class, 'getChapterInfo'], - 'data' => [ - 'n_id' => $intNId, - ] - ]; - $arrFilter = []; - foreach ($arrChapterList as $intKey => $arrChapter) { - $arrChapter['c_sort_num'] = $intPage * $intPageSize + $intKey + 1; - $arrChapter['c_source_id'] = supperExtractFilename($arrChapter['uri']); - $arrChapter['c_page'] = 0; - - $arrFilter[] = $arrChapter; - if ($intKey % $intSliceSize == 0) { - $arrTask['data']['filter'] = $arrFilter; - $this->QueueManage->set($arrTask); - $arrFilter = []; - } - } - - if (!empty($arrFilter)) { - $arrTask['data']['filter'] = $arrFilter; - $this->QueueManage->set($arrTask); - } - } - - /** - * Undocumented function - * - * @param array $arrFilter - * @return void - */ - public function getNovelInfo(array $arrFilter) - { - $Site = new Site; - - $arrNovelPageList = $Site->getNovelPageList($arrFilter); - - $arrCover = []; - foreach ($arrNovelPageList as $NovelPage) { - $arrNovel = $NovelPage->getNovelInfo(); - if (empty($arrNovel)) { - continue; - } - - $arrNextPageData = $NovelPage->getNextPageArgs(); - if (!empty($arrNextPageData)) { - $this->fetchNovel([ - [ - 'n_source_id' => $arrNextPageData['n_source_id'], - 'page' => $arrNextPageData['page'], - ] - ]); - } - - $arrCoverItem = $NovelPage->getDownCoverInfo(); - - if (!StorageCore::getInstance()->has($arrCoverItem['local_path'])) { - $arrCover[] = $arrCoverItem; - } - - $arrChapterList = $NovelPage->getChapters(); - $this->fetchChapter($arrNovel['n_id'], $NovelPage->getData()['page'], $arrChapterList); - } - - if ($arrCover) { - $Site->downRemoteImgToLocal($arrCover); - } - } - - /** - * Undocumented function - * - * @param array $arrData - * @return void - */ - public function getChapterInfo(array $arrData) - { - $Site = new Site; - - $ChapterModel = ChapterModel::getInstance(); - - foreach ($arrData['filter'] as $intKey => $arrChapterFilter) { - $strUUId = sprintf('%s-chapter-%s-%s', $Site->getSiteCode(), $arrChapterFilter['c_source_id'], $arrChapterFilter['c_page']); - $arrChapter = $ChapterModel->findOne(['c_site_uuid' => $strUUId]); - - if (!empty($arrChapter) && !empty($arrChapter['content'])) { - unset($arrData['filter'][$intKey]); - } - } - - $arrData['filter'] = array_values($arrData['filter']); - - $arrChapterPageList = $Site->getChapterPageList($arrData['filter']); - - $arrNextPageFilterList = []; - foreach ($arrChapterPageList as $ChapterPage) { - - $ChapterPage->saveChapter($arrData['n_id']); - - $arrNextPageFilter = $ChapterPage->getNextPage(); - - if (!empty($arrNextPageFilter)) { - $arrNextPageFilterList[] = $arrNextPageFilter; - } - } - - if (!empty($arrNextPageFilterList)) { - - $arrTask = $arrData; - $arrTask['filter'] = $arrNextPageFilterList; - $arrTask = [ - 'callback' => [self::class, 'getChapterInfo'], - 'data' => $arrTask - ]; - $this->QueueManage->set($arrTask); + if ($this->Site == NULL) { + $this->Site = new Site; } + return $this->Site; } } diff --git a/code/app/task/crawler/zw630/page/ChapterPage.php b/code/app/task/crawler/zw630/page/ChapterPage.php index 4d19fb7..b079ea0 100644 --- a/code/app/task/crawler/zw630/page/ChapterPage.php +++ b/code/app/task/crawler/zw630/page/ChapterPage.php @@ -4,214 +4,6 @@ declare(strict_types=1); namespace app\task\crawler\zw630\page; -use app\model\ChapterModel; -use storage\StorageCore; +use app\task\crawler\zw\page\ChapterPage as ZwChapterPage; -class ChapterPage extends BasePage -{ - /** - * Undocumented variable - * - * @var string - */ - protected $strChapterContent = ''; - - /** - * Undocumented variable - * - * @var ChapterModel - */ - protected $ChapterModel; - - /** - * Undocumented function - * - * @return ChapterModel - */ - public function getChapterModel(): ChapterModel - { - if ($this->ChapterModel == NULL) { - $this->ChapterModel = ChapterModel::getInstance(); - } - return $this->ChapterModel; - } - - static public $arrOptions = [ - 'typeMap' => [ - 'root' => 'array', - 'document' => 'array', - 'array' => 'array', - ], - ]; - - /** - * Undocumented function - * - * @return array - */ - public function getChapterInfo(): array - { - if ($this->arrChapterInfo === NULL) { - - $this->arrChapterInfo = []; - - $strPattern = "/qsbs\.bb\('([^']*)'\)/"; - - preg_match_all($strPattern, $this->getContent(), $arrMatches); - - $strContent = ''; - - foreach ($arrMatches[1] as $str) { - $strContent .= base64_decode($str); - } - - $this->strChapterContent = $strContent; - - $this->arrChapterInfo['c_page_title'] = $this->getQueryList()->find('title')->text(); - $this->arrChapterInfo['c_page_keywords'] = $this->getQueryList()->find('meta[name="keywords"]')->attr('content'); - $this->arrChapterInfo['c_page_description'] = $this->getQueryList()->find('meta[name="description"]')->attr('content'); - } - - return $this->arrChapterInfo; - } - - /** - * Undocumented variable - * - * @var array - */ - protected $arrChapterInfo; - - - public function filterOtherFields(&$arrChapter) - { - $arrFields = [ - 'n_id', - 'name', - 'c_sort_num', - 'c_page', - 'c_source_id', - 'c_site_code', - 'c_site_uuid', - 'c_content_path', - 'created_at', - 'updated_at', - - ]; - - foreach ($arrChapter as $strKey => $strVal) { - if (!in_array($strKey, $arrFields)) { - unset($arrChapter[$strKey]); - } - } - } - - public function saveChapter(int $intNId): array - { - - try { - $this->getChapterInfo(); - - $strUUId = $this->getSite()->getSiteUUId('chapter-' . $this->arrData['c_source_id'] . '-' . $this->arrData['c_page']); - $arrExisting = $this->getChapterModel()->findOne(['c_site_uuid' => $strUUId], self::$arrOptions); - - if ($arrExisting) { - - $arrExisting = (array) $arrExisting; - $this->arrChapterInfo = array_merge($arrExisting, $this->getChapterInfo()); - - $this->arrChapterInfo['updated_at'] = new \MongoDB\BSON\UTCDateTime(); - - $arrUpdate = $this->arrChapterInfo; - - $this->filterOtherFields($arrUpdate); - - $this->getChapterModel()->updateOne( - ['c_site_uuid' => $strUUId], - ['$set' => $arrUpdate], - ); - } else { - - $this->arrChapterInfo['n_id'] = $intNId; - $this->arrChapterInfo['name'] = $this->arrData['name']; - $this->arrChapterInfo['c_sort_num'] = $this->arrData['c_sort_num']; - $this->arrChapterInfo['c_page'] = $this->arrData['c_page']; - $this->arrChapterInfo['c_source_id'] = $this->arrData['c_source_id']; - $this->arrChapterInfo['c_site_code'] = $this->getSite()->getSiteCode(); - $this->arrChapterInfo['c_site_uuid'] = $strUUId; - $this->arrChapterInfo['c_content_path'] = '/novel/' . ($intNId % 100) . '/' . $intNId . '/' . $this->arrChapterInfo['c_sort_num'] . '_' . $this->arrChapterInfo['c_page'] . '.txt'; - $this->arrChapterInfo['created_at'] = new \MongoDB\BSON\UTCDateTime(); - $arrInsert = $this->arrChapterInfo; - - $this->filterOtherFields($arrInsert); - - $this->getChapterModel()->insert($arrInsert); - - $this->updateNovelHaveChapter($intNId); - } - - if (mb_strlen($this->strChapterContent) > 20) { - if (!StorageCore::getInstance()->has($this->arrChapterInfo['c_content_path'])) { - StorageCore::getInstance()->set($this->arrChapterInfo['c_content_path'], $this->strChapterContent); - } - } - return $this->arrChapterInfo; - } catch (\Exception $e) { - throw new \Exception("Upsert failed: " . $e->getMessage()); - } - } - - - /** - * Undocumented function - * - * @param integer $intNId - * @return void - */ - protected function updateNovelHaveChapter(int $intNId) - { - $arrFilter = [ - 'n_id' => $intNId, - ]; - - $ChapterModel = ChapterModel::getInstance(); - $arrLasteChapter = $ChapterModel->getLatestChapterByNId($intNId, false); - - if (!empty($arrLasteChapter)) { - $arrUpdate['$set']['n_have_chapter'] = 1; - $arrUpdate['$set']['og_novel_latest_chapter_name'] = $arrLasteChapter['name']; - $this->getMongoBase()->updateOne('novel', $arrFilter, $arrUpdate); - } - } - - protected $arrNextPage; - - /** - * Undocumented function - * - * @return array - */ - public function getNextPage(): array - { - if ($this->arrNextPage === NULL) { - $this->arrNextPage = []; - - if ($this->isMobile()) { - $strPattern = "/var\s+hhekgsv\s*=\s*['\"](.*?)['\"]/"; - preg_match_all($strPattern, $this->getContent(), $arrMatches); - $strNextPageUri = end($arrMatches[1]) ?: ''; - } else { - $strHref = $this->getQueryList()->find('.read_btn a:contains("下一章")')->attr('href'); - $strNextPageUri = $strHref ?: ''; - } - - if (is_numeric(strpos($strNextPageUri, '_'))) { - $this->arrNextPage = $this->arrData; - $this->arrNextPage['c_page'] += 1; - $this->arrNextPage['uri'] = $strNextPageUri; - } - } - - return $this->arrNextPage; - } -} +class ChapterPage extends ZwChapterPage {} diff --git a/code/app/task/crawler/zw630/page/LatestListPage.php b/code/app/task/crawler/zw630/page/LatestListPage.php index 1c06477..8d20997 100644 --- a/code/app/task/crawler/zw630/page/LatestListPage.php +++ b/code/app/task/crawler/zw630/page/LatestListPage.php @@ -4,63 +4,8 @@ declare(strict_types=1); namespace app\task\crawler\zw630\page; -use app\model\CategoryModel; -use app\model\CountersModel; + use app\model\NovelModel; +use app\task\crawler\zw\page\LatestListPage as ZwLatestListPage; - -class LatestListPage extends BasePage -{ - - /** - * Undocumented variable - * - * @var NovelModel - */ - protected $NovelModel; - - /** - * Undocumented function - * - * @return NovelModel - */ - public function getNovelModel(): NovelModel - { - if ($this->NovelModel == NULL) { - $this->NovelModel = NovelModel::getInstance(); - } - return $this->NovelModel; - } - - static public $arrOptions = [ - 'typeMap' => [ - 'root' => 'array', - 'document' => 'array', - 'array' => 'array', - ], - ]; - - /** - * Undocumented function - * - * @return array - */ - public function getNovelFilter(): array - { - $arrFilter = []; - if ($this->isMobile()) { - $arrFilter = $this->getQueryList()->find('div.wrap-box')->eq(0)->find('ul.sort_list li')->map(function ($li) { - $strUri = $li->find('a')->eq(0)->attr('href'); - return supperExtractFilename($strUri); - })->filter()->all(); - } else { - - $arrFilter = $this->getQueryList()->find('div.layout.layout2.layout-col2.fl ul.txt-list li')->map(function ($li) { - $strUri = $li->find('a')->eq(0)->attr('href'); - return supperExtractFilename($strUri); - })->filter()->all(); - } - - return $arrFilter; - } -} +class LatestListPage extends ZwLatestListPage {} diff --git a/code/app/task/crawler/zw630/page/NovelPage.php b/code/app/task/crawler/zw630/page/NovelPage.php index bf29444..0aff642 100644 --- a/code/app/task/crawler/zw630/page/NovelPage.php +++ b/code/app/task/crawler/zw630/page/NovelPage.php @@ -4,113 +4,24 @@ declare(strict_types=1); namespace app\task\crawler\zw630\page; -use app\model\CategoryModel; -use app\model\CountersModel; -use app\model\NovelModel; -class NovelPage extends BasePage +use app\task\crawler\zw\page\NovelPage as ZwNovelPage; + +class NovelPage extends ZwNovelPage { /** - * Undocumented variable + * 分析关联数据 * - * @var array + * @return void */ - protected $arrNovelInfo = []; - - /** - * Undocumented variable - * - * @var array - */ - protected $arrRelatedNovel = []; - - /** - * Undocumented variable - * - * @var array - */ - protected $arrForgeNovel = []; - - /** - * Undocumented variable - * - * @var NovelModel - */ - protected $NovelModel; - - /** - * Undocumented function - * - * @return NovelModel - */ - public function getNovelModel(): NovelModel + public function getReletedNovel() { - if ($this->NovelModel == NULL) { - $this->NovelModel = NovelModel::getInstance(); - } - return $this->NovelModel; - } - - static public $arrOptions = [ - 'typeMap' => [ - 'root' => 'array', - 'document' => 'array', - 'array' => 'array', - ], - ]; - - /** - * 查询看的小说 - * - * @param integer $intNSourceId - * @param string $strName - * @return array - */ - private function findOrCreateNovel(int $intNSourceId, string $strName): array - { - $strUUId = $this->getSite()->getSiteUUId($intNSourceId); - - $arrExisting = $this->getNovelModel()->findOne(['n_site_uuid' => $strUUId], self::$arrOptions); - - if ($arrExisting) { - $arrNovelInfo = (array) $arrExisting; - } else { - $arrNovelInfo = [ - 'og_novel_book_name' => $strName, - 'og_novel_pin_yin' => zhToPinYin($strName), - 'n_site_code' => $this->getSite()->getSiteCode(), - 'n_site_uuid' => $strUUId, - 'n_source_id' => $intNSourceId, - 'n_level' => 1, - 'n_have_chapter' => 0, - 'n_id' => CountersModel::getInstance()->getNextSequence('novel'), - ]; - $this->getNovelModel()->insert($arrNovelInfo); - } - return $arrNovelInfo; - } - - - /** - * Undocumented function - * - * @return array - */ - public function getNovelInfo($boolSave = true): array - { - if (empty($this->arrNovelInfo)) { - - # 获取head字段 - $this->getQueryList()->find('meta[property]')->each(function ($Item) { - $strKey = str_replace(':', '_', $Item->attr('property')); - $this->arrNovelInfo[$strKey] = $Item->attr('content'); - }); - - # 获取 kan 和 shu 的关联数据 + if (empty($this->arrRelatedNovel) || empty($this->arrForgeNovel)) { $this->getQueryList()->find('.first_txt')->eq(0)->find("a")->map(function ($A) { $strName = $A->text(); $strUri = $A->attr('href'); + if (is_numeric(strpos($strUri, 'shu'))) { $intNSourceId = (int)supperExtractFilename($strUri); $arrNovelInfo = $this->findOrCreateNovel($intNSourceId, $strName); @@ -127,244 +38,6 @@ class NovelPage extends BasePage ]; } }); - - $this->arrNovelInfo['n_related_novel'] = $this->arrRelatedNovel; - $this->arrNovelInfo['n_forge_novel'] = $this->arrForgeNovel; - - $this->arrNovelInfo['og_novel_pin_yin'] = zhToPinYin($this->arrNovelInfo['og_novel_book_name'] ?? ''); - - if (!empty($this->arrNovelInfo['og_novel_category'])) { - $this->arrNovelInfo['category_pinyin'] = CategoryModel::getPinYinByZh($this->arrNovelInfo['og_novel_category']); - $this->arrNovelInfo['category_sex'] = CategoryModel::checkSex($this->arrNovelInfo['og_novel_category']); - $this->arrNovelInfo['category_sex_zh'] = CategoryModel::checkSexZh($this->arrNovelInfo['og_novel_category']); - } - - $this->arrNovelInfo['n_site_code'] = $this->getSite()->getSiteCode(); - $this->arrNovelInfo['n_source_id'] = $this->arrData['n_source_id']; } - - if ($boolSave && !empty($this->arrNovelInfo)) { - $this->saveNovelInfo(); - } - - return $this->arrNovelInfo; - } - - /** - * Undocumented function - * - * @param string $strName - * @return string - */ - public function generateCoverPath($strName): string - { - return sprintf('/img/%s/%s.jpg', date('Y/m/d', time() - 24 * 3600 * 14), $strName); - } - - public function filterOtherFields(&$arrNovel) - { - $arrFields = [ - 'n_id', - 'og_novel_book_name', - 'og_novel_pin_yin', - 'og_novel_status', - 'og_novel_author', - 'og_description', - 'og_novel_category', - 'category_pinyin', - 'category_sex', - 'category_sex_zh', - 'n_cover_path', - 'og_novel_latest_chapter_name', - 'og_novel_update_time', - 'n_forge_novel', - 'n_related_novel', - 'og_site_code', - 'n_site_uuid', - 'n_source_id', - 'n_have_chapter', - 'created_at', - 'updated_at', - ]; - - foreach ($arrNovel as $strKey => $strVal) { - if (!in_array($strKey, $arrFields)) { - unset($arrNovel[$strKey]); - } - } - } - - /** - * Undocumented function - * - * @return void - */ - public function saveNovelInfo() - { - try { - $strUUId = $this->getSite()->getSiteUUId($this->arrData['n_source_id']); - - $arrExisting = $this->getNovelModel()->findOne(['n_site_uuid' => $strUUId], self::$arrOptions); - - if ($arrExisting) { - - $arrExisting = (array) $arrExisting; - $this->arrNovelInfo = array_merge($arrExisting, $this->arrNovelInfo); - - if ( - !isset($this->arrNovelInfo['n_cover_path']) || - empty($this->arrNovelInfo['n_cover_path']) || - !file_exists($this->arrNovelInfo['n_cover_path']) || - - (basename($this->arrNovelInfo['n_cover_path']) == '.jpg') - ) { - $this->arrNovelInfo['n_cover_path'] = $this->generateCoverPath($this->arrNovelInfo['og_novel_pin_yin']); - } - - $this->arrNovelInfo['updated_at'] = new \MongoDB\BSON\UTCDateTime(); - - $arrUpdate = $this->arrNovelInfo; - $this->filterOtherFields($arrUpdate); - - $this->getNovelModel()->updateOne( - ['n_site_uuid' => $strUUId], - ['$set' => $arrUpdate], - ); - } else { - $this->arrNovelInfo['n_site_uuid'] = $strUUId; - $this->arrNovelInfo['n_id'] = CountersModel::getInstance()->getNextSequence('novel'); - $this->arrNovelInfo['n_level'] = 1; - $this->arrNovelInfo['n_have_chapter'] = 0; - $this->arrNovelInfo['n_cover_path'] = $this->generateCoverPath($this->arrNovelInfo['og_novel_pin_yin']); - - $arrInsert = $this->arrNovelInfo; - $this->filterOtherFields($arrInsert); - - $this->getNovelModel()->insert($arrInsert); - } - return $this->arrNovelInfo; - } catch (\Exception $e) { - throw new \Exception("Upsert failed: " . $e->getMessage()); - } - } - - /** - * Undocumented function - * - * @return array - */ - public function getDownCoverInfo(): array - { - return [ - 'uri' => $this->arrNovelInfo['og_image'], - 'local_path' => $this->arrNovelInfo['n_cover_path'], - ]; - } - - /** - * Undocumented variable - * - * @var array - */ - protected $arrLatestChapters = NULL; - - /** - * Undocumented variable - * - * @var array - */ - protected $arrChapters = NULL; - - public function getLatestChapterByNIdsList(): array - { - if ($this->arrLatestChapters === NULL) { - $this->arrLatestChapters = []; - - if ($this->isMobile()) { - $strClass = '.chapter-list'; - } else { - $strClass = '.section-list'; - } - - if ($this->arrData['page'] == 0) { - $this->getQueryList()->find($strClass)->eq(0)->find("a")->map(function ($A) { - $this->arrLatestChapters[] = [ - 'name' => $A->text(), - 'uri' => $A->attr('href'), - ]; - }); - } - } - return $this->arrLatestChapters; - } - - /** - * Undocumented function - * - * @return array - */ - public function getChapters(): array - { - if ($this->arrChapters === NULL) { - $this->arrChapters = []; - if ($this->isMobile()) { - $strClass = '.chapter-list'; - } else { - $strClass = '.section-list'; - } - - $intDivIndex = 0; - if ($this->arrData['page'] == 0) { - $intDivIndex = 1; - } - - $this->getQueryList()->find($strClass)->eq($intDivIndex)->find("a")->map(function ($A) { - $this->arrChapters[] = [ - 'name' => $A->text(), - 'uri' => $A->attr('href'), - ]; - }); - } - return $this->arrChapters; - } - - - protected $arrNextPageArgs = NULL; - - /** - * Undocumented function - * - * @return array - */ - public function getNextPageArgs(): array - { - $this->arrNextPageArgs = []; - if ($this->arrData['page'] == 0) { - $this->getLatestChapterByNIdsList(); - $this->getChapters(); - - $arrLatestChapterFirstBlock = count($this->arrLatestChapters) > 0 ? $this->arrLatestChapters[0] : null; - $arrLatestChapterSecondBlock = count($this->arrChapters) > 0 ? $this->arrChapters[count($this->arrChapters) - 1] : null; - - if ( - !empty($arrLatestChapterFirstBlock) && !empty($arrLatestChapterSecondBlock) - && ($arrLatestChapterFirstBlock['uri'] != $arrLatestChapterSecondBlock['uri']) - - ) { - $this->arrNextPageArgs = [ - 'n_source_id' => $this->arrData['n_source_id'], - 'page' => $this->arrData['page'] + 1, - ]; - } - } else { - $strLastUri = $this->getQueryList()->find('select:first option:last')->attr('value'); - if ($strLastUri != '' && $strLastUri != $this->strUri) { - $this->arrNextPageArgs = [ - 'n_source_id' => $this->arrData['n_source_id'], - 'page' => $this->arrData['page'] + 1, - ]; - } - } - return $this->arrNextPageArgs; } } diff --git a/code/app/task/crawler/zw630/page/Site.php b/code/app/task/crawler/zw630/page/Site.php index 4220611..eb3cc5a 100644 --- a/code/app/task/crawler/zw630/page/Site.php +++ b/code/app/task/crawler/zw630/page/Site.php @@ -4,22 +4,14 @@ declare(strict_types=1); namespace app\task\crawler\zw630\page; -use GuzzleHttp\Client; +use app\task\crawler\zw\page\Site as ZwSite; use GuzzleHttp\Promise; -use storage\StorageCore; - +use GuzzleHttp\Client; /** * @mixin think\Model */ -class Site +class Site extends ZwSite { - /** - * Undocumented variable - * - * @var Client - */ - private $Client; - /** * Undocumented variable * @@ -27,33 +19,17 @@ class Site */ protected $strDomain = 'https://www.630zw.com'; + /** + * Undocumented variable + * + * @var string + */ protected $strSiteCode = 'zw630'; /** * Undocumented function - * - * @return string */ - public function getSiteCode(): string - { - return $this->strSiteCode; - } - - /** - * Undocumented function - * - * @param integer|string $strSign - * @return string - */ - public function getSiteUUId(int|string $strSign): string - { - return $this->getSiteCode() . '-' . $strSign; - } - - /** - * Undocumented function - */ - public function __construct() + public function initClient() { $arrConfig = [ 'http_version' => '1.1', // 强制用 HTTP/1.1，已有，保持 @@ -82,208 +58,90 @@ class Site 'sec-fetch-user' => '?1', 'upgrade-insecure-requests' => '1', ], - 'pool_size' => 5, - 'verify' => false, + 'pool_size' => 10, + 'verify' => false, ]; $arrProxy = config('proxy'); if ($arrProxy['proxy_status']) { - $intTime = time(); - $strOrderNo = $arrProxy['proxy_order']; + + $strCode = $arrProxy['proxy_code']; + $strProxyHost = $arrProxy['proxy_host']; + $strKey = $arrProxy['proxy_key']; $strSecret = $arrProxy['proxy_secret']; - $strTxt = "orderno=" . $strOrderNo . ",secret=" . $strSecret . ",timestamp=" . $intTime; - $strSign = strtoupper(md5($strTxt)); - $strAuth = 'sign=' . $strSign . '&orderno=' . $strOrderNo . '×tamp=' . $intTime; - $arrConfig['proxy'] = 'http://dtbf.xiongmaodaili.com:8089'; - $arrConfig['headers']['Proxy-Authorization'] = $strAuth; + switch ($strCode) { + case 'XIONGMAO': + $intTime = time(); + $strTxt = "orderno=" . $strKey . ",secret=" . $strSecret . ",timestamp=" . $intTime; + $strSign = strtoupper(md5($strTxt)); + $strAuth = 'sign=' . $strSign . '&orderno=' . $strKey . '×tamp=' . $intTime; + $arrConfig['proxy'] = $strProxyHost; + $arrConfig['headers']['Proxy-Authorization'] = $strAuth; + break; + case 'KUAI': + $arrConfig['proxy'] = $strProxyHost; + $arrConfig['curl'][CURLOPT_PROXYAUTH] = CURLAUTH_BASIC; + $arrConfig['curl'][CURLOPT_PROXYUSERPWD] = "$strKey:$strSecret"; + break; + } } $this->Client = new Client($arrConfig); } + + public function getNovelPageUri($intNSourceId, $intPage): string + { + $strUri = ''; + + if ($intPage == 0) { + + $arrItem['uri'] = sprintf('/shu/%s.html', $intNSourceId); + } else { + + $arrItem['uri'] = sprintf('/shu/%s_%s.html', $intNSourceId, $intPage); + } + + return $strUri; + } + + /** * Undocumented function * - * @return Client - */ - public function getClient(): Client - { - return $this->Client; - } - - /** - * retrieve novel page model - * + * @param object $Site * @param string $strUri * @return NovelPage */ - public function getNovelPage(string $strUri): NovelPage + public function getNovelPage($Site, $strUri) { - return new NovelPage($this, $strUri); + return new NovelPage($Site, $strUri); } - /** - * retrieve multiple novel page model + * Undocumented function * + * @param object $Site * @param string $strUri - * @return NovelPage[] + * @return ChapterPage */ - public function getNovelPageList(array $arrFilter): array + public function getChapterPage($Site, $strUri) { - - /** @var NovelPage[] */ - $arrNovelPage = []; - - foreach ($arrFilter as $intKey => $arrItem) { - - if ($arrItem['page'] == 0) { - $arrItem['uri'] = sprintf('/shu/%s.html', $arrItem['n_source_id']); - } else { - $arrItem['uri'] = sprintf('/shu/%s_%s.html', $arrItem['n_source_id'], $arrItem['page']); - } - $arrFilter[$intKey]['uri'] = $arrItem['uri']; - - $arrPromises[$intKey] = $this->getClient()->getAsync($arrItem['uri']); - } - - $arrResults = Promise\Utils::settle($arrPromises)->wait(); - - $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) { - return $arrResult['state'] === 'fulfilled'; - }); - - foreach ($arrSuccessfulResults as $intKey => $arrResult) { - $Response = $arrResult['value']; - - $strContent = $Response->getBody()->getContents(); - if ($strContent == '索引文件不存在！') { - continue; - } - - $NovelPage = new NovelPage($this, $arrFilter[$intKey]['uri']); - - $NovelPage->setContent($strContent); - $NovelPage->setData($arrFilter[$intKey]); - - $arrNovelPage[] = $NovelPage; - } - - - return $arrNovelPage; + return new ChapterPage($Site, $strUri); } /** - * retrieve multiple chapter page model + * Undocumented function * + * @param object $Site * @param string $strUri - * @return ChapterPage[] + * @return LatestListPage */ - public function getChapterPageList(array $arrFilter): array + public function getLatestListPage($Site, $strUri) { - - /** @var ChapterPage[] */ - $arrChapterPage = []; - - $arrPromises = []; - foreach ($arrFilter as $intKey => $arrItem) { - $arrPromises[$intKey] = $this->getClient()->getAsync($arrItem['uri']); - } - - $arrResults = Promise\Utils::settle($arrPromises)->wait(); - - $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) { - return $arrResult['state'] === 'fulfilled'; - }); - - foreach ($arrSuccessfulResults as $intKey => $arrResult) { - $Response = $arrResult['value']; - - $strContent = $Response->getBody()->getContents();; - - $ChapterPage = new ChapterPage($this, $arrFilter[$intKey]['uri']); - - $ChapterPage->setContent($strContent); - $ChapterPage->setData($arrFilter[$intKey]); - - $arrChapterPage[] = $ChapterPage; - } - - - return $arrChapterPage; - } - - - /** - * retrieve multiple chapter page model - * - * @param string $strUri - * @return LatestListPage[] - */ - public function getLatestListPageList(array $arrFilter): array - { - - /** @var LatestListPage[] */ - $arrLatestListPage = []; - - $arrPromises = []; - foreach ($arrFilter as $intKey => $strUri) { - $arrPromises[$intKey] = $this->getClient()->getAsync($strUri); - } - - $arrResults = Promise\Utils::settle($arrPromises)->wait(); - - $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) { - return $arrResult['state'] === 'fulfilled'; - }); - - foreach ($arrSuccessfulResults as $intKey => $arrResult) { - $Response = $arrResult['value']; - - $strContent = $Response->getBody()->getContents();; - - $LatestListPage = new LatestListPage($this, $arrFilter[$intKey]); - - $LatestListPage->setContent($strContent); - - $arrLatestListPage[] = $LatestListPage; - } - - - return $arrLatestListPage; - } - - /** - * download the image to the local system - * - * @param array $arrFilter - * @return void - */ - public function downRemoteImgToLocal(array $arrFilter) - { - $StorageCore = StorageCore::getInstance(); - - $arrPromises = []; - - foreach ($arrFilter as $intKey => $arrItem) { - if (!$StorageCore->has($arrItem['local_path'])) { - $arrPromises[$intKey] = $this->getClient()->getAsync($arrItem['uri']); - } - } - - $arrResults = Promise\Utils::settle($arrPromises)->wait(); - - $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) { - return $arrResult['state'] === 'fulfilled'; - }); - - foreach ($arrSuccessfulResults as $intKey => $arrResult) { - $Response = $arrResult['value']; - - $StorageCore->writeStream($arrFilter[$intKey]['local_path'], $Response->getBody()->detach());; - } + return new LatestListPage($Site, $strUri); } } diff --git a/code/app/task/crawler/zw74/Scheduler.php b/code/app/task/crawler/zw74/Scheduler.php new file mode 100644 index 0000000..6780f01 --- /dev/null +++ b/code/app/task/crawler/zw74/Scheduler.php @@ -0,0 +1,37 @@ +intSlice = 8; + } + + /** + * Undocumented function + * + * @return Site + */ + public function getSite() + { + if ($this->Site == NULL) { + $this->Site = new Site; + } + return $this->Site; + } +} diff --git a/code/app/task/crawler/zw74/page/ChapterPage.php b/code/app/task/crawler/zw74/page/ChapterPage.php new file mode 100644 index 0000000..c2d49b1 --- /dev/null +++ b/code/app/task/crawler/zw74/page/ChapterPage.php @@ -0,0 +1,9 @@ +arrRelatedNovel) || empty($this->arrForgeNovel)) { + $this->getQueryList()->find('.first_txt')->eq(0)->find("a")->map(function ($A) { + $strName = $A->text(); + $strUri = $A->attr('href'); + + if (is_numeric(strpos($strUri, 'http'))) { + return; + } + + $strPattern = '#^/\d+/(\d+)/$#'; + + if (is_numeric(strpos($strUri, 'info'))) { + $intNForgeId = (int)supperExtractFilename($strUri); + $this->arrForgeNovel[] = [ + "n_forge_id" => $intNForgeId, + "n_forge_title" => $strName, + ]; + } elseif (preg_match($strPattern, $strUri, $arrMatches)) { + $intNSourceId = (int)$arrMatches[1]; + $arrNovelInfo = $this->findOrCreateNovel($intNSourceId, $strName); + $this->arrRelatedNovel[] = [ + "n_id" => $arrNovelInfo["n_id"], + "n_source_id" => $arrNovelInfo["n_source_id"], + "og_novel_book_name" => $arrNovelInfo["og_novel_book_name"], + ]; + } + }); + } + } +} diff --git a/code/app/task/crawler/zw74/page/Site.php b/code/app/task/crawler/zw74/page/Site.php new file mode 100644 index 0000000..298a260 --- /dev/null +++ b/code/app/task/crawler/zw74/page/Site.php @@ -0,0 +1,152 @@ + '1.1', // 强制用 HTTP/1.1，已有，保持 + 'base_uri' => $this->strDomain, // 基础域名，已有，保持 + 'http_errors' => false, // 不抛 HTTP 错误，已有，保持 + 'timeout' => 60, // 从 160 秒改为 60 秒 + 'connect_timeout' => 10, // 从 30 秒改为 10 秒 + 'curl' => [ + CURLOPT_TCP_KEEPALIVE => 1, // 启用 TCP Keep-Alive，已有，保持 + CURLOPT_TCP_KEEPIDLE => 120, // Keep-Alive 空闲时间 + CURLOPT_TCP_KEEPINTVL => 60, // Keep-Alive 探测间隔 + // CURLOPT_SSL_VERIFYPEER => true, // 验证 SSL，已有，保持 + ], + 'headers' => [ + 'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36 Edg/135.0.0.0', + 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7', + 'Accept-Language' => 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6', + 'Referer' => $this->strDomain, + 'Connection' => 'keep-alive', + 'sec-ch-ua' => '"Microsoft Edge";v="135", "Not-A.Brand";v="8", "Chromium";v="135"', + 'sec-ch-ua-mobile' => '?0', + 'sec-ch-ua-platform' => '"Windows"', + 'sec-fetch-dest' => 'document', + 'sec-fetch-mode' => 'navigate', + 'sec-fetch-site' => 'same-origin', + 'sec-fetch-user' => '?1', + 'upgrade-insecure-requests' => '1', + ], + 'pool_size' => 10, + 'verify' => false, + ]; + + $arrProxy = config('proxy'); + if ($arrProxy['proxy_status']) { + + $strCode = $arrProxy['proxy_code']; + $strProxyHost = $arrProxy['proxy_host']; + $strKey = $arrProxy['proxy_key']; + $strSecret = $arrProxy['proxy_secret']; + + + switch ($strCode) { + case 'XIONGMAO': + $intTime = time(); + $strTxt = "orderno=" . $strKey . ",secret=" . $strSecret . ",timestamp=" . $intTime; + $strSign = strtoupper(md5($strTxt)); + $strAuth = 'sign=' . $strSign . '&orderno=' . $strKey . '×tamp=' . $intTime; + $arrConfig['proxy'] = $strProxyHost; + $arrConfig['headers']['Proxy-Authorization'] = $strAuth; + break; + case 'KUAI': + $arrConfig['proxy'] = $strProxyHost; + $arrConfig['curl'][CURLOPT_PROXYAUTH] = CURLAUTH_BASIC; + $arrConfig['curl'][CURLOPT_PROXYUSERPWD] = "$strKey:$strSecret"; + break; + } + } + + + $this->Client = new Client($arrConfig); + } + + /** + * Undocumented function + * + * @param int $intNSourceId + * @param int $intPage + * @return string + */ + public function getNovelPageUri($intNSourceId, $intPage): string + { + $intUriFirst = intdiv($intNSourceId, 1000); + + $strUri = ''; + + if ($intPage == 0) { + $strUri = sprintf('/%s/%s/', $intUriFirst, $intNSourceId); + } else { + $strUri = sprintf('/book/%s/list%s.html', $intNSourceId, $intPage); + } + + return $strUri; + } + + /** + * Undocumented function + * + * @param object $Site + * @param string $strUri + * @return NovelPage + */ + public function getNovelPage($Site, $strUri) + { + return new NovelPage($Site, $strUri); + } + + /** + * Undocumented function + * + * @param object $Site + * @param string $strUri + * @return ChapterPage + */ + public function getChapterPage($Site, $strUri) + { + return new ChapterPage($Site, $strUri); + } + + /** + * Undocumented function + * + * @param object $Site + * @param string $strUri + * @return LatestListPage + */ + public function getLatestListPage($Site, $strUri) + { + return new LatestListPage($Site, $strUri); + } +} diff --git a/code/app/task/module/PlanTask.php b/code/app/task/module/PlanTask.php index 8807682..f0c873c 100644 --- a/code/app/task/module/PlanTask.php +++ b/code/app/task/module/PlanTask.php @@ -6,6 +6,7 @@ namespace app\task\module; use app\model\PlanTaskModel; use app\task\crawler\zw630\Scheduler as Zw630Scheduler; +use app\task\crawler\zw74\Scheduler as Zw74Scheduler; use microserver\QueueManage; use microserver\ProcessSanitizer; @@ -34,6 +35,13 @@ class PlanTask case 'PULL_630_ZW_NOVEL_LATEST': self::pull630ZwNovelLatest(); break; + + case 'PULL_74_ZW_NOVEL': + self::pull74ZwNovel(); + break; + case 'PULL_74_ZW_NOVEL_LATEST': + self::pull74ZwNovelLatest(); + break; default: break; } @@ -71,4 +79,37 @@ class PlanTask $QueueManage = new QueueManage(); $QueueManage->set($arrTask); } + + + /** + * Undocumented function + * + * @return void + */ + public static function pull74ZwNovel() + { + $arrTask = [ + 'callback' => [Zw74Scheduler::class, 'crawlFull'], + 'data' => [] + ]; + + $QueueManage = new QueueManage(); + $QueueManage->set($arrTask); + } + + /** + * Undocumented function + * + * @return void + */ + public static function pull74ZwNovelLatest() + { + $arrTask = [ + 'callback' => [Zw74Scheduler::class, 'crawLatest'], + 'data' => [] + ]; + + $QueueManage = new QueueManage(); + $QueueManage->set($arrTask); + } } diff --git a/code/config/proxy.php b/code/config/proxy.php index 6a7bc15..483e797 100644 --- a/code/config/proxy.php +++ b/code/config/proxy.php @@ -6,6 +6,7 @@ return [ 'proxy_status' => env('PROXY_STATUS', false), 'proxy_code' => env('PROXY_CODE', ''), - 'proxy_order' => env('PROXY_ORDER', ''), + 'proxy_host' => env('PROXY_HOST', ''), + 'proxy_key' => env('PROXY_KEY', ''), 'proxy_secret' => env('PROXY_SECRET', ''), ]; diff --git a/code/database/mongo/migrations/novel.php b/code/database/mongo/migrations/novel.php index 9029029..55df22f 100644 --- a/code/database/mongo/migrations/novel.php +++ b/code/database/mongo/migrations/novel.php @@ -4,14 +4,14 @@ return [ 'indexes' => [ [ 'key' => ['n_id' => 1], + 'options' => ['unique' => true] + ], + [ + 'key' => ['n_name' => 1], 'options' => ['unique' => false] ], [ - 'key' => ['og_novel_book_name' => 1], - 'options' => ['unique' => false] - ], - [ - 'key' => ['og_novel_status' => 1], + 'key' => ['n_status' => 1], 'options' => ['unique' => false] ], [ diff --git a/code/database/seeders/PlanTaskSeeder.php b/code/database/seeders/PlanTaskSeeder.php index e59e5ca..a8520a3 100644 --- a/code/database/seeders/PlanTaskSeeder.php +++ b/code/database/seeders/PlanTaskSeeder.php @@ -20,13 +20,28 @@ class PlanTaskSeeder [ 'pt_name' => '630小说全量扫描', 'pt_code' => 'PULL_630_ZW_NOVEL', - 'pt_enable' => 1, + 'pt_enable' => 0, 'pt_limit' => 10 * 24 * 3600 * 3, ], [ 'pt_name' => '630小说每日增量', 'pt_code' => 'PULL_630_ZW_NOVEL_LATEST', - 'pt_enable' => 1, + 'pt_enable' => 0, + 'pt_limit' => 1 * 24 * 3600, + ], + + + + [ + 'pt_name' => '74小说全量扫描', + 'pt_code' => 'PULL_74_ZW_NOVEL', + 'pt_enable' => 0, + 'pt_limit' => 10 * 24 * 3600 * 3, + ], + [ + 'pt_name' => '74小说每日增量', + 'pt_code' => 'PULL_74_ZW_NOVEL_LATEST', + 'pt_enable' => 0, 'pt_limit' => 1 * 24 * 3600, ], ]; diff --git a/code/extend/storage/StorageCore.php b/code/extend/storage/StorageCore.php index bb2d579..4557008 100644 --- a/code/extend/storage/StorageCore.php +++ b/code/extend/storage/StorageCore.php @@ -75,10 +75,14 @@ class StorageCore public function read($strKey) { - $Stream = $this->Filesystem->readStream($strKey); - $strContent = stream_get_contents($Stream); - $strContent = zstd_uncompress($strContent); - return $strContent; + try { + $Stream = $this->Filesystem->readStream($strKey); + $strContent = stream_get_contents($Stream); + $strContent = zstd_uncompress($strContent); + return $strContent; + } catch (\Exception $e) { + return ''; + } } public function has($strKey) From d19ea974817549526d417053c70d9c620af22f24 Mon Sep 17 00:00:00 2001 From: Default Date: Tue, 15 Apr 2025 21:16:14 +0800 Subject: [PATCH 2/3] debug --- code/app/home/controller/Index.php | 8 +- code/app/home/controller/Novel.php | 111 ++++---- code/app/home/view/default/Index/index.html | 10 +- code/app/home/view/default/baseH5.html | 2 +- .../view/kuangYu/novel/getNovelCatalog.html | 50 ++-- .../view/kuangYu/novel/getNovelChapter.html | 86 +++--- .../home/view/kuangYu/novel/getNovelInfo.html | 90 +++---- .../view/kuangYu/novel/getSearchNovel.html | 2 +- .../home/view/kuangYu/pc/getKanNovelInfo.html | 106 ++++---- .../home/view/kuangYu/pc/getNovelCatalog.html | 54 ++-- .../home/view/kuangYu/pc/getNovelChapter.html | 90 +++---- .../home/view/kuangYu/pc/getNovelInfo.html | 108 ++++---- .../home/view/kuangYu/pc/getSearchNovel.html | 2 +- code/app/home/view/kuangYu/pc/index.html | 8 +- .../view/kuangYu/pc/public/endNovelList.html | 8 +- .../view/kuangYu/pc/public/hostListImg.html | 10 +- .../view/kuangYu/pc/public/hostListText.html | 6 +- .../kuangYu/pc/public/indexBoyNewsList.html | 16 +- .../kuangYu/pc/public/indexBoyNewsOne.html | 16 +- .../kuangYu/pc/public/indexBoyRankList.html | 14 +- .../kuangYu/pc/public/indexBoyRankOne.html | 12 +- .../pc/public/indexNewsUpdateList.html | 18 +- .../kuangYu/pc/public/shukuNovelList.html | 16 +- .../view/kuangYu/pc/public/swiperListImg.html | 4 +- .../kuangYu/pc/public/swiperListText.html | 4 +- code/app/home/view/kuangYu/pc/user/index.html | 68 ++--- .../home/view/kuangYu/public/articleList.html | 2 +- .../home/view/kuangYu/public/novelListH.html | 12 +- .../view/kuangYu/public/novelListRank.html | 2 +- .../home/view/kuangYu/public/novelListS.html | 8 +- code/app/model/CategoryModel.php | 6 +- code/app/model/ChapterModel.php | 18 +- code/app/model/NovelClicksModel.php | 2 +- code/app/model/NovelModel.php | 18 +- code/app/task/crawler/zw/Scheduler.php | 16 +- code/app/task/crawler/zw/page/ChapterPage.php | 135 +++++----- code/app/task/crawler/zw/page/NovelPage.php | 246 +++++++++++------- code/app/task/crawler/zw/page/Site.php | 66 ++++- .../app/task/crawler/zw630/page/NovelPage.php | 2 +- code/app/task/crawler/zw74/page/NovelPage.php | 2 +- code/composer.json | 4 +- code/composer.lock | 154 +++++------ code/database/mongo/migrations/chapter.php | 2 +- code/database/mongo/migrations/novel.php | 6 +- code/extend/db/NovelManage.php | 10 +- code/extend/storage/StorageCore.php | 6 + .../template/kuangyu/public/js/common.js | 40 +-- 47 files changed, 907 insertions(+), 769 deletions(-) diff --git a/code/app/home/controller/Index.php b/code/app/home/controller/Index.php index 910ba42..5a70e4b 100644 --- a/code/app/home/controller/Index.php +++ b/code/app/home/controller/Index.php @@ -42,7 +42,7 @@ class Index extends BaseController break; case 'kuangYu': $strKey = sprintf('%s:H5:INDEX:ZUIJIAN:SEX:%s:%s', $Request->DomainModel->d_domain, 0, 8); - $arrNewsNovel = $NovelModel->getCommonNovelOnCache($strKey, $intTTL, 8, 0, NULL, ['og_novel_update_time' => -1]); + $arrNewsNovel = $NovelModel->getCommonNovelOnCache($strKey, $intTTL, 8, 0, NULL, ['n_update_time' => -1]); $strKey = sprintf('%s:H5:INDEX:RANK:%s:SEX:%s', $Request->DomainModel->d_domain, 'daily', 0); $arrDayRankNovel = $NovelModel->getRankNovelOnCache($strKey, 10, 0, 'daily'); @@ -141,7 +141,7 @@ class Index extends BaseController // 男生频道，最新的 随机5个小说 $strKey = sprintf('%s:PC:INDEX:ZUIJIAN:SEX:%s:%s', $Request->DomainModel->d_domain, 1, 5); - $arrBoyNewsNovel = $NovelModel->getCommonNovelOnCache($strKey, $intTTL, 5, 1, NULL, ['og_novel_update_time' => -1]); + $arrBoyNewsNovel = $NovelModel->getCommonNovelOnCache($strKey, $intTTL, 5, 1, NULL, ['n_update_time' => -1]); // 男生频道，周排行榜 随机10个小说 @@ -158,7 +158,7 @@ class Index extends BaseController // 女生频道，最新的 随机5个小说 $strKey = sprintf('%s:PC:INDEX:ZUIJIAN:SEX:%s:%s', $Request->DomainModel->d_domain, 2, 5); - $arrGirlNewsNovel = $NovelModel->getCommonNovelOnCache($strKey, $intTTL, 5, 2, NULL, ['og_novel_update_time' => -1]); + $arrGirlNewsNovel = $NovelModel->getCommonNovelOnCache($strKey, $intTTL, 5, 2, NULL, ['n_update_time' => -1]); // 女生频道，周排行榜 随机10个小说 $strKey = sprintf('%s:PC:INDEX:RANK:%s:SEX:%s', $Request->DomainModel->d_domain, 'weekly', 2); @@ -174,7 +174,7 @@ class Index extends BaseController // 最近更新-随机50 $strKey = sprintf('%s:PC:INDEX:ZUIJIAN:SEX:%s:%s', $Request->DomainModel->d_domain, 0, 50); - $arrNewsUpdateNovel = $NovelModel->getCommonNovelOnCache($strKey, $intTTL, 50, 0, NULL, ['og_novel_update_time' => -1]); + $arrNewsUpdateNovel = $NovelModel->getCommonNovelOnCache($strKey, $intTTL, 50, 0, NULL, ['n_update_time' => -1]); // Title 模板 ConverterMovel::setVal([ diff --git a/code/app/home/controller/Novel.php b/code/app/home/controller/Novel.php index 0224a04..c85cb65 100644 --- a/code/app/home/controller/Novel.php +++ b/code/app/home/controller/Novel.php @@ -248,11 +248,11 @@ class Novel extends BaseController // 模拟数据-最近更新-随机50 $strKey = sprintf('%s:PC:Novel:Channel:Latest:%s', $Request->DomainModel->d_domain, $intType); $intCount = 20; - $arrNewsUpdateNovel = $NovelModel->getCommonNovelOnCache($strKey, $intLifeTime, $intCount, $intType, NULL, ['og_novel_update_time' => -1]); + $arrNewsUpdateNovel = $NovelModel->getCommonNovelOnCache($strKey, $intLifeTime, $intCount, $intType, NULL, ['n_update_time' => -1]); // echo "
";
         // var_dump($arrNewsUpdateNovel);exit;
-        // TODO   og_description  改成 og_description
+        // TODO   n_description  改成 n_description
         // 更新值
         ConverterMovel::setVal([
             'intPage' => 1,
@@ -388,14 +388,14 @@ class Novel extends BaseController
 
         foreach ($arrNovel['n_forge_novel'] as $arrForgeNovel) {
             if ($arrForgeNovel['n_forge_id'] == $intForgeId) {
-                $arrNovel['og_novel_book_name'] = $arrForgeNovel['n_forge_title'];
+                $arrNovel['n_name'] = $arrForgeNovel['n_forge_title'];
                 break;
             }
         }
 
         // 分类拼音
-        $arrNovel['category_pinyin'] = CategoryModel::getPinYinByZh($arrNovel['og_novel_category']);
-        $arrNovel['category_sex'] = $arrNovel['category_sex'] ?? 'man';
+        $arrNovel['n_category_pinyin'] = CategoryModel::getPinYin($arrNovel['n_category']);
+        $arrNovel['n_category_sex_en'] = $arrNovel['n_category_sex_en'] ?? 'man';
 
         // 最后章节
         $ChapterModel = ChapterModel::getInstance();
@@ -411,7 +411,7 @@ class Novel extends BaseController
         $strFocus = '';
         // 要点
         if ($arrLatestChapter) {
-            $strFocus =  strip_tags($arrLatestChapter['content']);
+            $strFocus =  strip_tags($arrLatestChapter['c_content']);
             $intCount = mb_strlen($strFocus);
             if ($intCount > 300) {
                 $intPos = mt_rand(0, $intCount - 300);
@@ -439,12 +439,12 @@ class Novel extends BaseController
         // 更新值
         ConverterMovel::setVal([
             'intPage' => 1,
-            'strNovelName' => $arrNovel['og_novel_book_name'],
-            'strNovelAuthor' => $arrNovel['og_novel_author'],
-            'strNovelCategoryName' => $arrNovel['og_novel_category'],
-            'strNovelSex' => $arrNovel['category_sex'] == 'man' ? '男生' : '女生',
-            'strNovelStatus' => $arrNovel['og_novel_status'],
-            'strNovelDescription' => $arrNovel['og_description'],
+            'strNovelName' => $arrNovel['n_name'],
+            'strNovelAuthor' => $arrNovel['n_author'],
+            'strNovelCategoryName' => $arrNovel['n_category'],
+            'strNovelSex' => $arrNovel['n_category_sex_en'] == 'man' ? '男生' : '女生',
+            'strNovelStatus' => $arrNovel['n_status'],
+            'strNovelDescription' => $arrNovel['n_description'],
         ]);
 
         $strTitleTemplate =  $Request->DomainModel->getFomartSubject('NOVEL@GETNOVELINFO@TITLE',$intForgeId);
@@ -457,7 +457,7 @@ class Novel extends BaseController
 
         // js使用到
         $arrNovelJs = $arrNovel;
-        unset($arrNovelJs['n_related_novel'], $arrNovelJs['n_forge_novel'], $arrNovelJs['og_description']);
+        unset($arrNovelJs['n_related_novel'], $arrNovelJs['n_forge_novel'], $arrNovelJs['n_description']);
 
         View::assign([
             'arrRelateveNovel' => $arrRelateveNovel,
@@ -498,8 +498,8 @@ class Novel extends BaseController
         $arrNovel = $NovelModel->getNovelByNId($intNId);
 
         // 分类拼音
-        $arrNovel['category_pinyin'] = CategoryModel::getPinYinByZh($arrNovel['og_novel_category']);
-        $arrNovel['category_sex'] = $arrNovel['category_sex'] ?? 'man';
+        $arrNovel['n_category_pinyin'] = CategoryModel::getPinYin($arrNovel['n_category']);
+        $arrNovel['n_category_sex_en'] = $arrNovel['n_category_sex_en'] ?? 'man';
 
         // 最后章节
         $ChapterModel = ChapterModel::getInstance();
@@ -515,7 +515,7 @@ class Novel extends BaseController
         $strFocus = '';
         // 要点
         if ($arrLatestChapter) {
-            $strFocus =  strip_tags($arrLatestChapter['content']);
+            $strFocus =  strip_tags($arrLatestChapter['c_content']);
             $intCount = mb_strlen($strFocus);
             if ($intCount > 300) {
                 $intPos = mt_rand(0, $intCount - 300);
@@ -543,12 +543,12 @@ class Novel extends BaseController
         // 更新值
         ConverterMovel::setVal([
             'intPage' => 1,
-            'strNovelName' => $arrNovel['og_novel_book_name'],
-            'strNovelAuthor' => $arrNovel['og_novel_author'],
-            'strNovelCategoryName' => $arrNovel['og_novel_category'],
-            'strNovelSex' => $arrNovel['category_sex'] == 'man' ? '男生' : '女生',
-            'strNovelStatus' => $arrNovel['og_novel_status'],
-            'strNovelDescription' => $arrNovel['og_description'],
+            'strNovelName' => $arrNovel['n_name'],
+            'strNovelAuthor' => $arrNovel['n_author'],
+            'strNovelCategoryName' => $arrNovel['n_category'],
+            'strNovelSex' => $arrNovel['n_category_sex_en'] == 'man' ? '男生' : '女生',
+            'strNovelStatus' => $arrNovel['n_status'],
+            'strNovelDescription' => $arrNovel['n_description'],
         ]);
 
         $strTitleTemplate =  $Request->DomainModel->getFomartSubject(getFomartSubjectKey(__METHOD__) . '@TITLE',$intNId);
@@ -595,8 +595,8 @@ class Novel extends BaseController
         $NovelModel = NovelModel::getInstance();
         $arrNovel = $NovelModel->getNovelByNId($intNId);
 
-        $arrNovel['category_pinyin'] =  $arrNovel['category_pinyin'] ?? 'all';
-        $arrNovel['category_sex'] = $arrNovel['category_sex'] ?? 'man';
+        $arrNovel['n_category_pinyin'] =  $arrNovel['n_category_pinyin'] ?? 'all';
+        $arrNovel['n_category_sex_en'] = $arrNovel['n_category_sex_en'] ?? 'man';
 
         // 最新章节
         $ChapterModel = ChapterModel::getInstance();
@@ -611,9 +611,11 @@ class Novel extends BaseController
             $arrNextChapter = $ChapterModel->getNextChapter($arrChapter['n_id'], $arrChapter['c_sort_num'], $arrChapter['c_page']);
 
             // 要点-先从章节随机截取，后面如果用上ai，看看使用ai 提炼 精简内容
-            $strChapterDescription = strip_tags($arrChapter['content']);
+            $strChapterDescription = strip_tags($arrChapter['c_content']);
             $intCount = mb_strlen($strChapterDescription);
             $strChapterDescription = ($intCount > 50) ? mb_substr($strChapterDescription, 0, 50) : $strChapterDescription;
+
+            $arrChapter['c_content'] = base64_encode($arrChapter['c_content']);
         }
 
         $arrCategoryAll = CategoryModel::$arrCategoryAll;
@@ -621,13 +623,13 @@ class Novel extends BaseController
         // 更新值
         ConverterMovel::setVal([
             'intPage' => 1,
-            'strNovelName' => $arrNovel['og_novel_book_name'],
-            'strNovelAuthor' => $arrNovel['og_novel_author'],
-            'strNovelCategoryName' => $arrNovel['og_novel_category'],
-            'strNovelStatus' => $arrNovel['og_novel_status'],
-            'strNovelDescription' => $arrNovel['og_description'],
+            'strNovelName' => $arrNovel['n_name'],
+            'strNovelAuthor' => $arrNovel['n_author'],
+            'strNovelCategoryName' => $arrNovel['n_category'],
+            'strNovelStatus' => $arrNovel['n_status'],
+            'strNovelDescription' => $arrNovel['n_description'],
             'strChapterDescription' => $strChapterDescription,
-            'strNovelChapterName' => $arrChapter['name'] ?? '',
+            'strNovelChapterName' => $arrChapter['c_name'] ?? '',
             'strNovelChapterSort' => $arrChapter['c_sort_num'] ?? '',
         ]);
 
@@ -640,6 +642,7 @@ class Novel extends BaseController
         $strKeywords = ConverterMovel::convert($strKeywordsTemplate);
         $strDescription = ConverterMovel::convert($strDescriptionTemplate);
 
+
         View::assign([
             'arrNovel' => $arrNovel,
             'arrCategoryAll' => $arrCategoryAll,
@@ -679,8 +682,8 @@ class Novel extends BaseController
         $intNId = $intNovelId;
         $arrNovel = $NovelModel->getNovelByNId($intNId);
 
-        $arrNovel['category_pinyin'] =  $arrNovel['category_pinyin'] ?? 'all';
-        $arrNovel['category_sex'] = $arrNovel['category_sex'] ?? 'man';
+        $arrNovel['n_category_pinyin'] =  $arrNovel['n_category_pinyin'] ?? 'all';
+        $arrNovel['n_category_sex_en'] = $arrNovel['n_category_sex_en'] ?? 'man';
 
         // 章节
         $ChapterModel = ChapterModel::getInstance();
@@ -697,11 +700,11 @@ class Novel extends BaseController
             $arrNextChapter = $ChapterModel->getNextChapter($arrChapter['n_id'], $arrChapter['c_sort_num'], $arrChapter['c_page']);
 
             // 要点-先从章节随机截取，后面如果用上ai，看看使用ai 提炼 精简内容
-            $strChapterDescription = strip_tags($arrChapter['content']);
+            $strChapterDescription = strip_tags($arrChapter['c_content']);
             $intCount = mb_strlen($strChapterDescription);
             $strChapterDescription = ($intCount > 50) ? mb_substr($strChapterDescription, 0, 50) : $strChapterDescription;
 
-            $arrChapter['content'] = base64_encode($arrChapter['content']);
+            $arrChapter['c_content'] = base64_encode($arrChapter['c_content']);
         }
 
         $arrCategoryAll = CategoryModel::$arrCategoryAll;
@@ -709,13 +712,13 @@ class Novel extends BaseController
         // 更新值
         ConverterMovel::setVal([
             'intPage' => 1,
-            'strNovelName' => $arrNovel['og_novel_book_name'],
-            'strNovelAuthor' => $arrNovel['og_novel_author'],
-            'strNovelCategoryName' => $arrNovel['og_novel_category'],
-            'strNovelStatus' => $arrNovel['og_novel_status'],
-            'strNovelDescription' => $arrNovel['og_description'],
+            'strNovelName' => $arrNovel['n_name'],
+            'strNovelAuthor' => $arrNovel['n_author'],
+            'strNovelCategoryName' => $arrNovel['n_category'],
+            'strNovelStatus' => $arrNovel['n_status'],
+            'strNovelDescription' => $arrNovel['n_description'],
             'strChapterDescription' => $strChapterDescription,
-            'strNovelChapterName' => $arrChapter['name'],
+            'strNovelChapterName' => $arrChapter['c_name'],
             'strNovelChapterSort' => $arrChapter['c_sort_num'],
         ]);
 
@@ -765,8 +768,8 @@ class Novel extends BaseController
         $intNId = $intNovelId;
         $arrNovel = $NovelModel->getNovelByNId($intNId);
 
-        $arrNovel['category_pinyin'] =  $arrNovel['category_pinyin'] ?? 'all';
-        $arrNovel['category_sex'] = $arrNovel['category_sex'] ?? 'man';
+        $arrNovel['n_category_pinyin'] =  $arrNovel['n_category_pinyin'] ?? 'all';
+        $arrNovel['n_category_sex_en'] = $arrNovel['n_category_sex_en'] ?? 'man';
 
         $ChapterModel = ChapterModel::getInstance();
 
@@ -789,18 +792,18 @@ class Novel extends BaseController
             $intButtomNum = 10;
         }
 
-        $strBaseUrl = sprintf('/pc/xiaoshuo/%s-%s/mulu/%s/{page}', $arrNovel['og_novel_pin_yin'], $intNId, $strOrder);
+        $strBaseUrl = sprintf('/pc/xiaoshuo/%s-%s/mulu/%s/{page}', $arrNovel['n_name_pinyin'], $intNId, $strOrder);
         $arrPaginator = CusteomPage02::generate($intPage,  $arrPage['total'], $intLimit, $strBaseUrl, $intButtomNum);
 
         // 更新值
         ConverterMovel::setVal([
             'intPage' => $intPage,
-            'strNovelSex' => $arrNovel['category_sex'] == 'man' ? '男生' : '女生',
-            'strNovelName' => $arrNovel['og_novel_book_name'],
-            'strNovelAuthor' => $arrNovel['og_novel_author'],
-            'strNovelCategoryName' => $arrNovel['og_novel_category'],
-            'strNovelStatus' => $arrNovel['og_novel_status'],
-            'strNovelDescription' => $arrNovel['og_description'],
+            'strNovelSex' => $arrNovel['n_category_sex_en'] == 'man' ? '男生' : '女生',
+            'strNovelName' => $arrNovel['n_name'],
+            'strNovelAuthor' => $arrNovel['n_author'],
+            'strNovelCategoryName' => $arrNovel['n_category'],
+            'strNovelStatus' => $arrNovel['n_status'],
+            'strNovelDescription' => $arrNovel['n_description'],
         ]);
 
         $strTitleTemplate =  $Request->DomainModel->getFomartSubject(getFomartSubjectKey(__METHOD__) . '@TITLE',$intNId);
@@ -861,10 +864,10 @@ class Novel extends BaseController
 
         $arrFilter = [
             '$or' => [
-                ['og_novel_book_name' => ['$regex' => $strSearchKeywords, '$options' => 'i']],
-                ['og_novel_author' => ['$regex' => $strSearchKeywords, '$options' => 'i']],
-                ['og_novel_category' => ['$regex' => $strSearchKeywords, '$options' => 'i']],
-                ['og_description' => ['$regex' => $strSearchKeywords, '$options' => 'i']],
+                ['n_name' => ['$regex' => $strSearchKeywords, '$options' => 'i']],
+                ['n_author' => ['$regex' => $strSearchKeywords, '$options' => 'i']],
+                ['n_category' => ['$regex' => $strSearchKeywords, '$options' => 'i']],
+                ['n_description' => ['$regex' => $strSearchKeywords, '$options' => 'i']],
             ]
         ];
 
diff --git a/code/app/home/view/default/Index/index.html b/code/app/home/view/default/Index/index.html
index 12d6a61..c4c95d5 100644
--- a/code/app/home/view/default/Index/index.html
+++ b/code/app/home/view/default/Index/index.html
@@ -16,15 +16,15 @@
         {foreach $arrRecommendNovel as $key=>$Novel }
         

             

-                
-                    {$Novel.og_novel_book_name ?? ''} - {$Novel.og_novel_category ?? ''}最新章节在线免费阅读
+                    {$Novel.n_name ?? ''} - {$Novel.n_category ?? ''}最新章节在线免费阅读
                     
                 

             

-                
{$Novel.og_novel_book_name}

-                
作者：{$Novel.og_novel_author}

-                
{$Novel.og_description}

+                
{$Novel.n_name}

+                
作者：{$Novel.n_author}

+                
{$Novel.n_description}

             

         

         {/foreach}
diff --git a/code/app/home/view/default/baseH5.html b/code/app/home/view/default/baseH5.html
index 64e913f..32f2351 100644
--- a/code/app/home/view/default/baseH5.html
+++ b/code/app/home/view/default/baseH5.html
@@ -68,7 +68,7 @@
         
最新标签

         

             {foreach $arrRecommendNovel as $key=>$vo }
-            {$vo.og_novel_book_name}  
+            {$vo.n_name}  
             {/foreach}
             XML地图  
         

diff --git a/code/app/home/view/kuangYu/novel/getNovelCatalog.html b/code/app/home/view/kuangYu/novel/getNovelCatalog.html
index 0d49627..421b923 100644
--- a/code/app/home/view/kuangYu/novel/getNovelCatalog.html
+++ b/code/app/home/view/kuangYu/novel/getNovelCatalog.html
@@ -4,26 +4,26 @@
 {block name="description"}{$strDescription??''}{/block}
 {block name="head"}
 
-
+
 
 
+    content='https://{$DomainModel->d_domain}{site:nurl n_id="$arrNovel[n_id]" n_py="$arrNovel[n_name_pinyin]" /}' />
 
 
 
-
-
-
+
+
+
 
-
-
-
+    content='https://{$DomainModel->d_domain}{site:cpurl n_id="$arrNovel[n_id]" n_py="$arrNovel[n_name_pinyin]" c_sort="$intFirstChapterSort"/}'>
+
+
+
 
+    content='https://{$DomainModel->d_domain}{site:lcpurl n_id="$arrNovel[n_id]" n_py="$arrNovel[n_name_pinyin]" /}'>
 
 
-
+
 
 
 
@@ -33,23 +33,23 @@
 {
     "@context": "https://schema.org",
     "@type": "Book",
-    "name": "{$arrNovel['og_novel_book_name'] ?? '未知小说'}",
+    "name": "{$arrNovel['n_name'] ?? '未知小说'}",
     "author": {
         "@type": "Person",
-        "name": "{$arrNovel['og_novel_author'] ??'佚名'}"
+        "name": "{$arrNovel['n_author'] ??'佚名'}"
     },
     "publisher": {
         "@type": "Organization",
         "name": "{$DomainModel->d_name}",
         "url": "https://{$DomainModel->d_domain}"
     },
-    "url": 'https://{$DomainModel->d_domain}{site:nclurl n_id="$arrNovel[n_id]" n_py="$arrNovel[og_novel_pin_yin]" order="zheng" page="$intPage"/}',
+    "url": 'https://{$DomainModel->d_domain}{site:nclurl n_id="$arrNovel[n_id]" n_py="$arrNovel[n_name_pinyin]" order="zheng" page="$intPage"/}',
     "inLanguage": "zh-CN",
-    "genre": "{$arrNovel['og_novel_category'] ?? '未知分类'}",
+    "genre": "{$arrNovel['n_category'] ?? '未知分类'}",
     "hasPart": {
         "@type": "WebPage",
-        "name": "{$arrNovel['og_novel_book_name'] ?? '未知小说'}章节目录 第{$intPage}页",
-        "url": 'https://{$DomainModel->d_domain}{site:nclurl n_id="$arrNovel[n_id]" n_py="$arrNovel[og_novel_pin_yin]" order="zheng" page="$intPage"/}'
+        "name": "{$arrNovel['n_name'] ?? '未知小说'}章节目录 第{$intPage}页",
+        "url": 'https://{$DomainModel->d_domain}{site:nclurl n_id="$arrNovel[n_id]" n_py="$arrNovel[n_name_pinyin]" order="zheng" page="$intPage"/}'
     }
 }
 
@@ -72,15 +72,15 @@
                 
             
             
  • 
-                
-                
-                    {$arrNovel.og_novel_category}
+                
+                
+                    {$arrNovel.n_category}
                 
                 
             
    • 
             
  • 
-                
-                    {$arrNovel['og_novel_book_name']}
+                
+                    {$arrNovel['n_name']}
                 
                 
             
    • 
@@ -91,7 +91,7 @@
     
    
         {$arrGRM[0]|raw}
         
    
-            
    {$arrNovel['og_novel_book_name']}章节目录 - {$arrNovel['og_novel_author']} - {$DomainModel->d_name} 第{$intPage}页
    
+            
    {$arrNovel['n_name']}章节目录 - {$arrNovel['n_author']} - {$DomainModel->d_name} 第{$intPage}页
    
         
    
         
 
@@ -103,8 +103,8 @@
                 {volist name="arrPage.data" id="Chapter" key="key"}
                 
  • 
                     {$arrGRM[$key]|raw}
-                    {$Chapter.name}
+                    {$Chapter.c_name}
                 
    • 
                 {/volist}
             
diff --git a/code/app/home/view/kuangYu/novel/getNovelChapter.html b/code/app/home/view/kuangYu/novel/getNovelChapter.html
index 38098f7..1ada4bc 100644
--- a/code/app/home/view/kuangYu/novel/getNovelChapter.html
+++ b/code/app/home/view/kuangYu/novel/getNovelChapter.html
@@ -20,26 +20,26 @@
 	
 
 	
-	
+	
 	
 	
+		content='https://{$DomainModel->d_domain}{site:nurl n_id="$arrNovel[n_id]" n_py="$arrNovel[n_name_pinyin]" /}'  />
 	
 	
 	
-	
-	
-	
+	
+	
+	
 	
-	
-	
-	
+		content='https://{$DomainModel->d_domain}{site:cpurl n_id="$arrNovel[n_id]" n_py="$arrNovel[n_name_pinyin]" c_sort="$$arrChapter[c_sort_num]"/}'>
+	
+	
+	
 	
+		content='https://{$DomainModel->d_domain}{site:lcpurl n_id="$arrNovel[n_id]" n_py="$arrNovel[n_name_pinyin]" /}'>
 	
 	
-	
+	
 	
 	
 	
@@ -47,10 +47,10 @@
 		{
 			"@context": "https://schema.org",
 			"@type": "Book",
-			"name": "{$arrNovel['og_novel_book_name'] ?? '未知小说'}",
+			"name": "{$arrNovel['n_name'] ?? '未知小说'}",
 			"author": {
 				"@type": "Person",
-				"name": "{$arrNovel['og_novel_author'] ??'佚名'}"
+				"name": "{$arrNovel['n_author'] ??'佚名'}"
 			},
 			"publisher": {
 				"@type": "Organization",
@@ -58,14 +58,14 @@
 				"url": "https://{$DomainModel->d_domain}"
 			},
 			"image": "{$arrNovel.n_cover_path ?? ''}",
-			"url":'https://{$DomainModel->d_domain}{site:cpurl n_id="$arrNovel[n_id]" n_py="$arrNovel[og_novel_pin_yin]" c_sort="$arrChapter[c_sort_num]" c_page="0"/}',
+			"url":'https://{$DomainModel->d_domain}{site:cpurl n_id="$arrNovel[n_id]" n_py="$arrNovel[n_name_pinyin]" c_sort="$arrChapter[c_sort_num]" c_page="0"/}',
 			"inLanguage": "zh-CN",
-			"genre": "{$arrNovel.og_novel_category ??''}",
-			"datePublished": "{$arrNovel['og_novel_update_time'] ?? time()}",
+			"genre": "{$arrNovel.n_category ??''}",
+			"datePublished": "{$arrNovel['n_update_time'] ?? time()}",
 			"hasPart": {
 				"@type": "CreativeWork",
-				"name": "{$arrChapter.name ?? ''}",
-				"url":'https://{$DomainModel->d_domain}{site:cpurl n_id="$arrNovel[n_id]" n_py="$arrNovel[og_novel_pin_yin]" c_sort="$arrChapter[c_sort_num]" c_page="0"/}',
+				"name": "{$arrChapter.c_name ?? ''}",
+				"url":'https://{$DomainModel->d_domain}{site:cpurl n_id="$arrNovel[n_id]" n_py="$arrNovel[n_name_pinyin]" c_sort="$arrChapter[c_sort_num]" c_page="0"/}',
 				"description": "{$strDescription}"
 			}
 		}
@@ -104,12 +104,12 @@
 				上一章
 				{else}
 				{$arrGRM[4]|raw}
-				上一章
 				{/if}
 
 				{if $arrNextChapter && $arrNextChapter.c_sort_num}
-				下一章
 				{/if}
 				
@@ -117,7 +117,7 @@
 			
    
 				
    
 					
-					
+					
 				
    
 			
    
 			{$arrGRM[5]|raw}
@@ -126,12 +126,12 @@
 				
 				上一章
 				{else}
-				上一章
 				{/if}
 
 				{if $arrNextChapter && $arrNextChapter.c_sort_num}
-				下一章
 				{/if}
 
@@ -150,25 +150,25 @@
 						
 					
 					
  • 
-						
-						
-							{$arrNovel.og_novel_category}
+						
+						
+							{$arrNovel.n_category}
 						
 						
 					
    • 
 					
  • 
-						
-							{$arrNovel.og_novel_book_name}
+						
+							{$arrNovel.n_name}
 						
 						
 					
    • 
 				
 			
 			
    
-				
-				{$arrNovel.og_novel_book_name}
+				
+				{$arrNovel.n_name}
 				
-				{$arrChapter.name}
+				{$arrChapter.c_name}
 				
 			
    
 		
    
@@ -202,7 +202,7 @@
 	    			
 	    			离线章节
 	    		 -->
-			
+			
 				
 				书籍详情
 			
@@ -299,7 +299,7 @@