diff --git a/code/.example.env b/code/.example.env
index 9d54ea7..927e93d 100644
--- a/code/.example.env
+++ b/code/.example.env
@@ -2,7 +2,6 @@
APP_DEBUG = true
SHOW_ERROR_MSG = true
DEFAULT_LANG = zh-cn
-DEFAULT_
# 数据库
DB_TYPE = mysql
@@ -26,6 +25,13 @@ REDIS_PASSWORD =
REDIS_PREFIX =
REDIS_DATABASE = 0
+#MongoDB
+MONGO_HOST = 127.0.0.1
+MONGO_PORT = 27017
+MONGO_USER = admin
+MONGO_PASSWORD = qq123123
+MONGO_DB = novel
+
# TASK
TASK_PROCESS_NUM_01 = 4
TASK_PROCESS_NUM_02 = 2
diff --git a/code/123.html b/code/123.html
deleted file mode 100644
index a96cc61..0000000
--- a/code/123.html
+++ /dev/null
@@ -1,359 +0,0 @@
-
-
-
-
-
-
-
-
-
-
- 张柠檬2最新章节_张柠檬2伯未有_张柠檬2小说免费全文阅读_恋上你中文
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
张柠檬2
-
-
作 者：伯未有
-
类 别：都市小说
-
状 态：连载中
-
动 作：加入书架、章节目录、开始阅读
-
最后更新：2025-03-19 17:06:56
-
最新章节：二重奏
-
-
-
- 这仍是一部探索欲望与心理SM现实主义小说，主人公张柠檬以自述的方式讲述了自己的生活。作者试图写出一篇严肃的色情小说，表达一种独特的“异化美感”，希望从冷酷中塑造一种黑暗的诗意。（我也不知道我在说什么）—————————介绍：本文描绘了大量SM场景元素，存在各式各样的拘束，奴役、羞辱和强迫的内容。前作换过很多名字，最早名为《我是如何成为富二代的的玩物的》，又更名为《玩具》、《张柠檬》等还有一些我也忘记的名字，看过就是缘。————————不知道诸位看官究竟喜欢什么内容，故事的内容是无限的，但作者的精力属实是有限的，众所周知本作断更已然是一种常态~_~;所以大伙儿们喜欢什么请一定要告诉我，我好更多的去侧重呀。诸如：A五花八门的调教??B云朝雨暮的性爱C风花雪月的交往D刻骨铭心的爱情如果还有别的也请补充
-
-
-
-
-
-
-
-
-
';
+ // print_r($arrRecommendSerializationNovel);
+ // exit;
$arrRecommendSerializationNovel = [
['id' => 1, 'name' => '小说1', 'author' => '作者1'],
['id' => 2, 'name' => '小说2', 'author' => '作者2'],
@@ -133,27 +141,36 @@ class Index extends BaseController
['id' => 8, 'name' => '小说3', 'author' => '作者3'],
['id' => 8, 'name' => '小说3', 'author' => '作者3'],
['id' => 8, 'name' => '小说3', 'author' => '作者3'],
-
+
];
// 模拟数据-推荐等级为9的完结的 随机12个小说
- $arrRecommendEndNovel = [
- ['id' => 1, 'name' => '小说1', 'author' => '作者1'],
- ['id' => 2, 'name' => '小说2', 'author' => '作者2'],
- ['id' => 3, 'name' => '小说3', 'author' => '作者3'],
- ['id' => 4, 'name' => '小说3', 'author' => '作者3'],
- ['id' => 5, 'name' => '小说3', 'author' => '作者3'],
- ['id' => 6, 'name' => '小说3', 'author' => '作者3'],
- ['id' => 7, 'name' => '小说3', 'author' => '作者3'],
- ['id' => 8, 'name' => '小说3', 'author' => '作者3'],
- ['id' => 9, 'name' => '小说3', 'author' => '作者3'],
- ['id' => 10, 'name' => '小说3', 'author' => '作者3'],
- ['id' => 11, 'name' => '小说3', 'author' => '作者3'],
- ['id' => 12, 'name' => '小说3', 'author' => '作者3'],
-
- ];
+
+ $arrRecommendEndNovel = $NovelModel->getFinishedNovelOnCache($Request->DomainModel->d_domain, 13);
+ // echo '';
+ // print_r($arrRecommendEndNovel);
+ // exit;
+ // $arrRecommendEndNovel = [
+ // ['id' => 1, 'name' => '小说1', 'author' => '作者1'],
+ // ['id' => 2, 'name' => '小说2', 'author' => '作者2'],
+ // ['id' => 3, 'name' => '小说3', 'author' => '作者3'],
+ // ['id' => 4, 'name' => '小说3', 'author' => '作者3'],
+ // ['id' => 5, 'name' => '小说3', 'author' => '作者3'],
+ // ['id' => 6, 'name' => '小说3', 'author' => '作者3'],
+ // ['id' => 7, 'name' => '小说3', 'author' => '作者3'],
+ // ['id' => 8, 'name' => '小说3', 'author' => '作者3'],
+ // ['id' => 9, 'name' => '小说3', 'author' => '作者3'],
+ // ['id' => 10, 'name' => '小说3', 'author' => '作者3'],
+ // ['id' => 11, 'name' => '小说3', 'author' => '作者3'],
+ // ['id' => 12, 'name' => '小说3', 'author' => '作者3'],
+
+ // ];
// 模拟数据-男生频道，最新的 随机5个小说
+ $arrBoyNewsNovel = $NovelModel->getSexNovelOnCache($Request->DomainModel->d_domain, 5, 1);
+ // echo '';
+ // print_r($arrBoyNewsNovel);
+ // exit;
$arrBoyNewsNovel = [
['id' => 1, 'name' => '小说1', 'author' => '作者1'],
['id' => 2, 'name' => '小说2', 'author' => '作者2'],
@@ -163,6 +180,10 @@ class Index extends BaseController
];
// 模拟数据-男生频道，周排行榜 随机10个小说
+ $arrBoyWeekRankNovel = $NovelModel->getRankNovelOnCache($Request->DomainModel->d_domain, 5, 1, 'weekly');
+ // echo '';
+ // print_r($arrBoyWeekRankNovel);
+ // exit;
$arrBoyWeekRankNovel = [
['id' => 1, 'name' => '小说1', 'author' => '作者1'],
['id' => 2, 'name' => '小说2', 'author' => '作者2'],
@@ -177,6 +198,10 @@ class Index extends BaseController
];
// 模拟数据-男生频道，月排行榜 随机10个小说
+ $arrBoyMoonRankNovel = $NovelModel->getRankNovelOnCache($Request->DomainModel->d_domain, 5, 1, 'monthly');
+ // echo '';
+ // print_r($arrBoyMoonRankNovel);
+ // exit;
$arrBoyMoonRankNovel = [
['id' => 1, 'name' => '小说1', 'author' => '作者1'],
['id' => 2, 'name' => '小说2', 'author' => '作者2'],
@@ -191,6 +216,10 @@ class Index extends BaseController
];
// 模拟数据-男生频道，总排行榜 随机10个小说
+ $arrBoyTotalRankNovel = $NovelModel->getRankNovelOnCache($Request->DomainModel->d_domain, 5, 1, 'total');
+ // echo '';
+ // print_r($arrBoyTotalRankNovel);
+ // exit;
$arrBoyTotalRankNovel = [
['id' => 1, 'name' => '小说1', 'author' => '作者1'],
['id' => 2, 'name' => '小说2', 'author' => '作者2'],
@@ -203,8 +232,12 @@ class Index extends BaseController
['id' => 9, 'name' => '小说3', 'author' => '作者3'],
['id' => 10, 'name' => '小说3', 'author' => '作者3'],
];
-
+
// 模拟数据-女生频道，最新的 随机5个小说
+ $arrGirlNewsNovel = $NovelModel->getSexNovelOnCache($Request->DomainModel->d_domain, 5, 2);
+ // echo '';
+ // print_r($arrGirlNewsNovel);
+ // exit;
$arrGirlNewsNovel = [
['id' => 1, 'name' => '小说1', 'author' => '作者1'],
['id' => 2, 'name' => '小说2', 'author' => '作者2'],
@@ -212,8 +245,12 @@ class Index extends BaseController
['id' => 4, 'name' => '小说3', 'author' => '作者3'],
['id' => 5, 'name' => '小说3', 'author' => '作者3'],
];
-
+
// 模拟数据-女生频道，周排行榜 随机10个小说
+ $arrGirlWeekRankNovel = $NovelModel->getRankNovelOnCache($Request->DomainModel->d_domain, 5, 2, 'weekly');
+ // echo '';
+ // print_r($arrGirlWeekRankNovel);
+ // exit;
$arrGirlWeekRankNovel = [
['id' => 1, 'name' => '小说1', 'author' => '作者1'],
['id' => 2, 'name' => '小说2', 'author' => '作者2'],
@@ -226,7 +263,12 @@ class Index extends BaseController
['id' => 9, 'name' => '小说3', 'author' => '作者3'],
['id' => 10, 'name' => '小说3', 'author' => '作者3'],
];
+
// 模拟数据-女生频道，月排行榜 随机10个小说
+ $arrGirlMoonRankNovel = $NovelModel->getRankNovelOnCache($Request->DomainModel->d_domain, 5, 2, 'monthly');
+ // echo '';
+ // print_r($arrGirlMoonRankNovel);
+ // exit;
$arrGirlMoonRankNovel = [
['id' => 1, 'name' => '小说1', 'author' => '作者1'],
['id' => 2, 'name' => '小说2', 'author' => '作者2'],
@@ -240,6 +282,10 @@ class Index extends BaseController
['id' => 10, 'name' => '小说3', 'author' => '作者3'],
];
// 模拟数据-女生频道，总排行榜 随机10个小说
+ $arrBGirlTotalRankNovel = $NovelModel->getRankNovelOnCache($Request->DomainModel->d_domain, 5, 2, 'total');
+ // echo '';
+ // print_r($arrBGirlTotalRankNovel);
+ // exit;
$arrBGirlTotalRankNovel = [
['id' => 1, 'name' => '小说1', 'author' => '作者1'],
['id' => 2, 'name' => '小说2', 'author' => '作者2'],
@@ -254,6 +300,10 @@ class Index extends BaseController
];
// 模拟数据-最近更新-随机50
+ $arrNewsUpdateNovel = $NovelModel->getSexNovelOnCache($Request->DomainModel->d_domain, 50);
+ // echo '';
+ // print_r($arrNewsUpdateNovel);
+ // exit;
$arrNewsUpdateNovel = [
['id' => 1, 'name' => '小说1', 'author' => '作者1'],
['id' => 2, 'name' => '小说2', 'author' => '作者2'],
@@ -300,25 +350,4 @@ class Index extends BaseController
}
}
-
-
- public function test()
- {
-
- $strNovelKey = "novels/12/chapters/3001.txt";
- $StorageCore = StorageCore::getInstance();
-
- // $content = '我是一个大傻逼';
-
- $content = file_get_contents('123.txt');
-
- if ($StorageCore->has($strNovelKey)) {
- // $MinIO->update($strNovelKey, $content); // 更新已有文件
- } else {
- $StorageCore->write($strNovelKey, $content); // 创建新文件
- }
-
- echo $StorageCore->read($strNovelKey);
- return;
- }
}
diff --git a/code/app/home/view/kuangyu/Pc/index.html b/code/app/home/view/kuangyu/Pc/index.html
index 5ee7c9d..36f129d 100644
--- a/code/app/home/view/kuangyu/Pc/index.html
+++ b/code/app/home/view/kuangyu/Pc/index.html
@@ -52,7 +52,17 @@
{foreach $arrRecommendEndNovel as $key=>$Novel}
- {include file="Pc/Public/endNovelList" /}
+ -
+
+
+
+
+
+
{$Novel->og_novel_author}
+
{/foreach}
diff --git a/code/app/model/CategoryModel.php b/code/app/model/CategoryModel.php
new file mode 100644
index 0000000..8ef6dec
--- /dev/null
+++ b/code/app/model/CategoryModel.php
@@ -0,0 +1,61 @@
+ '玄幻修真',
+ 2 => '军史小说',
+ 3 => '网游小说',
+ 4 => '科幻小说',
+ 5 => '重生穿越',
+ 6 => '都市小说',
+ 7 => '灵异小说',
+ 8 => '言情小说',
+ 9 => '其他小说',
+ ];
+
+ static $arrMan = [
+ 1 => '玄幻修真',
+ 2 => '军史小说',
+ 3 => '网游小说',
+ 4 => '科幻小说',
+ ];
+
+ static $arrWomen = [
+ 5 => '重生穿越',
+ 6 => '都市小说',
+ 7 => '灵异小说',
+ 8 => '言情小说',
+ 9 => '其他小说',
+ ];
+
+
+ static public function getCategory():array
+ {
+ return self::$arrCategory;
+ }
+
+ static public function getManCategory():array
+ {
+ return self::$arrMan;
+ }
+
+ static public function getWomenCategory():array
+ {
+ return self::$arrWomen;
+ }
+}
diff --git a/code/app/model/ChapterModel.php b/code/app/model/ChapterModel.php
new file mode 100644
index 0000000..403cda1
--- /dev/null
+++ b/code/app/model/ChapterModel.php
@@ -0,0 +1,17 @@
+MongoBase = MongoBase::getInstance();
+ }
+
+ /**
+ * Undocumented function
+ *
+ * @return MongoBase
+ */
+ public function getMongoBase(): MongoBase
+ {
+ return $this->MongoBase;
+ }
+
+ /**
+ * Undocumented function
+ *
+ * @return Collection
+ */
+ public function getCol(): Collection
+ {
+ return $this->getMongoBase()->getDb()->selectCollection($this->getCollectionName());
+ }
+
+ /**
+ * Undocumented function
+ *
+ * @return string
+ */
+ public function getCollectionName(): string
+ {
+ return $this->strCollection;
+ }
+}
diff --git a/code/app/model/NovelClicksModel.php b/code/app/model/NovelClicksModel.php
new file mode 100644
index 0000000..09c2ae8
--- /dev/null
+++ b/code/app/model/NovelClicksModel.php
@@ -0,0 +1,67 @@
+getCol()->findOne([
+ "n_id" => $intNId,
+ ]);
+
+ if (empty($Novel)) {
+ return false;
+ }
+
+ $Now = new DateTime();
+
+ $strDailyDate = $Now->format('Y-m-d');
+ $strWeeklyDate = $Now->modify('monday this week')->format('Y-m-d');
+ $MonthlyDate = $Now->format('Y-m-01');
+
+ $arrClickTypes = [
+ 'daily' => $strDailyDate,
+ 'weekly' => $strWeeklyDate,
+ 'monthly' => $MonthlyDate,
+ 'total' => null,
+ ];
+
+ foreach ($arrClickTypes as $strType => $strDate) {
+ $this->getCol()->updateOne(
+ [
+ 'n_id' => $intNId,
+ 'nc_date' => $strDate,
+ 'nc_type' => $strType,
+ ],
+ [
+ '$inc' => ['nc_clicks' => 1],
+ '$set' => ['n_category' => $Novel->og_novel_category]
+ ],
+ [
+ 'upsert' => true
+ ]
+ );
+ }
+ return true;
+ }
+}
diff --git a/code/app/model/NovelModel.php b/code/app/model/NovelModel.php
new file mode 100644
index 0000000..05e9320
--- /dev/null
+++ b/code/app/model/NovelModel.php
@@ -0,0 +1,329 @@
+ ['$regex' => trim($strKeyword), '$arrOptions' => 'i']];
+ }, $arrKeywords);
+
+ $UpdateResult = $this->getCol()->updateMany(
+ ['$or' => $arrConditions],
+ ['$set' => ['n_level' => $intLevel]]
+ );
+
+ $intUpdateNum = $UpdateResult->getModifiedCount();
+ } catch (Exception $e) {
+
+ return false;
+ }
+
+ return true;
+ }
+
+
+ /**
+ * Undocumented function
+ *
+ * @param string $strSign
+ * @param integer $intCount
+ * @param integer $intType
+ * @param integer $intLevel
+ * @return array
+ */
+ public function getHighLevelNovelOnCache(string $strSign, int $intCount = 10, int $intType = 0): array
+ {
+ try {
+ $strKey = sprintf('%s:PC:HOME:FIRST', $strSign);
+
+ $intLifeTime = 24 * 3600;
+
+ $arrResult = Cache::get($strKey);
+
+ // if (empty($arrResult)) {
+ if (true) {
+ if ($intType == 1) {
+ $arrCategory = array_values(CategoryModel::getManCategory());
+ $arrFilter = [
+ 'og_novel_category' => ['$in' => $arrCategory]
+ ];
+ } elseif ($intType == 2) {
+ $arrCategory = array_values(CategoryModel::getWomenCategory());
+ $arrFilter = [
+ 'og_novel_category' => ['$in' => $arrCategory]
+ ];
+ } else {
+ $arrFilter = [];
+ }
+
+ $arrOptions = [
+ 'sort' => ['n_level' => -1],
+ 'limit' => $intCount * 3,
+ 'projection' => [
+ 'n_id' => 1,
+ 'og_novel_book_name' => 1,
+ 'og_novel_pin_yin' => 1,
+ 'n_cover_path' => 1,
+ 'n_novel_author' => 1,
+ 'og_novel_category' => 1,
+ 'og_description' => 1,
+ ],
+ ];
+
+ $Cursor = $this->getCol()->find($arrFilter, $arrOptions);
+ $arrResult = iterator_to_array($Cursor);
+
+ shuffle($arrResult);
+
+ $arrResult = array_slice($arrResult, 0, $intCount);
+
+ if (!empty($arrResult)) {
+ Cache::set($strKey, $arrResult, $intLifeTime);
+ }
+ }
+
+ return $arrResult;
+ } catch (\Exception $e) {
+ return [];
+ }
+ }
+
+
+ /**
+ * Undocumented function
+ *
+ * @param string $strSign
+ * @param integer $intCount
+ * @return array
+ */
+ public function getFinishedNovelOnCache(string $strSign, int $intCount = 10): array
+ {
+ try {
+ $strKey = sprintf('%s:PC:HOME:FINISHED', $strSign);
+
+ $intLifeTime = 24 * 3600;
+
+ $arrResult = Cache::get($strKey);
+
+ // if (empty($arrResult)) {
+ if (true) {
+ $arrFilter = [
+ 'og_novel_status' => "已完结",
+ ];
+
+ $arrOptions = [
+ 'sort' => ['og_novel_update_time' => -1],
+ 'limit' => $intCount * 3,
+ 'projection' => [
+ 'n_id' => 1,
+ 'og_novel_book_name' => 1,
+ 'og_novel_pin_yin' => 1,
+ 'n_cover_path' => 1,
+ 'og_novel_author' => 1,
+ 'og_novel_category' => 1,
+ 'og_description' => 1,
+ ],
+ ];
+
+ $Cursor = $this->getCol()->find($arrFilter, $arrOptions);
+ $arrResult = iterator_to_array($Cursor);
+
+ shuffle($arrResult);
+
+ $arrResult = array_slice($arrResult, 0, $intCount);
+
+ if (!empty($arrResult)) {
+ Cache::set($strKey, $arrResult, $intLifeTime);
+ }
+ }
+
+ return $arrResult;
+ } catch (\Exception $e) {
+ return [];
+ }
+ }
+
+ /**
+ * Undocumented function
+ *
+ * @param string $strSign
+ * @param integer $intCount
+ * @param integer $intType
+ * @param integer $intLevel
+ * @return array
+ */
+ public function getSexNovelOnCache(string $strSign, int $intCount = 10, int $intType = 0): array
+ {
+ try {
+ $strKey = sprintf('%s:PC:HOME:SIX:%s', $strSign, $intType);
+
+ $intLifeTime = 24 * 3600;
+
+ $arrResult = Cache::get($strKey);
+
+ // if (empty($arrResult)) {
+ if (true) {
+ if ($intType == 1) {
+ $arrCategory = array_values(CategoryModel::getManCategory());
+ $arrFilter = [
+ 'og_novel_category' => ['$in' => $arrCategory]
+ ];
+ } elseif ($intType == 2) {
+ $arrCategory = array_values(CategoryModel::getWomenCategory());
+ $arrFilter = [
+ 'og_novel_category' => ['$in' => $arrCategory]
+ ];
+ } else {
+ $arrFilter = [];
+ }
+
+ $arrOptions = [
+ 'sort' => ['og_novel_update_time' => -1],
+ 'limit' => $intCount * 3,
+ 'projection' => [
+ 'n_id' => 1,
+ 'og_novel_book_name' => 1,
+ 'og_novel_pin_yin' => 1,
+ 'n_cover_path' => 1,
+ 'n_novel_author' => 1,
+ 'og_novel_category' => 1,
+ 'og_description' => 1,
+ ],
+ ];
+
+ $Cursor = $this->getCol()->find($arrFilter, $arrOptions);
+ $arrResult = iterator_to_array($Cursor);
+
+ shuffle($arrResult);
+
+ $arrResult = array_slice($arrResult, 0, $intCount);
+
+ if (!empty($arrResult)) {
+ Cache::set($strKey, $arrResult, $intLifeTime);
+ }
+ }
+
+ return $arrResult;
+ } catch (\Exception $e) {
+ return [];
+ }
+ }
+
+
+
+ /**
+ * Undocumented function
+ *
+ * @param string $strSign
+ * @param integer $intCount
+ * @param integer $intType
+ * @param string strDateType
+ * * daily 天
+ * * weekly 周
+ * * monthly 月
+ * * total 总
+ * @return array
+ */
+ public function getRankNovelOnCache(string $strSign, int $intCount = 10, int $intType = 0, $strDateType = 'daily'): array
+ {
+ try {
+ $strKey = sprintf('%s:PC:HOME:SIX:%s:TYPE:%s', $strSign, $intType, $strDateType);
+
+ $intLifeTime = 24 * 3600;
+
+ $arrResult = Cache::get($strKey);
+
+ // if (empty($arrResult)) {
+ if (true) {
+ $arrFilter = [
+ 'nc_type' => $strDateType,
+ ];
+ if ($intType == 1) {
+ $arrCategory = array_values(CategoryModel::getManCategory());
+ $arrFilter['n_category'] = ['$in' => $arrCategory];
+ } elseif ($intType == 2) {
+ $arrCategory = array_values(CategoryModel::getWomenCategory());
+ $arrFilter['n_category'] = ['$in' => $arrCategory];
+ }
+ $arrOptions = [
+ 'sort' => ['nc_clicks' => -1],
+ 'limit' => $intCount * 3,
+ 'projection' => [
+ 'n_id' => 1,
+ ],
+ ];
+
+ $NovelClicksModel = new NovelClicksModel;
+
+ $Cursor = $NovelClicksModel->getCol()->find($arrFilter, $arrOptions);
+
+ $arrNId = [];
+ foreach ($Cursor as $Doc) {
+ $arrNId[] = $Doc['n_id'];
+ }
+
+ $Cursor = $this->getCol()->find(
+ ['n_id' => ['$in' => $arrNId]],
+ [
+ 'projection' => [
+ 'n_id' => 1,
+ 'og_novel_book_name' => 1,
+ 'og_novel_pin_yin' => 1,
+ 'n_cover_path' => 1,
+ 'n_novel_author' => 1,
+ 'og_novel_category' => 1,
+ 'og_description' => 1,
+ ],
+ ]
+ );
+
+
+ $arrResult = iterator_to_array($Cursor);
+
+ shuffle($arrResult);
+
+ $arrResult = array_slice($arrResult, 0, $intCount);
+
+ if (!empty($arrResult)) {
+ Cache::set($strKey, $arrResult, $intLifeTime);
+ }
+ }
+
+ return $arrResult;
+ } catch (\Exception $e) {
+ return [];
+ }
+ }
+}
diff --git a/code/app/task/collection/PullDataColBase.php b/code/app/task/collection/PullDataColBase.php
deleted file mode 100644
index 38144a8..0000000
--- a/code/app/task/collection/PullDataColBase.php
+++ /dev/null
@@ -1,48 +0,0 @@
-Client = new Client();
-
- $this->QueueManage = new QueueManage();
- }
-
- public function getAilisiDomain()
- {
- if (!key_exists('AILISI_DOMAIN', $this->arrCaiJiPeiZhi)) {
- $this->arrCaiJiPeiZhi['AILISI_DOMAIN'] = CaiJiPeiZhiModel::getValByCode('AILISI_DOMAIN');
- }
-
- return $this->arrCaiJiPeiZhi['AILISI_DOMAIN'];
- }
-}
diff --git a/code/app/task/collection/zw630/Zw630Col.php b/code/app/task/collection/zw630/Zw630Col.php
deleted file mode 100644
index 2f46683..0000000
--- a/code/app/task/collection/zw630/Zw630Col.php
+++ /dev/null
@@ -1,611 +0,0 @@
-MongoBase = MongoBase::getInstance();
-
- $this->Client = new Client([
- 'base_uri' => $this->strDomain,
- 'http_errors' => false,
- 'version' => 2.0, // 强制使用 HTTP/2
- 'timeout' => 160,
- 'connect_timeout' => 30, // 连接超时 10 秒
- 'curl' => [
- CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_2_0, // 强制 HTTP/2
- CURLOPT_TCP_KEEPALIVE => 1, // 启用 TCP Keep-Alive
- CURLOPT_SSL_VERIFYPEER => true, // 验证 SSL
- ],
- 'headers' => [
- 'User-Agent' => 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Mobile Safari/537.36 Edg/134.0.0.0',
- 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
- 'Accept-Language' => 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
- 'Referer' => $this->strDomain,
- 'sec-ch-ua-mobile' => '?1',
- 'sec-ch-ua-platform' => '"Android"',
- 'sec-fetch-dest' => 'document',
- 'sec-fetch-mode' => 'navigate',
- 'sec-fetch-site' => 'same-origin',
- 'sec-fetch-user' => '?1',
- 'upgrade-insecure-requests' => '1',
- ],
- ]);
-
- $this->QueueManage = new QueueManage();
- }
-
-
- public function pull630ZwNovel()
- {
- // 全局扫描任务
- $QueueManage = $this->QueueManage;
- // for ($intNovelId = 1; $intNovelId <= 531208; $intNovelId++) {
- // for ($intNovelId = 1; $intNovelId <= 10000; $intNovelId++) {
- for ($intNovelId = 1; $intNovelId <= 1; $intNovelId++) {
- $arrTask = [
- 'callback' => [self::class, 'getNovelInfo'],
- 'data' => [
- 'n_id' => $intNovelId,
- 'page' => 0,
- ]
- ];
-
- // print_r($arrTask);
- $QueueManage->set($arrTask);
- }
- }
-
-
- public function getNovelInfo($arrData)
- {
- // content="mobile" 有这个的是手机，否则是PC
- print_r($arrData);
-
- if ($arrData['page'] == 0) {
- $strUri = sprintf('/shu/%s.html', $arrData['n_id']);
- } else {
- $strUri = sprintf('/shu/%s_%s.html', $arrData['n_id']);
- }
-
- $ResponseBase = $this->Client->get($strUri);
- $strNovelContent = $ResponseBase->getBody()->getContents();
-
- $NovelQuery = QueryList::html($strNovelContent);
-
- $arrNovel = $this->getNovelHead($NovelQuery, $arrData['n_id'], $arrData['page']);
-
- if (empty($arrNovel)) {
- return false;
- }
-
-
- print_r($arrNovel);
- // $strResult = (string)$Response->getBody();
- }
-
- private function getNovelHead(QueryList $QueryList, $intSourceNovelId, $intPage = 0)
- {
- $arrResult = [];
- $QueryList->find('meta[property]')->each(function ($Item) use (&$arrResult) {
- $property = str_replace(':', '_', $Item->attr('property'));
- $arrResult[$property] = $Item->attr('content');
- });
-
- if (empty($arrResult)) {
- return false;
- }
-
- // print_r($arrResult);
-
- $arrResult['novel_source_id'] = 'zw630-' . $intSourceNovelId;
-
- try {
- $Col = $this->MongoBase->getDb()->selectCollection('novel');
-
- $arrExisting = $Col->findOne(['novel_source_id' => $arrResult['novel_source_id']]);
-
- // use novel
- // db.novel.createIndex({ "novel_source_id": 1 }, { unique: true })
-
- if ($arrExisting) {
-
- $arrExisting = (array) $arrExisting; // MongoDB\BSON\Document 转为数组
- $arrResult = array_merge($arrExisting, $arrResult); // 合并，保留原有字段
-
- $arrResult['updated_at'] = new \MongoDB\BSON\UTCDateTime(); // 添加更新时间
- $Col->updateOne(
- ['novel_source_id' => $arrResult['novel_source_id']],
- ['$set' => $arrResult],
- // ['upsert' => false]
- );
- } else {
- $arrResult['novel_id'] = $this->MongoBase->getNextSequence('novel');
- $this->MongoBase->insertOne('novel', $arrResult);
- }
- return $arrResult;
- } catch (\MongoDB\Exception\Exception $e) {
- throw new \Exception("Upsert failed: " . $e->getMessage());
- }
- }
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- public function getNovelInfo1($arrData)
- {
- //var_dump('getNovelFengMian');
- //var_dump($arrData);
- $Client = $this->Client;
- $QueueManage = $this->QueueManage;
-
- // 获取 Redis 实例
- $redis = Cache::store('redis')->handler();
- $intXiaoShuoSourceId = $arrData['xiaoshuo_id'];
- $intXiaoShuoSourceSeoId = $arrData['xiaoshuo_seo_id'];
- $redisKey = "task:novel_source_id:" . $intXiaoShuoSourceId;
-
- try {
- $arrCookie = $this->getAilisiCookie();
- $strHost = getDomainFromUrl($arrData['xiaoshuo_url']);
- $CookieJar = CookieJar::fromArray($arrCookie, $strHost);
-
- $strSiteUrl = $this->getAilisiDomain();
-
-
- $strCover = $QueryList->find('#fmimg img')->eq(0)->attr("src");
-
- if (strpos($strCover, 'http') === false) {
- $strCover = $strSiteUrl . $strCover;
- }
- $strCoverUri = $strCover;
-
- try {
- $strCoverUri = "";
- $strCoverUri = ImageProcessor::getInstance()->downloadAndEncryptImage($strCover, 'xs');
- $arrCover = [
- 'code' => 'AI_LI_SI_COVER',
- // 'uri' => getUriByUrl($strCover),
- 'uri' => $strCoverUri,
-
- ];
-
- $arrXiaoShuoInfo = [
- 'xsxq_feng_mian' => json_encode($arrCover),
-
- ];
- $XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
- } catch (\Throwable $t) {
- $strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
- echo $strErr;
- }
- } catch (\Throwable $T) {
- throw $T;
- }
- }
-
-
- // public function getNovelInfo($arrData)
- // {
- // $QueueManage = $this->QueueManage;
- // $Client = $this->Client;
- // $strSiteUrl = $this->getBiquxsDomain();
-
- // $strUrl = 'http://www.biquxs.com/book/' . $arrData['xs_source_id'] ; //'http://www.biquxs.com/book/20053';
- // $intXiaoShuoSourceId = $arrData['xs_source_id']; //20053;
- // $intXiaoShuoSourceSeoId = $arrData['xs_source_seo_id']; //1 ;
- // try {
- // $arrCookie = $this->getAilisiCookie();
- // $strHost = getDomainFromUrl($strUrl);
-
- // $CookieJar = CookieJar::fromArray($arrCookie, $strHost);
-
- // $proxy = getRandomProxies();
- // $userAgent = getRandomUserAgent();
- // $Response = $Client->request('GET', $strUrl, [
- // 'headers' => [
- // 'User-Agent' => $userAgent,
- // 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
- // 'Accept-Language' => 'en-US,en;q=0.5',
- // ],
- // 'cookies' => $CookieJar,
- // // 'proxy' => $proxy,
- // 'timeout' => 30,
- // ]);
-
- // $strResult = (string)$Response->getBody();
-
- // $QueryList = QueryList::html($strResult);
- // // var_dump($QueryList);
- // // 提取所有 标签的属性
- // $result = $QueryList->find('meta[property^="og:"]')->map(function ($item) {
- // return [
- // 'property' => $item->attr('property'),
- // 'content' => $item->attr('content'),
- // ];
- // });
-
- // // 打印结果
- // // print_r($result->all());
- // $strZuoZhe = '';
- // $strMingZi = '';
- // $strLastDate = '';
- // $strJieShao = '';
- // $strCover = '';
- // $strFenlei = '';
- // $strZiShu = 0;
- // $strZhuangTai = $QueryList->find('#info p')->eq(1)->text();
- // $strZhuangTai = getStringAfterWord($strZhuangTai, '状态：');
-
- // foreach ($result as $item) {
- // switch ($item['property']) {
- // case 'og:title':
- // $strMingZi = $item['content'];
- // break;
- // case 'og:description':
- // $strJieShao = $item['content'];
- // break;
- // case 'og:image':
- // $strCover = $item['content'];
- // break;
- // case 'og:book:category':
- // $strFenlei = $item['content'];
- // break;
- // case 'og:book:author':
- // $strZuoZhe = $item['content'];
- // break;
- // case 'og:book:update_time':
- // $strLastDate = $item['content'];
- // break;
- // }
- // }
-
-
- // $strLastDate = trim((string)$strLastDate);
- // if (!isValidDateTime($strLastDate)) {
- // $strLastDate = date('Y-m-d H:i:s');
- // }
-
- // $arrTag = [];
-
- // if (strpos($strCover, 'http') === false) {
- // $strCover = $strSiteUrl . $strCover;
- // }
- // $strCoverUri = $strCover;
-
- // try {
- // $strCoverUri = "";
- // $strCoverUri = ImageProcessor::getInstance()->downloadImage( $intXiaoShuoSourceId,$strCover, 'xs');
- // } catch (\Throwable $t) {
- // $strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
- // echo $strErr;
- // }
-
- // $arrCover = [
- // 'code' => 'BI_QU_XS_COVER',
- // // 'uri' => getUriByUrl($strCover),
- // 'uri' => $strCoverUri,
-
- // ];
-
- // $strFenleiIId = 1 ;
- // switch ($strFenlei) {
- // case '玄幻小说':
- // $strName = "玄幻";
- // $strFenleiIId = 1;
- // break;
- // case '修真小说':
- // $strName = "修真";
- // $strFenleiIId = 2;
- // break;
- // case '都市小说':
- // $strName = "都市";
- // $strFenleiIId = 3;
- // break;
- // case '历史小说':
- // $strName = "历史";
- // $strFenleiIId = 4;
- // break;
- // case '网游小说':
- // $strName = "网游";
- // $strFenleiIId = 5;
- // break;
- // case '科幻小说':
- // $strName = "科幻";
- // $strFenleiIId = 6;
- // break;
- // case '女频小说':
- // $strName = "女频";
- // $strFenleiIId = 7;
- // break;
- // case '其它小说':
- // $strName = "其它";
- // $strFenleiIId = 8;
- // break;
- // default:
- // $strName = "其它";
- // $strFenleiIId = 8;
- // break;
- // }
-
- // $arrXiaoShuoInfo = [
- // 'xsxq_ming_zi' => $strMingZi,
- // 'xsxq_zuozhe' => $strZuoZhe , //$arrData['xiaoshuo_zuozhe'],
- // 'xsfl_id' => $strFenleiIId,
- // 'xsxq_zhuang_tai' => $strZhuangTai,
- // 'xsxq_zi_shu' => $strZiShu,
- // 'xsxq_jie_shao' => $strJieShao,
- // 'xsxq_feng_mian' => json_encode($arrCover),
- // 'xsxq_geng_xin_shi_jian' => $strLastDate,
- // 'xsxq_tag' => json_encode($arrTag, JSON_UNESCAPED_UNICODE),
- // 'xsxq_source_id' => $intXiaoShuoSourceId,
- // 'xsxq_source_seo_id' => $intXiaoShuoSourceId.'1',
- // 'xsxq_source_code' => 'BIQU_XIAOSHUO_' . $intXiaoShuoSourceId . '_' . $intXiaoShuoSourceSeoId,
- // ];
- // // print_r($arrXiaoShuoInfo);
- // var_dump($arrXiaoShuoInfo['xsxq_source_code']);
-
- // $XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
-
- // $arrZhangjie = [];
- // $intChapterIndex = 0; // 定义一个计数器变量
-
- // // 检查是否找到元素
- // if ($QueryList->find('.listmain a')->count() > 0) {
- // $QueryList->find('.listmain a')->map(function ($ZhangJie) use (&$arrZhangjie, &$intChapterIndex) {
- // echo "Processing index $intChapterIndex: " . $ZhangJie->text() . PHP_EOL;
-
- // // 从索引6开始
- // if ($intChapterIndex >= 5) {
- // $arrZhangjie[] = [
- // 'mingzi' => $ZhangJie->text(),
- // 'url' => $ZhangJie->attr('href'),
- // ];
- // }
- // $intChapterIndex++;
- // });
-
- // // 输出最终结果
- // var_dump($arrZhangjie);
- // } else {
- // echo "No elements found for .listmain a" . PHP_EOL;
- // }
-
- // //unset($QueryList);
-
- // // 获取最大章节索引
- // $intMaxXiaoShuoZhangJiePaixu = XiaoShuoZhangJieModel::where('xsxq_id', $intXiaoShuoSourceId)
- // ->max('xszj_pai_xu'); // 获取最大值
- // var_dump('获取小说章节最大值-'.$intMaxXiaoShuoZhangJiePaixu);
-
- // // 如果没有结果，max 返回 null，可以设置默认值
- // $intMaxXiaoShuoZhangJiePaixu = $intMaxXiaoShuoZhangJiePaixu ?? 0;
-
- // foreach ($arrZhangjie as $intZhangJieNum => $arrZhangjieOne) {
- // var_dump($intZhangJieNum);
-
- // // 如果数据库里的章节最大索引大于当前索引，说明已经入库，不需要入队列了
- // if($intMaxXiaoShuoZhangJiePaixu > $intZhangJieNum){
- // var_dump("小说章节已经入库，不再推送任务");
- // continue;// 如果任务已存在，直接跳过 break
- // }
-
- // $strZhangJieUrl = $strSiteUrl . $arrZhangjieOne['url'];
-
- // $strZhangJieId = extractFilename($strZhangJieUrl);
-
- // # 'AILISI_XIAOSHUO_ZHANGJIE_小说源id_小说源章节id'
- // $strXszjSourceCode = 'AILISI_XIAOSHUO_ZHANGJIE_'. $XiaoShuoXiangQingModel->xsxq_source_id . '_' . $strZhangJieId;
-
- // $arrTask = [
- // 'callback' => [self::class, 'getNovelReader'],
- // 'data' => [
- // 'xszj_source_id' => $strZhangJieId,
- // 'xszj_paixu' => $intZhangJieNum,
- // 'xszj_name' => $arrZhangjieOne['mingzi'],
- // 'xszj_url' => $strZhangJieUrl,
- // 'xszj_source_code' => $strXszjSourceCode,
- // 'xsxq_source_id' => $XiaoShuoXiangQingModel->xsxq_source_id,
- // 'xsfl_id' => $strFenleiIId,
- // ]
- // ];
- // $QueueManage->set($arrTask);
-
- // }
- // var_dump("章节任务投递成功");
-
- // } catch (\Throwable $T) {
- // throw $T;
- // }
- // }
-
- // public function getNovelFengMian($arrData)
- // {
- // //var_dump('getNovelFengMian');
- // //var_dump($arrData);
- // $Client = $this->Client;
- // $QueueManage = $this->QueueManage;
-
- // // 获取 Redis 实例
- // $redis = Cache::store('redis')->handler();
- // $intXiaoShuoSourceId = $arrData['xiaoshuo_id'];
- // $intXiaoShuoSourceSeoId = $arrData['xiaoshuo_seo_id'];
- // $redisKey = "task:novel_source_id:" . $intXiaoShuoSourceId;
-
- // try {
- // $arrCookie = $this->getAilisiCookie();
- // $strHost = getDomainFromUrl($arrData['xiaoshuo_url']);
- // $CookieJar = CookieJar::fromArray($arrCookie, $strHost);
-
- // $strSiteUrl = $this->getAilisiDomain();
-
-
- // $strCover = $QueryList->find('#fmimg img')->eq(0)->attr("src");
-
- // if (strpos($strCover, 'http') === false) {
- // $strCover = $strSiteUrl . $strCover;
- // }
- // $strCoverUri = $strCover;
-
- // try {
- // $strCoverUri = "";
- // $strCoverUri = ImageProcessor::getInstance()->downloadAndEncryptImage($strCover, 'xs');
- // $arrCover = [
- // 'code' => 'AI_LI_SI_COVER',
- // // 'uri' => getUriByUrl($strCover),
- // 'uri' => $strCoverUri,
-
- // ];
-
- // $arrXiaoShuoInfo = [
- // 'xsxq_feng_mian' => json_encode($arrCover),
-
- // ];
- // $XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
- // } catch (\Throwable $t) {
- // $strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
- // echo $strErr;
- // }
-
-
-
- // } catch (\Throwable $T) {
- // throw $T;
- // }
- // }
-
- // public function getNovelReader($arrData)
- // {
- // var_dump('getNovelReader');
- // var_dump($arrData['xszj_url']);
- // $Client = $this->Client;
- // $strXszjSourceCode = $arrData['xszj_source_code'];
- // $strZhangJieUrl = $arrData['xszj_url'];
- // $intZhangJiePaiXu = $arrData['xszj_paixu'];
- // $intFenleiId = $arrData['xsfl_id'];
- // $intXiaoShuoSourceID = $arrData['xsxq_source_id'];
- // $strZhangJieSourceId = $arrData['xszj_source_id'];
- // $strZhangJieName = $arrData['xszj_name'];
- // try {
- // $arrCookie = $this->getAilisiCookie();
- // $strHost = getDomainFromUrl($strZhangJieUrl);
- // $CookieJar = CookieJar::fromArray($arrCookie, $strHost);
-
- // if (XiaoShuoZhangJieModel::checkZhangJieExists($strXszjSourceCode)) {
- // return true;
- // }
-
- // $proxy = getRandomProxies();
- // $userAgent = getRandomUserAgent();
- // $Response = $Client->request('GET', $strZhangJieUrl, [
- // 'headers' => [
- // 'User-Agent' => $userAgent,
- // 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
- // 'Accept-Language' => 'en-US,en;q=0.5',
- // ],
- // 'cookies' => $CookieJar,
- // // 'proxy' => $proxy,
- // 'timeout' => 30,
- // ]);
-
- // $strResult = (string)$Response->getBody();
- // unset($Response);
- // $ZhangJieQueryList = QueryList::html($strResult);
-
-
- // // 获取原始 HTML
- // $strNeiRong = $ZhangJieQueryList->find('#content')->eq(0)->html();
-
- // // 解码 HTML 实体
- // $strNeiRong = html_entity_decode($strNeiRong, ENT_QUOTES, 'UTF-8');
-
- // // 去掉 标签中的 class="content_detail" 属性
- // $strNeiRong = preg_replace('/');
-
- // // 替换不间断空格 (\xc2\xa0) 和 为普通空格
- // $strNeiRong = preg_replace('/\xc2\xa0/', ' ', $strNeiRong); // 替换 UTF-8 不间断空格
- // $strNeiRong = preg_replace('/ /', ' ', $strNeiRong); // 替换
-
- // // 清理多余空格和换行符
- // $strNeiRong = preg_replace('/\s+/', ' ', $strNeiRong);
- // $strNeiRong = preg_replace('/\s+/', '
', $strNeiRong); // 清理
标签内部多余空格
- // $strNeiRong = preg_replace('/\s+<\/p>/', '
', $strNeiRong); // 清理 标签结尾多余空格
-
- // // 输出结果
- // // var_dump($strNeiRong);
-
-
- // # '/novel/123yqw/分类id/小说源id/小说源章节id.txt'
- // $strNeiRongTxtPath = config("filesystem.novel").'/novel/biquxs/'.$intFenleiId.'/'.$intXiaoShuoSourceID.'/'.$strZhangJieSourceId.'.txt';
- // //$strNeiRongTxtPath = "/www/novel/123yqw/1/11/111.txt";
- // // 调用函数
- // $result = saveCompressAndDeleteTxt($strNeiRongTxtPath, $strNeiRong);
-
- // // 输出状态
- // if ($result['status']) {
- // var_dump("Success: " . $result['message']) ;
- // } else {
- // var_dump("Error: " . $result['message']) ;
- // }
-
- // $arrXiaoShuoZhangJie = [
- // 'xsxq_id' => $intXiaoShuoSourceID,
- // 'xszj_pai_xu' => $intZhangJiePaiXu,
- // 'xszj_ming_zi' => $strZhangJieName,
- // 'xszj_nei_rong' => $strNeiRongTxtPath,
- // 'xszj_source_id' => $strZhangJieSourceId,
- // 'xszj_source_code' => $strXszjSourceCode,
- // ];
-
- // XiaoShuoZhangJieModel::addXiaoShuoZhangJie($arrXiaoShuoZhangJie);
- // unset($ZhangJieQueryList);
- // // 延时 3 秒
- // //sleep(3);
-
- // } catch (\Throwable $T) {
- // throw $T;
- // }
- // }
-}
diff --git a/code/app/task/crawler/zw630/Scheduler.php b/code/app/task/crawler/zw630/Scheduler.php
new file mode 100644
index 0000000..d467624
--- /dev/null
+++ b/code/app/task/crawler/zw630/Scheduler.php
@@ -0,0 +1,213 @@
+QueueManage = new QueueManage();
+ }
+
+ /**
+ * 全局扫描任务
+ *
+ * @return void
+ */
+ public function crawlFull()
+ {
+ $intSliceSize = 6;
+
+ $arrFilter = [];
+ // for ($intNId = 1; $intNId <= 531208; $intNId++) {
+ // for ($intNId = 1; $intNId <= 10000; $intNId++) {
+ for ($intNId = 1; $intNId <= 20; $intNId++) {
+
+ $arrFilter[] = [
+ 'n_source_id' => $intNId,
+ 'page' => 0,
+ ];
+
+ if ($intNId % $intSliceSize == 0) {
+ $this->fetchNovel($arrFilter);
+ $arrFilter = [];
+ }
+ }
+ if (!empty($arrFilter)) {
+ $this->fetchNovel($arrFilter);
+ }
+ return;
+
+
+
+ // $arrTest = [
+ // 'n_id' => 9973,
+ // 'filter' => [
+ // [
+ // 'name' => '第十四章 神兽空青蛇妖',
+ // 'uri' => '/shu/1/33.html',
+ // 'c_sort_num' => 32,
+ // 'c_source_id' => 33,
+ // 'c_page' => 0,
+ // ],
+ // ]
+ // ];
+
+ // $arrTask = [
+ // 'callback' => [self::class, 'getChapterInfo'],
+ // 'data' => $arrTest
+ // ];
+ // $this->QueueManage->set($arrTask);
+ }
+
+ /**
+ * Undocumented function
+ *
+ * @param array $arrFilter
+ * @return void
+ */
+ public function fetchNovel(array $arrFilter) //(int $intNSourceId, int $intPage = 0)
+ {
+ $arrTask = [
+ 'callback' => [self::class, 'getNovelInfo'],
+ 'data' => $arrFilter,
+ // 'data' => [
+ // 'n_source_id' => $intNSourceId,
+ // 'page' => $intPage,
+ // ]
+ ];
+ $this->QueueManage->set($arrTask);
+ }
+
+ /**
+ * Undocumented function
+ *
+ * @param integer $intNId
+ * @param integer $intPage
+ * @param array $arrChapterList
+ * @return void
+ */
+ public function fetchChapter(int $intNId, int $intPage, array $arrChapterList = [])
+ {
+ $intSliceSize = 6;
+ $intPageSize = 100;
+
+ $arrTask = [
+ 'callback' => [self::class, 'getChapterInfo'],
+ 'data' => [
+ 'n_id' => $intNId,
+ // 'filter' => $arrFilter,
+ ]
+ ];
+ $arrFilter = [];
+ foreach ($arrChapterList as $intKey => $arrChapter) {
+ $arrChapter['c_sort_num'] = $intPage * $intPageSize + $intKey + 1;
+ $arrChapter['c_source_id'] = supperExtractFilename($arrChapter['uri']);
+ $arrChapter['c_page'] = 0;
+
+ $arrFilter[] = $arrChapter;
+ if ($intKey % $intSliceSize == 0) {
+ $arrTask['data']['filter'] = $arrFilter;
+ $this->QueueManage->set($arrTask);
+ $arrFilter = [];
+ }
+ }
+
+ if (!empty($arrFilter)) {
+ $arrTask['data']['filter'] = $arrFilter;
+ $this->QueueManage->set($arrTask);
+ }
+ }
+
+ /**
+ * Undocumented function
+ *
+ * @param array $arrFilter
+ * @return void
+ */
+ public function getNovelInfo(array $arrFilter)
+ {
+ $Site = new Site;
+
+ $arrNovelPageList = $Site->getNovelPageList($arrFilter);
+
+ $arrCover = [];
+ foreach ($arrNovelPageList as $NovelPage) {
+ $arrNovel = $NovelPage->getNovelInfo();
+ if (empty($arrNovel)) {
+ continue;
+ }
+
+
+ $arrNextPageData = $NovelPage->getNextPageArgs();
+ if (!empty($arrNextPageData)) {
+ $this->fetchNovel([
+ [
+ 'n_source_id' => $arrNextPageData['n_source_id'],
+ 'page' => $arrNextPageData['page'],
+ ]
+ ]);
+ }
+
+ // $arrCover[] = $NovelPage->getDownCoverInfo();
+ // $arrChapterList = $NovelPage->getChapters();
+
+ // $this->fetchChapter($arrNovel['n_id'], $NovelPage->getData()['page'], $arrChapterList);
+ }
+ if ($arrCover) {
+ $Site->downRemoteImgToLocal($arrCover);
+ }
+ }
+
+ /**
+ * Undocumented function
+ *
+ * @param array $arrData
+ * @return void
+ */
+ public function getChapterInfo(array $arrData)
+ {
+ $Site = new Site;
+
+ $arrChapterPageList = $Site->getChapterPageList($arrData['filter']);
+
+ $arrNextPageFilterList = [];
+ foreach ($arrChapterPageList as $ChapterPage) {
+
+ $ChapterPage->saveChapter($arrData['n_id']);
+
+ $arrNextPageFilter = $ChapterPage->getNextPage();
+
+ if (!empty($arrNextPageFilter)) {
+ $arrNextPageFilterList[] = $arrNextPageFilter;
+ }
+ }
+
+ if (!empty($arrNextPageFilterList)) {
+
+ $arrTask = $arrData;
+ $arrTask['filter'] = $arrNextPageFilterList;
+ $arrTask = [
+ 'callback' => [self::class, 'getChapterInfo'],
+ 'data' => $arrTask
+ ];
+ $this->QueueManage->set($arrTask);
+ }
+ }
+}
diff --git a/code/app/task/crawler/zw630/page/BasePage.php b/code/app/task/crawler/zw630/page/BasePage.php
new file mode 100644
index 0000000..491e5a6
--- /dev/null
+++ b/code/app/task/crawler/zw630/page/BasePage.php
@@ -0,0 +1,258 @@
+setSite($Site);
+
+ $this->strUri = $strUri;
+ }
+
+ /**
+ * Undocumented function
+ *
+ * @param array $arrData
+ * @return void
+ */
+ public function setData(array $arrData)
+ {
+ $this->arrData = $arrData;
+ }
+
+ /**
+ * Undocumented function
+ *
+ * @return array
+ */
+ public function getData(): array
+ {
+ return $this->arrData;
+ }
+
+ /**
+ * Undocumented function
+ *
+ * @return QueueManage
+ */
+ public function getQueueManage(): QueueManage
+ {
+ if ($this->QueueManage == NULL) {
+ $this->QueueManage = QueueManage::getInstance();
+ }
+
+ return $this->QueueManage;
+ }
+
+ /**
+ * Undocumented function
+ *
+ * @return MongoBase
+ */
+ public function getMongoBase(): MongoBase
+ {
+ if ($this->MongoBase == NULL) {
+ DDLManage::load('MongoBase');
+ $this->MongoBase = MongoBase::getInstance();
+ }
+
+ return $this->MongoBase;
+ }
+
+ public function getQueryList(): QueryList
+ {
+ if ($this->QueryList == NULL) {
+ $this->QueryList = QueryList::html($this->getContent());
+ }
+
+ return $this->QueryList;
+ }
+
+
+ /**
+ * Undocumented function
+ *
+ * @return string
+ */
+ public function getContent(): string
+ {
+ if ($this->strContent == NULL) {
+ $this->Response = $this->getClient()->get($this->strUri);
+ $this->strContent = $this->Response->getBody()->getContents();
+ }
+
+ return $this->strContent;
+ }
+
+ /**
+ * force set page content
+ *
+ * @param string $strContent
+ * @return void
+ */
+ public function setContent($strContent)
+ {
+ $this->strContent = $strContent;
+ }
+
+
+ /**
+ * Undocumented function
+ *
+ * @return boolean
+ */
+ public function isMobile($strMark = 'content="mobile"'): bool
+ {
+ if ($this->boolIsMobile === NULL) {
+ $this->boolIsMobile = is_numeric(strpos($this->getContent(), $strMark));
+ }
+ return $this->boolIsMobile;
+ }
+
+
+ /**
+ * Undocumented function
+ *
+ * @return Client
+ */
+ protected function getClient(): Client
+ {
+ if ($this->Client == NULL) {
+ $this->Client = $this->getSite()->getClient();
+ }
+ return $this->Client;
+ }
+
+
+ /**
+ * Undocumented variable
+ *
+ * @var Site
+ */
+ protected $Site;
+
+ public function setSite(Site $Site)
+ {
+ $this->Site = $Site;
+ }
+
+ public function getSite()
+ {
+ return $this->Site;
+ }
+
+ /**
+ * Undocumented variable
+ *
+ * @var boolean
+ */
+ protected $boolIsFirstSave = false;
+
+ /**
+ * Undocumented function
+ *
+ * @return boolean
+ */
+ public function getIsFirstSave(): bool
+ {
+ return $this->boolIsFirstSave;
+ }
+}
diff --git a/code/app/task/crawler/zw630/page/ChapterPage.php b/code/app/task/crawler/zw630/page/ChapterPage.php
new file mode 100644
index 0000000..f91a330
--- /dev/null
+++ b/code/app/task/crawler/zw630/page/ChapterPage.php
@@ -0,0 +1,139 @@
+arrChapterInfo === NULL) {
+
+ $this->arrChapterInfo = [];
+
+ $strPattern = "/qsbs\.bb\('([^']*)'\)/";
+
+ preg_match_all($strPattern, $this->getContent(), $arrMatches);
+
+ $strContent = '';
+
+ foreach ($arrMatches[1] as $str) {
+ $strContent .= base64_decode($str);
+ }
+
+ // $this->arrChapterInfo['c_content'] = $strContent;
+ $this->strChapterContent = $strContent;
+
+ $this->arrChapterInfo['c_page_title'] = $this->getQueryList()->find('title')->text();
+ $this->arrChapterInfo['c_page_keywords'] = $this->getQueryList()->find('meta[name="keywords"]')->attr('content');
+ $this->arrChapterInfo['c_page_description'] = $this->getQueryList()->find('meta[name="description"]')->attr('content');
+ }
+
+ return $this->arrChapterInfo;
+ }
+
+ /**
+ * Undocumented variable
+ *
+ * @var array
+ */
+ protected $arrChapterInfo;
+
+ public function saveChapter(int $intNId): array
+ {
+
+ try {
+ $this->getChapterInfo();
+
+ $strUUId = 'zw630-chapter-' . $this->arrData['c_source_id'] . '-' . $this->arrData['c_page'];
+ $Col = $this->getMongoBase()->getDb()->selectCollection('chapter');
+
+ $arrExisting = $Col->findOne(['c_site_uuid' => $strUUId]);
+
+ if ($arrExisting) {
+
+ $arrExisting = (array) $arrExisting;
+ $this->arrChapterInfo = array_merge($arrExisting, $this->getChapterInfo());
+
+ $this->arrChapterInfo['updated_at'] = new \MongoDB\BSON\UTCDateTime(); // 添加更新时间
+ $Col->updateOne(
+ ['c_site_uuid' => $strUUId],
+ ['$set' => $this->arrChapterInfo],
+ // ['upsert' => false]
+ );
+ } else {
+
+ $this->arrChapterInfo['n_id'] = $intNId;
+
+ $this->arrChapterInfo['name'] = $this->arrData['name'];
+
+ $this->arrChapterInfo['c_sort_num'] = $this->arrData['c_sort_num'];
+ $this->arrChapterInfo['c_page'] = $this->arrData['c_page'];
+
+ $this->arrChapterInfo['c_source_id'] = $this->arrData['c_source_id'];
+ $this->arrChapterInfo['c_source_uri'] = $this->arrData['uri'];
+ $this->arrChapterInfo['c_site_name'] = '恋上你中文';
+ $this->arrChapterInfo['c_site_code'] = 'zw630';
+ $this->arrChapterInfo['c_site_uuid'] = $strUUId;
+ $this->arrChapterInfo['c_content_path'] = '/novel/' . ($intNId % 100) . '/' . $intNId . '/' . $this->arrChapterInfo['c_sort_num'] . '_' . $this->arrChapterInfo['c_page'] . '.txt';
+
+ $this->getMongoBase()->insertOne('chapter', $this->arrChapterInfo);
+
+ if (!StorageCore::getInstance()->has($this->arrChapterInfo['c_content_path'])) {
+ StorageCore::getInstance()->set($this->arrChapterInfo['c_content_path'], $this->strChapterContent);
+ }
+
+ $this->boolIsFirstSave = true;
+ }
+ return $this->arrChapterInfo;
+ } catch (\MongoDB\Exception\Exception $e) {
+ throw new \Exception("Upsert failed: " . $e->getMessage());
+ }
+ }
+
+ protected $arrNextPage;
+
+ /**
+ * Undocumented function
+ *
+ * @return array
+ */
+ public function getNextPage(): array
+ {
+ if ($this->arrNextPage === NULL) {
+ $this->arrNextPage = [];
+
+ if ($this->isMobile()) {
+ $strPattern = "/var\s+hhekgsv\s*=\s*['\"](.*?)['\"]/";
+ preg_match_all($strPattern, $this->getContent(), $arrMatches);
+ $strNextPageUri = end($arrMatches[1]) ?: '';
+ } else {
+ $strHref = $this->getQueryList()->find('.read_btn a:contains("下一章")')->attr('href');
+ $strNextPageUri = $strHref ?: '';
+ }
+
+ if (is_numeric(strpos($strNextPageUri, '_'))) {
+ $this->arrNextPage = $this->arrData;
+ $this->arrNextPage['c_page'] += 1;
+ $this->arrNextPage['uri'] = $strNextPageUri;
+ }
+ }
+
+ return $this->arrNextPage;
+ }
+}
diff --git a/code/app/task/crawler/zw630/page/NovelPage.php b/code/app/task/crawler/zw630/page/NovelPage.php
new file mode 100644
index 0000000..7b44797
--- /dev/null
+++ b/code/app/task/crawler/zw630/page/NovelPage.php
@@ -0,0 +1,304 @@
+getMongoBase()->getDb()->selectCollection('novel');
+
+ $arrExisting = $Col->findOne(['n_site_uuid' => $strUUId]);
+
+ if ($arrExisting) {
+ $arrNovelInfo = (array) $arrExisting;
+ } else {
+ $arrNovelInfo['og_title'] = $strName;
+ $arrNovelInfo['og_novel_book_name'] = $strName;
+ $arrNovelInfo['og_novel_pin_yin'] = zhToPinYin($strName);
+ $arrNovelInfo['n_site_name'] = '恋上你中文';
+ $arrNovelInfo['n_site_code'] = 'zw630';
+ $arrNovelInfo['n_site_uuid'] = $strUUId;
+ $arrNovelInfo['n_source_id'] = $intNSourceId;
+ $arrNovelInfo['n_level'] = 1;
+ $arrNovelInfo['n_id'] = $this->getMongoBase()->getNextSequence('novel');
+ $this->getMongoBase()->insertOne('novel', $arrNovelInfo);
+ }
+ return $arrNovelInfo;
+ }
+
+ /**
+ * Find or create a category by name, handling concurrency safely.
+ *
+ * @param string $strCategory
+ * @return array|null The category document (existing or newly created)
+ */
+ public function findOrCreateCategory(string $strCategory)
+ {
+ if (empty($strCategory)) {
+ return false;
+ }
+ $Col = $this->getMongoBase()->getDb()->selectCollection('category');
+
+ $arrCategory = ['ca_name' => $strCategory];
+ try {
+ $Result = $Col->findOneAndUpdate(
+ $arrCategory,
+ ['$set' => $arrCategory],
+ [
+ 'upsert' => true,
+ 'returnDocument' => \MongoDB\Operation\FindOneAndUpdate::RETURN_DOCUMENT_AFTER
+ ]
+ );
+ return $Result;
+ } catch (\MongoDB\Driver\Exception\Exception $e) {
+ return null;
+ }
+ }
+
+
+ /**
+ * Undocumented function
+ *
+ * @return array
+ */
+ public function getNovelInfo($boolSave = true): array
+ {
+ if (empty($this->arrNovelInfo)) {
+ $this->getQueryList()->find('meta[property]')->each(function ($Item) {
+ $strKey = str_replace(':', '_', $Item->attr('property'));
+ $this->arrNovelInfo[$strKey] = $Item->attr('content');
+ });
+
+ $this->arrNovelInfo['n_page_title'] = $this->getQueryList()->find('title')->text();
+ $this->arrNovelInfo['n_page_keywords'] = $this->getQueryList()->find('meta[name="keywords"]')->attr('content');
+ $this->arrNovelInfo['n_page_description'] = $this->getQueryList()->find('meta[name="description"]')->attr('content');
+
+ $this->getQueryList()->find('.first_txt')->eq(0)->find("a")->map(function ($A) {
+ $strName = $A->text();
+ $strUri = $A->attr('href');
+ if (is_numeric(strpos($strUri, 'shu'))) {
+ $intNSourceId = (int)supperExtractFilename($strUri);
+ $arrNovelInfo = $this->findOrCreateNovel($intNSourceId, $strName);
+ $this->arrRelatedNovel[] = [
+ "n_id" => $arrNovelInfo["n_id"],
+ "n_source_id" => $arrNovelInfo["n_source_id"],
+ "og_title" => $arrNovelInfo["og_title"],
+ ];
+ } elseif (is_numeric(strpos($strUri, 'kan'))) {
+ $intNForgeId = (int)supperExtractFilename($strUri);
+ $this->arrForgeNovel[] = [
+ "n_forge_id" => $intNForgeId,
+ "n_forge_title" => $strName,
+ ];
+ }
+ });
+ $this->arrNovelInfo['n_related_novel'] = $this->arrRelatedNovel;
+ $this->arrNovelInfo['n_forge_novel'] = $this->arrForgeNovel;
+ }
+
+
+ if ($boolSave && !empty($this->arrNovelInfo)) {
+ $this->saveNovelInfo();
+
+ $this->findOrCreateCategory($this->arrNovelInfo['og_novel_category']);
+ }
+ return $this->arrNovelInfo;
+ }
+
+ /**
+ * Undocumented function
+ *
+ * @return void
+ */
+ public function saveNovelInfo()
+ {
+ try {
+ $strUUId = 'zw630-' . $this->arrData['n_source_id'];
+ $Col = $this->getMongoBase()->getDb()->selectCollection('novel');
+
+ $arrExisting = $Col->findOne(['n_site_uuid' => $strUUId]);
+
+ if ($arrExisting) {
+
+ $arrExisting = (array) $arrExisting;
+ $this->arrNovelInfo = array_merge($arrExisting, $this->arrNovelInfo);
+
+ if (!isset($this->arrNovelInfo['og_novel_pin_yin']) || empty($this->arrNovelInfo['og_novel_pin_yin'])) {
+ $this->arrNovelInfo['og_novel_pin_yin'] = zhToPinYin($this->arrNovelInfo['og_novel_book_name'] ?? '');
+ }
+
+ if (!isset($this->arrNovelInfo['n_cover_path']) || empty($this->arrNovelInfo['n_cover_path'])) {
+ $this->arrNovelInfo['n_cover_path'] = 'img/' . date('Y/m/d', time() - 24 * 3600 * 14) . '/' . $this->arrNovelInfo['n_id'] . '.jpg';
+ }
+
+ $this->arrNovelInfo['updated_at'] = new \MongoDB\BSON\UTCDateTime();
+ $Col->updateOne(
+ ['n_site_uuid' => $strUUId],
+ ['$set' => $this->arrNovelInfo],
+ // ['upsert' => false]
+ );
+ } else {
+ $this->arrNovelInfo['og_novel_pin_yin'] = zhToPinYin($this->arrNovelInfo['og_novel_book_name'] ?? '');
+ $this->arrNovelInfo['n_site_name'] = '恋上你中文';
+ $this->arrNovelInfo['n_site_code'] = 'zw630';
+ $this->arrNovelInfo['n_site_uuid'] = $strUUId;
+ $this->arrNovelInfo['n_source_id'] = $this->arrData['n_source_id'];
+ $this->arrNovelInfo['n_id'] = $this->getMongoBase()->getNextSequence('novel');
+ $this->arrNovelInfo['n_level'] = 1;
+ $this->arrNovelInfo['n_cover_path'] = '/img/' . date('Y/m/d', time() - 24 * 3600 * 14) . '/' . $this->arrNovelInfo['n_id'] . '.jpg';
+ $this->getMongoBase()->insertOne('novel', $this->arrNovelInfo);
+
+ $this->boolIsFirstSave = true;
+ }
+ return $this->arrNovelInfo;
+ } catch (\MongoDB\Exception\Exception $e) {
+ throw new \Exception("Upsert failed: " . $e->getMessage());
+ }
+ }
+
+ /**
+ * Undocumented function
+ *
+ * @return array
+ */
+ public function getDownCoverInfo(): array
+ {
+ return [
+ 'uri' => $this->arrNovelInfo['og_image'],
+ 'local_path' => $this->arrNovelInfo['n_cover_path'],
+ ];
+ }
+
+ /**
+ * Undocumented variable
+ *
+ * @var array
+ */
+ protected $arrLatestChapters = NULL;
+
+ /**
+ * Undocumented variable
+ *
+ * @var array
+ */
+ protected $arrChapters = NULL;
+
+ public function getLatestChaptersList(): array
+ {
+ if ($this->arrLatestChapters === NULL) {
+ $this->arrLatestChapters = [];
+
+ if ($this->isMobile()) {
+ $strClass = '.chapter-list';
+ } else {
+ $strClass = '.section-list';
+ }
+
+ if ($this->arrData['page'] == 0) {
+ $this->getQueryList()->find($strClass)->eq(0)->find("a")->map(function ($A) {
+ $this->arrLatestChapters[] = [
+ 'name' => $A->text(),
+ 'uri' => $A->attr('href'),
+ ];
+ });
+ }
+ }
+ return $this->arrLatestChapters;
+ }
+
+ public function getChapters(): array
+ {
+ if ($this->arrChapters === NULL) {
+ $this->arrChapters = [];
+ if ($this->isMobile()) {
+ $strClass = '.chapter-list';
+ } else {
+ $strClass = '.section-list';
+ }
+
+ $intDivIndex = 0;
+ if ($this->arrData['page'] == 0) {
+ $intDivIndex = 1;
+ }
+
+ $this->getQueryList()->find($strClass)->eq($intDivIndex)->find("a")->map(function ($A) {
+ $this->arrChapters[] = [
+ 'name' => $A->text(),
+ 'uri' => $A->attr('href'),
+ ];
+ });
+ }
+ return $this->arrChapters;
+ }
+
+
+ protected $arrNextPageArgs = NULL;
+
+ /**
+ * Undocumented function
+ *
+ * @return array
+ */
+ public function getNextPageArgs(): array
+ {
+ $this->arrNextPageArgs = [];
+ if ($this->arrData['page'] == 0) {
+ $this->getLatestChaptersList();
+ $this->getChapters();
+
+ $arrLatestChapterFirstBlock = count($this->arrLatestChapters) > 0 ? $this->arrLatestChapters[0] : null;
+ $arrLatestChapterSecondBlock = count($this->arrChapters) > 0 ? $this->arrChapters[count($this->arrChapters) - 1] : null;
+
+ if (
+ !empty($arrLatestChapterFirstBlock) && !empty($arrLatestChapterSecondBlock)
+ && ($arrLatestChapterFirstBlock['uri'] != $arrLatestChapterSecondBlock['uri'])
+
+ ) {
+ $this->arrNextPageArgs = [
+ 'n_source_id' => $this->arrData['n_source_id'],
+ 'page' => $this->arrData['page'] + 1,
+ ];
+ }
+ } else {
+ $strLastUri = $this->getQueryList()->find('select:first option:last')->attr('value');
+ if ($strLastUri != '' && $strLastUri != $this->strUri) {
+ $this->arrNextPageArgs = [
+ 'n_source_id' => $this->arrData['n_source_id'],
+ 'page' => $this->arrData['page'] + 1,
+ ];
+ }
+ }
+ return $this->arrNextPageArgs;
+ }
+}
diff --git a/code/app/task/crawler/zw630/page/Site.php b/code/app/task/crawler/zw630/page/Site.php
new file mode 100644
index 0000000..86adbb7
--- /dev/null
+++ b/code/app/task/crawler/zw630/page/Site.php
@@ -0,0 +1,257 @@
+Client = new Client([
+ // 'http_version' => '1.1', // 强制用 HTTP/1.1
+ // 'base_uri' => $this->strDomain,
+ // 'http_errors' => false,
+ // // 'version' => 2.0,
+ // 'timeout' => 160,
+ // 'connect_timeout' => 30,
+ // 'curl' => [
+ // // CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_2_0,
+ // CURLOPT_TCP_KEEPALIVE => 1,
+ // CURLOPT_SSL_VERIFYPEER => true,
+ // ],
+ // 'headers' => [
+ // 'User-Agent' => 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Mobile Safari/537.36 Edg/134.0.0.0',
+ // 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
+ // 'Accept-Language' => 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
+ // 'Referer' => $this->strDomain,
+ // 'sec-ch-ua-mobile' => '?1',
+ // 'sec-ch-ua-platform' => '"Android"',
+ // 'sec-fetch-dest' => 'document',
+ // 'sec-fetch-mode' => 'navigate',
+ // 'sec-fetch-site' => 'same-origin',
+ // 'sec-fetch-user' => '?1',
+ // 'upgrade-insecure-requests' => '1',
+ // ],
+ // ]);
+
+ $this->Client = new Client([
+ 'http_version' => '1.1', // 强制用 HTTP/1.1，已有，保持
+ 'base_uri' => $this->strDomain, // 基础域名，已有，保持
+ 'http_errors' => false, // 不抛 HTTP 错误，已有，保持
+ 'timeout' => 60, // 从 160 秒改为 60 秒
+ 'connect_timeout' => 10, // 从 30 秒改为 10 秒
+ 'curl' => [
+ CURLOPT_TCP_KEEPALIVE => 1, // 启用 TCP Keep-Alive，已有，保持
+ CURLOPT_TCP_KEEPIDLE => 120, // Keep-Alive 空闲时间
+ CURLOPT_TCP_KEEPINTVL => 60, // Keep-Alive 探测间隔
+ CURLOPT_SSL_VERIFYPEER => true, // 验证 SSL，已有，保持
+ CURLOPT_SSL_VERIFYHOST => 2, // 确保主机名匹配
+ ],
+ 'headers' => [
+ // Mobile
+ // 'User-Agent' => 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Mobile Safari/537.36 Edg/134.0.0.0',
+ // 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
+
+ //PC
+ 'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36 Edg/134.0.0.0',
+ 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
+
+ 'Accept-Language' => 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
+ 'Referer' => $this->strDomain,
+ 'Connection' => 'keep-alive', // 显式启用 Keep-Alive
+
+
+ // Mobile
+ // 'sec-ch-ua-mobile' => '?1',
+ // 'sec-ch-ua-platform' => '"Android"',
+ // 'sec-fetch-dest' => 'document',
+ // 'sec-fetch-mode' => 'navigate',
+ // 'sec-fetch-site' => 'same-origin',
+ // 'sec-fetch-user' => '?1',
+
+
+ // PC
+ 'sec-ch-ua' => '"Chromium";v="134", "Not:A-Brand";v="24", "Microsoft Edge";v="134"',
+ 'sec-ch-ua-mobile' => '?0',
+ 'sec-ch-ua-platform' => '"Windows"',
+ 'sec-fetch-dest' => 'document',
+ 'sec-fetch-mode' => 'navigate',
+ 'sec-fetch-site' => 'same-origin',
+ 'sec-fetch-user' => '?1',
+
+ 'upgrade-insecure-requests' => '1',
+ ],
+ 'pool_size' => 40, // 并发连接池大小（视需求调整）
+ ]);
+ }
+
+ /**
+ * Undocumented function
+ *
+ * @return Client
+ */
+ public function getClient(): Client
+ {
+ return $this->Client;
+ }
+
+ /**
+ * Undocumented function
+ *
+ * @param string $strUri
+ * @return NovelPage
+ */
+ public function getNovelPage(string $strUri): NovelPage
+ {
+ return new NovelPage($this, $strUri);
+ }
+
+
+ /**
+ * Undocumented function
+ *
+ * @param string $strUri
+ * @return NovelPage[]
+ */
+ public function getNovelPageList(array $arrFilter): array
+ {
+
+ /** @var NovelPage[] */
+ $arrNovelPage = [];
+
+ foreach ($arrFilter as $intKey => $arrItem) {
+
+ if ($arrItem['page'] == 0) {
+ $arrItem['uri'] = sprintf('/shu/%s.html', $arrItem['n_source_id']);
+ } else {
+ $arrItem['uri'] = sprintf('/shu/%s_%s.html', $arrItem['n_source_id'], $arrItem['page']);
+ }
+ $arrFilter[$intKey]['uri'] = $arrItem['uri'];
+
+ $arrPromises[$intKey] = $this->getClient()->getAsync($arrItem['uri']);
+ }
+
+ $arrResults = Promise\Utils::settle($arrPromises)->wait();
+
+ $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
+ return $arrResult['state'] === 'fulfilled';
+ });
+
+ foreach ($arrSuccessfulResults as $intKey => $arrResult) {
+ $Response = $arrResult['value'];
+
+ $strContent = $Response->getBody()->getContents();
+ if ($strContent == '索引文件不存在！') {
+ continue;
+ }
+
+ $NovelPage = new NovelPage($this, $arrItem['uri']);
+
+ $NovelPage->setContent($strContent);
+ $NovelPage->setData($arrFilter[$intKey]);
+
+ $arrNovelPage[] = $NovelPage;
+ }
+
+
+ return $arrNovelPage;
+ }
+
+ /**
+ * Undocumented function
+ *
+ * @param string $strUri
+ * @return ChapterPage[]
+ */
+ public function getChapterPageList(array $arrFilter): array
+ {
+
+ /** @var ChapterPage[] */
+ $arrChapterPage = [];
+
+ $arrPromises = [];
+ foreach ($arrFilter as $intKey => $arrItem) {
+ $arrPromises[$intKey] = $this->getClient()->getAsync($arrItem['uri']);
+ }
+
+ $arrResults = Promise\Utils::settle($arrPromises)->wait();
+
+ $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
+ return $arrResult['state'] === 'fulfilled';
+ });
+
+ foreach ($arrSuccessfulResults as $intKey => $arrResult) {
+ $Response = $arrResult['value'];
+
+ $strContent = $Response->getBody()->getContents();;
+
+ $ChapterPage = new ChapterPage($this, $arrItem['uri']);
+
+ $ChapterPage->setContent($strContent);
+ $ChapterPage->setData($arrFilter[$intKey]);
+
+ $arrChapterPage[] = $ChapterPage;
+ }
+
+
+ return $arrChapterPage;
+ }
+
+ /**
+ * Undocumented function
+ *
+ * @param array $arrFilter
+ * @return void
+ */
+ public function downRemoteImgToLocal(array $arrFilter)
+ {
+ $StorageCore = StorageCore::getInstance();
+
+ $arrPromises = [];
+
+ foreach ($arrFilter as $intKey => $arrItem) {
+ if (!$StorageCore->has($arrItem['local_path'])) {
+ $arrPromises[$intKey] = $this->getClient()->getAsync($arrItem['uri']);
+ }
+ }
+
+ $arrResults = Promise\Utils::settle($arrPromises)->wait();
+
+ $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
+ return $arrResult['state'] === 'fulfilled';
+ });
+
+ foreach ($arrSuccessfulResults as $intKey => $arrResult) {
+ $Response = $arrResult['value'];
+
+ // $strContent = $Response->getBody()->getContents();
+
+ $StorageCore->writeStream($arrFilter[$intKey]['local_path'], $Response->getBody()->detach());;
+ }
+ }
+}
diff --git a/code/app/task/module/Listen.php b/code/app/task/module/Listen.php
index bab833e..b9130df 100644
--- a/code/app/task/module/Listen.php
+++ b/code/app/task/module/Listen.php
@@ -4,8 +4,8 @@ declare(strict_types=1);
namespace app\task\module;
-use minserver\ProcessSanitizer;
-use minserver\QueueManage;
+use microserver\ProcessSanitizer;
+use microserver\QueueManage;
class Listen
{
diff --git a/code/app/task/module/PlanTask.php b/code/app/task/module/PlanTask.php
index 2fb6fc7..db73150 100644
--- a/code/app/task/module/PlanTask.php
+++ b/code/app/task/module/PlanTask.php
@@ -5,9 +5,9 @@ declare(strict_types=1);
namespace app\task\module;
use app\model\PlanTaskModel;
-use app\task\collection\zw630\Zw630Col;
-use minserver\QueueManage;
-use minserver\ProcessSanitizer;
+use app\task\crawler\zw630\Scheduler as Zw630Scheduler;
+use microserver\QueueManage;
+use microserver\ProcessSanitizer;
class PlanTask
@@ -41,7 +41,7 @@ class PlanTask
public static function pull630ZwNovel()
{
$arrTask = [
- 'callback' => [Zw630Col::class, 'pull630ZwNovel'],
+ 'callback' => [Zw630Scheduler::class, 'crawlFull'],
'data' => []
];
diff --git a/code/composer.json b/code/composer.json
index f065d4e..16d9146 100644
--- a/code/composer.json
+++ b/code/composer.json
@@ -34,7 +34,9 @@
"aws/aws-sdk-php": "^3.337",
"league/flysystem": "^1.1",
"league/flysystem-aws-s3-v3": "^1.0",
- "mongodb/mongodb": "^1.21"
+ "mongodb/mongodb": "^1.21",
+ "topthink/think-throttle": "^2.0",
+ "phpoffice/phpspreadsheet": "^4.1"
},
"require-dev": {
"topthink/think-dumper": "^1.0",
diff --git a/code/config/app.php b/code/config/app.php
index c3a318d..6c56163 100644
--- a/code/config/app.php
+++ b/code/config/app.php
@@ -26,6 +26,7 @@ return [
'domain_bind' => [
'*' => 'home',
'adapi' => 'admin',
+ 'api' => 'api',
],
// 禁止URL访问的应用列表（自动多应用模式有效）
diff --git a/code/config/console.php b/code/config/console.php
index d64ab68..98da0ec 100644
--- a/code/config/console.php
+++ b/code/config/console.php
@@ -2,10 +2,14 @@
// +----------------------------------------------------------------------
// | 控制台配置
// +----------------------------------------------------------------------
+
+use ddl\DDLManage;
+
+DDLManage::load('MicroServer');
+
return [
- // 指令定义
'commands' => [
- 'task' => 'minserver\ServerConsole',
+ 'task' => 'microserver\ServerConsole',
'db:manage' => 'db\DatabaseManage',
],
];
diff --git a/code/config/throttle.php b/code/config/throttle.php
new file mode 100644
index 0000000..575662e
--- /dev/null
+++ b/code/config/throttle.php
@@ -0,0 +1,33 @@
+ 'throttle_',
+ // 缓存的键，true 表示使用来源ip
+ 'key' => true,
+ // 要被限制的请求类型, eg: GET POST PUT DELETE HEAD 等
+ 'visit_method' => ['GET', 'HEAD'],
+ // 设置访问频率，例如 '10/m' 指的是允许每分钟请求10次;'10/60'指允许每60秒请求10次。值 null 表示不限制， eg: null 10/m 20/h 300/d 200/300
+ 'visit_rate' => '100/m',
+ /*
+ * 设置节流算法，组件提供了四种算法：
+ * - CounterFixed ：计数固定窗口
+ * - CounterSlider: 滑动窗口
+ * - TokenBucket : 令牌桶算法
+ * - LeakyBucket : 漏桶限流算法
+ */
+ 'driver_name' => CounterFixed::class,
+ // 响应体中设置速率限制的头部信息
+ 'visit_enable_show_rate_limit' => true,
+ // 访问受限时返回的响应
+ 'visit_fail_response' => function (Throttle $throttle, Request $request, int $wait_seconds) {
+ return Response::create('Too many requests, try again after ' . $wait_seconds . ' seconds.')->code(429);
+ },
+];
diff --git a/code/extend/db/DatabaseManage.php b/code/extend/db/DatabaseManage.php
new file mode 100644
index 0000000..89dc728
--- /dev/null
+++ b/code/extend/db/DatabaseManage.php
@@ -0,0 +1,114 @@
+setName('db:manage')
+ ->addArgument('action', \think\console\input\Argument::OPTIONAL, 'The action to perform: backup or init or seed')
+ ->setDescription('Manage the database: backup or initialize');
+ }
+
+ protected function execute(Input $input, Output $Output)
+ {
+ $strAction = $input->getArgument('action');
+
+ if ($strAction === 'backup') {
+ $this->backupDatabase($Output);
+ } elseif ($strAction === 'init') {
+ $this->initializeDatabase($Output);
+ } elseif ($strAction === 'seed') {
+ $this->seed($Output);
+ } else {
+ $Output->writeln("Invalid action. Use 'backup' or 'init'. or 'seed'");
+ }
+ }
+
+ private function seed(Output $Output)
+ {
+ $SeedManager = new SeedManager;
+ $Output->writeln("开始数据播种" . PHP_EOL);
+ $SeedManager->handle();
+ $Output->writeln("数据播种结束");
+ }
+
+ private function backupDatabase(Output $Output)
+ {
+ $arrConfig = config('database.connections.mysql');
+ $strDb = $arrConfig['database'];
+ $strUserName = $arrConfig['username'];
+ $strPwd = $arrConfig['password'];
+ $strHost = $arrConfig['hostname'];
+ $strBackupFile = app()->getRootPath() . 'database/schema/mysql-schema.dump';
+
+ $strTempFile = tempnam(sys_get_temp_dir(), 'mysql-schema-') . '.dump';
+
+ $strCmd = "mysqldump -h{$strHost} -u{$strUserName} -p{$strPwd} --no-data {$strDb} > {$strTempFile}";
+
+ system($strCmd, $intResultCode);
+
+ if ($intResultCode === 0) {
+ if (file_exists($strBackupFile)) {
+ unlink($strBackupFile);
+ }
+
+ if (rename($strTempFile, $strBackupFile)) {
+ $Output->writeln("Database backup successful: {$strBackupFile}");
+ } else {
+ $Output->writeln("Failed to move backup file to destination.");
+ }
+ } else {
+ $Output->writeln("Database backup failed.");
+ unlink($strTempFile);
+ }
+ }
+
+ private function initializeDatabase(Output $Output)
+ {
+ $arrConfig = config('database.connections.mysql');
+ $strDb = $arrConfig['database'];
+ $strUserName = $arrConfig['username'];
+ $strPwd = $arrConfig['password'];
+ $strHost = $arrConfig['hostname'];
+ $strPort = $arrConfig['hostport'];
+ $strBackupFile = app()->getRootPath() . 'database/schema/mysql-schema.dump';
+
+ $checkTableQuery = "SHOW TABLES LIKE 'migrations'";
+ $boolMigrateExists = Db::query($checkTableQuery);
+
+ if (!empty($boolMigrateExists)) {
+ $Output->writeln("数据库已存在，跳过初始化导入");
+ return;
+ }
+
+ $strTempFile = tempnam(sys_get_temp_dir(), 'mysql-schema-') . '.dump';
+
+ if (!copy($strBackupFile, $strTempFile)) {
+ $Output->writeln("创建临时导入文件失败.");
+ return;
+ }
+
+ $strCmd = "mysql -h{$strHost} -P{$strPort} -u{$strUserName} -p{$strPwd} {$strDb} < {$strTempFile}";
+
+ system($strCmd, $intResultCode);
+
+ if ($intResultCode === 0) {
+ $Output->writeln("成功导入初始化文件");
+ } else {
+ $Output->writeln("初始化文件导入失败");
+ }
+
+ unlink($strTempFile);
+ }
+}
diff --git a/code/extend/ddl/DDLManage.php b/code/extend/ddl/DDLManage.php
index 4326d1e..80a453b 100644
--- a/code/extend/ddl/DDLManage.php
+++ b/code/extend/ddl/DDLManage.php
@@ -11,7 +11,6 @@ class DDLManage
private static $arrModuleFiles = [
'MicroServer' => 'extend/ddl/MicroCore.ddl',
'MongoBase' => 'extend/ddl/MongoBase.ddl',
- 'DatabaseManage' => 'extend/ddl/DatabaseManage.ddl',
];
/**
diff --git a/code/extend/ddl/DatabaseManage.ddl b/code/extend/ddl/DatabaseManage.ddl
deleted file mode 100644
index e00387a..0000000
Binary files a/code/extend/ddl/DatabaseManage.ddl and /dev/null differ
diff --git a/code/extend/ddl/MicroCore.ddl b/code/extend/ddl/MicroCore.ddl
index 6648f17..4a8fe8f 100644
Binary files a/code/extend/ddl/MicroCore.ddl and b/code/extend/ddl/MicroCore.ddl differ
diff --git a/code/extend/ddl/MongoBase.ddl b/code/extend/ddl/MongoBase.ddl
index aeff78d..e06fd09 100644
Binary files a/code/extend/ddl/MongoBase.ddl and b/code/extend/ddl/MongoBase.ddl differ
diff --git a/code/extend/storage/StorageCore.php b/code/extend/storage/StorageCore.php
index 62a5ae2..bb2d579 100644
--- a/code/extend/storage/StorageCore.php
+++ b/code/extend/storage/StorageCore.php
@@ -56,6 +56,10 @@ class StorageCore
return $this->Filesystem->write($strKey, $strContent);
}
+ public function writeStream(string $strPath, $Resource)
+ {
+ $this->Filesystem->writeStream($strPath, $Resource);
+ }
public function write($strKey, $strContent, array $arrConfig = [])
{
diff --git a/code/extend/tools/pinyin-tool b/code/extend/tools/pinyin-tool
new file mode 100644
index 0000000..fc3f78e
Binary files /dev/null and b/code/extend/tools/pinyin-tool differ
diff --git a/code/fenxixq.php b/code/fenxixq.php
deleted file mode 100644
index 78f2e15..0000000
--- a/code/fenxixq.php
+++ /dev/null
@@ -1,21 +0,0 @@
-find('meta[property]')->each(function ($item) use (&$result) {
- $property = str_replace(':', '_', $item->attr('property')); // 冒号替换为下划线
- $result[$property] = $item->attr('content');
-});
-
-// 输出结果
-print_r($result);
\ No newline at end of file
- 
-
- 
-
- 
-
- 
-
- 
-
- 
-
-