admin/SEONexus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

123

Browse Source
This commit is contained in:
Default
2025-03-27 23:10:03 +08:00
parent 81b573c296
commit dd0d52f693
45 changed files with 2502 additions and 1279 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
code/app/admin/config/router.php
View File

@@ -2,6 +2,8 @@
declare(strict_types=1);
use app\admin\controller\Domain;
use app\admin\controller\Novel;
use app\admin\middleware\AdminAuth;
use think\facade\Route;
@@ -24,37 +26,19 @@ Route::group("/system", function () {
Route::get("/cjpz/list", "SystemConfig/getCaiJiPeiZhiList")->name("SystemConfig@getCaiJiPeiZhiList");
Route::post("/cjpz/save", "SystemConfig/saveCaiJiPeiZhi")->name("SystemConfig@saveCaiJiPeiZhi");
Route::get("/zydd/list", "SystemConfig/getZiYuanDuanDianList")->name("SystemConfig@getZiYuanDuanDianList");
Route::post("/zydd/save", "SystemConfig/saveZiYuanDuanDian")->name("SystemConfig@saveZiYuanDuanDian");
Route::get("/plan/list", "SystemConfig/getPlanTaskList")->name("SystemConfig@getPlanTaskList");
Route::post("/plan/save", "SystemConfig/savePlanTask")->name("SystemConfig@savePlanTask");
Route::get("/keyword/list", "SeoKeywords/getSeoKeywordList")->name("SeoKeywords@getSeoKeywordList");
Route::post("/keyword/save", "SeoKeywords/saveSeoKeyword")->name("SeoKeywords@saveSeoKeyword");
Route::post("/keyword/del", "SeoKeywords/delSeoKeyword")->name("SeoKeywords@delSeoKeyword");
})->middleware(AdminAuth::class);
# 广告
Route::group("/guanggao", function () {
# 主
Route::get("/main/group/list", "GuangGao/getGuangGaoFenZuList")->name("GuangGao@getGuangGaoFenZuList");
Route::post("/main/group/save", "GuangGao/saveGuangGaoFenZu")->name("GuangGao@saveGuangGaoFenZu");
Route::post("/main/group/del", "GuangGao/delGuangGaoFenZu")->name("GuangGao@delGuangGaoFenZu");
Route::get("/main/list", "GuangGao/getGuangGaoList")->name("GuangGao@getGuangGaoList");
Route::post("/main/save", "GuangGao/saveGuangGao")->name("GuangGao@saveGuangGao");
Route::post("/main/del", "GuangGao/delGuangGao")->name("GuangGao@delGuangGao");
# 广告配置
Route::get("/ggpz/list", "GuangGao/getGuangGaoPeiZhiList")->name("GuangGao@getGuangGaoPeiZhiList");
Route::post("/ggpz/save", "GuangGao/saveGuangGaoPeiZhi")->name("GuangGao@saveGuangGaoPeiZhi");
})->middleware(AdminAuth::class);
# 小说
Route::group("/xiaoshuo", function () {
Route::get("/main/list", "XiaoShuo/getXiaoShuoList")->name("XiaoShuo@getXiaoShuoList");
Route::get("/zhangjie/list", "XiaoShuo/getXiaoShuoZhangJieList")->name("XiaoShuo@getXiaoShuoZhangJieList");
Route::get("/fenlei/list", "XiaoShuo/getXiaoShuoFenLeiList")->name("XiaoShuo@getXiaoShuoFenLeiList");
Route::group("/novel", function () {
Route::get("/list", [Novel::class, "getNovelList"])->name("Domain@getNovelList");
Route::post("/level/set", [Novel::class, "updateNovelLevel"])->name("Novel@updateNovelLevel");
Route::get("/category/list", [Novel::class, "getCategory"])->name("Domain@getCategory");
})->middleware(AdminAuth::class);
# 站点
Route::group("/site", function () {
Route::get("/domain/list", [Domain::class, "getDomainList"])->name("Domain@getDomainList");
Route::post("/domain/upload", [Domain::class, "uploadDomain"])->name("Domain@uploadDomain");
})->middleware(AdminAuth::class);

118
code/app/admin/controller/Domain.php Normal file
View File

@@ -0,0 +1,118 @@
<?php
declare(strict_types=1);
namespace app\admin\controller;
use app\admin\BaseController;
use app\model\AdminUserModel;
use app\model\DomainModel;
use app\Request;
use PhpOffice\PhpSpreadsheet\IOFactory;
use think\Response;
class Domain extends BaseController
{
/**
* upload site domain info file
*
* @param Request $Request
* @param AdminUserModel|null $AdminUserModel
* @return Response
*/
public function uploadDomain(Request $Request, ?AdminUserModel $AdminUserModel)
{
$File = $Request->file('excel_file');
if (!$File) {
return $this->error("9999", '请上传 Excel 文件');
}
try {
$arrAllowedExt = ['xls', 'xlsx'];
$strExt = $File->getOriginalExtension();
if (!in_array($strExt, $arrAllowedExt)) {
return $this->error("9999", '仅支持 xls 或 xlsx 文件');
}
$FilePath = $File->getPathname();
$Spreadsheet = IOFactory::load($FilePath);
$Sheet = $Spreadsheet->getActiveSheet();
$arrData = $Sheet->toArray();
array_shift($arrData);
foreach ($arrData as $arrRows) {
$arrDomain = [
'd_domain' => $arrRows[0],
't_id' => $arrRows[1],
'd_title' => $arrRows[2],
'd_keywords' => $arrRows[3],
'd_description' => $arrRows[4],
'd_statis' => $arrRows[5],
'd_logo_type' => (int)$arrRows[6],
'd_text_log' => $arrRows[7],
'd_img_log' => $arrRows[8],
];
$DomainModel = DomainModel::where('d_domain', $arrDomain['d_domain'])->findOrEmpty();
$DomainModel->fill($arrDomain);
$DomainModel->save();
}
return $this->success();
} catch (\Exception $e) {
return $this->error("9999", '导入失败: ' . $e->getMessage());
}
}
/**
* 列表
*
* @param Request $Request
* @param AdminUserModel|null $AdminUserModel
* @return Response
*/
public function getDomainList(Request $Request, ?AdminUserModel $AdminUserModel)
{
$arrData = [
"page" => $Request->get('page', 1),
"limit" => $Request->get('limit', 10),
"key" => $Request->get('key'),
];
$arrRule = [
'page' => 'require|number',
'limit' => 'require|number',
];
$this->validate($arrData, $arrRule);
$DomainModel = DomainModel::alias('d');
if (!empty($arrData['key'])) {
$strKey = $arrData['key'];
$DomainModel->where(function ($DomainModel) use ($strKey) {
$DomainModel->whereOr([
['d.d_domain', 'like', "%" . $strKey . "%"],
['d.d_id', 'like', "%" . $strKey . "%"],
]);
});
}
$Paginate = $DomainModel->paginate([
'list_rows' => $arrData['limit'],
'page' => $arrData['page'],
]);
$arrResult = [
'items' => $Paginate->items(),
'total' => $Paginate->total(),
'current_page' => $Paginate->currentPage(),
'total_pages' => $Paginate->lastPage(),
];
return $this->success($arrResult);
}
}

115
code/app/admin/controller/Novel.php Normal file
View File

@@ -0,0 +1,115 @@
<?php
declare(strict_types=1);
namespace app\admin\controller;
use app\admin\BaseController;
use app\model\AdminUserModel;
use app\model\CategoryModel;
use app\model\FenLeiModel;
use app\model\GuangGaoFenZuModel;
use app\model\GuangGaoModel;
use app\model\GuangGaoPeiZhiModel;
use app\model\NovelModel;
use app\model\NvYouModel;
use app\model\ShiPinModel;
use app\Request;
use think\Response;
class Novel extends BaseController
{
/**
* 根据关键词修改小说评级
*
* @param Request $Request
* @param AdminUserModel|null $AdminUserModel
* @return Response
*/
public function updateNovelLevel(Request $Request, ?AdminUserModel $AdminUserModel)
{
$arrData = [
"keywords" => $Request->post('keywords'),
"n_level" => (int)$Request->post('n_level'),
];
$arrRule = [
'keywords' => 'require',
'n_level' => 'require',
];
$this->validate($arrData, $arrRule);
$arrKeywords = explode(',', $arrData['keywords']);
$NovelModel = new NovelModel;
$NovelModel->updateByKeywords($arrKeywords, $arrData['n_level']);
return $this->success();
}
/**
* 小说分页
*
* @param Request $Request
* @param AdminUserModel|null $AdminUserModel
* @return Response
*/
public function getNovelList(Request $Request, ?AdminUserModel $AdminUserModel)
{
$intPage = $Request->param('page', 1, 'intval');
$intLimit = $Request->param('limit', 10, 'intval');
try {
$NovelModel = new NovelModel;
$Col = $NovelModel->getCol();
// 查询条件（可选）
$arrFilter = [];
// 示例：如果需要筛选，例如 status=1
// $arrFilter['status'] = 1;
$arrOptions = [
'sort' => ['_id' => -1],
'skip' => ($intPage - 1) * $intLimit,
'limit' => $intLimit,
];
$Cursor = $Col->find($arrFilter, $arrOptions);
$arrData = iterator_to_array($Cursor);
$intTotal = $Col->countDocuments($arrFilter);
$arrResult = [
'code' => 200,
'msg' => '查询成功',
'data' => $arrData,
'total' => $intTotal,
'page' => $intPage,
'limit' => $intLimit,
'pages' => ceil($intTotal / $intLimit)
];
return $this->success($arrResult);
} catch (\Exception $e) {
return $this->error("9999", $e->getMessage());
}
}
/**
* get category data
*
* @param Request $Request
* @param AdminUserModel|null $AdminUserModel
* @return void
*/
public function getCategory(Request $Request, ?AdminUserModel $AdminUserModel)
{
$arrResult = [
'all' => CategoryModel::getCategory(),
'man' => CategoryModel::getManCategory(),
'women' => CategoryModel::getWomenCategory(),
];
return $this->success($arrResult);
}
}

98
code/app/api/BaseController.php Normal file
View File

@@ -0,0 +1,98 @@
<?php
declare(strict_types=1);
namespace app\api;
use ddl\DDLManage;
use think\App;
use think\exception\ValidateException;
use think\facade\View;
use think\Validate;
/**
* 控制器基础类
*/
abstract class BaseController
{
/**
* Request实例
* @var \think\Request
*/
protected $request;
/**
* 应用实例
* @var \think\App
*/
protected $app;
/**
* 是否批量验证
* @var bool
*/
protected $batchValidate = false;
/**
* 控制器中间件
* @var array
*/
protected $middleware = [];
/**
* 构造方法
* @access public
* @param App $app 应用对象
*/
public function __construct(App $app)
{
$this->app = $app;
$this->request = $this->app->request;
// 控制器初始化
$this->initialize();
// View::assign('')
}
// 初始化
protected function initialize() {}
/**
* 验证数据
* @access protected
* @param array $data 数据
* @param string|array $validate 验证器名或者验证规则数组
* @param array $message 提示信息
* @param bool $batch 是否批量验证
* @return array|string|true
* @throws ValidateException
*/
protected function validate(array $data, string|array $validate, array $message = [], bool $batch = false)
{
if (is_array($validate)) {
$v = new Validate();
$v->rule($validate);
} else {
if (strpos($validate, '.')) {
// 支持场景
[$validate, $scene] = explode('.', $validate);
}
$class = false !== strpos($validate, '\\') ? $validate : $this->app->parseClass('validate', $validate);
$v = new $class();
if (!empty($scene)) {
$v->scene($scene);
}
}
$v->message($message);
// 是否批量验证
if ($batch || $this->batchValidate) {
$v->batch(true);
}
return $v->failException(true)->check($data);
}
}

2
code/app/api/common.php Normal file
View File

@@ -0,0 +1,2 @@
<?php
// 这是系统自动生成的公共文件

33
code/app/api/config/router.php Normal file
View File

@@ -0,0 +1,33 @@
<?php
declare(strict_types=1);
use app\api\controller\Novel;
use app\api\controller\Statis;
use think\facade\Route;
Route::post("/publish", [Statis::class, 'publish'])
// ->middleware(\think\middleware\Throttle::class, [
// 'visit_rate' => '1/m',
// 'key' => '__CONTROLLER__/__ACTION__/__IP__',
// ])
->name('Statis@publish');
Route::get("/search", [Novel::class, 'search'])
// ->middleware(\think\middleware\Throttle::class, [
// 'visit_rate' => '1/m',
// 'key' => '__CONTROLLER__/__ACTION__/__IP__',
// ])
->name('Novel@search');;
Route::get("/test", [Statis::class, 'test'])
// ->middleware(\think\middleware\Throttle::class, [
// 'visit_rate' => '1/m',
// 'key' => '__CONTROLLER__/__ACTION__/__IP__',
// ])
->name('Statis@test');

12
code/app/api/config/throttle.php Normal file
View File

@@ -0,0 +1,12 @@
<?php
use think\middleware\Throttle;
use think\Response;
return [
'prefix' => 'api:throttle_',
'visit_method' => ["GET", "POST", "PUT", "DELETE", "HEAD"],
'visit_fail_response' => function (Throttle $throttle, $request, $wait_seconds) {
return Response::create('抱歉，请求过于频繁, 请 ' . $wait_seconds . ' 秒后重新操作 。')->code(429);
},
];

14
code/app/api/controller/Novel.php Normal file
View File

@@ -0,0 +1,14 @@
<?php
declare (strict_types = 1);
namespace app\api\controller;
use app\api\BaseController;
class Novel extends BaseController
{
public function search()
{
return '您好！这是一个[api]示例应用';
}
}

66
code/app/api/controller/Statis.php Normal file
View File

@@ -0,0 +1,66 @@
<?php
declare(strict_types=1);
namespace app\api\controller;
use app\api\BaseController;
use app\model\CategoryModel;
use app\model\NovelClicksModel;
use app\model\NovelModel;
use DateTime;
use db\mongo\MongoBase;
use ddl\DDLManage;
use Exception;
use think\Request;
use think\Response;
class Statis extends BaseController
{
/**
* novel statis
*
* @param Request $Request
* @return Response
*/
public function publish(Request $Request): Response
{
$intNId = $Request->post('n_id/d');
if ($intNId <= 0) {
return Response::create('非法请求！')->code(400);
}
$NovelClicksModel = new NovelClicksModel;
try {
// for ($intNId = 1; $intNId <= 20; $intNId++) {
// $NovelClicksModel->addStatis($intNId);
// }
$NovelClicksModel->addStatis($intNId);
} catch (\Exception $e) {
// throw $e;
}
return Response::create()->code(204);
}
/**
* test code
*
* @param Request $Request
* @return Response
*/
public function test(Request $Request): Response
{
// $arrCategory = CategoryModel::getCategory();
// print_r($arrCategory);
$NovelModel = new NovelModel;
$NovelModel->updateByKeywords(["夜", "宝"], 9);
exit;
return Response::create();
}
}

5
code/app/api/event.php Normal file
View File

@@ -0,0 +1,5 @@
<?php
// 这是系统自动生成的event定义文件
return [
];

4
code/app/api/middleware.php Normal file
View File

@@ -0,0 +1,4 @@
<?php
// 这是系统自动生成的middleware定义文件
return [
];

45
code/app/common.php
View File

@@ -762,4 +762,49 @@ namespace {
return false;
}
/**
* 从 URL、URI 或路径中提取文件名（不带后缀）
* @param string $path 输入的路径，比如 "/shu/1/4313672.html" 或 "https://example.com/files/test.pdf"
* @return string|null 返回文件名部分，如果提取失败返回 null
*/
function supperExtractFilename($path)
{
if (empty($path) || !is_string($path)) {
return null;
}
$slashPos = strrpos($path, '/');
if ($slashPos === false) {
$slashPos = -1;
}
$dotPos = strrpos($path, '.');
if ($dotPos === false || $dotPos <= $slashPos) {
return substr($path, $slashPos + 1);
}
$start = $slashPos + 1;
$length = $dotPos - $start;
return substr($path, $start, $length);
}
/**
* Undocumented function
*
* @param string $strZh
* @return string
*/
function zhToPinYin(string $strZh): string
{
$strPinYinToolsPath = root_path() . '/extend/tools/pinyin-tool';
$strPinYin = shell_exec(sprintf("%s %s", $strPinYinToolsPath, escapeshellcmd($strZh)));
$strPinYin = trim((string) $strPinYin);
return $strPinYin;
}
}

12
code/app/home/BaseController.php
View File

@@ -1,5 +1,6 @@
<?php
declare (strict_types = 1);
declare(strict_types=1);
namespace app\home;
@@ -54,8 +55,7 @@ abstract class BaseController
}
// 初始化
protected function initialize()
{}
protected function initialize() {}
/**
* 验证数据
@@ -94,4 +94,10 @@ abstract class BaseController
return $v->failException(true)->check($data);
}
public function rending($strTemplate = '')
{
View::assign("DomainModel", $this->request->DomainModel);
View::assign("TemplatesModel", $this->request->TemplatesModel);
return View::fetch($strTemplate);
}
}

109
code/app/home/controller/Index.php
View File

@@ -5,6 +5,7 @@ declare(strict_types=1);
namespace app\home\controller;
use app\home\BaseController;
use app\model\NovelModel;
use storage\StorageCore;
use think\facade\View;
use app\Request;
@@ -27,7 +28,7 @@ class Index extends BaseController
['id' => 6, 'name' => '小说3', 'author' => '作者3'],
['id' => 7, 'name' => '小说3', 'author' => '作者3'],
['id' => 8, 'name' => '小说3', 'author' => '作者3'],
];
// 模拟数据-最新 随机8个小说
@@ -118,7 +119,14 @@ class Index extends BaseController
// 获取路由附加的参数
$boolIsMobile = $Request->param('boolIsMobile');
$NovelModel = new NovelModel;
// 模拟数据-推荐等级为9的连载的 随机15个小说
$arrRecommendSerializationNovel = $NovelModel->getHighLevelNovelOnCache($Request->DomainModel->d_domain, 13);
// echo '<pre>';
// print_r($arrRecommendSerializationNovel);
// exit;
$arrRecommendSerializationNovel = [
['id' => 1, 'name' => '小说1', 'author' => '作者1'],
['id' => 2, 'name' => '小说2', 'author' => '作者2'],
@@ -133,27 +141,36 @@ class Index extends BaseController
['id' => 8, 'name' => '小说3', 'author' => '作者3'],
['id' => 8, 'name' => '小说3', 'author' => '作者3'],
['id' => 8, 'name' => '小说3', 'author' => '作者3'],
];
// 模拟数据-推荐等级为9的完结的 随机12个小说
$arrRecommendEndNovel = [
['id' => 1, 'name' => '小说1', 'author' => '作者1'],
['id' => 2, 'name' => '小说2', 'author' => '作者2'],
['id' => 3, 'name' => '小说3', 'author' => '作者3'],
['id' => 4, 'name' => '小说3', 'author' => '作者3'],
['id' => 5, 'name' => '小说3', 'author' => '作者3'],
['id' => 6, 'name' => '小说3', 'author' => '作者3'],
['id' => 7, 'name' => '小说3', 'author' => '作者3'],
['id' => 8, 'name' => '小说3', 'author' => '作者3'],
['id' => 9, 'name' => '小说3', 'author' => '作者3'],
['id' => 10, 'name' => '小说3', 'author' => '作者3'],
['id' => 11, 'name' => '小说3', 'author' => '作者3'],
['id' => 12, 'name' => '小说3', 'author' => '作者3'],
];
$arrRecommendEndNovel = $NovelModel->getFinishedNovelOnCache($Request->DomainModel->d_domain, 13);
// echo '<pre>';
// print_r($arrRecommendEndNovel);
// exit;
// $arrRecommendEndNovel = [
// ['id' => 1, 'name' => '小说1', 'author' => '作者1'],
// ['id' => 2, 'name' => '小说2', 'author' => '作者2'],
// ['id' => 3, 'name' => '小说3', 'author' => '作者3'],
// ['id' => 4, 'name' => '小说3', 'author' => '作者3'],
// ['id' => 5, 'name' => '小说3', 'author' => '作者3'],
// ['id' => 6, 'name' => '小说3', 'author' => '作者3'],
// ['id' => 7, 'name' => '小说3', 'author' => '作者3'],
// ['id' => 8, 'name' => '小说3', 'author' => '作者3'],
// ['id' => 9, 'name' => '小说3', 'author' => '作者3'],
// ['id' => 10, 'name' => '小说3', 'author' => '作者3'],
// ['id' => 11, 'name' => '小说3', 'author' => '作者3'],
// ['id' => 12, 'name' => '小说3', 'author' => '作者3'],
// ];
// 模拟数据-男生频道，最新的 随机5个小说
$arrBoyNewsNovel = $NovelModel->getSexNovelOnCache($Request->DomainModel->d_domain, 5, 1);
// echo '<pre>';
// print_r($arrBoyNewsNovel);
// exit;
$arrBoyNewsNovel = [
['id' => 1, 'name' => '小说1', 'author' => '作者1'],
['id' => 2, 'name' => '小说2', 'author' => '作者2'],
@@ -163,6 +180,10 @@ class Index extends BaseController
];
// 模拟数据-男生频道，周排行榜 随机10个小说
$arrBoyWeekRankNovel = $NovelModel->getRankNovelOnCache($Request->DomainModel->d_domain, 5, 1, 'weekly');
// echo '<pre>';
// print_r($arrBoyWeekRankNovel);
// exit;
$arrBoyWeekRankNovel = [
['id' => 1, 'name' => '小说1', 'author' => '作者1'],
['id' => 2, 'name' => '小说2', 'author' => '作者2'],
@@ -177,6 +198,10 @@ class Index extends BaseController
];
// 模拟数据-男生频道，月排行榜 随机10个小说
$arrBoyMoonRankNovel = $NovelModel->getRankNovelOnCache($Request->DomainModel->d_domain, 5, 1, 'monthly');
// echo '<pre>';
// print_r($arrBoyMoonRankNovel);
// exit;
$arrBoyMoonRankNovel = [
['id' => 1, 'name' => '小说1', 'author' => '作者1'],
['id' => 2, 'name' => '小说2', 'author' => '作者2'],
@@ -191,6 +216,10 @@ class Index extends BaseController
];
// 模拟数据-男生频道，总排行榜 随机10个小说
$arrBoyTotalRankNovel = $NovelModel->getRankNovelOnCache($Request->DomainModel->d_domain, 5, 1, 'total');
// echo '<pre>';
// print_r($arrBoyTotalRankNovel);
// exit;
$arrBoyTotalRankNovel = [
['id' => 1, 'name' => '小说1', 'author' => '作者1'],
['id' => 2, 'name' => '小说2', 'author' => '作者2'],
@@ -203,8 +232,12 @@ class Index extends BaseController
['id' => 9, 'name' => '小说3', 'author' => '作者3'],
['id' => 10, 'name' => '小说3', 'author' => '作者3'],
];
// 模拟数据-女生频道，最新的 随机5个小说
$arrGirlNewsNovel = $NovelModel->getSexNovelOnCache($Request->DomainModel->d_domain, 5, 2);
// echo '<pre>';
// print_r($arrGirlNewsNovel);
// exit;
$arrGirlNewsNovel = [
['id' => 1, 'name' => '小说1', 'author' => '作者1'],
['id' => 2, 'name' => '小说2', 'author' => '作者2'],
@@ -212,8 +245,12 @@ class Index extends BaseController
['id' => 4, 'name' => '小说3', 'author' => '作者3'],
['id' => 5, 'name' => '小说3', 'author' => '作者3'],
];
// 模拟数据-女生频道，周排行榜 随机10个小说
$arrGirlWeekRankNovel = $NovelModel->getRankNovelOnCache($Request->DomainModel->d_domain, 5, 2, 'weekly');
// echo '<pre>';
// print_r($arrGirlWeekRankNovel);
// exit;
$arrGirlWeekRankNovel = [
['id' => 1, 'name' => '小说1', 'author' => '作者1'],
['id' => 2, 'name' => '小说2', 'author' => '作者2'],
@@ -226,7 +263,12 @@ class Index extends BaseController
['id' => 9, 'name' => '小说3', 'author' => '作者3'],
['id' => 10, 'name' => '小说3', 'author' => '作者3'],
];
// 模拟数据-女生频道，月排行榜 随机10个小说
$arrGirlMoonRankNovel = $NovelModel->getRankNovelOnCache($Request->DomainModel->d_domain, 5, 2, 'monthly');
// echo '<pre>';
// print_r($arrGirlMoonRankNovel);
// exit;
$arrGirlMoonRankNovel = [
['id' => 1, 'name' => '小说1', 'author' => '作者1'],
['id' => 2, 'name' => '小说2', 'author' => '作者2'],
@@ -240,6 +282,10 @@ class Index extends BaseController
['id' => 10, 'name' => '小说3', 'author' => '作者3'],
];
// 模拟数据-女生频道，总排行榜 随机10个小说
$arrBGirlTotalRankNovel = $NovelModel->getRankNovelOnCache($Request->DomainModel->d_domain, 5, 2, 'total');
// echo '<pre>';
// print_r($arrBGirlTotalRankNovel);
// exit;
$arrBGirlTotalRankNovel = [
['id' => 1, 'name' => '小说1', 'author' => '作者1'],
['id' => 2, 'name' => '小说2', 'author' => '作者2'],
@@ -254,6 +300,10 @@ class Index extends BaseController
];
// 模拟数据-最近更新-随机50
$arrNewsUpdateNovel = $NovelModel->getSexNovelOnCache($Request->DomainModel->d_domain, 50);
// echo '<pre>';
// print_r($arrNewsUpdateNovel);
// exit;
$arrNewsUpdateNovel = [
['id' => 1, 'name' => '小说1', 'author' => '作者1'],
['id' => 2, 'name' => '小说2', 'author' => '作者2'],
@@ -300,25 +350,4 @@ class Index extends BaseController
}
}
public function test()
{
$strNovelKey = "novels/12/chapters/3001.txt";
$StorageCore = StorageCore::getInstance();
// $content = '我是一个大傻逼';
$content = file_get_contents('123.txt');
if ($StorageCore->has($strNovelKey)) {
// $MinIO->update($strNovelKey, $content); // 更新已有文件
} else {
$StorageCore->write($strNovelKey, $content); // 创建新文件
}
echo $StorageCore->read($strNovelKey);
return;
}
}

12
code/app/home/view/kuangyu/Pc/index.html
View File

@@ -52,7 +52,17 @@
</div>
<ul class="f2_list" mod="hot_tj">
{foreach $arrRecommendEndNovel as $key=>$Novel}
{include file="Pc/Public/endNovelList" /}
<li>
<a href="/pc/novel/334.html" title="{$Novel->og_novel_book_name}" target="_blank" class="book_cov">
<img class="lazyload_book_cover lazyload-img"
src="/static/img/default.jpg" data-src="{$Novel->n_cover_path}"
alt="{$Novel->og_novel_book_name}">
</a>
<div class="book_inf">
<h3><a href="/pc/novel/334.html" title="{$Novel->og_novel_book_name}" target="_blank">{$Novel->og_novel_book_name}</a></h3>
<span class="aut">{$Novel->og_novel_author}</span>
</div>
</li>
{/foreach}
</ul>

61
code/app/model/CategoryModel.php Normal file
View File

@@ -0,0 +1,61 @@
<?php
declare(strict_types=1);
namespace app\model;
use DateTime;
use think\facade\Cache;
use think\Model;
/**
* @mixin think\Model
*/
class CategoryModel extends MongoModel
{
protected $strCollection = 'category';
static $arrCategory = [
1 => '玄幻修真',
2 => '军史小说',
3 => '网游小说',
4 => '科幻小说',
5 => '重生穿越',
6 => '都市小说',
7 => '灵异小说',
8 => '言情小说',
9 => '其他小说',
];
static $arrMan = [
1 => '玄幻修真',
2 => '军史小说',
3 => '网游小说',
4 => '科幻小说',
];
static $arrWomen = [
5 => '重生穿越',
6 => '都市小说',
7 => '灵异小说',
8 => '言情小说',
9 => '其他小说',
];
static public function getCategory():array
{
return self::$arrCategory;
}
static public function getManCategory():array
{
return self::$arrMan;
}
static public function getWomenCategory():array
{
return self::$arrWomen;
}
}

17
code/app/model/ChapterModel.php Normal file
View File

@@ -0,0 +1,17 @@
<?php
declare(strict_types=1);
namespace app\model;
use DateTime;
use think\facade\Cache;
use think\Model;
/**
* @mixin think\Model
*/
class ChapterModel extends MongoModel
{
protected $strCollection = 'chapter';
}

71
code/app/model/MongoModel.php Normal file
View File

@@ -0,0 +1,71 @@
<?php
declare(strict_types=1);
namespace app\model;
use db\mongo\MongoBase;
use ddl\DDLManage;
use MongoDB\Collection;
use think\facade\Cache;
use think\Model;
/**
* @mixin think\Model
*/
abstract class MongoModel
{
/**
* collection name
*
* @var string
*/
protected $strCollection;
/**
* Undocumented variable
*
* @var MongoBase
*/
private $MongoBase;
/**
* Undocumented function
*/
public function __construct()
{
DDLManage::load('MongoBase');
$this->MongoBase = MongoBase::getInstance();
}
/**
* Undocumented function
*
* @return MongoBase
*/
public function getMongoBase(): MongoBase
{
return $this->MongoBase;
}
/**
* Undocumented function
*
* @return Collection
*/
public function getCol(): Collection
{
return $this->getMongoBase()->getDb()->selectCollection($this->getCollectionName());
}
/**
* Undocumented function
*
* @return string
*/
public function getCollectionName(): string
{
return $this->strCollection;
}
}

67
code/app/model/NovelClicksModel.php Normal file
View File

@@ -0,0 +1,67 @@
<?php
declare(strict_types=1);
namespace app\model;
use DateTime;
use think\facade\Cache;
use think\Model;
/**
* @mixin think\Model
*/
class NovelClicksModel extends MongoModel
{
protected $strCollection = 'novel_clicks';
/**
* Undocumented function
*
* @param integer $intNId
* @return void
*/
public function addStatis(int $intNId)
{
$NovelModel = new NovelModel;
$Novel = $NovelModel->getCol()->findOne([
"n_id" => $intNId,
]);
if (empty($Novel)) {
return false;
}
$Now = new DateTime();
$strDailyDate = $Now->format('Y-m-d');
$strWeeklyDate = $Now->modify('monday this week')->format('Y-m-d');
$MonthlyDate = $Now->format('Y-m-01');
$arrClickTypes = [
'daily' => $strDailyDate,
'weekly' => $strWeeklyDate,
'monthly' => $MonthlyDate,
'total' => null,
];
foreach ($arrClickTypes as $strType => $strDate) {
$this->getCol()->updateOne(
[
'n_id' => $intNId,
'nc_date' => $strDate,
'nc_type' => $strType,
],
[
'$inc' => ['nc_clicks' => 1],
'$set' => ['n_category' => $Novel->og_novel_category]
],
[
'upsert' => true
]
);
}
return true;
}
}

329
code/app/model/NovelModel.php Normal file
View File

@@ -0,0 +1,329 @@
<?php
declare(strict_types=1);
namespace app\model;
use DateTime;
use Exception;
use storage\StorageCore;
use think\facade\Cache;
use think\Model;
/**
* @mixin think\Model
*/
class NovelModel extends MongoModel
{
/**
* Undocumented variable
*
* @var string
*/
protected $strCollection = 'novel';
/**
* 根据关键词数组更新小说推荐等级
* @param array $arrKeywords 关键词数组，例如 ["斗罗", "遮天", "凡人"]
* @param int $intLevel 推荐等级1~9，例如 3
* @return array 更新结果
*/
public function updateByKeywords(array $arrKeywords, int $intLevel)
{
if (empty($arrKeywords)) {
return false;
}
try {
$arrConditions = array_map(function ($strKeyword) {
return ['og_title' => ['$regex' => trim($strKeyword), '$arrOptions' => 'i']];
}, $arrKeywords);
$UpdateResult = $this->getCol()->updateMany(
['$or' => $arrConditions],
['$set' => ['n_level' => $intLevel]]
);
$intUpdateNum = $UpdateResult->getModifiedCount();
} catch (Exception $e) {
return false;
}
return true;
}
/**
* Undocumented function
*
* @param string $strSign
* @param integer $intCount
* @param integer $intType
* @param integer $intLevel
* @return array
*/
public function getHighLevelNovelOnCache(string $strSign, int $intCount = 10, int $intType = 0): array
{
try {
$strKey = sprintf('%s:PC:HOME:FIRST', $strSign);
$intLifeTime = 24 * 3600;
$arrResult = Cache::get($strKey);
// if (empty($arrResult)) {
if (true) {
if ($intType == 1) {
$arrCategory = array_values(CategoryModel::getManCategory());
$arrFilter = [
'og_novel_category' => ['$in' => $arrCategory]
];
} elseif ($intType == 2) {
$arrCategory = array_values(CategoryModel::getWomenCategory());
$arrFilter = [
'og_novel_category' => ['$in' => $arrCategory]
];
} else {
$arrFilter = [];
}
$arrOptions = [
'sort' => ['n_level' => -1],
'limit' => $intCount * 3,
'projection' => [
'n_id' => 1,
'og_novel_book_name' => 1,
'og_novel_pin_yin' => 1,
'n_cover_path' => 1,
'n_novel_author' => 1,
'og_novel_category' => 1,
'og_description' => 1,
],
];
$Cursor = $this->getCol()->find($arrFilter, $arrOptions);
$arrResult = iterator_to_array($Cursor);
shuffle($arrResult);
$arrResult = array_slice($arrResult, 0, $intCount);
if (!empty($arrResult)) {
Cache::set($strKey, $arrResult, $intLifeTime);
}
}
return $arrResult;
} catch (\Exception $e) {
return [];
}
}
/**
* Undocumented function
*
* @param string $strSign
* @param integer $intCount
* @return array
*/
public function getFinishedNovelOnCache(string $strSign, int $intCount = 10): array
{
try {
$strKey = sprintf('%s:PC:HOME:FINISHED', $strSign);
$intLifeTime = 24 * 3600;
$arrResult = Cache::get($strKey);
// if (empty($arrResult)) {
if (true) {
$arrFilter = [
'og_novel_status' => "已完结",
];
$arrOptions = [
'sort' => ['og_novel_update_time' => -1],
'limit' => $intCount * 3,
'projection' => [
'n_id' => 1,
'og_novel_book_name' => 1,
'og_novel_pin_yin' => 1,
'n_cover_path' => 1,
'og_novel_author' => 1,
'og_novel_category' => 1,
'og_description' => 1,
],
];
$Cursor = $this->getCol()->find($arrFilter, $arrOptions);
$arrResult = iterator_to_array($Cursor);
shuffle($arrResult);
$arrResult = array_slice($arrResult, 0, $intCount);
if (!empty($arrResult)) {
Cache::set($strKey, $arrResult, $intLifeTime);
}
}
return $arrResult;
} catch (\Exception $e) {
return [];
}
}
/**
* Undocumented function
*
* @param string $strSign
* @param integer $intCount
* @param integer $intType
* @param integer $intLevel
* @return array
*/
public function getSexNovelOnCache(string $strSign, int $intCount = 10, int $intType = 0): array
{
try {
$strKey = sprintf('%s:PC:HOME:SIX:%s', $strSign, $intType);
$intLifeTime = 24 * 3600;
$arrResult = Cache::get($strKey);
// if (empty($arrResult)) {
if (true) {
if ($intType == 1) {
$arrCategory = array_values(CategoryModel::getManCategory());
$arrFilter = [
'og_novel_category' => ['$in' => $arrCategory]
];
} elseif ($intType == 2) {
$arrCategory = array_values(CategoryModel::getWomenCategory());
$arrFilter = [
'og_novel_category' => ['$in' => $arrCategory]
];
} else {
$arrFilter = [];
}
$arrOptions = [
'sort' => ['og_novel_update_time' => -1],
'limit' => $intCount * 3,
'projection' => [
'n_id' => 1,
'og_novel_book_name' => 1,
'og_novel_pin_yin' => 1,
'n_cover_path' => 1,
'n_novel_author' => 1,
'og_novel_category' => 1,
'og_description' => 1,
],
];
$Cursor = $this->getCol()->find($arrFilter, $arrOptions);
$arrResult = iterator_to_array($Cursor);
shuffle($arrResult);
$arrResult = array_slice($arrResult, 0, $intCount);
if (!empty($arrResult)) {
Cache::set($strKey, $arrResult, $intLifeTime);
}
}
return $arrResult;
} catch (\Exception $e) {
return [];
}
}
/**
* Undocumented function
*
* @param string $strSign
* @param integer $intCount
* @param integer $intType
* @param string strDateType
* * daily 天
* * weekly 周
* * monthly 月
* * total 总
* @return array
*/
public function getRankNovelOnCache(string $strSign, int $intCount = 10, int $intType = 0, $strDateType = 'daily'): array
{
try {
$strKey = sprintf('%s:PC:HOME:SIX:%s:TYPE:%s', $strSign, $intType, $strDateType);
$intLifeTime = 24 * 3600;
$arrResult = Cache::get($strKey);
// if (empty($arrResult)) {
if (true) {
$arrFilter = [
'nc_type' => $strDateType,
];
if ($intType == 1) {
$arrCategory = array_values(CategoryModel::getManCategory());
$arrFilter['n_category'] = ['$in' => $arrCategory];
} elseif ($intType == 2) {
$arrCategory = array_values(CategoryModel::getWomenCategory());
$arrFilter['n_category'] = ['$in' => $arrCategory];
}
$arrOptions = [
'sort' => ['nc_clicks' => -1],
'limit' => $intCount * 3,
'projection' => [
'n_id' => 1,
],
];
$NovelClicksModel = new NovelClicksModel;
$Cursor = $NovelClicksModel->getCol()->find($arrFilter, $arrOptions);
$arrNId = [];
foreach ($Cursor as $Doc) {
$arrNId[] = $Doc['n_id'];
}
$Cursor = $this->getCol()->find(
['n_id' => ['$in' => $arrNId]],
[
'projection' => [
'n_id' => 1,
'og_novel_book_name' => 1,
'og_novel_pin_yin' => 1,
'n_cover_path' => 1,
'n_novel_author' => 1,
'og_novel_category' => 1,
'og_description' => 1,
],
]
);
$arrResult = iterator_to_array($Cursor);
shuffle($arrResult);
$arrResult = array_slice($arrResult, 0, $intCount);
if (!empty($arrResult)) {
Cache::set($strKey, $arrResult, $intLifeTime);
}
}
return $arrResult;
} catch (\Exception $e) {
return [];
}
}
}

48
code/app/task/collection/PullDataColBase.php
View File

@@ -1,48 +0,0 @@
<?php
declare(strict_types=1);
namespace app\task\collection;
use app\model\CaiJiPeiZhiModel;
use app\model\SystemConfigModel;
use minserver\QueueManage;
use GuzzleHttp\Client;
/**
* @mixin think\Model
*/
class PullDataColBase
{
/**
* Undocumented variable
*
* @var \GuzzleHttp\Client
*/
protected $Client = NULL;
/**
* Undocumented variable
*
* @var \minserver\QueueManage
*/
protected $QueueManage = NULL;
protected $arrCaiJiPeiZhi = [];
public function __construct()
{
$this->Client = new Client();
$this->QueueManage = new QueueManage();
}
public function getAilisiDomain()
{
if (!key_exists('AILISI_DOMAIN', $this->arrCaiJiPeiZhi)) {
$this->arrCaiJiPeiZhi['AILISI_DOMAIN'] = CaiJiPeiZhiModel::getValByCode('AILISI_DOMAIN');
}
return $this->arrCaiJiPeiZhi['AILISI_DOMAIN'];
}
}

611
code/app/task/collection/zw630/Zw630Col.php
View File

@@ -1,611 +0,0 @@
<?php
declare(strict_types=1);
namespace app\task\collection\zw630;
use app\model\XiaoShuoXiangQingModel;
use app\task\collection\PullDataColBase;
use db\mongo\MongoBase;
use GuzzleHttp\Cookie\CookieJar;
use processor\ImageProcessor;
use think\facade\Cache;
use QL\QueryList;
use minserver\QueueManage;
use GuzzleHttp\Client;
/**
* @mixin think\Model
*/
class Zw630Col extends PullDataColBase
{
private $strDomain = 'https://www.630zw.org';
/**
* Undocumented variable
*
* @var MongoBase
*/
private $MongoBase;
public function __construct()
{
$this->MongoBase = MongoBase::getInstance();
$this->Client = new Client([
'base_uri' => $this->strDomain,
'http_errors' => false,
'version' => 2.0, // 强制使用 HTTP/2
'timeout' => 160,
'connect_timeout' => 30, // 连接超时 10 秒
'curl' => [
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_2_0, // 强制 HTTP/2
CURLOPT_TCP_KEEPALIVE => 1, // 启用 TCP Keep-Alive
CURLOPT_SSL_VERIFYPEER => true, // 验证 SSL
],
'headers' => [
'User-Agent' => 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Mobile Safari/537.36 Edg/134.0.0.0',
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'Accept-Language' => 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
'Referer' => $this->strDomain,
'sec-ch-ua-mobile' => '?1',
'sec-ch-ua-platform' => '"Android"',
'sec-fetch-dest' => 'document',
'sec-fetch-mode' => 'navigate',
'sec-fetch-site' => 'same-origin',
'sec-fetch-user' => '?1',
'upgrade-insecure-requests' => '1',
],
]);
$this->QueueManage = new QueueManage();
}
public function pull630ZwNovel()
{
// 全局扫描任务
$QueueManage = $this->QueueManage;
// for ($intNovelId = 1; $intNovelId <= 531208; $intNovelId++) {
// for ($intNovelId = 1; $intNovelId <= 10000; $intNovelId++) {
for ($intNovelId = 1; $intNovelId <= 1; $intNovelId++) {
$arrTask = [
'callback' => [self::class, 'getNovelInfo'],
'data' => [
'n_id' => $intNovelId,
'page' => 0,
]
];
// print_r($arrTask);
$QueueManage->set($arrTask);
}
}
public function getNovelInfo($arrData)
{
// content="mobile" 有这个的是手机，否则是PC
print_r($arrData);
if ($arrData['page'] == 0) {
$strUri = sprintf('/shu/%s.html', $arrData['n_id']);
} else {
$strUri = sprintf('/shu/%s_%s.html', $arrData['n_id']);
}
$ResponseBase = $this->Client->get($strUri);
$strNovelContent = $ResponseBase->getBody()->getContents();
$NovelQuery = QueryList::html($strNovelContent);
$arrNovel = $this->getNovelHead($NovelQuery, $arrData['n_id'], $arrData['page']);
if (empty($arrNovel)) {
return false;
}
print_r($arrNovel);
// $strResult = (string)$Response->getBody();
}
private function getNovelHead(QueryList $QueryList, $intSourceNovelId, $intPage = 0)
{
$arrResult = [];
$QueryList->find('meta[property]')->each(function ($Item) use (&$arrResult) {
$property = str_replace(':', '_', $Item->attr('property'));
$arrResult[$property] = $Item->attr('content');
});
if (empty($arrResult)) {
return false;
}
// print_r($arrResult);
$arrResult['novel_source_id'] = 'zw630-' . $intSourceNovelId;
try {
$Col = $this->MongoBase->getDb()->selectCollection('novel');
$arrExisting = $Col->findOne(['novel_source_id' => $arrResult['novel_source_id']]);
// use novel
// db.novel.createIndex({ "novel_source_id": 1 }, { unique: true })
if ($arrExisting) {
$arrExisting = (array) $arrExisting; // MongoDB\BSON\Document 转为数组
$arrResult = array_merge($arrExisting, $arrResult); // 合并，保留原有字段
$arrResult['updated_at'] = new \MongoDB\BSON\UTCDateTime(); // 添加更新时间
$Col->updateOne(
['novel_source_id' => $arrResult['novel_source_id']],
['$set' => $arrResult],
// ['upsert' => false]
);
} else {
$arrResult['novel_id'] = $this->MongoBase->getNextSequence('novel');
$this->MongoBase->insertOne('novel', $arrResult);
}
return $arrResult;
} catch (\MongoDB\Exception\Exception $e) {
throw new \Exception("Upsert failed: " . $e->getMessage());
}
}
public function getNovelInfo1($arrData)
{
//var_dump('getNovelFengMian');
//var_dump($arrData);
$Client = $this->Client;
$QueueManage = $this->QueueManage;
// 获取 Redis 实例
$redis = Cache::store('redis')->handler();
$intXiaoShuoSourceId = $arrData['xiaoshuo_id'];
$intXiaoShuoSourceSeoId = $arrData['xiaoshuo_seo_id'];
$redisKey = "task:novel_source_id:" . $intXiaoShuoSourceId;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($arrData['xiaoshuo_url']);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$strSiteUrl = $this->getAilisiDomain();
$strCover = $QueryList->find('#fmimg img')->eq(0)->attr("src");
if (strpos($strCover, 'http') === false) {
$strCover = $strSiteUrl . $strCover;
}
$strCoverUri = $strCover;
try {
$strCoverUri = "";
$strCoverUri = ImageProcessor::getInstance()->downloadAndEncryptImage($strCover, 'xs');
$arrCover = [
'code' => 'AI_LI_SI_COVER',
// 'uri' => getUriByUrl($strCover),
'uri' => $strCoverUri,
];
$arrXiaoShuoInfo = [
'xsxq_feng_mian' => json_encode($arrCover),
];
$XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
} catch (\Throwable $t) {
$strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
echo $strErr;
}
} catch (\Throwable $T) {
throw $T;
}
}
// public function getNovelInfo($arrData)
// {
// $QueueManage = $this->QueueManage;
// $Client = $this->Client;
// $strSiteUrl = $this->getBiquxsDomain();
// $strUrl = 'http://www.biquxs.com/book/' . $arrData['xs_source_id'] ; //'http://www.biquxs.com/book/20053';
// $intXiaoShuoSourceId = $arrData['xs_source_id']; //20053;
// $intXiaoShuoSourceSeoId = $arrData['xs_source_seo_id']; //1 ;
// try {
// $arrCookie = $this->getAilisiCookie();
// $strHost = getDomainFromUrl($strUrl);
// $CookieJar = CookieJar::fromArray($arrCookie, $strHost);
// $proxy = getRandomProxies();
// $userAgent = getRandomUserAgent();
// $Response = $Client->request('GET', $strUrl, [
// 'headers' => [
// 'User-Agent' => $userAgent,
// 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
// 'Accept-Language' => 'en-US,en;q=0.5',
// ],
// 'cookies' => $CookieJar,
// // 'proxy' => $proxy,
// 'timeout' => 30,
// ]);
// $strResult = (string)$Response->getBody();
// $QueryList = QueryList::html($strResult);
// // var_dump($QueryList);
// // 提取所有 <meta> 标签的属性
// $result = $QueryList->find('meta[property^="og:"]')->map(function ($item) {
// return [
// 'property' => $item->attr('property'),
// 'content' => $item->attr('content'),
// ];
// });
// // 打印结果
// // print_r($result->all());
// $strZuoZhe = '';
// $strMingZi = '';
// $strLastDate = '';
// $strJieShao = '';
// $strCover = '';
// $strFenlei = '';
// $strZiShu = 0;
// $strZhuangTai = $QueryList->find('#info p')->eq(1)->text();
// $strZhuangTai = getStringAfterWord($strZhuangTai, '状态：');
// foreach ($result as $item) {
// switch ($item['property']) {
// case 'og:title':
// $strMingZi = $item['content'];
// break;
// case 'og:description':
// $strJieShao = $item['content'];
// break;
// case 'og:image':
// $strCover = $item['content'];
// break;
// case 'og:book:category':
// $strFenlei = $item['content'];
// break;
// case 'og:book:author':
// $strZuoZhe = $item['content'];
// break;
// case 'og:book:update_time':
// $strLastDate = $item['content'];
// break;
// }
// }
// $strLastDate = trim((string)$strLastDate);
// if (!isValidDateTime($strLastDate)) {
// $strLastDate = date('Y-m-d H:i:s');
// }
// $arrTag = [];
// if (strpos($strCover, 'http') === false) {
// $strCover = $strSiteUrl . $strCover;
// }
// $strCoverUri = $strCover;
// try {
// $strCoverUri = "";
// $strCoverUri = ImageProcessor::getInstance()->downloadImage( $intXiaoShuoSourceId,$strCover, 'xs');
// } catch (\Throwable $t) {
// $strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
// echo $strErr;
// }
// $arrCover = [
// 'code' => 'BI_QU_XS_COVER',
// // 'uri' => getUriByUrl($strCover),
// 'uri' => $strCoverUri,
// ];
// $strFenleiIId = 1 ;
// switch ($strFenlei) {
// case '玄幻小说':
// $strName = "玄幻";
// $strFenleiIId = 1;
// break;
// case '修真小说':
// $strName = "修真";
// $strFenleiIId = 2;
// break;
// case '都市小说':
// $strName = "都市";
// $strFenleiIId = 3;
// break;
// case '历史小说':
// $strName = "历史";
// $strFenleiIId = 4;
// break;
// case '网游小说':
// $strName = "网游";
// $strFenleiIId = 5;
// break;
// case '科幻小说':
// $strName = "科幻";
// $strFenleiIId = 6;
// break;
// case '女频小说':
// $strName = "女频";
// $strFenleiIId = 7;
// break;
// case '其它小说':
// $strName = "其它";
// $strFenleiIId = 8;
// break;
// default:
// $strName = "其它";
// $strFenleiIId = 8;
// break;
// }
// $arrXiaoShuoInfo = [
// 'xsxq_ming_zi' => $strMingZi,
// 'xsxq_zuozhe' => $strZuoZhe , //$arrData['xiaoshuo_zuozhe'],
// 'xsfl_id' => $strFenleiIId,
// 'xsxq_zhuang_tai' => $strZhuangTai,
// 'xsxq_zi_shu' => $strZiShu,
// 'xsxq_jie_shao' => $strJieShao,
// 'xsxq_feng_mian' => json_encode($arrCover),
// 'xsxq_geng_xin_shi_jian' => $strLastDate,
// 'xsxq_tag' => json_encode($arrTag, JSON_UNESCAPED_UNICODE),
// 'xsxq_source_id' => $intXiaoShuoSourceId,
// 'xsxq_source_seo_id' => $intXiaoShuoSourceId.'1',
// 'xsxq_source_code' => 'BIQU_XIAOSHUO_' . $intXiaoShuoSourceId . '_' . $intXiaoShuoSourceSeoId,
// ];
// // print_r($arrXiaoShuoInfo);
// var_dump($arrXiaoShuoInfo['xsxq_source_code']);
// $XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
// $arrZhangjie = [];
// $intChapterIndex = 0; // 定义一个计数器变量
// // 检查是否找到元素
// if ($QueryList->find('.listmain a')->count() > 0) {
// $QueryList->find('.listmain a')->map(function ($ZhangJie) use (&$arrZhangjie, &$intChapterIndex) {
// echo "Processing index $intChapterIndex: " . $ZhangJie->text() . PHP_EOL;
// // 从索引6开始
// if ($intChapterIndex >= 5) {
// $arrZhangjie[] = [
// 'mingzi' => $ZhangJie->text(),
// 'url' => $ZhangJie->attr('href'),
// ];
// }
// $intChapterIndex++;
// });
// // 输出最终结果
// var_dump($arrZhangjie);
// } else {
// echo "No elements found for .listmain a" . PHP_EOL;
// }
// //unset($QueryList);
// // 获取最大章节索引
// $intMaxXiaoShuoZhangJiePaixu = XiaoShuoZhangJieModel::where('xsxq_id', $intXiaoShuoSourceId)
// ->max('xszj_pai_xu'); // 获取最大值
// var_dump('获取小说章节最大值-'.$intMaxXiaoShuoZhangJiePaixu);
// // 如果没有结果，max 返回 null，可以设置默认值
// $intMaxXiaoShuoZhangJiePaixu = $intMaxXiaoShuoZhangJiePaixu ?? 0;
// foreach ($arrZhangjie as $intZhangJieNum => $arrZhangjieOne) {
// var_dump($intZhangJieNum);
// // 如果数据库里的章节最大索引大于当前索引，说明已经入库，不需要入队列了
// if($intMaxXiaoShuoZhangJiePaixu > $intZhangJieNum){
// var_dump("小说章节已经入库，不再推送任务");
// continue;// 如果任务已存在，直接跳过 break
// }
// $strZhangJieUrl = $strSiteUrl . $arrZhangjieOne['url'];
// $strZhangJieId = extractFilename($strZhangJieUrl);
// # 'AILISI_XIAOSHUO_ZHANGJIE_小说源id_小说源章节id'
// $strXszjSourceCode = 'AILISI_XIAOSHUO_ZHANGJIE_'. $XiaoShuoXiangQingModel->xsxq_source_id . '_' . $strZhangJieId;
// $arrTask = [
// 'callback' => [self::class, 'getNovelReader'],
// 'data' => [
// 'xszj_source_id' => $strZhangJieId,
// 'xszj_paixu' => $intZhangJieNum,
// 'xszj_name' => $arrZhangjieOne['mingzi'],
// 'xszj_url' => $strZhangJieUrl,
// 'xszj_source_code' => $strXszjSourceCode,
// 'xsxq_source_id' => $XiaoShuoXiangQingModel->xsxq_source_id,
// 'xsfl_id' => $strFenleiIId,
// ]
// ];
// $QueueManage->set($arrTask);
// }
// var_dump("章节任务投递成功");
// } catch (\Throwable $T) {
// throw $T;
// }
// }
// public function getNovelFengMian($arrData)
// {
// //var_dump('getNovelFengMian');
// //var_dump($arrData);
// $Client = $this->Client;
// $QueueManage = $this->QueueManage;
// // 获取 Redis 实例
// $redis = Cache::store('redis')->handler();
// $intXiaoShuoSourceId = $arrData['xiaoshuo_id'];
// $intXiaoShuoSourceSeoId = $arrData['xiaoshuo_seo_id'];
// $redisKey = "task:novel_source_id:" . $intXiaoShuoSourceId;
// try {
// $arrCookie = $this->getAilisiCookie();
// $strHost = getDomainFromUrl($arrData['xiaoshuo_url']);
// $CookieJar = CookieJar::fromArray($arrCookie, $strHost);
// $strSiteUrl = $this->getAilisiDomain();
// $strCover = $QueryList->find('#fmimg img')->eq(0)->attr("src");
// if (strpos($strCover, 'http') === false) {
// $strCover = $strSiteUrl . $strCover;
// }
// $strCoverUri = $strCover;
// try {
// $strCoverUri = "";
// $strCoverUri = ImageProcessor::getInstance()->downloadAndEncryptImage($strCover, 'xs');
// $arrCover = [
// 'code' => 'AI_LI_SI_COVER',
// // 'uri' => getUriByUrl($strCover),
// 'uri' => $strCoverUri,
// ];
// $arrXiaoShuoInfo = [
// 'xsxq_feng_mian' => json_encode($arrCover),
// ];
// $XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
// } catch (\Throwable $t) {
// $strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
// echo $strErr;
// }
// } catch (\Throwable $T) {
// throw $T;
// }
// }
// public function getNovelReader($arrData)
// {
// var_dump('getNovelReader');
// var_dump($arrData['xszj_url']);
// $Client = $this->Client;
// $strXszjSourceCode = $arrData['xszj_source_code'];
// $strZhangJieUrl = $arrData['xszj_url'];
// $intZhangJiePaiXu = $arrData['xszj_paixu'];
// $intFenleiId = $arrData['xsfl_id'];
// $intXiaoShuoSourceID = $arrData['xsxq_source_id'];
// $strZhangJieSourceId = $arrData['xszj_source_id'];
// $strZhangJieName = $arrData['xszj_name'];
// try {
// $arrCookie = $this->getAilisiCookie();
// $strHost = getDomainFromUrl($strZhangJieUrl);
// $CookieJar = CookieJar::fromArray($arrCookie, $strHost);
// if (XiaoShuoZhangJieModel::checkZhangJieExists($strXszjSourceCode)) {
// return true;
// }
// $proxy = getRandomProxies();
// $userAgent = getRandomUserAgent();
// $Response = $Client->request('GET', $strZhangJieUrl, [
// 'headers' => [
// 'User-Agent' => $userAgent,
// 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
// 'Accept-Language' => 'en-US,en;q=0.5',
// ],
// 'cookies' => $CookieJar,
// // 'proxy' => $proxy,
// 'timeout' => 30,
// ]);
// $strResult = (string)$Response->getBody();
// unset($Response);
// $ZhangJieQueryList = QueryList::html($strResult);
// // 获取原始 HTML
// $strNeiRong = $ZhangJieQueryList->find('#content')->eq(0)->html();
// // 解码 HTML 实体
// $strNeiRong = html_entity_decode($strNeiRong, ENT_QUOTES, 'UTF-8');
// // 去掉 <p> 标签中的 class="content_detail" 属性
// $strNeiRong = preg_replace('/<p\s+class="content_detail"/i', '<p', $strNeiRong);
// // 清理多余的 HTML 标签
// $strNeiRong = strip_tags($strNeiRong, '<p>');
// // 替换不间断空格 (\xc2\xa0) 和 &nbsp; 为普通空格
// $strNeiRong = preg_replace('/\xc2\xa0/', ' ', $strNeiRong); // 替换 UTF-8 不间断空格
// $strNeiRong = preg_replace('/&nbsp;/', ' ', $strNeiRong); // 替换 &nbsp;
// // 清理多余空格和换行符
// $strNeiRong = preg_replace('/\s+/', ' ', $strNeiRong);
// $strNeiRong = preg_replace('/<p>\s+/', '<p>', $strNeiRong); // 清理 <p> 标签内部多余空格
// $strNeiRong = preg_replace('/\s+<\/p>/', '</p>', $strNeiRong); // 清理 <p> 标签结尾多余空格
// // 输出结果
// // var_dump($strNeiRong);
// # '/novel/123yqw/分类id/小说源id/小说源章节id.txt'
// $strNeiRongTxtPath = config("filesystem.novel").'/novel/biquxs/'.$intFenleiId.'/'.$intXiaoShuoSourceID.'/'.$strZhangJieSourceId.'.txt';
// //$strNeiRongTxtPath = "/www/novel/123yqw/1/11/111.txt";
// // 调用函数
// $result = saveCompressAndDeleteTxt($strNeiRongTxtPath, $strNeiRong);
// // 输出状态
// if ($result['status']) {
// var_dump("Success: " . $result['message']) ;
// } else {
// var_dump("Error: " . $result['message']) ;
// }
// $arrXiaoShuoZhangJie = [
// 'xsxq_id' => $intXiaoShuoSourceID,
// 'xszj_pai_xu' => $intZhangJiePaiXu,
// 'xszj_ming_zi' => $strZhangJieName,
// 'xszj_nei_rong' => $strNeiRongTxtPath,
// 'xszj_source_id' => $strZhangJieSourceId,
// 'xszj_source_code' => $strXszjSourceCode,
// ];
// XiaoShuoZhangJieModel::addXiaoShuoZhangJie($arrXiaoShuoZhangJie);
// unset($ZhangJieQueryList);
// // 延时 3 秒
// //sleep(3);
// } catch (\Throwable $T) {
// throw $T;
// }
// }
}

213
code/app/task/crawler/zw630/Scheduler.php Normal file
View File

@@ -0,0 +1,213 @@
<?php
declare(strict_types=1);
namespace app\task\crawler\zw630;
use app\task\crawler\zw630\page\NovelPage;
use app\task\crawler\zw630\page\Site;
use microserver\QueueManage;
/**
* @mixin think\Model
*/
class Scheduler
{
/**
* Undocumented variable
*
* @var \microserver\QueueManage
*/
protected $QueueManage = NULL;
public function __construct()
{
$this->QueueManage = new QueueManage();
}
/**
* 全局扫描任务
*
* @return void
*/
public function crawlFull()
{
$intSliceSize = 6;
$arrFilter = [];
// for ($intNId = 1; $intNId <= 531208; $intNId++) {
// for ($intNId = 1; $intNId <= 10000; $intNId++) {
for ($intNId = 1; $intNId <= 20; $intNId++) {
$arrFilter[] = [
'n_source_id' => $intNId,
'page' => 0,
];
if ($intNId % $intSliceSize == 0) {
$this->fetchNovel($arrFilter);
$arrFilter = [];
}
}
if (!empty($arrFilter)) {
$this->fetchNovel($arrFilter);
}
return;
// $arrTest = [
// 'n_id' => 9973,
// 'filter' => [
// [
// 'name' => '第十四章 神兽空青蛇妖',
// 'uri' => '/shu/1/33.html',
// 'c_sort_num' => 32,
// 'c_source_id' => 33,
// 'c_page' => 0,
// ],
// ]
// ];
// $arrTask = [
// 'callback' => [self::class, 'getChapterInfo'],
// 'data' => $arrTest
// ];
// $this->QueueManage->set($arrTask);
}
/**
* Undocumented function
*
* @param array $arrFilter
* @return void
*/
public function fetchNovel(array $arrFilter) //(int $intNSourceId, int $intPage = 0)
{
$arrTask = [
'callback' => [self::class, 'getNovelInfo'],
'data' => $arrFilter,
// 'data' => [
// 'n_source_id' => $intNSourceId,
// 'page' => $intPage,
// ]
];
$this->QueueManage->set($arrTask);
}
/**
* Undocumented function
*
* @param integer $intNId
* @param integer $intPage
* @param array $arrChapterList
* @return void
*/
public function fetchChapter(int $intNId, int $intPage, array $arrChapterList = [])
{
$intSliceSize = 6;
$intPageSize = 100;
$arrTask = [
'callback' => [self::class, 'getChapterInfo'],
'data' => [
'n_id' => $intNId,
// 'filter' => $arrFilter,
]
];
$arrFilter = [];
foreach ($arrChapterList as $intKey => $arrChapter) {
$arrChapter['c_sort_num'] = $intPage * $intPageSize + $intKey + 1;
$arrChapter['c_source_id'] = supperExtractFilename($arrChapter['uri']);
$arrChapter['c_page'] = 0;
$arrFilter[] = $arrChapter;
if ($intKey % $intSliceSize == 0) {
$arrTask['data']['filter'] = $arrFilter;
$this->QueueManage->set($arrTask);
$arrFilter = [];
}
}
if (!empty($arrFilter)) {
$arrTask['data']['filter'] = $arrFilter;
$this->QueueManage->set($arrTask);
}
}
/**
* Undocumented function
*
* @param array $arrFilter
* @return void
*/
public function getNovelInfo(array $arrFilter)
{
$Site = new Site;
$arrNovelPageList = $Site->getNovelPageList($arrFilter);
$arrCover = [];
foreach ($arrNovelPageList as $NovelPage) {
$arrNovel = $NovelPage->getNovelInfo();
if (empty($arrNovel)) {
continue;
}
$arrNextPageData = $NovelPage->getNextPageArgs();
if (!empty($arrNextPageData)) {
$this->fetchNovel([
[
'n_source_id' => $arrNextPageData['n_source_id'],
'page' => $arrNextPageData['page'],
]
]);
}
// $arrCover[] = $NovelPage->getDownCoverInfo();
// $arrChapterList = $NovelPage->getChapters();
// $this->fetchChapter($arrNovel['n_id'], $NovelPage->getData()['page'], $arrChapterList);
}
if ($arrCover) {
$Site->downRemoteImgToLocal($arrCover);
}
}
/**
* Undocumented function
*
* @param array $arrData
* @return void
*/
public function getChapterInfo(array $arrData)
{
$Site = new Site;
$arrChapterPageList = $Site->getChapterPageList($arrData['filter']);
$arrNextPageFilterList = [];
foreach ($arrChapterPageList as $ChapterPage) {
$ChapterPage->saveChapter($arrData['n_id']);
$arrNextPageFilter = $ChapterPage->getNextPage();
if (!empty($arrNextPageFilter)) {
$arrNextPageFilterList[] = $arrNextPageFilter;
}
}
if (!empty($arrNextPageFilterList)) {
$arrTask = $arrData;
$arrTask['filter'] = $arrNextPageFilterList;
$arrTask = [
'callback' => [self::class, 'getChapterInfo'],
'data' => $arrTask
];
$this->QueueManage->set($arrTask);
}
}
}

258
code/app/task/crawler/zw630/page/BasePage.php Normal file
View File

@@ -0,0 +1,258 @@
<?php
declare(strict_types=1);
namespace app\task\crawler\zw630\page;
use db\mongo\MongoBase;
use ddl\DDLManage;
use GuzzleHttp\Client;
use microserver\QueueManage;
use QL\QueryList;
use Psr\Http\Message\ResponseInterface;
/**
* @mixin think\Model
*/
abstract class BasePage
{
/**
* Undocumented variable
*
* @var string
*/
// protected $strDomain;
/**
* Undocumented variable
*
* @var string
*/
protected $strUri;
/**
* Undocumented variable
*
* @var MongoBase
*/
protected $MongoBase;
/**
* Undocumented variable
*
* @var \GuzzleHttp\Client
*/
protected $Client = NULL;
/**
* Undocumented variable
*
* @var \microserver\QueueManage
*/
protected $QueueManage = NULL;
/**
* Undocumented variable
*
* @var array
*/
protected $arrCaiJiPeiZhi = [];
/**
* Undocumented variable
*
* @var string
*/
protected $strContent;
/**
* Undocumented variable
*
* @var QueryList
*/
protected $QueryList;
/**
* Undocumented variable
*
* @var array
*/
protected $arrData = [];
/**
* Undocumented variable
*
* @var bool
*/
protected $boolIsMobile;
/**
* Undocumented variable
*
* @var ResponseInterface
*/
private $Response;
/**
* Undocumented function
*
* @param string $strUri
*/
public function __construct(Site $Site, string $strUri)
{
$this->setSite($Site);
$this->strUri = $strUri;
}
/**
* Undocumented function
*
* @param array $arrData
* @return void
*/
public function setData(array $arrData)
{
$this->arrData = $arrData;
}
/**
* Undocumented function
*
* @return array
*/
public function getData(): array
{
return $this->arrData;
}
/**
* Undocumented function
*
* @return QueueManage
*/
public function getQueueManage(): QueueManage
{
if ($this->QueueManage == NULL) {
$this->QueueManage = QueueManage::getInstance();
}
return $this->QueueManage;
}
/**
* Undocumented function
*
* @return MongoBase
*/
public function getMongoBase(): MongoBase
{
if ($this->MongoBase == NULL) {
DDLManage::load('MongoBase');
$this->MongoBase = MongoBase::getInstance();
}
return $this->MongoBase;
}
public function getQueryList(): QueryList
{
if ($this->QueryList == NULL) {
$this->QueryList = QueryList::html($this->getContent());
}
return $this->QueryList;
}
/**
* Undocumented function
*
* @return string
*/
public function getContent(): string
{
if ($this->strContent == NULL) {
$this->Response = $this->getClient()->get($this->strUri);
$this->strContent = $this->Response->getBody()->getContents();
}
return $this->strContent;
}
/**
* force set page content
*
* @param string $strContent
* @return void
*/
public function setContent($strContent)
{
$this->strContent = $strContent;
}
/**
* Undocumented function
*
* @return boolean
*/
public function isMobile($strMark = 'content="mobile"'): bool
{
if ($this->boolIsMobile === NULL) {
$this->boolIsMobile = is_numeric(strpos($this->getContent(), $strMark));
}
return $this->boolIsMobile;
}
/**
* Undocumented function
*
* @return Client
*/
protected function getClient(): Client
{
if ($this->Client == NULL) {
$this->Client = $this->getSite()->getClient();
}
return $this->Client;
}
/**
* Undocumented variable
*
* @var Site
*/
protected $Site;
public function setSite(Site $Site)
{
$this->Site = $Site;
}
public function getSite()
{
return $this->Site;
}
/**
* Undocumented variable
*
* @var boolean
*/
protected $boolIsFirstSave = false;
/**
* Undocumented function
*
* @return boolean
*/
public function getIsFirstSave(): bool
{
return $this->boolIsFirstSave;
}
}

139
code/app/task/crawler/zw630/page/ChapterPage.php Normal file
View File

@@ -0,0 +1,139 @@
<?php
declare(strict_types=1);
namespace app\task\crawler\zw630\page;
use storage\StorageCore;
class ChapterPage extends BasePage
{
/**
* Undocumented variable
*
* @var string
*/
protected $strChapterContent = '';
/**
* Undocumented function
*
* @return array
*/
public function getChapterInfo(): array
{
if ($this->arrChapterInfo === NULL) {
$this->arrChapterInfo = [];
$strPattern = "/qsbs\.bb\('([^']*)'\)/";
preg_match_all($strPattern, $this->getContent(), $arrMatches);
$strContent = '';
foreach ($arrMatches[1] as $str) {
$strContent .= base64_decode($str);
}
// $this->arrChapterInfo['c_content'] = $strContent;
$this->strChapterContent = $strContent;
$this->arrChapterInfo['c_page_title'] = $this->getQueryList()->find('title')->text();
$this->arrChapterInfo['c_page_keywords'] = $this->getQueryList()->find('meta[name="keywords"]')->attr('content');
$this->arrChapterInfo['c_page_description'] = $this->getQueryList()->find('meta[name="description"]')->attr('content');
}
return $this->arrChapterInfo;
}
/**
* Undocumented variable
*
* @var array
*/
protected $arrChapterInfo;
public function saveChapter(int $intNId): array
{
try {
$this->getChapterInfo();
$strUUId = 'zw630-chapter-' . $this->arrData['c_source_id'] . '-' . $this->arrData['c_page'];
$Col = $this->getMongoBase()->getDb()->selectCollection('chapter');
$arrExisting = $Col->findOne(['c_site_uuid' => $strUUId]);
if ($arrExisting) {
$arrExisting = (array) $arrExisting;
$this->arrChapterInfo = array_merge($arrExisting, $this->getChapterInfo());
$this->arrChapterInfo['updated_at'] = new \MongoDB\BSON\UTCDateTime(); // 添加更新时间
$Col->updateOne(
['c_site_uuid' => $strUUId],
['$set' => $this->arrChapterInfo],
// ['upsert' => false]
);
} else {
$this->arrChapterInfo['n_id'] = $intNId;
$this->arrChapterInfo['name'] = $this->arrData['name'];
$this->arrChapterInfo['c_sort_num'] = $this->arrData['c_sort_num'];
$this->arrChapterInfo['c_page'] = $this->arrData['c_page'];
$this->arrChapterInfo['c_source_id'] = $this->arrData['c_source_id'];
$this->arrChapterInfo['c_source_uri'] = $this->arrData['uri'];
$this->arrChapterInfo['c_site_name'] = '恋上你中文';
$this->arrChapterInfo['c_site_code'] = 'zw630';
$this->arrChapterInfo['c_site_uuid'] = $strUUId;
$this->arrChapterInfo['c_content_path'] = '/novel/' . ($intNId % 100) . '/' . $intNId . '/' . $this->arrChapterInfo['c_sort_num'] . '_' . $this->arrChapterInfo['c_page'] . '.txt';
$this->getMongoBase()->insertOne('chapter', $this->arrChapterInfo);
if (!StorageCore::getInstance()->has($this->arrChapterInfo['c_content_path'])) {
StorageCore::getInstance()->set($this->arrChapterInfo['c_content_path'], $this->strChapterContent);
}
$this->boolIsFirstSave = true;
}
return $this->arrChapterInfo;
} catch (\MongoDB\Exception\Exception $e) {
throw new \Exception("Upsert failed: " . $e->getMessage());
}
}
protected $arrNextPage;
/**
* Undocumented function
*
* @return array
*/
public function getNextPage(): array
{
if ($this->arrNextPage === NULL) {
$this->arrNextPage = [];
if ($this->isMobile()) {
$strPattern = "/var\s+hhekgsv\s*=\s*['\"](.*?)['\"]/";
preg_match_all($strPattern, $this->getContent(), $arrMatches);
$strNextPageUri = end($arrMatches[1]) ?: '';
} else {
$strHref = $this->getQueryList()->find('.read_btn a:contains("下一章")')->attr('href');
$strNextPageUri = $strHref ?: '';
}
if (is_numeric(strpos($strNextPageUri, '_'))) {
$this->arrNextPage = $this->arrData;
$this->arrNextPage['c_page'] += 1;
$this->arrNextPage['uri'] = $strNextPageUri;
}
}
return $this->arrNextPage;
}
}

304
code/app/task/crawler/zw630/page/NovelPage.php Normal file
View File

@@ -0,0 +1,304 @@
<?php
declare(strict_types=1);
namespace app\task\crawler\zw630\page;
use ddl\DDLManage;
use QL\QueryList;
class NovelPage extends BasePage
{
/**
* Undocumented variable
*
* @var array
*/
protected $arrNovelInfo = [];
/**
* Undocumented variable
*
* @var array
*/
protected $arrRelatedNovel = [];
/**
* Undocumented variable
*
* @var array
*/
protected $arrForgeNovel = [];
private function findOrCreateNovel(int $intNSourceId, string $strName): array
{
$strUUId = 'zw630-' . $intNSourceId;
$Col = $this->getMongoBase()->getDb()->selectCollection('novel');
$arrExisting = $Col->findOne(['n_site_uuid' => $strUUId]);
if ($arrExisting) {
$arrNovelInfo = (array) $arrExisting;
} else {
$arrNovelInfo['og_title'] = $strName;
$arrNovelInfo['og_novel_book_name'] = $strName;
$arrNovelInfo['og_novel_pin_yin'] = zhToPinYin($strName);
$arrNovelInfo['n_site_name'] = '恋上你中文';
$arrNovelInfo['n_site_code'] = 'zw630';
$arrNovelInfo['n_site_uuid'] = $strUUId;
$arrNovelInfo['n_source_id'] = $intNSourceId;
$arrNovelInfo['n_level'] = 1;
$arrNovelInfo['n_id'] = $this->getMongoBase()->getNextSequence('novel');
$this->getMongoBase()->insertOne('novel', $arrNovelInfo);
}
return $arrNovelInfo;
}
/**
* Find or create a category by name, handling concurrency safely.
*
* @param string $strCategory
* @return array|null The category document (existing or newly created)
*/
public function findOrCreateCategory(string $strCategory)
{
if (empty($strCategory)) {
return false;
}
$Col = $this->getMongoBase()->getDb()->selectCollection('category');
$arrCategory = ['ca_name' => $strCategory];
try {
$Result = $Col->findOneAndUpdate(
$arrCategory,
['$set' => $arrCategory],
[
'upsert' => true,
'returnDocument' => \MongoDB\Operation\FindOneAndUpdate::RETURN_DOCUMENT_AFTER
]
);
return $Result;
} catch (\MongoDB\Driver\Exception\Exception $e) {
return null;
}
}
/**
* Undocumented function
*
* @return array
*/
public function getNovelInfo($boolSave = true): array
{
if (empty($this->arrNovelInfo)) {
$this->getQueryList()->find('meta[property]')->each(function ($Item) {
$strKey = str_replace(':', '_', $Item->attr('property'));
$this->arrNovelInfo[$strKey] = $Item->attr('content');
});
$this->arrNovelInfo['n_page_title'] = $this->getQueryList()->find('title')->text();
$this->arrNovelInfo['n_page_keywords'] = $this->getQueryList()->find('meta[name="keywords"]')->attr('content');
$this->arrNovelInfo['n_page_description'] = $this->getQueryList()->find('meta[name="description"]')->attr('content');
$this->getQueryList()->find('.first_txt')->eq(0)->find("a")->map(function ($A) {
$strName = $A->text();
$strUri = $A->attr('href');
if (is_numeric(strpos($strUri, 'shu'))) {
$intNSourceId = (int)supperExtractFilename($strUri);
$arrNovelInfo = $this->findOrCreateNovel($intNSourceId, $strName);
$this->arrRelatedNovel[] = [
"n_id" => $arrNovelInfo["n_id"],
"n_source_id" => $arrNovelInfo["n_source_id"],
"og_title" => $arrNovelInfo["og_title"],
];
} elseif (is_numeric(strpos($strUri, 'kan'))) {
$intNForgeId = (int)supperExtractFilename($strUri);
$this->arrForgeNovel[] = [
"n_forge_id" => $intNForgeId,
"n_forge_title" => $strName,
];
}
});
$this->arrNovelInfo['n_related_novel'] = $this->arrRelatedNovel;
$this->arrNovelInfo['n_forge_novel'] = $this->arrForgeNovel;
}
if ($boolSave && !empty($this->arrNovelInfo)) {
$this->saveNovelInfo();
$this->findOrCreateCategory($this->arrNovelInfo['og_novel_category']);
}
return $this->arrNovelInfo;
}
/**
* Undocumented function
*
* @return void
*/
public function saveNovelInfo()
{
try {
$strUUId = 'zw630-' . $this->arrData['n_source_id'];
$Col = $this->getMongoBase()->getDb()->selectCollection('novel');
$arrExisting = $Col->findOne(['n_site_uuid' => $strUUId]);
if ($arrExisting) {
$arrExisting = (array) $arrExisting;
$this->arrNovelInfo = array_merge($arrExisting, $this->arrNovelInfo);
if (!isset($this->arrNovelInfo['og_novel_pin_yin']) || empty($this->arrNovelInfo['og_novel_pin_yin'])) {
$this->arrNovelInfo['og_novel_pin_yin'] = zhToPinYin($this->arrNovelInfo['og_novel_book_name'] ?? '');
}
if (!isset($this->arrNovelInfo['n_cover_path']) || empty($this->arrNovelInfo['n_cover_path'])) {
$this->arrNovelInfo['n_cover_path'] = 'img/' . date('Y/m/d', time() - 24 * 3600 * 14) . '/' . $this->arrNovelInfo['n_id'] . '.jpg';
}
$this->arrNovelInfo['updated_at'] = new \MongoDB\BSON\UTCDateTime();
$Col->updateOne(
['n_site_uuid' => $strUUId],
['$set' => $this->arrNovelInfo],
// ['upsert' => false]
);
} else {
$this->arrNovelInfo['og_novel_pin_yin'] = zhToPinYin($this->arrNovelInfo['og_novel_book_name'] ?? '');
$this->arrNovelInfo['n_site_name'] = '恋上你中文';
$this->arrNovelInfo['n_site_code'] = 'zw630';
$this->arrNovelInfo['n_site_uuid'] = $strUUId;
$this->arrNovelInfo['n_source_id'] = $this->arrData['n_source_id'];
$this->arrNovelInfo['n_id'] = $this->getMongoBase()->getNextSequence('novel');
$this->arrNovelInfo['n_level'] = 1;
$this->arrNovelInfo['n_cover_path'] = '/img/' . date('Y/m/d', time() - 24 * 3600 * 14) . '/' . $this->arrNovelInfo['n_id'] . '.jpg';
$this->getMongoBase()->insertOne('novel', $this->arrNovelInfo);
$this->boolIsFirstSave = true;
}
return $this->arrNovelInfo;
} catch (\MongoDB\Exception\Exception $e) {
throw new \Exception("Upsert failed: " . $e->getMessage());
}
}
/**
* Undocumented function
*
* @return array
*/
public function getDownCoverInfo(): array
{
return [
'uri' => $this->arrNovelInfo['og_image'],
'local_path' => $this->arrNovelInfo['n_cover_path'],
];
}
/**
* Undocumented variable
*
* @var array
*/
protected $arrLatestChapters = NULL;
/**
* Undocumented variable
*
* @var array
*/
protected $arrChapters = NULL;
public function getLatestChaptersList(): array
{
if ($this->arrLatestChapters === NULL) {
$this->arrLatestChapters = [];
if ($this->isMobile()) {
$strClass = '.chapter-list';
} else {
$strClass = '.section-list';
}
if ($this->arrData['page'] == 0) {
$this->getQueryList()->find($strClass)->eq(0)->find("a")->map(function ($A) {
$this->arrLatestChapters[] = [
'name' => $A->text(),
'uri' => $A->attr('href'),
];
});
}
}
return $this->arrLatestChapters;
}
public function getChapters(): array
{
if ($this->arrChapters === NULL) {
$this->arrChapters = [];
if ($this->isMobile()) {
$strClass = '.chapter-list';
} else {
$strClass = '.section-list';
}
$intDivIndex = 0;
if ($this->arrData['page'] == 0) {
$intDivIndex = 1;
}
$this->getQueryList()->find($strClass)->eq($intDivIndex)->find("a")->map(function ($A) {
$this->arrChapters[] = [
'name' => $A->text(),
'uri' => $A->attr('href'),
];
});
}
return $this->arrChapters;
}
protected $arrNextPageArgs = NULL;
/**
* Undocumented function
*
* @return array
*/
public function getNextPageArgs(): array
{
$this->arrNextPageArgs = [];
if ($this->arrData['page'] == 0) {
$this->getLatestChaptersList();
$this->getChapters();
$arrLatestChapterFirstBlock = count($this->arrLatestChapters) > 0 ? $this->arrLatestChapters[0] : null;
$arrLatestChapterSecondBlock = count($this->arrChapters) > 0 ? $this->arrChapters[count($this->arrChapters) - 1] : null;
if (
!empty($arrLatestChapterFirstBlock) && !empty($arrLatestChapterSecondBlock)
&& ($arrLatestChapterFirstBlock['uri'] != $arrLatestChapterSecondBlock['uri'])
) {
$this->arrNextPageArgs = [
'n_source_id' => $this->arrData['n_source_id'],
'page' => $this->arrData['page'] + 1,
];
}
} else {
$strLastUri = $this->getQueryList()->find('select:first option:last')->attr('value');
if ($strLastUri != '' && $strLastUri != $this->strUri) {
$this->arrNextPageArgs = [
'n_source_id' => $this->arrData['n_source_id'],
'page' => $this->arrData['page'] + 1,
];
}
}
return $this->arrNextPageArgs;
}
}

257
code/app/task/crawler/zw630/page/Site.php Normal file
View File

@@ -0,0 +1,257 @@
<?php
declare(strict_types=1);
namespace app\task\crawler\zw630\page;
use GuzzleHttp\Client;
use GuzzleHttp\Promise;
use storage\StorageCore;
/**
* @mixin think\Model
*/
class Site
{
/**
* Undocumented variable
*
* @var Client
*/
private $Client;
/**
* Undocumented variable
*
* @var string
*/
protected $strDomain = 'https://www.630zw.org';
/**
* Undocumented function
*/
public function __construct()
{
// $this->Client = new Client([
// 'http_version' => '1.1', // 强制用 HTTP/1.1
// 'base_uri' => $this->strDomain,
// 'http_errors' => false,
// // 'version' => 2.0,
// 'timeout' => 160,
// 'connect_timeout' => 30,
// 'curl' => [
// // CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_2_0,
// CURLOPT_TCP_KEEPALIVE => 1,
// CURLOPT_SSL_VERIFYPEER => true,
// ],
// 'headers' => [
// 'User-Agent' => 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Mobile Safari/537.36 Edg/134.0.0.0',
// 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
// 'Accept-Language' => 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
// 'Referer' => $this->strDomain,
// 'sec-ch-ua-mobile' => '?1',
// 'sec-ch-ua-platform' => '"Android"',
// 'sec-fetch-dest' => 'document',
// 'sec-fetch-mode' => 'navigate',
// 'sec-fetch-site' => 'same-origin',
// 'sec-fetch-user' => '?1',
// 'upgrade-insecure-requests' => '1',
// ],
// ]);
$this->Client = new Client([
'http_version' => '1.1', // 强制用 HTTP/1.1，已有，保持
'base_uri' => $this->strDomain, // 基础域名，已有，保持
'http_errors' => false, // 不抛 HTTP 错误，已有，保持
'timeout' => 60, // 从 160 秒改为 60 秒
'connect_timeout' => 10, // 从 30 秒改为 10 秒
'curl' => [
CURLOPT_TCP_KEEPALIVE => 1, // 启用 TCP Keep-Alive，已有，保持
CURLOPT_TCP_KEEPIDLE => 120, // Keep-Alive 空闲时间
CURLOPT_TCP_KEEPINTVL => 60, // Keep-Alive 探测间隔
CURLOPT_SSL_VERIFYPEER => true, // 验证 SSL，已有，保持
CURLOPT_SSL_VERIFYHOST => 2, // 确保主机名匹配
],
'headers' => [
// Mobile
// 'User-Agent' => 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Mobile Safari/537.36 Edg/134.0.0.0',
// 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
//PC
'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36 Edg/134.0.0.0',
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'Accept-Language' => 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
'Referer' => $this->strDomain,
'Connection' => 'keep-alive', // 显式启用 Keep-Alive
// Mobile
// 'sec-ch-ua-mobile' => '?1',
// 'sec-ch-ua-platform' => '"Android"',
// 'sec-fetch-dest' => 'document',
// 'sec-fetch-mode' => 'navigate',
// 'sec-fetch-site' => 'same-origin',
// 'sec-fetch-user' => '?1',
// PC
'sec-ch-ua' => '"Chromium";v="134", "Not:A-Brand";v="24", "Microsoft Edge";v="134"',
'sec-ch-ua-mobile' => '?0',
'sec-ch-ua-platform' => '"Windows"',
'sec-fetch-dest' => 'document',
'sec-fetch-mode' => 'navigate',
'sec-fetch-site' => 'same-origin',
'sec-fetch-user' => '?1',
'upgrade-insecure-requests' => '1',
],
'pool_size' => 40, // 并发连接池大小（视需求调整）
]);
}
/**
* Undocumented function
*
* @return Client
*/
public function getClient(): Client
{
return $this->Client;
}
/**
* Undocumented function
*
* @param string $strUri
* @return NovelPage
*/
public function getNovelPage(string $strUri): NovelPage
{
return new NovelPage($this, $strUri);
}
/**
* Undocumented function
*
* @param string $strUri
* @return NovelPage[]
*/
public function getNovelPageList(array $arrFilter): array
{
/** @var NovelPage[] */
$arrNovelPage = [];
foreach ($arrFilter as $intKey => $arrItem) {
if ($arrItem['page'] == 0) {
$arrItem['uri'] = sprintf('/shu/%s.html', $arrItem['n_source_id']);
} else {
$arrItem['uri'] = sprintf('/shu/%s_%s.html', $arrItem['n_source_id'], $arrItem['page']);
}
$arrFilter[$intKey]['uri'] = $arrItem['uri'];
$arrPromises[$intKey] = $this->getClient()->getAsync($arrItem['uri']);
}
$arrResults = Promise\Utils::settle($arrPromises)->wait();
$arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
return $arrResult['state'] === 'fulfilled';
});
foreach ($arrSuccessfulResults as $intKey => $arrResult) {
$Response = $arrResult['value'];
$strContent = $Response->getBody()->getContents();
if ($strContent == '索引文件不存在！') {
continue;
}
$NovelPage = new NovelPage($this, $arrItem['uri']);
$NovelPage->setContent($strContent);
$NovelPage->setData($arrFilter[$intKey]);
$arrNovelPage[] = $NovelPage;
}
return $arrNovelPage;
}
/**
* Undocumented function
*
* @param string $strUri
* @return ChapterPage[]
*/
public function getChapterPageList(array $arrFilter): array
{
/** @var ChapterPage[] */
$arrChapterPage = [];
$arrPromises = [];
foreach ($arrFilter as $intKey => $arrItem) {
$arrPromises[$intKey] = $this->getClient()->getAsync($arrItem['uri']);
}
$arrResults = Promise\Utils::settle($arrPromises)->wait();
$arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
return $arrResult['state'] === 'fulfilled';
});
foreach ($arrSuccessfulResults as $intKey => $arrResult) {
$Response = $arrResult['value'];
$strContent = $Response->getBody()->getContents();;
$ChapterPage = new ChapterPage($this, $arrItem['uri']);
$ChapterPage->setContent($strContent);
$ChapterPage->setData($arrFilter[$intKey]);
$arrChapterPage[] = $ChapterPage;
}
return $arrChapterPage;
}
/**
* Undocumented function
*
* @param array $arrFilter
* @return void
*/
public function downRemoteImgToLocal(array $arrFilter)
{
$StorageCore = StorageCore::getInstance();
$arrPromises = [];
foreach ($arrFilter as $intKey => $arrItem) {
if (!$StorageCore->has($arrItem['local_path'])) {
$arrPromises[$intKey] = $this->getClient()->getAsync($arrItem['uri']);
}
}
$arrResults = Promise\Utils::settle($arrPromises)->wait();
$arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
return $arrResult['state'] === 'fulfilled';
});
foreach ($arrSuccessfulResults as $intKey => $arrResult) {
$Response = $arrResult['value'];
// $strContent = $Response->getBody()->getContents();
$StorageCore->writeStream($arrFilter[$intKey]['local_path'], $Response->getBody()->detach());;
}
}
}

4
code/app/task/module/Listen.php
View File

@@ -4,8 +4,8 @@ declare(strict_types=1);
namespace app\task\module;
use minserver\ProcessSanitizer;
use minserver\QueueManage;
use microserver\ProcessSanitizer;
use microserver\QueueManage;
class Listen
{

8
code/app/task/module/PlanTask.php
View File

@@ -5,9 +5,9 @@ declare(strict_types=1);
namespace app\task\module;
use app\model\PlanTaskModel;
use app\task\collection\zw630\Zw630Col;
use minserver\QueueManage;
use minserver\ProcessSanitizer;
use app\task\crawler\zw630\Scheduler as Zw630Scheduler;
use microserver\QueueManage;
use microserver\ProcessSanitizer;
class PlanTask
@@ -41,7 +41,7 @@ class PlanTask
public static function pull630ZwNovel()
{
$arrTask = [
'callback' => [Zw630Col::class, 'pull630ZwNovel'],
'callback' => [Zw630Scheduler::class, 'crawlFull'],
'data' => []
];