This commit is contained in:
Default
2025-03-15 20:12:21 +08:00
parent 105ffe11c7
commit eb9035a7ae
134 changed files with 15450 additions and 119 deletions

View File

@@ -0,0 +1,36 @@
<?php
declare(strict_types=1);
namespace app\admin;
use app\admin\model\AdminUserModel;
use app\admin\service\AdminUserAuth;
use app\Request;
use think\Container;
class AdminServiceProvider extends \think\Service
{
/**
* 注册服务
*
* @return mixed
*/
public function register()
{
Container::getInstance()->bind(AdminUserModel::class, function (Request $Request) {
return AdminUserModel::getUserByToken($Request->header("admin-token"));
});
}
/**
* 执行服务
*
* @return mixed
*/
public function boot()
{
//
}
}

View File

@@ -0,0 +1,107 @@
<?php
declare(strict_types=1);
namespace app\admin;
use app\admin\middleware\AdminResponse;
use think\App;
use think\exception\ValidateException;
use think\Validate;
/**
* 控制器基础类
*/
abstract class BaseController
{
/**
* Request实例
* @var \think\Request
*/
protected $request;
/**
* 应用实例
* @var \think\App
*/
protected $app;
/**
* 是否批量验证
* @var bool
*/
protected $batchValidate = false;
/**
* 控制器中间件
* @var array
*/
protected $middleware = [];
/**
* 构造方法
* @access public
* @param App $app 应用对象
*/
public function __construct(App $app)
{
$this->app = $app;
$this->request = $this->app->request;
// 控制器初始化
$this->initialize();
}
// 初始化
protected function initialize()
{
}
/**
* 验证数据
* @access protected
* @param array $data 数据
* @param string|array $validate 验证器名或者验证规则数组
* @param array $message 提示信息
* @param bool $batch 是否批量验证
* @return array|string|true
* @throws ValidateException
*/
protected function validate(array $data, $validate, array $message = [], bool $batch = false)
{
if (is_array($validate)) {
$v = new Validate();
$v->rule($validate);
} else {
if (strpos($validate, '.')) {
// 支持场景
[$validate, $scene] = explode('.', $validate);
}
$class = false !== strpos($validate, '\\') ? $validate : $this->app->parseClass('validate', $validate);
$v = new $class();
if (!empty($scene)) {
$v->scene($scene);
}
}
$v->message($message);
// 是否批量验证
if ($batch || $this->batchValidate) {
$v->batch(true);
}
return $v->failException(true)->check($data);
}
public function success($arrData = [], $strMsg = "", $strCode = "1000")
{
return AdminResponse::create($strCode, $strMsg, $arrData);
}
public function error($strCode = "9999", $strMsg = "", $arrData = [])
{
return AdminResponse::create($strCode, $strMsg, $arrData);
}
}

View File

@@ -0,0 +1,38 @@
<?php
declare(strict_types=1);
return [
#用户组
'1000' => '成功',
'1001' => '用户不存在!',
'1002' => '密码错误!',
'1003' => '用户名不能为空!',
'1004' => '密码不能为空!',
#权限节点组
'1011' => '权限节点路径不能为空!',
'1012' => '权限节点名不能为空!',
'1013' => '权限节点是否显示不能为空!',
'1014' => '权限节点ID不能为空',
'1015' => '权限节点集合不能为空!',
'1016' => '该部门下存在成员无法删除!',
'1017' => '角色名不能为空!',
'1018' => '角色ID不能为空',
#其他组
'9900' => '参数错误!',
'9901' => '获取PHP程序位置失败',
'9991' => '你没有这个权限!',
'9992' => '远程密钥错误!',
'9993' => '未查询到相关数据!',
'9994' => '信息输入有误!',
'9995' => '请先登录!',
'9996' => '无效令牌!',
'9997' => '地址错误!',
'9998' => '请求对象未定义!',
'9999' => '系统异常!',
];

View File

@@ -0,0 +1,60 @@
<?php
declare(strict_types=1);
use app\admin\middleware\AdminAuth;
use think\facade\Route;
Route::group("/guest", function () {
Route::post("/login", "Guest/Login")->name("Guest@Login");
Route::post("/logout", "Guest/Logout")->name("Guest@Logout");
});
# 系统配置
Route::group("/system", function () {
Route::get("/user/info", "AdminUser/Info")->name("AdminUser@Info");
Route::get("/user/list", "AdminUser/getAdminUserList")->name("AdminUser@getAdminUserList");
Route::post("/user/save", "AdminUser/saveAdminUser")->name("AdminUser@saveAdminUser");
Route::post("/user/del", "AdminUser/delAdminUser")->name("AdminUser@delAdminUser");
Route::get("/config/list", "SystemConfig/getSystemConfigList")->name("SystemConfig@getSystemConfigList");
Route::post("/config/save", "SystemConfig/saveSystemConfig")->name("SystemConfig@saveSystemConfig");
Route::get("/cjpz/list", "SystemConfig/getCaiJiPeiZhiList")->name("SystemConfig@getCaiJiPeiZhiList");
Route::post("/cjpz/save", "SystemConfig/saveCaiJiPeiZhi")->name("SystemConfig@saveCaiJiPeiZhi");
Route::get("/zydd/list", "SystemConfig/getZiYuanDuanDianList")->name("SystemConfig@getZiYuanDuanDianList");
Route::post("/zydd/save", "SystemConfig/saveZiYuanDuanDian")->name("SystemConfig@saveZiYuanDuanDian");
Route::get("/plan/list", "SystemConfig/getPlanTaskList")->name("SystemConfig@getPlanTaskList");
Route::post("/plan/save", "SystemConfig/savePlanTask")->name("SystemConfig@savePlanTask");
Route::get("/keyword/list", "SeoKeywords/getSeoKeywordList")->name("SeoKeywords@getSeoKeywordList");
Route::post("/keyword/save", "SeoKeywords/saveSeoKeyword")->name("SeoKeywords@saveSeoKeyword");
Route::post("/keyword/del", "SeoKeywords/delSeoKeyword")->name("SeoKeywords@delSeoKeyword");
})->middleware(AdminAuth::class);
# 广告
Route::group("/guanggao", function () {
# 主
Route::get("/main/group/list", "GuangGao/getGuangGaoFenZuList")->name("GuangGao@getGuangGaoFenZuList");
Route::post("/main/group/save", "GuangGao/saveGuangGaoFenZu")->name("GuangGao@saveGuangGaoFenZu");
Route::post("/main/group/del", "GuangGao/delGuangGaoFenZu")->name("GuangGao@delGuangGaoFenZu");
Route::get("/main/list", "GuangGao/getGuangGaoList")->name("GuangGao@getGuangGaoList");
Route::post("/main/save", "GuangGao/saveGuangGao")->name("GuangGao@saveGuangGao");
Route::post("/main/del", "GuangGao/delGuangGao")->name("GuangGao@delGuangGao");
# 广告配置
Route::get("/ggpz/list", "GuangGao/getGuangGaoPeiZhiList")->name("GuangGao@getGuangGaoPeiZhiList");
Route::post("/ggpz/save", "GuangGao/saveGuangGaoPeiZhi")->name("GuangGao@saveGuangGaoPeiZhi");
})->middleware(AdminAuth::class);
# 小说
Route::group("/xiaoshuo", function () {
Route::get("/main/list", "XiaoShuo/getXiaoShuoList")->name("XiaoShuo@getXiaoShuoList");
Route::get("/zhangjie/list", "XiaoShuo/getXiaoShuoZhangJieList")->name("XiaoShuo@getXiaoShuoZhangJieList");
Route::get("/fenlei/list", "XiaoShuo/getXiaoShuoFenLeiList")->name("XiaoShuo@getXiaoShuoFenLeiList");
})->middleware(AdminAuth::class);

View File

@@ -0,0 +1,153 @@
<?php
declare(strict_types=1);
namespace app\admin\controller;
use app\admin\BaseController;
use app\admin\model\AdminUserModel;
use app\admin\service\AdminUserAuth;
use app\Request;
use think\facade\Cache;
use think\Response;
class AdminUser extends BaseController
{
/**
* 详情
*
* @param Request $Request
* @param AdminUserModel|null $AdminUserModel
* @return Response
*/
public function Info(Request $Request, ?AdminUserModel $AdminUserModel)
{
$arrResult = [
'admin_user' => $AdminUserModel,
];
return $this->success($arrResult);
}
/**
* 列表
*
* @param Request $Request
* @param AdminUserModel|null $AdminUserModel
* @return Response
*/
public function getAdminUserList(Request $Request, ?AdminUserModel $AdminUserModel)
{
$arrData = [
"page" => $Request->get('page', 1),
"limit" => $Request->get('limit', 10),
"key" => $Request->get('key'),
];
$arrRule = [
'page' => 'require|number',
'limit' => 'require|number',
];
$this->validate($arrData, $arrRule);
$AdminUserModel = AdminUserModel::alias('au');
if (!empty($arrData['key'])) {
$strKey = $arrData['key'];
$AdminUserModel->where(function ($AdminUserModel) use ($strKey) {
$AdminUserModel->whereOr([
['au.au_name', 'like', "%" . $strKey . "%"],
['au.au_id', 'like', "%" . $strKey . "%"],
]);
});
}
$Paginate = $AdminUserModel->paginate([
'list_rows' => $arrData['limit'],
'page' => $arrData['page'],
]);
$arrResult = [
'items' => $Paginate->items(),
'total' => $Paginate->total(),
'current_page' => $Paginate->currentPage(),
'total_pages' => $Paginate->lastPage(),
];
return $this->success($arrResult);
}
/**
* 设置管理员
*
* @param Request $Request
* @param AdminUserModel|null $AdminUserModel
* @return Response
*/
public function saveAdminUser(Request $Request, ?AdminUserModel $AdminUserModel)
{
$arrData = [
"au_id" => $Request->post('au_id'),
"au_status" => $Request->post('au_status'),
"au_name" => $Request->post('au_name'),
"au_pwd" => $Request->post('au_pwd'),
];
$arrRule = [
// 'au_id' => 'require',
'au_status' => 'require',
'au_name' => 'require',
];
$this->validate($arrData, $arrRule);
if ($arrData['au_id'] == NULL) {
$AdminUserModel = new AdminUserModel;
} else {
$AdminUserModel = AdminUserModel::where('au_id', $arrData['au_id'])->find();
if (empty($AdminUserModel)) {
return $this->error('9993');
}
}
if (isset($arrData['au_pwd']) && !empty($arrData['au_pwd'])) {
$arrData['au_pwd'] = md5($arrData['au_pwd']);
} else {
unset($arrData['au_pwd']);
}
$AdminUserModel->fill($arrData);
$AdminUserModel->save();
return $this->success();
}
/**
* 删除管理员
*
* @param Request $Request
* @param AdminUserModel|null $AdminUserModel
* @return Response
*/
public function delAdminUser(Request $Request, ?AdminUserModel $AdminUserModel)
{
$arrData = [
"au_id" => $Request->post('au_id'),
];
$arrRule = [
'au_id' => 'require',
];
$this->validate($arrData, $arrRule);
$arrData['au_id'] = explode(',', $arrData['au_id']);
AdminUserModel::whereIn('au_id', $arrData['au_id'])->delete();
return $this->success();
}
}

View File

@@ -0,0 +1,359 @@
<?php
declare(strict_types=1);
namespace app\admin\controller;
use app\admin\BaseController;
use app\admin\model\AdminUserModel;
use app\admin\model\FenLeiModel;
use app\admin\model\GuangGaoFenZuModel;
use app\admin\model\GuangGaoModel;
use app\admin\model\GuangGaoPeiZhiModel;
use app\admin\model\NvYouModel;
use app\admin\model\ShiPinModel;
use app\Request;
use think\Response;
class GuangGao extends BaseController
{
/**
* 廣告分組列表
*
* @param Request $Request
* @param AdminUserModel|null $AdminUserModel
* @return Response
*/
public function getGuangGaoFenZuList(Request $Request, ?AdminUserModel $AdminUserModel)
{
$arrData = [
"page" => $Request->get('page', 1),
"limit" => $Request->get('limit', 10),
"key" => $Request->get('key'),
];
$arrRule = [
'page' => 'require|number',
'limit' => 'require|number',
];
$this->validate($arrData, $arrRule);
$GuangGaoFenZuModel = GuangGaoFenZuModel::alias('ggfz');
if (!empty($arrData['key'])) {
$strKey = $arrData['key'];
$GuangGaoFenZuModel->where(function ($GuangGaoFenZuModel) use ($strKey) {
$GuangGaoFenZuModel->whereOr([
['ggfz.ggfz_name', 'like', "%" . $strKey . "%"],
['ggfz.ggfz_id', 'like', "%" . $strKey . "%"],
]);
});
}
$Paginate = $GuangGaoFenZuModel->paginate([
'list_rows' => $arrData['limit'],
'page' => $arrData['page'],
]);
$arrResult = [
'items' => $Paginate->items(),
'total' => $Paginate->total(),
'current_page' => $Paginate->currentPage(),
'total_pages' => $Paginate->lastPage(),
];
return $this->success($arrResult);
}
/**
* 设置广告分组
*
* @param Request $Request
* @param AdminUserModel|null $AdminUserModel
* @return Response
*/
public function saveGuangGaoFenZu(Request $Request, ?AdminUserModel $AdminUserModel)
{
$arrData = [
"ggfz_id" => $Request->post('ggfz_id'),
"ggfz_code" => $Request->post('ggfz_code'),
"ggfz_name" => $Request->post('ggfz_name'),
];
$arrRule = [
// 'ggfz_id' => 'require',
'ggfz_code' => 'require',
'ggfz_name' => 'require',
];
$this->validate($arrData, $arrRule);
if ($arrData['ggfz_id'] == NULL) {
$GuangGaoFenZuModel = new GuangGaoFenZuModel;
} else {
$GuangGaoFenZuModel = GuangGaoFenZuModel::where('ggfz_id', $arrData['ggfz_id'])->find();
if (empty($GuangGaoFenZuModel)) {
return $this->error('9993');
}
}
$GuangGaoFenZuModel->fill($arrData);
$GuangGaoFenZuModel->save();
GuangGaoModel::flushCache();
return $this->success();
}
/**
* 删除广告分组
*
* @param Request $Request
* @param AdminUserModel|null $AdminUserModel
* @return Response
*/
public function delGuangGaoFenZu(Request $Request, ?AdminUserModel $AdminUserModel)
{
$arrData = [
"ggfz_id" => $Request->post('ggfz_id'),
];
$arrRule = [
'ggfz_id' => 'require',
];
$this->validate($arrData, $arrRule);
$arrData['ggfz_id'] = explode(',', $arrData['ggfz_id']);
GuangGaoFenZuModel::whereIn('ggfz_id', $arrData['ggfz_id'])->delete();
GuangGaoModel::whereIn('ggfz_id', $arrData['ggfz_id'])->delete();
GuangGaoModel::flushCache();
return $this->success();
}
/**
* 列表
*
* @param Request $Request
* @param AdminUserModel|null $AdminUserModel
* @return Response
*/
public function getGuangGaoList(Request $Request, ?AdminUserModel $AdminUserModel)
{
$arrData = [
"page" => $Request->get('page', 1),
"limit" => $Request->get('limit', 10),
"key" => $Request->get('key'),
"ggfz_id" => $Request->get('ggfz_id'),
];
$arrRule = [
'page' => 'require|number',
'limit' => 'require|number',
];
$this->validate($arrData, $arrRule);
$GuangGaoModel = GuangGaoModel::alias('gg')
->leftJoin('guang_gao_fen_zu ggfz', 'ggfz.ggfz_id = gg.ggfz_id')
->field(['gg.*', 'ggfz.ggfz_name'])
->order('ggfz.ggfz_id','desc')
->order('gg.gg_sort','desc');
if ($arrData['ggfz_id'] !== NULL) {
$GuangGaoModel = $GuangGaoModel->where('gg.ggfz_id', $arrData['ggfz_id']);
}
if (!empty($arrData['key'])) {
$strKey = $arrData['key'];
$GuangGaoModel->where(function ($GuangGaoModel) use ($strKey) {
$GuangGaoModel = $GuangGaoModel->whereOr([
['gg.gg_wen_zi', 'like', "%" . $strKey . "%"],
['gg.gg_id', 'like', "%" . $strKey . "%"],
['ggfz.ggfz_name', 'like', "%" . $strKey . "%"],
]);
});
}
$Paginate = $GuangGaoModel->paginate([
'list_rows' => $arrData['limit'],
'page' => $arrData['page'],
]);
$arrResult = [
'items' => $Paginate->items(),
'total' => $Paginate->total(),
'current_page' => $Paginate->currentPage(),
'total_pages' => $Paginate->lastPage(),
];
return $this->success($arrResult);
}
/**
* 设置广告
*
* @param Request $Request
* @param AdminUserModel|null $AdminUserModel
* @return Response
*/
public function saveGuangGao(Request $Request, ?AdminUserModel $AdminUserModel)
{
$arrData = [
"gg_id" => $Request->post('gg_id'),
"ggfz_id" => $Request->post('ggfz_id'),
"gg_lei_xing" => $Request->post('gg_lei_xing'),
"gg_tu_pian" => $Request->post('gg_tu_pian'),
"gg_wen_zi" => $Request->post('gg_wen_zi'),
"gg_tiao_zhuan_di_zhi" => $Request->post('gg_tiao_zhuan_di_zhi'),
"gg_sort" => $Request->post('gg_sort'),
"gg_status" => $Request->post('gg_status'),
];
$arrRule = [
// 'gg_id' => 'require',
'ggfz_id' => 'require',
'gg_lei_xing' => 'require',
'gg_status' => 'require',
];
$this->validate($arrData, $arrRule);
if ($arrData['gg_id'] == NULL) {
$GuangGaoModel = new GuangGaoModel;
} else {
$GuangGaoModel = GuangGaoModel::where('gg_id', $arrData['gg_id'])->find();
if (empty($GuangGaoModel)) {
return $this->error('9993');
}
}
$GuangGaoModel->fill($arrData);
$GuangGaoModel->save();
GuangGaoModel::flushCache();
return $this->success();
}
/**
* 删除广告
*
* @param Request $Request
* @param AdminUserModel|null $AdminUserModel
* @return Response
*/
public function delGuangGao(Request $Request, ?AdminUserModel $AdminUserModel)
{
$arrData = [
"gg_id" => $Request->post('gg_id'),
];
$arrRule = [
'gg_id' => 'require',
];
$this->validate($arrData, $arrRule);
$arrData['gg_id'] = explode(',', $arrData['gg_id']);
GuangGaoModel::whereIn('gg_id', $arrData['gg_id'])->delete();
GuangGaoModel::flushCache();
return $this->success();
}
/**
* 广告配置列表
*
* @param Request $Request
* @param AdminUserModel|null $AdminUserModel
* @return Response
*/
public function getGuangGaoPeiZhiList(Request $Request, ?AdminUserModel $AdminUserModel)
{
$arrData = [
"page" => $Request->get('page', 1),
"limit" => $Request->get('limit', 10),
"key" => $Request->get('key'),
];
$arrRule = [
'page' => 'require|number',
'limit' => 'require|number',
];
$this->validate($arrData, $arrRule);
$GuangGaoPeiZhiModel = GuangGaoPeiZhiModel::alias('ggpz');
if (!empty($arrData['key'])) {
$strKey = $arrData['key'];
$GuangGaoPeiZhiModel->where(function ($GuangGaoPeiZhiModel) use ($strKey) {
$GuangGaoPeiZhiModel->whereOr([
['ggpz.ggpz_code', 'like', "%" . $strKey . "%"],
['ggpz.ggpz_id', 'like', "%" . $strKey . "%"],
]);
});
}
$Paginate = $GuangGaoPeiZhiModel->paginate([
'list_rows' => $arrData['limit'],
'page' => $arrData['page'],
]);
$arrResult = [
'items' => $Paginate->items(),
'total' => $Paginate->total(),
'current_page' => $Paginate->currentPage(),
'total_pages' => $Paginate->lastPage(),
];
return $this->success($arrResult);
}
/**
* 设置广告配置
*
* @param Request $Request
* @param AdminUserModel|null $AdminUserModel
* @return Response
*/
public function saveGuangGaoPeiZhi(Request $Request, ?AdminUserModel $AdminUserModel)
{
$arrData = [
"ggpz_id" => $Request->post('ggpz_id'),
"ggpz_val" => $Request->post('ggpz_val'),
];
$arrRule = [
'ggpz_id' => 'require',
'ggpz_val' => 'require',
];
$this->validate($arrData, $arrRule);
$GuangGaoPeiZhiModel = GuangGaoPeiZhiModel::where('ggpz_id', $arrData['ggpz_id'])->find();
if (empty($GuangGaoPeiZhiModel)) {
return $this->error('9993');
}
$GuangGaoPeiZhiModel->fill($arrData);
$GuangGaoPeiZhiModel->save();
GuangGaoPeiZhiModel::flushCache();
return $this->success();
}
}

View File

@@ -0,0 +1,70 @@
<?php
declare(strict_types=1);
namespace app\admin\controller;
use app\admin\BaseController;
use app\admin\model\AdminUserModel;
use app\admin\service\AdminUserAuth;
use app\Request;
use think\facade\Cache;
use think\Response;
class Guest extends BaseController
{
/**
* 登录
*
* @param Request $Request
* @param AdminUserModel|null $AdminUserModel
* @return Response
*/
public function Login(Request $Request, ?AdminUserModel $AdminUserModel)
{
$arrData = [
"au_name" => $Request->post('au_name'),
"au_pwd" => $Request->post('au_pwd'),
];
$arrRule = [
'au_name' => 'require|max:25',
'au_pwd' => 'require|min:6',
];
$this->validate($arrData, $arrRule);
$arrData['au_pwd'] = md5(trim($arrData['au_pwd']));
$AdminUserModel = AdminUserModel::where('au_name', $arrData['au_name'])
->where('au_pwd', $arrData['au_pwd'])
->where('au_status', 0)
->find();
if (empty($AdminUserModel)) {
return $this->error("1002");
}
$arrResult = [
'admin-token' => $AdminUserModel->createToken(),
];
return $this->success($arrResult);
}
/**
* 退出
*
* @param Request $Request
* @param AdminUserModel|null $AdminUserModel
* @return Response
*/
public function Logout(Request $Request, ?AdminUserModel $AdminUserModel)
{
$strToken = $Request->header("admin-token");
if ($strToken) {
Cache::delete($strToken);
}
}
}

View File

@@ -0,0 +1,154 @@
<?php
declare(strict_types=1);
namespace app\admin\controller;
use app\admin\BaseController;
use app\admin\model\AdminUserModel;
use app\admin\model\BaoYangRuZhuModel;
use app\admin\model\CaiJiPeiZhiModel;
use app\admin\model\DaLiWanFanKuiModel;
use app\admin\model\DaLiWanModel;
use app\admin\model\GuanFangZiYingModel;
use app\admin\model\LouFengModel;
use app\admin\model\PlanTaskModel;
use app\admin\model\SeoKeywordsModel;
use app\admin\model\SystemConfigModel;
use app\admin\model\ZiYuanDuanDianModel;
use app\Request;
use think\Response;
class SeoKeywords extends BaseController
{
/**
* 系统配置列表
*
* @param Request $Request
* @param AdminUserModel|null $AdminUserModel
* @return Response
*/
public function getSeoKeywordList(Request $Request, ?AdminUserModel $AdminUserModel)
{
$arrData = [
"page" => $Request->get('page', 1),
"limit" => $Request->get('limit', 10),
"key" => $Request->get('key'),
];
$arrRule = [
'page' => 'require|number',
'limit' => 'require|number',
];
$this->validate($arrData, $arrRule);
$SeoKeywordsModel = SeoKeywordsModel::alias('sw');
if (!empty($arrData['key'])) {
$strKey = $arrData['key'];
$SeoKeywordsModel->where(function ($SeoKeywordsModel) use ($strKey) {
$SeoKeywordsModel->whereOr([
['sw.sw_title', 'like', "%" . $strKey . "%"],
['sw.sw_id', 'like', "%" . $strKey . "%"],
]);
});
}
$Paginate = $SeoKeywordsModel->paginate([
'list_rows' => $arrData['limit'],
'page' => $arrData['page'],
]);
$arrResult = [
'items' => $Paginate->items(),
'total' => $Paginate->total(),
'current_page' => $Paginate->currentPage(),
'total_pages' => $Paginate->lastPage(),
];
return $this->success($arrResult);
}
/**
* 设置系统配置
*
* @param Request $Request
* @param AdminUserModel|null $AdminUserModel
* @return Response
*/
public function saveSeoKeyword(Request $Request, ?AdminUserModel $AdminUserModel)
{
$arrData = [
"sw_id" => $Request->post('sw_id'),
"sw_title" => $Request->post('sw_title'),
"sw_status" => $Request->post('sw_status'),
"sw_html_path" => $Request->post('sw_html_path') ?? '',
];
$arrRule = [
// 'sw_id' => 'require',
'sw_title' => 'require',
];
$this->validate($arrData, $arrRule);
if ($arrData['sw_id'] == NULL) {
$SeoKeywordsModel = new SeoKeywordsModel;
} else {
$SeoKeywordsModel = SeoKeywordsModel::where('sw_id', $arrData['sw_id'])->find();
if (empty($SeoKeywordsModel)) {
return $this->error('9993');
}
}
if (isset($arrData['sw_status']) && empty($arrData['sw_status']) && $arrData['sw_status'] != 0) {
unset($arrData['sw_status']);
}
if (isset($arrData['sw_html_path']) && empty($arrData['sw_html_path'])) {
unset($arrData['sw_html_path']);
}
$SeoKeywordsModel->fill($arrData);
$SeoKeywordsModel->save();
$strHtmlPath = $SeoKeywordsModel->sw_html_path;
if(!empty($strHtmlPath)){
SeoKeywordsModel::flushCache($strHtmlPath);
}
return $this->success();
}
/**
* 删除
*
* @param Request $Request
* @param SeoKeywordsModel|null $SeoKeywordsModel
* @return Response
*/
public function delSeoKeyword(Request $Request, ?SeoKeywordsModel $SeoKeywordsModel)
{
$arrData = [
"sw_id" => $Request->post('sw_id'),
];
$arrRule = [
'sw_id' => 'require',
];
$this->validate($arrData, $arrRule);
$arrData['sw_id'] = explode(',', $arrData['sw_id']);
SeoKeywordsModel::whereIn('sw_id', $arrData['sw_id'])->delete();
return $this->success();
}
}

View File

@@ -0,0 +1,367 @@
<?php
declare(strict_types=1);
namespace app\admin\controller;
use app\admin\BaseController;
use app\admin\model\AdminUserModel;
use app\admin\model\BaoYangRuZhuModel;
use app\admin\model\CaiJiPeiZhiModel;
use app\admin\model\DaLiWanFanKuiModel;
use app\admin\model\DaLiWanModel;
use app\admin\model\GuanFangZiYingModel;
use app\admin\model\LouFengModel;
use app\admin\model\PlanTaskModel;
use app\admin\model\SystemConfigModel;
use app\admin\model\ZiYuanDuanDianModel;
use app\Request;
use think\Response;
class SystemConfig extends BaseController
{
/**
* 采集配置列表
*
* @param Request $Request
* @param AdminUserModel|null $AdminUserModel
* @return Response
*/
public function getCaiJiPeiZhiList(Request $Request, ?AdminUserModel $AdminUserModel)
{
$arrData = [
"page" => $Request->get('page', 1),
"limit" => $Request->get('limit', 10),
"key" => $Request->get('key'),
];
$arrRule = [
'page' => 'require|number',
'limit' => 'require|number',
];
$this->validate($arrData, $arrRule);
$CaiJiPeiZhiModel = CaiJiPeiZhiModel::alias('cjpz');
if (!empty($arrData['key'])) {
$strKey = $arrData['key'];
$CaiJiPeiZhiModel->where(function ($CaiJiPeiZhiModel) use ($strKey) {
$CaiJiPeiZhiModel->whereOr([
['cjpz.cjpz_code', 'like', "%" . $strKey . "%"],
['cjpz.cjpz_id', 'like', "%" . $strKey . "%"],
]);
});
}
$Paginate = $CaiJiPeiZhiModel->paginate([
'list_rows' => $arrData['limit'],
'page' => $arrData['page'],
]);
$arrResult = [
'items' => $Paginate->items(),
'total' => $Paginate->total(),
'current_page' => $Paginate->currentPage(),
'total_pages' => $Paginate->lastPage(),
];
return $this->success($arrResult);
}
/**
* 设置采集配置
*
* @param Request $Request
* @param AdminUserModel|null $AdminUserModel
* @return Response
*/
public function saveCaiJiPeiZhi(Request $Request, ?AdminUserModel $AdminUserModel)
{
$arrData = [
"cjpz_id" => $Request->post('cjpz_id'),
"cjpz_val" => $Request->post('cjpz_val'),
];
$arrRule = [
'cjpz_id' => 'require',
'cjpz_val' => 'require',
];
$this->validate($arrData, $arrRule);
$CaiJiPeiZhiModel = CaiJiPeiZhiModel::where('cjpz_id', $arrData['cjpz_id'])->find();
if (empty($CaiJiPeiZhiModel)) {
return $this->error('9993');
}
$CaiJiPeiZhiModel->fill($arrData);
$CaiJiPeiZhiModel->save();
CaiJiPeiZhiModel::flushCache();
return $this->success();
}
/**
* 系统配置列表
*
* @param Request $Request
* @param AdminUserModel|null $AdminUserModel
* @return Response
*/
public function getSystemConfigList(Request $Request, ?AdminUserModel $AdminUserModel)
{
$arrData = [
"page" => $Request->get('page', 1),
"limit" => $Request->get('limit', 10),
"key" => $Request->get('key'),
];
$arrRule = [
'page' => 'require|number',
'limit' => 'require|number',
];
$this->validate($arrData, $arrRule);
$SystemConfigModel = SystemConfigModel::alias('sc');
if (!empty($arrData['key'])) {
$strKey = $arrData['key'];
$SystemConfigModel->where(function ($SystemConfigModel) use ($strKey) {
$SystemConfigModel->whereOr([
['sc.sc_code', 'like', "%" . $strKey . "%"],
['sc.sc_id', 'like', "%" . $strKey . "%"],
]);
});
}
$Paginate = $SystemConfigModel->paginate([
'list_rows' => $arrData['limit'],
'page' => $arrData['page'],
]);
$arrResult = [
'items' => $Paginate->items(),
'total' => $Paginate->total(),
'current_page' => $Paginate->currentPage(),
'total_pages' => $Paginate->lastPage(),
];
return $this->success($arrResult);
}
/**
* 设置系统配置
*
* @param Request $Request
* @param AdminUserModel|null $AdminUserModel
* @return Response
*/
public function saveSystemConfig(Request $Request, ?AdminUserModel $AdminUserModel)
{
$arrData = [
"sc_id" => $Request->post('sc_id'),
"sc_val" => $Request->post('sc_val'),
];
$arrRule = [
'sc_id' => 'require',
'sc_val' => 'require',
];
$this->validate($arrData, $arrRule);
$SystemConfigModel = SystemConfigModel::where('sc_id', $arrData['sc_id'])->find();
if (empty($SystemConfigModel)) {
return $this->error('9993');
}
$SystemConfigModel->fill($arrData);
$SystemConfigModel->save();
SystemConfigModel::flushCache();
return $this->success();
}
/**
* 资源端点列表
*
* @param Request $Request
* @param AdminUserModel|null $AdminUserModel
* @return Response
*/
public function getZiYuanDuanDianList(Request $Request, ?AdminUserModel $AdminUserModel)
{
$arrData = [
"page" => $Request->get('page', 1),
"limit" => $Request->get('limit', 10),
"key" => $Request->get('key'),
];
$arrRule = [
'page' => 'require|number',
'limit' => 'require|number',
];
$this->validate($arrData, $arrRule);
$ZiYuanDuanDianModel = ZiYuanDuanDianModel::alias('zydd');
if (!empty($arrData['key'])) {
$strKey = $arrData['key'];
$ZiYuanDuanDianModel->where(function ($ZiYuanDuanDianModel) use ($strKey) {
$ZiYuanDuanDianModel->whereOr([
['zydd.zydd_ming_zi', 'like', "%" . $strKey . "%"],
['zydd.zydd_id', 'like', "%" . $strKey . "%"],
]);
});
}
$Paginate = $ZiYuanDuanDianModel->paginate([
'list_rows' => $arrData['limit'],
'page' => $arrData['page'],
]);
$arrResult = [
'items' => $Paginate->items(),
'total' => $Paginate->total(),
'current_page' => $Paginate->currentPage(),
'total_pages' => $Paginate->lastPage(),
];
return $this->success($arrResult);
}
/**
* 设置资源端点
*
* @param Request $Request
* @param AdminUserModel|null $AdminUserModel
* @return Response
*/
public function saveZiYuanDuanDian(Request $Request, ?AdminUserModel $AdminUserModel)
{
$arrData = [
"zydd_id" => $Request->post('zydd_id'),
"zydd_domain" => $Request->post('zydd_domain'),
];
$arrRule = [
'zydd_id' => 'require',
'zydd_domain' => 'require',
];
$this->validate($arrData, $arrRule);
$ZiYuanDuanDianModel = ZiYuanDuanDianModel::where('zydd_id', $arrData['zydd_id'])->find();
if (empty($ZiYuanDuanDianModel)) {
return $this->error('9993');
}
$ZiYuanDuanDianModel->fill($arrData);
$ZiYuanDuanDianModel->save();
ZiYuanDuanDianModel::flushCache();
return $this->success();
}
/**
* 计划任务列表
*
* @param Request $Request
* @param AdminUserModel|null $AdminUserModel
* @return Response
*/
public function getPlanTaskList(Request $Request, ?AdminUserModel $AdminUserModel)
{
$arrData = [
"page" => $Request->get('page', 1),
"limit" => $Request->get('limit', 10),
"key" => $Request->get('key'),
];
$arrRule = [
'page' => 'require|number',
'limit' => 'require|number',
];
$this->validate($arrData, $arrRule);
$PlanTaskModel = PlanTaskModel::alias('pt');
if (!empty($arrData['key'])) {
$strKey = $arrData['key'];
$PlanTaskModel->where(function ($PlanTaskModel) use ($strKey) {
$PlanTaskModel->whereOr([
['pt.pt_name', 'like', "%" . $strKey . "%"],
['pt.pt_id', 'like', "%" . $strKey . "%"],
]);
});
}
$Paginate = $PlanTaskModel->paginate([
'list_rows' => $arrData['limit'],
'page' => $arrData['page'],
]);
$arrResult = [
'items' => $Paginate->items(),
'total' => $Paginate->total(),
'current_page' => $Paginate->currentPage(),
'total_pages' => $Paginate->lastPage(),
];
return $this->success($arrResult);
}
/**
* 设置计划任务
*
* @param Request $Request
* @param AdminUserModel|null $AdminUserModel
* @return Response
*/
public function savePlanTask(Request $Request, ?AdminUserModel $AdminUserModel)
{
$arrData = [
"pt_id" => $Request->post('pt_id'),
"pt_enable" => $Request->post('pt_enable'),
"pt_limit" => $Request->post('pt_limit'),
];
$arrRule = [
'pt_id' => 'require|number',
'pt_enable' => 'require|number',
'pt_limit' => 'require|number',
];
$this->validate($arrData, $arrRule);
$PlanTaskModel = PlanTaskModel::where('pt_id', $arrData['pt_id'])->find();
if (empty($PlanTaskModel)) {
return $this->error('9993');
}
$PlanTaskModel->fill($arrData);
$PlanTaskModel->save();
return $this->success();
}
}

View File

@@ -0,0 +1,179 @@
<?php
declare(strict_types=1);
namespace app\admin\controller;
use app\admin\BaseController;
use app\admin\model\AdminUserModel;
use app\admin\model\BaoYangRuZhuModel;
use app\admin\model\DaLiWanFanKuiModel;
use app\admin\model\DaLiWanModel;
use app\admin\model\GuanFangZiYingModel;
use app\admin\model\LouFengModel;
use app\admin\model\SystemConfigModel;
use app\admin\model\XiaoShuoFenLeiModel;
use app\admin\model\XiaoShuoModel;
use app\admin\model\XiaoShuoXiangQingModel;
use app\admin\model\XiaoShuoZhangJieModel;
use app\Request;
use think\Response;
class XiaoShuo extends BaseController
{
/**
* 列表
*
* @param Request $Request
* @param AdminUserModel|null $AdminUserModel
* @return Response
*/
public function getXiaoShuoList(Request $Request, ?AdminUserModel $AdminUserModel)
{
$arrData = [
"page" => $Request->get('page', 1),
"limit" => $Request->get('limit', 10),
"key" => $Request->get('key'),
];
$arrRule = [
'page' => 'require|number',
'limit' => 'require|number',
];
$this->validate($arrData, $arrRule);
$XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::alias('xsxq');
if (!empty($arrData['key'])) {
$strKey = $arrData['key'];
$XiaoShuoXiangQingModel->where(function ($XiaoShuoXiangQingModel) use ($strKey) {
$XiaoShuoXiangQingModel->whereOr([
['xsxq.xsxq_ming_zi', 'like', "%" . $strKey . "%"],
['xsxq.xsxq_id', 'like', "%" . $strKey . "%"],
]);
});
}
$Paginate = $XiaoShuoXiangQingModel->paginate([
'list_rows' => $arrData['limit'],
'page' => $arrData['page'],
]);
$arrResult = [
'items' => $Paginate->items(),
'total' => $Paginate->total(),
'current_page' => $Paginate->currentPage(),
'total_pages' => $Paginate->lastPage(),
];
return $this->success($arrResult);
}
/**
* 列表
*
* @param Request $Request
* @param AdminUserModel|null $AdminUserModel
* @return Response
*/
public function getXiaoShuoZhangJieList(Request $Request, ?AdminUserModel $AdminUserModel)
{
$arrData = [
"page" => $Request->get('page', 1),
"limit" => $Request->get('limit', 10),
"key" => $Request->get('key'),
"xsxq_id" => $Request->get('xsxq_id'),
];
$arrRule = [
'page' => 'require|number',
'limit' => 'require|number',
];
$this->validate($arrData, $arrRule);
$XiaoShuoZhangJieModel = XiaoShuoZhangJieModel::alias('xszj');
if (!empty($arrData['key'])) {
$strKey = $arrData['key'];
$XiaoShuoZhangJieModel->where(function ($XiaoShuoZhangJieModel) use ($strKey) {
$XiaoShuoZhangJieModel->whereOr([
['xszj.xszj_ming_zi', 'like', "%" . $strKey . "%"],
['xszj.xszj_id', 'like', "%" . $strKey . "%"],
]);
});
}
if (!empty($arrData['xsxq_id'])) {
$XiaoShuoZhangJieModel->where('xszj.xsxq_id', $arrData['xsxq_id']);
$XiaoShuoZhangJieModel->order('xszj.xszj_pai_xu', 'desc');
}
$Paginate = $XiaoShuoZhangJieModel->paginate([
'list_rows' => $arrData['limit'],
'page' => $arrData['page'],
]);
$arrResult = [
'items' => $Paginate->items(),
'total' => $Paginate->total(),
'current_page' => $Paginate->currentPage(),
'total_pages' => $Paginate->lastPage(),
];
return $this->success($arrResult);
}
/**
* 列表
*
* @param Request $Request
* @param AdminUserModel|null $AdminUserModel
* @return Response
*/
public function getXiaoShuoFenLeiList(Request $Request, ?AdminUserModel $AdminUserModel)
{
$arrData = [
"page" => $Request->get('page', 1),
"limit" => $Request->get('limit', 10),
"key" => $Request->get('key'),
];
$arrRule = [
'page' => 'require|number',
'limit' => 'require|number',
];
$this->validate($arrData, $arrRule);
$XiaoShuoFenLeiModel = XiaoShuoFenLeiModel::alias('xsfl');
if (!empty($arrData['key'])) {
$strKey = $arrData['key'];
$XiaoShuoFenLeiModel->where(function ($XiaoShuoFenLeiModel) use ($strKey) {
$XiaoShuoFenLeiModel->whereOr([
['xsfl.xsfl_name', 'like', "%" . $strKey . "%"],
['xsfl.xsfl_id', 'like', "%" . $strKey . "%"],
]);
});
}
$Paginate = $XiaoShuoFenLeiModel->paginate([
'list_rows' => $arrData['limit'],
'page' => $arrData['page'],
]);
$arrResult = [
'items' => $Paginate->items(),
'total' => $Paginate->total(),
'current_page' => $Paginate->currentPage(),
'total_pages' => $Paginate->lastPage(),
];
return $this->success($arrResult);
}
}

View File

@@ -1,5 +1,9 @@
<?php
// 这是系统自动生成的middleware定义文件
return [
use app\admin\middleware\AdminCors;
return [
AdminCors::class,
think\middleware\LoadLangPack::class,
];

View File

@@ -0,0 +1,34 @@
<?php
declare(strict_types=1);
namespace app\admin\middleware;
use app\admin\model\AdminUserModel;
use app\admin\service\AdminUserAuth;
use app\Request;
use think\Response;
use think\facade\App;
class AdminAuth
{
public function handle(Request $Request, \Closure $Next)
{
try {
/**@var AdminUserModel */
$AdminUserModel = App::make(AdminUserModel::class);
if (empty($AdminUserModel)) {
return AdminResponse::create("9995");
}
if ($AdminUserModel->checkRole() == false) {
return AdminResponse::create("9991");
}
return $Next($Request);
} catch (\Throwable $T) {
return AdminResponse::createByErr($T);
}
}
}

View File

@@ -0,0 +1,22 @@
<?php
declare(strict_types=1);
namespace app\admin\middleware;
use app\admin\model\AdminUserModel;
use app\admin\service\AdminUserAuth;
use app\Request;
use think\Response;
use think\facade\App;
use think\middleware\AllowCrossDomain;
class AdminCors extends AllowCrossDomain
{
protected $header = [
'Access-Control-Allow-Credentials' => 'true',
'Access-Control-Max-Age' => 3600,
'Access-Control-Allow-Methods' => '*',
'Access-Control-Allow-Headers' => '*',
];
}

View File

@@ -0,0 +1,17 @@
<?php
declare(strict_types=1);
namespace app\admin\middleware;
use think\exception\Handle;
use think\Response;
use Throwable;
class AdminException extends Handle
{
public function render($Request, Throwable $T): Response
{
return AdminResponse::createByErr($T);
}
}

View File

@@ -0,0 +1,57 @@
<?php
declare(strict_types=1);
namespace app\admin\middleware;
use app\admin\service\AdminUserAuth;
use app\Request;
use think\exception\ValidateException;
use think\Response;
use think\facade\App;
class AdminResponse
{
static public function create($strCode = "1000", $strMsg = "", $arrData = [])
{
$arrError = config("error");
if (!key_exists($strCode, $arrError)) {
$strCode = "9999";
}
if ($strMsg == "") {
$strMsg = $arrError[$strCode];
}
$arrData = [
'code' => $strCode,
'msg' => $strMsg,
'data' => $arrData,
];
return json($arrData, 200, [], [JSON_UNESCAPED_UNICODE]);
}
static $arrExceptionMap = [
ValidateException::class => "9900",
];
static public function createByErr(\Throwable $T)
{
$strCode = $T->getCode();
if ($T instanceof ValidateException) {
$strCode = "9900";
}
foreach (self::$arrExceptionMap as $ExceptionClass => $strExceptonCode) {
if ($T instanceof $ExceptionClass) {
$strCode = $strExceptonCode;
break;
}
}
return self::create($strCode, $T->getMessage());
}
}

View File

@@ -0,0 +1,15 @@
<?php
declare (strict_types = 1);
namespace app\admin\model;
use think\Model;
/**
* @mixin think\Model
*/
class AdminNodeModel extends BaseModel
{
protected $name = 'admin_node';
protected $pk = 'an_id';
}

View File

@@ -0,0 +1,47 @@
<?php
declare (strict_types = 1);
namespace app\admin\model;
use think\Model;
use think\facade\Cache;
/**
* @mixin think\Model
*/
class AdminRoleModel extends BaseModel
{
protected $name = 'admin_role';
protected $pk = 'ar_id';
public function getRoleNodeByCache($intArId)
{
$strCacheKey = 'Admin:Role:Node:'.$intArId;
$arrRoleNode = Cache::get($strCacheKey);
if($arrRoleNode === NULL)
{
$AdminRoleNodeModel = new AdminRoleNodeModel();
$AdminRoleNodeModel = $AdminRoleNodeModel->alias('arn')
->field('an.an_url')
->leftjoin('admin_node an','an.an_id=arn.an_id');
if($intArId>0)
{
$AdminRoleNodeModel = $AdminRoleNodeModel->where('arn.ar_id',$intArId);
}
$arrRoleNode = $AdminRoleNodeModel
->select()
->toArray();
$arrRoleNode = array_column($arrRoleNode, 'an_url');
// Cache::set($strCacheKey,$arrRoleNode);
}
return $arrRoleNode;
}
}

View File

@@ -0,0 +1,15 @@
<?php
declare (strict_types = 1);
namespace app\admin\model;
use think\Model;
/**
* @mixin think\Model
*/
class AdminRoleNodeModel extends BaseModel
{
protected $name = 'admin_role_node';
protected $pk = 'arn_id';
}

View File

@@ -0,0 +1,92 @@
<?php
declare(strict_types=1);
namespace app\admin\model;
use think\facade\Cache;
use think\Model;
use think\helper\Str;
/**
* @mixin think\Model
*/
class AdminUserModel extends BaseModel
{
protected $name = 'admin_user';
protected $pk = 'au_id';
protected $token = "";
// protected $hidden = ['au_pwd'];
/**
* Undocumented function
*
* @param string $strToken
* @return AdminUserModel|NULL
*/
static public function getUserByToken($strToken = "")
{
if ($strToken == "") {
return NULL;
}
$strKey = "AdminToken:" . $strToken;
$intAuId = Cache::get($strKey);
if (empty($intAuId)) {
return NULL;
}
$AdminUserModel = self::getAdminUserModelByCache($intAuId);
$AdminUserModel->token = $strToken;
$AdminUserModel->refreshToken();
return $AdminUserModel;
}
public function checkRole()
{
return true;
}
public function createToken()
{
$this->token = Str::random(32);
$this->refreshToken();
return $this->token;
}
public function refreshToken()
{
Cache::set("AdminToken:" . $this->token, $this->au_id, 3600);
}
static public function getAdminUserModelByCache($intAuId, $boolForceUpdate = false)
{
$strKey = "AdminUser:Id:" . $intAuId;
$AdminUserModel = Cache::get($strKey);
if (empty($AdminUserModel) || $boolForceUpdate == true) {
$AdminUserModel = self::where('au_id', $intAuId)->find();
if (empty($AdminUserModel)) {
throw new \Exception("用户不存在", 9999);
}
$AdminUserModel->au_pwd = "";
Cache::set($strKey, $AdminUserModel);
}
return $AdminUserModel;
}
}

View File

@@ -0,0 +1,21 @@
<?php
declare(strict_types=1);
namespace app\admin\model;
use think\facade\Cache;
use think\Model;
/**
* @mixin think\Model
*/
class BaseModel extends Model
{
public function fill($arrData)
{
foreach ($arrData as $strKey => $strVal) {
$this->$strKey = $strVal;
}
}
}

View File

@@ -0,0 +1,39 @@
<?php
declare(strict_types=1);
namespace app\admin\model;
use think\facade\Cache;
use think\Model;
/**
* @mixin think\Model
*/
class CaiJiPeiZhiModel extends BaseModel
{
protected $name = 'cai_ji_pei_zhi';
protected $pk = 'cjpz_id';
static $arrCaiJiPeiZhi = [];
static public function getValByCode($strCode)
{
if (self::$arrCaiJiPeiZhi == []) {
self::$arrCaiJiPeiZhi = Cache::get('CaiJiPeiZhi');
if (empty(self::$arrCaiJiPeiZhi)) {
self::flushCache();
self::$arrCaiJiPeiZhi = Cache::get('CaiJiPeiZhi');
}
}
return self::$arrCaiJiPeiZhi[$strCode] ?? NULL;
}
static public function flushCache()
{
$arrCaiJiPeiZhi = self::column('cjpz_val', 'cjpz_code');
Cache::tag('CaiJiPeiZhi')->set('CaiJiPeiZhi', $arrCaiJiPeiZhi);
}
}

View File

@@ -0,0 +1,70 @@
<?php
declare(strict_types=1);
namespace app\admin\model;
use think\facade\Cache;
use think\Model;
/**
* @mixin think\Model
*/
class ContentStatsModel extends BaseModel
{
protected $name = 'content_stats';
protected $pk = 'cs_id';
static public function add($arrData): self
{
try {
$existingRecord = self::where([
'cs_md5' => $arrData['cs_md5'],
])->find();
if ($existingRecord) {
$existingRecord->cs_pv = $arrData['cs_pv'];
$existingRecord->cs_uv = $arrData['cs_uv'];
$existingRecord->updated_at = $arrData['updated_at'];
// 尝试保存数据
if (!$existingRecord->save()) {
// 数据保存失败,抛出异常
throw new \RuntimeException('数据保存失败:' . json_encode($data, JSON_UNESCAPED_UNICODE));
}
// 返回模型实例
return $existingRecord;
// return $existingRecord->save();
// return $existingRecord->save($arrData);
} else {
// 插入新记录
return self::create($arrData);
}
// if ($existingRecord) {
// // 更新记录
// $existingRecord->cs_pv = $arrData['cs_pv'];
// $existingRecord->cs_uv = $arrData['cs_uv'];
// $existingRecord->updated_at = $arrData['updated_at'];
// $existingRecord->save();
// return $existingRecord;
// }
// $existingRecord = new self;
// $existingRecord->save($arrData);
// return $existingRecord;
} catch (\Throwable $T) {
var_dump($T);
if ($T->getCode() == 10501) {
//return self::addVideoLeixin($arrData);
}
throw $T;
}
}
}

View File

@@ -0,0 +1,133 @@
<?php
declare(strict_types=1);
namespace app\admin\model;
use think\facade\Cache;
use think\Model;
/**
* @mixin think\Model
*/
class FenLeiModel extends BaseModel
{
protected $name = 'fen_lei';
protected $pk = 'fl_id';
static $arrFenLei = [];
static public function addFenlei($intFlId, $intFlPId = 0, $strFlMingzi)
{
try {
if (empty($strFlMingzi)) {
return true;
}
$FenLeiModel = self::where('fl_pid',$intFlPId)->where('fl_mingzi',$strFlMingzi)->find();
if ($FenLeiModel) {
return $FenLeiModel;
}
$FenLeiModel = new self;
// $FenLeiModel->fl_id = $intFlId;
$FenLeiModel->fl_leixing = 0;
$FenLeiModel->fl_pid = $intFlPId;
$FenLeiModel->fl_mingzi = $strFlMingzi;
$FenLeiModel->save();
return $FenLeiModel;
} catch (\Throwable $T) {
}
if ($T->getCode() == 10501) {
return self::addFenlei($intFlId, $intFlPId, $strFlMingzi);
}
throw $T;
}
static public function getAllFenLei()
{
if (self::$arrFenLei == []) {
self::$arrFenLei = Cache::get('SHI_PIN_FEN_LEI');
if (empty(self::$arrFenLei)) {
self::flushCache();
self::$arrFenLei = Cache::get('SHI_PIN_FEN_LEI');
}
}
return self::$arrFenLei;
}
static public function getFenLeiByFlId($intFlId)
{
if (self::$arrFenLei == []) {
self::$arrFenLei = Cache::get('SHI_PIN_FEN_LEI');
if (empty(self::$arrFenLei)) {
self::flushCache();
self::$arrFenLei = Cache::get('SHI_PIN_FEN_LEI');
}
}
return self::$arrFenLei[$intFlId] ?? "";
}
static public function getFenLeiMingZiByFlId($intFlId, $intZiFlId)
{
if (empty(self::$arrFenLei)) {
self::$arrFenLei = Cache::get('SHI_PIN_FEN_LEI');
if (empty(self::$arrFenLei)) {
self::flushCache();
self::$arrFenLei = Cache::get('SHI_PIN_FEN_LEI');
}
}
if ($intZiFlId == 0) {
return self::$arrFenLei[$intFlId]['fl_mingzi'] ?? '';
} else {
$arrFenLeiChild = self::$arrFenLei[$intFlId]['child'] ?? [];
foreach ($arrFenLeiChild as $item) {
if ($item['fl_id'] == $intZiFlId) {
return $item['fl_mingzi'];
}
}
}
return '';
}
static public function flushCache()
{
// $arrFenLei = self::select()->toArray();
$arrFenLei = self::order('fl_sort', 'desc')->select()->toArray();
$arrFenLei = static::buildTree($arrFenLei);
Cache::tag('SHI_PIN_FEN_LEI')->set('SHI_PIN_FEN_LEI', $arrFenLei);
}
static public function buildTree(array $elements, $parentId = 0)
{
$branch = [];
foreach ($elements as $element) {
if ($element['fl_pid'] == $parentId) {
$children = static::buildTree($elements, $element['fl_id']);
if ($children) {
$element['child'] = $children;
}
$branch[$element['fl_id']] = $element;
}
}
return $branch;
}
}

View File

@@ -0,0 +1,76 @@
<?php
declare(strict_types=1);
namespace app\admin\model;
use think\facade\Cache;
use think\Model;
/**
* @mixin think\Model
*/
class GanRaoMaModel extends BaseModel
{
protected $name = 'gan_rao_ma';
protected $pk = 'grm_id';
/**
* 根据 HTML 路径获取值
* @param string $strHtmlPath
* @return string|null
*/
public static function getValByHtmlPath(string $strHtmlPath): ?string
{
// 尝试从缓存获取
$strGamRaoMa = Cache::get($strHtmlPath);
// 如果缓存不存在,刷新缓存
if (!$strGamRaoMa) {
self::flushCache($strHtmlPath);
$strGamRaoMa = Cache::get($strHtmlPath);
}
return $strGamRaoMa ?? null;
}
/**
* 刷新缓存数据
* @param string $strHtmlPath
* @return void
*/
public static function flushCache(string $strHtmlPath): void
{
// 初始化缓存
$cache = Cache::tag('GanRaoMaTag');
// 尝试从数据库获取
$strGamRaoMa = self::where('site_id',config('app.default_app_id'))->where('grm_html_path', $strHtmlPath)->value('grm_neirong');
if ($strGamRaoMa) {
// 设置缓存
$cache->set($strHtmlPath, $strGamRaoMa);
} else {
// 如果数据库中没有记录,生成随机内容
$fallbackKeyword = generateRandomHTML(3, 15);
if ($fallbackKeyword) {
$arrGanRaoMa = [
'grm_html_path' => $strHtmlPath,
'grm_neirong' => $fallbackKeyword,
'site_id' => config('app.default_app_id'),
];
// 保存到数据库
$model = new self();
$model->save($arrGanRaoMa);
// 设置缓存
$cache->set($strHtmlPath, $fallbackKeyword);
} else {
// 记录日志或其他处理逻辑
// Log::warning("No fallback keyword generated for path: $strHtmlPath");
}
}
}
}

View File

@@ -0,0 +1,17 @@
<?php
declare(strict_types=1);
namespace app\admin\model;
use think\facade\Cache;
use think\Model;
/**
* @mixin think\Model
*/
class GuangGaoFenZuModel extends BaseModel
{
protected $name = 'guang_gao_fen_zu';
protected $pk = 'ggfz_id';
}

View File

@@ -0,0 +1,59 @@
<?php
declare(strict_types=1);
namespace app\admin\model;
use think\facade\Cache;
use think\Model;
/**
* @mixin think\Model
*/
class GuangGaoModel extends BaseModel
{
protected $name = 'guang_gao';
protected $pk = 'gg_id';
static $arrGuangGao = [];
static public function getAdByCache($strGGFZCode, $intLimit = 10, $strSort = 'asc')
{
if (self::$arrGuangGao == []) {
self::$arrGuangGao = Cache::get('GuangGao');
if (empty(self::$arrGuangGao)) {
self::flushCache();
self::$arrGuangGao = Cache::get('GuangGao');
}
}
$arrTemp = self::$arrGuangGao[$strGGFZCode] ?? [];
if ($strSort == 'rand') {
shuffle($arrTemp);
}
return array_slice($arrTemp, 0, $intLimit);
}
static public function flushCache()
{
$arrGuangGao = [];
$arrGGFZCode = GuangGaoFenZuModel::column('ggfz_code', 'ggfz_id');
$GuangGaoModelAll = self::where('site_id',config('app.default_app_id'))->order('ggfz_id', 'desc')->order('gg_sort', 'desc')->where('gg_status',0)->select();
foreach ($GuangGaoModelAll as $GuangGaoModel) {
$GuangGaoModel->ggfz_id;
if (!key_exists($GuangGaoModel->ggfz_id, $arrGGFZCode)) {
continue;
}
if (!key_exists($arrGGFZCode[$GuangGaoModel->ggfz_id], $arrGuangGao)) {
$arrGuangGao[$arrGGFZCode[$GuangGaoModel->ggfz_id]] = [];
}
$arrGuangGao[$arrGGFZCode[$GuangGaoModel->ggfz_id]][] = $GuangGaoModel->toArray();
}
Cache::tag('GuangGao')->set('GuangGao', $arrGuangGao);
}
}

View File

@@ -0,0 +1,40 @@
<?php
declare(strict_types=1);
namespace app\admin\model;
use think\facade\Cache;
use think\Model;
/**
* @mixin think\Model
*/
class GuangGaoPeiZhiModel extends BaseModel
{
protected $name = 'guang_gao_pei_zhi';
protected $pk = 'ggpz_id';
static $arrGuangGaoPeiZhi = [];
static public function getValByCode($strCode)
{
self::flushCache();
if (self::$arrGuangGaoPeiZhi == []) {
self::$arrGuangGaoPeiZhi = Cache::get('GuangGaoPeiZhi');
if (empty(self::$arrGuangGaoPeiZhi)) {
self::flushCache();
self::$arrGuangGaoPeiZhi = Cache::get('GuangGaoPeiZhi');
}
}
return self::$arrGuangGaoPeiZhi[$strCode] ?? NULL;
}
static public function flushCache()
{
$arrGuangGaoPeiZhi = self::column('ggpz_val', 'ggpz_code');
Cache::tag('GuangGaoPeiZhi')->set('GuangGaoPeiZhi', $arrGuangGaoPeiZhi);
}
}

View File

@@ -0,0 +1,41 @@
<?php
declare(strict_types=1);
namespace app\admin\model;
use think\facade\Cache;
use think\Model;
/**
* @mixin think\Model
*/
class NovelInfoSeoModel extends BaseModel
{
protected $name = 'novel_info_seo';
protected $pk = 'nis_id';
static public function addXiaoShuoXiangQingSeo($arrData): self
{
try {
$XiaoShuoXiangQingModel = self::where('nis_md5', $arrData['nis_md5'])->find();
if ($XiaoShuoXiangQingModel) {
return $XiaoShuoXiangQingModel;
}
$XiaoShuoXiangQingModel = new self;
$XiaoShuoXiangQingModel->save($arrData);
return $XiaoShuoXiangQingModel;
} catch (\Throwable $T) {
throw $T; // 超过重试次数后抛出异常
}
}
}

View File

@@ -0,0 +1,17 @@
<?php
declare(strict_types=1);
namespace app\admin\model;
use think\facade\Cache;
use think\Model;
/**
* @mixin think\Model
*/
class PlanTaskModel extends BaseModel
{
protected $name = 'plan_task';
protected $pk = 'pt_id';
}

View File

@@ -0,0 +1,42 @@
<?php
declare(strict_types=1);
namespace app\admin\model;
use think\Model;
/**
* @mixin think\Model
*/
class ReSouFenZuModel extends BaseModel
{
protected $name = 're_sou_fen_zu';
protected $pk = 'rsfz_id';
static public function addReSouFenZu($arrData): self
{
try {
if (empty($arrData['rsfz_ming_zi'])) {
throw new \Exception("热搜词分组信息不全!");
}
$ReSouFenZuModel = self::where('rsfz_ming_zi', $arrData['rsfz_ming_zi'])->find();
if ($ReSouFenZuModel) {
return $ReSouFenZuModel;
}
$ReSouFenZuModel = new self;
$ReSouFenZuModel->rsfz_pai_xu = $arrData['rsfz_pai_xu'];
$ReSouFenZuModel->rsfz_ming_zi = $arrData['rsfz_ming_zi'];
$ReSouFenZuModel->save();
return $ReSouFenZuModel;
} catch (\Throwable $T) {
if ($T->getCode() == 10501) {
return self::addReSouFenZu($arrData);
}
throw $T;
}
}
}

View File

@@ -0,0 +1,101 @@
<?php
declare(strict_types=1);
namespace app\admin\model;
use think\facade\Cache;
use think\Model;
/**
* @mixin think\Model
*/
class ReSouModel extends BaseModel
{
protected $name = 're_sou';
protected $pk = 'rs_id';
static $arrReSou = [];
static $arrReSouAll = [];
static public function addReSou($arrData): self
{
try {
if (empty($arrData['rs_ming_zi']) || empty($arrData['rsfz_id'])) {
throw new \Exception("热搜词信息不全!");
}
$ReSouModel = self::where('rs_ming_zi', $arrData['rs_ming_zi'])->find();
if ($ReSouModel) {
return $ReSouModel;
}
$ReSouModel = new self;
$ReSouModel->rs_ming_zi = $arrData['rs_ming_zi'];
$ReSouModel->rsfz_id = $arrData['rsfz_id'];
$ReSouModel->save();
return $ReSouModel;
} catch (\Throwable $T) {
if ($T->getCode() == 10501) {
return self::addReSou($arrData);
}
throw $T;
}
}
static public function getRsByCache($intLimit = 2, $strSort = 'asc')
{
if (self::$arrReSouAll == []) {
self::$arrReSouAll = Cache::get('RE_SOU_ALL');
if (empty(self::$arrReSouAll)) {
self::flushCache();
self::$arrReSouAll = Cache::get('RE_SOU_ALL');
}
}
$arrTemp = self::$arrReSouAll;
if ($strSort == 'rand') {
shuffle($arrTemp);
}
return array_slice($arrTemp, 0, $intLimit);
}
static public function getFomartDataByCache()
{
if (self::$arrReSou == []) {
self::$arrReSou = Cache::get('RE_SOU');
if (empty(self::$arrReSou)) {
self::flushCache();
self::$arrReSou = Cache::get('RE_SOU');
}
}
return self::$arrReSou;
}
static public function flushCache()
{
$FenZu = ReSouFenZuModel::order('rsfz_pai_xu', 'asc')->select();
$arrFenZu = [];
foreach ($FenZu as $FenZuOne) {
$arrFenZu[$FenZuOne->rsfz_id] = $FenZuOne->toArray();
$arrFenZu[$FenZuOne->rsfz_id]['child'] = [];
}
$ReSou = self::select();
foreach ($ReSou as $ReSouOne) {
$arrFenZu[$ReSouOne->rsfz_id]['child'][] = $ReSouOne->toArray();
}
$arrFenZu = array_values($arrFenZu);
Cache::tag('RE_SOU')->set('RE_SOU', $arrFenZu);
Cache::tag('RE_SOU_ALL')->set('RE_SOU_ALL', $ReSou->toArray());
}
}

View File

@@ -0,0 +1,63 @@
<?php
declare(strict_types=1);
namespace app\admin\model;
use think\facade\Cache;
use think\Model;
/**
* @mixin think\Model
*/
class SeoHtmlModel extends BaseModel
{
protected $name = 'seo_html';
protected $pk = 'sh_id';
public static function getValByHtmlPath(string $strHtmlPath): ?string
{
$strSeoKeyword = Cache::get($strHtmlPath);;
if (!$strSeoKeyword) {
self::flushCache($strHtmlPath);
$strSeoKeyword = Cache::get($strHtmlPath);;
}
return $strSeoKeyword ?? null;
}
public static function flushCache(string $strHtmlPath): void
{
$cache = Cache::tag('SeoKeywords');
$strSeoKeyword = self::where('sw_html_path', $strHtmlPath)->value('sw_title');
if ($strSeoKeyword) {
$cache->set($strHtmlPath, $strSeoKeyword);
} else {
$fallbackKeyword = self::where('sw_status', 0)->find();
if ($fallbackKeyword) {
$fallbackKeyword->sw_status = 1;
$fallbackKeyword->sw_html_path = $strHtmlPath;
$fallbackKeyword->save();
$cache->set($strHtmlPath, $fallbackKeyword->sw_title);
} else {
// 如果连 sw_status = 0 的记录也没有,则可以选择不进行任何操作或进行日志记录
// Log::warning("No available SEO keyword found for fallback.");
}
}
}
}

View File

@@ -0,0 +1,63 @@
<?php
declare(strict_types=1);
namespace app\admin\model;
use think\facade\Cache;
use think\Model;
/**
* @mixin think\Model
*/
class SeoKeywordsModel extends BaseModel
{
protected $name = 'seo_keywords';
protected $pk = 'sw_id';
public static function getValByHtmlPath(string $strHtmlPath): ?string
{
$strSeoKeyword = Cache::get($strHtmlPath);;
if (!$strSeoKeyword) {
self::flushCache($strHtmlPath);
$strSeoKeyword = Cache::get($strHtmlPath);;
}
return $strSeoKeyword ?? null;
}
public static function flushCache(string $strHtmlPath): void
{
$cache = Cache::tag('SeoKeywords');
$strSeoKeyword = self::where('sw_html_path', $strHtmlPath)->value('sw_title');
if ($strSeoKeyword) {
$cache->set($strHtmlPath, $strSeoKeyword);
} else {
$fallbackKeyword = self::where('sw_status', 0)->find();
if ($fallbackKeyword) {
$fallbackKeyword->sw_status = 1;
$fallbackKeyword->sw_html_path = $strHtmlPath;
$fallbackKeyword->save();
$cache->set($strHtmlPath, $fallbackKeyword->sw_title);
} else {
// 如果连 sw_status = 0 的记录也没有,则可以选择不进行任何操作或进行日志记录
// Log::warning("No available SEO keyword found for fallback.");
}
}
}
}

View File

@@ -0,0 +1,63 @@
<?php
declare(strict_types=1);
namespace app\admin\model;
use think\facade\Cache;
use think\Model;
/**
* @mixin think\Model
*/
class SeoTdkConfigModel extends BaseModel
{
protected $name = 'seo_tdk_config';
protected $pk = 'stc_id';
static $arrSeoTdkConfig = [];
static public function getValByCode($strCode)
{
//self::flushCache();
if (self::$arrSeoTdkConfig == []) {
self::$arrSeoTdkConfig = Cache::get('seoTdkTonfig');
if (empty(self::$arrSeoTdkConfig)) {
self::flushCache();
self::$arrSeoTdkConfig = Cache::get('seoTdkTonfig');
}
}
return self::$arrSeoTdkConfig[$strCode] ?? NULL;
}
// 标签使用
static public function getValByCodeByValCode($strCode,$strValCode)
{
if (self::$arrSeoTdkConfig == []) {
self::$arrSeoTdkConfig = Cache::get('seoTdkTonfig');
if (empty(self::$arrSeoTdkConfig)) {
self::flushCache();
self::$arrSeoTdkConfig = Cache::get('seoTdkTonfig');
}
}
return self::$arrSeoTdkConfig[$strCode][$strValCode] ?? NULL;
}
static public function flushCache()
{
// 查询所有数据
$arrSeoTdkConfig = self::where('site_id',config('app.default_app_id'))->select()->toArray();
// 按照 'stc_code' 进行键值映射
$arrSeoTdkConfig = array_column($arrSeoTdkConfig, null, 'stc_code');
// var_dump($arrSeoTdkConfig);
// $arrSeoTdkConfig = self::select();
// $arrSeoTdkConfig = self::column('stc_title', 'stc_code');
// var_dump($arrSeoTdkConfig);
Cache::tag('seoTdkTonfig')->set('seoTdkTonfig', $arrSeoTdkConfig);
}
}

View File

@@ -0,0 +1,82 @@
<?php
declare(strict_types=1);
namespace app\admin\model;
use think\facade\Cache;
use think\Model;
/**
* @mixin think\Model
*/
class SiteModel extends BaseModel
{
protected $name = 'site';
protected $pk = 'site_id';
static $arrSite = [];
static public function addSite($arrData): self
{
try {
$SiteModel = self::where('site_name', $arrData['site_name'])->find();
if ($SiteModel) {
return $SiteModel;
}
$SiteModel = new self;
$SiteModel->save($arrData);
return $SiteModel;
} catch (\Throwable $T) {
throw $T; // 超过重试次数后抛出异常
}
}
static public function getValById($intSiteId)
{
//self::flushCache();
if (self::$arrSite == []) {
self::$arrSite = Cache::get('site');
if (empty(self::$arrSite)) {
self::flushCache();
self::$arrSite = Cache::get('site');
}
}
return self::$arrSite[$intSiteId] ?? NULL;
}
// 标签使用
static public function getValBySiteIdByValCode($strValCode)
{
if (self::$arrSite == []) {
self::$arrSite = Cache::get('site');
if (empty(self::$arrSite)) {
self::flushCache();
self::$arrSite = Cache::get('site');
}
}
return self::$arrSite[config('app.default_app_id')][$strValCode] ?? NULL;
}
static public function flushCache()
{
// 查询所有数据 where('site_id',config('app.default_app_id'))
$arrSite = self::select()->toArray();
// 按照 'stc_code' 进行键值映射
$arrSite = array_column($arrSite, null, 'site_id');
Cache::tag('site')->set('site', $arrSite);
}
}

View File

@@ -0,0 +1,39 @@
<?php
declare(strict_types=1);
namespace app\admin\model;
use think\facade\Cache;
use think\Model;
/**
* @mixin think\Model
*/
class SystemConfigModel extends BaseModel
{
protected $name = 'system_config';
protected $pk = 'sc_id';
static $arrSystemConfig = [];
static public function getValByCode($strCode)
{
if (self::$arrSystemConfig == []) {
self::$arrSystemConfig = Cache::get('Config');
if (empty(self::$arrSystemConfig)) {
self::flushCache();
self::$arrSystemConfig = Cache::get('Config');
}
}
return self::$arrSystemConfig[$strCode] ?? NULL;
}
static public function flushCache()
{
$arrSystemConfig = self::column('sc_val', 'sc_code');
Cache::tag('Config')->set('Config', $arrSystemConfig);
}
}

View File

@@ -0,0 +1,112 @@
<?php
declare(strict_types=1);
namespace app\admin\model;
use think\facade\Cache;
use think\Model;
/**
* @mixin think\Model
*/
class VideoClassModel extends BaseModel
{
protected $name = 'video_class';
protected $pk = 'vc_id';
static $arrVideoShuoFenLei = [];
static public function addVideoFenLei($arrData): self
{
try {
if (empty($arrData['vc_name'])) {
throw new \Exception("分类信息不全!");
}
$VideoFenLeiModel = self::where('vc_name', $arrData['vc_name'])->find();
if ($VideoFenLeiModel) {
return $VideoFenLeiModel;
}
$VideoFenLeiModel = new self;
$VideoFenLeiModel->vc_name = $arrData['vc_name'];
$VideoFenLeiModel->save();
return $VideoFenLeiModel;
} catch (\Throwable $T) {
if ($T->getCode() == 10501) {
return self::addVideoFenLei($arrData);
}
throw $T;
}
}
static public function getIdByName($strName)
{
$intId = 0;
foreach (self::$arrVideoShuoFenLei as $arrFenLei) {
if ($arrFenLei['vc_name'] == $strName) {
$intId = $arrFenLei['vc_id'];
}
}
return $intId;
}
static public function getNameById($intId)
{
static::getDataByCache();
$strName = '未知';
foreach (self::$arrVideoShuoFenLei as $arrFenLei) {
if ($arrFenLei['vc_id'] === $intId) {
$strName = $arrFenLei['vc_name'];
}
}
return $strName;
}
static public function getInfoById($intId)
{
$arrDefault = ['vc_id' => 1, 'vc_name' => '电影' , 'vc_necheng' => 'dy'];
foreach (self::$arrVideoShuoFenLei as $arrFenLei) {
if ($arrFenLei['vc_id'] == $intId) {
$arrDefault = $arrFenLei;
}
}
return $arrDefault;
}
static public function getInfoByNeCheng($strNeCheng)
{
$arrDefault = ['vc_id' => 1, 'vc_source_id' => 1,'vc_name' => '电影' , 'vc_necheng' => 'dy'];
foreach (self::$arrVideoShuoFenLei as $arrFenLei) {
if ($arrFenLei['vc_nicheng'] == $strNeCheng) {
$arrDefault = $arrFenLei;
}
}
return $arrDefault;
}
static public function getDataByCache()
{
if (self::$arrVideoShuoFenLei == []) {
self::$arrVideoShuoFenLei = Cache::get('VIDEO_FEN_LEI');
if (empty(self::$arrVideoShuoFenLei)) {
self::flushCache();
self::$arrVideoShuoFenLei = Cache::get('VIDEO_FEN_LEI');
}
}
return self::$arrVideoShuoFenLei;
}
static public function flushCache()
{
$arrVideoShuoFenLei = self::where('vc_status', 1)->where('site_id',config('app.default_app_id'))->order('vc_sort', 'asc')->select()->toArray();
self::$arrVideoShuoFenLei = $arrVideoShuoFenLei;
Cache::tag('VIDEO_FEN_LEI')->set('VIDEO_FEN_LEI', $arrVideoShuoFenLei);
}
}

View File

@@ -0,0 +1,90 @@
<?php
declare(strict_types=1);
namespace app\admin\model;
use think\facade\Cache;
use think\Model;
/**
* @mixin think\Model
*/
class VideoDiquModel extends BaseModel
{
protected $name = 'video_diqu';
protected $pk = 'vd_id';
static $arrVideoShuoFenLei = [];
static public function add($arrData): self
{
try {
if (empty($arrData['vd_name'])) {
throw new \Exception("地区信息不全!");
}else{
$VideoFenLeiModel = self::where('vd_name', $arrData['vd_name'])->where('vc_id', $arrData['vc_id'])->find();
if ($VideoFenLeiModel) {
return $VideoFenLeiModel;
}
$VideoFenLeiModel = new self;
$VideoFenLeiModel->save($arrData);
return $VideoFenLeiModel;
}
} catch (\Throwable $T) {
if ($T->getCode() == 10501) {
//return self::add($arrData);
}
throw $T;
}
}
static public function getIdByName($strName)
{
$intId = 0;
foreach (self::$arrVideoShuoFenLei as $arrFenLei) {
if ($arrFenLei['vd_name'] == $strName) {
$intId = $arrFenLei['vd_id'];
}
}
return $intId;
}
static public function getNameById($intId)
{
static::getDataByCache();
$strName = '未知';
// var_dump(self::$arrXiaoShuoFenLei);
foreach (self::$arrVideoShuoFenLei as $arrFenLei) {
if ($arrFenLei['vd_id'] === $intId) {
$strName = $arrFenLei['vd_name'];
}
}
return $strName;
}
static public function getDataByCache()
{
if (self::$arrVideoShuoFenLei == []) {
self::$arrVideoShuoFenLei = Cache::get('VIDEO_DI_QU');
if (empty(self::$arrVideoShuoFenLei)) {
self::flushCache();
self::$arrVideoShuoFenLei = Cache::get('VIDEO_DI_QU');
}
}
return self::$arrVideoShuoFenLei;
}
static public function flushCache()
{
$arrVideoShuoFenLei = self::select()->toArray();
Cache::tag('VIDEO_DI_QU')->set('VIDEO_DI_QU', $arrVideoShuoFenLei);
}
}

View File

@@ -0,0 +1,119 @@
<?php
declare(strict_types=1);
namespace app\admin\model;
use think\facade\Cache;
use think\Model;
use app\admin\model\XiaoShuoFenLeiModel;
/**
* @mixin think\Model
*/
class VideoInfoModel extends BaseModel
{
protected $name = 'video_info';
protected $pk = 'v_id';
protected $append = [
'video_feng_mian_url',
'vc_name',
'arr_daoyan',
'arr_zhuyan',
'arr_juqing',
];
public function getArrDaoyanAttr($value, $data)
{
if(!empty($data['v_daoyan'])){
$arrDaoyan = explode(",", $data['v_daoyan']);
return $arrDaoyan;
}
}
public function getArrZhuyanAttr($value, $data)
{
if(!empty($data['v_zhuyan'])){
$arrZhuyan = explode(",", $data['v_zhuyan']);
return $arrZhuyan;
}
}
public function getArrJuqingAttr($value, $data)
{
if(!empty($data['v_juqing'])){
$arrJuqing = explode(",", $data['v_juqing']);
return $arrJuqing;
}
}
public function getVideoFengMianUrlAttr($value, $data)
{
// 检查 v_cover 字段是否存在并解析 JSON 数据
$arrUrl = isset($data['v_cover']) ? json_decode($data['v_cover'], true) : null;
if (json_last_error() !== JSON_ERROR_NONE || !is_array($arrUrl)) {
// 如果 JSON 解析失败或结果不是数组,返回默认值
return null; // 或者返回一个默认图片地址,例如: '/static/images/default.png'
}
// 解析域名 (可替换成动态域名获取逻辑)
// $strDomain = ZiYuanDuanDianModel::getDomainByCode($arrUrl['code']);
// $strDomain = "https://www.123yqw.com";
// 构造完整的封面 URL
$uri = $arrUrl['uri'] ?? null;
if (!$uri) {
// 如果没有 URI 字段,返回默认值
return null; // 或默认图片地址
}
// 返回完整 URL (如果需要加域名,可在此处理)
return $uri; // 或 "{$strDomain}{$uri}" 如果域名需要拼接
}
public function getXsxqTagAttr($value, $data)
{
return json_decode($data['xsxq_tag'], true);
}
public function getXsflNameAttr($value, $data)
{
$strXiaoShuoFenLeiName = XiaoShuoFenLeiModel::getNameById($data['xsfl_id']);
return $strXiaoShuoFenLeiName;
}
static public function addVideoXiangQing($arrData): array
{
try {
$VideoXiangQingModel = self::where('v_md5', $arrData['v_md5'])->find();
if ($VideoXiangQingModel) {
return [
'model' => $VideoXiangQingModel,
'isNew' => false, // 已存在
];
}
$VideoXiangQingModel = new self;
$VideoXiangQingModel->save($arrData);
return [
'model' => $VideoXiangQingModel,
'isNew' => true, // 新增
];
} catch (\Throwable $T) {
throw $T; // 超过重试次数后抛出异常
}
}
}

View File

@@ -0,0 +1,41 @@
<?php
declare(strict_types=1);
namespace app\admin\model;
use think\facade\Cache;
use think\Model;
/**
* @mixin think\Model
*/
class VideoInfoSeoModel extends BaseModel
{
protected $name = 'video_info_seo';
protected $pk = 'vis_id';
static public function addVideoSeo($arrData): self
{
try {
$VideoModel = self::where('vis_md5', $arrData['vis_md5'])->find();
if ($VideoModel) {
return $VideoModel;
}
$VideoModel = new self;
$VideoModel->save($arrData);
return $VideoModel;
} catch (\Throwable $T) {
throw $T; // 超过重试次数后抛出异常
}
}
}

View File

@@ -0,0 +1,90 @@
<?php
declare(strict_types=1);
namespace app\admin\model;
use think\facade\Cache;
use think\Model;
/**
* @mixin think\Model
*/
class VideoJuqingModel extends BaseModel
{
protected $name = 'video_juqing';
protected $pk = 'vj_id';
static $arrVideoShuoFenLei = [];
static public function add($arrData): self
{
try {
if (empty($arrData['vj_name'])) {
throw new \Exception("剧情信息不全!");
}else{
$VideoFenLeiModel = self::where('vj_name', $arrData['vj_name'])->where('vc_id', $arrData['vc_id'])->find();
if ($VideoFenLeiModel) {
return $VideoFenLeiModel;
}
$VideoFenLeiModel = new self;
$VideoFenLeiModel->save($arrData);
return $VideoFenLeiModel;
}
} catch (\Throwable $T) {
if ($T->getCode() == 10501) {
//return self::add($arrData);
}
throw $T;
}
}
static public function getIdByName($strName)
{
$intId = 0;
foreach (self::$arrVideoShuoFenLei as $arrFenLei) {
if ($arrFenLei['vj_name'] == $strName) {
$intId = $arrFenLei['vj_id'];
}
}
return $intId;
}
static public function getNameById($intId)
{
static::getDataByCache();
$strName = '未知';
// var_dump(self::$arrXiaoShuoFenLei);
foreach (self::$arrVideoShuoFenLei as $arrFenLei) {
if ($arrFenLei['vj_id'] === $intId) {
$strName = $arrFenLei['vj_name'];
}
}
return $strName;
}
static public function getDataByCache()
{
if (self::$arrVideoShuoFenLei == []) {
self::$arrVideoShuoFenLei = Cache::get('VIDEO_JU_QING');
if (empty(self::$arrVideoShuoFenLei)) {
self::flushCache();
self::$arrVideoShuoFenLei = Cache::get('VIDEO_JU_QING');
}
}
return self::$arrVideoShuoFenLei;
}
static public function flushCache()
{
$arrVideoShuoFenLei = self::select()->toArray();
Cache::tag('VIDEO_JU_QING')->set('VIDEO_JU_QING', $arrVideoShuoFenLei);
}
}

View File

@@ -0,0 +1,90 @@
<?php
declare(strict_types=1);
namespace app\admin\model;
use think\facade\Cache;
use think\Model;
/**
* @mixin think\Model
*/
class VideoLeixinModel extends BaseModel
{
protected $name = 'video_leixin';
protected $pk = 'vl_id';
static $arrVideoShuoFenLei = [];
static public function addVideoLeixin($arrData): self
{
try {
if (empty($arrData['vl_name'])) {
throw new \Exception("类型信息不全!");
}else{
$VideoFenLeiModel = self::where('vl_name', $arrData['vl_name'])->where('vc_id', $arrData['vc_id'])->find();
if ($VideoFenLeiModel) {
return $VideoFenLeiModel;
}
$VideoFenLeiModel = new self;
$VideoFenLeiModel->save($arrData);
return $VideoFenLeiModel;
}
} catch (\Throwable $T) {
if ($T->getCode() == 10501) {
//return self::addVideoLeixin($arrData);
}
throw $T;
}
}
static public function getIdByName($strName)
{
$intId = 0;
foreach (self::$arrVideoShuoFenLei as $arrFenLei) {
if ($arrFenLei['vl_name'] == $strName) {
$intId = $arrFenLei['vl_id'];
}
}
return $intId;
}
static public function getNameById($intId)
{
static::getDataByCache();
$strName = '未知';
// var_dump(self::$arrXiaoShuoFenLei);
foreach (self::$arrVideoShuoFenLei as $arrFenLei) {
if ($arrFenLei['vl_id'] === $intId) {
$strName = $arrFenLei['vl_name'];
}
}
return $strName;
}
static public function getDataByCache()
{
if (self::$arrVideoShuoFenLei == []) {
self::$arrVideoShuoFenLei = Cache::get('VIDEO_LEI_XIN');
if (empty(self::$arrVideoShuoFenLei)) {
self::flushCache();
self::$arrVideoShuoFenLei = Cache::get('VIDEO_LEI_XIN');
}
}
return self::$arrVideoShuoFenLei;
}
static public function flushCache()
{
$arrVideoShuoFenLei = self::select()->toArray();
Cache::tag('VIDEO_LEI_XIN')->set('VIDEO_LEI_XIN', $arrVideoShuoFenLei);
}
}

View File

@@ -0,0 +1,96 @@
<?php
declare(strict_types=1);
namespace app\admin\model;
use think\facade\Cache;
use think\Model;
/**
* @mixin think\Model
*/
class VideoPlayurlModel extends BaseModel
{
protected $name = 'video_playurl';
protected $pk = 'vp_id';
protected $append = [
'http_m3u8_url',
];
public function getHttpM3u8UrlAttr($value, $data)
{
$arrUrl = json_decode($data['vp_url'], true);
// $strDomain = ZiYuanDuanDianModel::getDomainByCode($arrUrl['code']);
// $strDomain = "https://www.123yqw.com";
return $arrUrl['uri'] ?? '';
}
static public function add($arrData): array
{
try {
// 查询是否已存在
$XiaoShuoZhangJieModel = self::where('vp_md5', $arrData['vp_md5'])->find();
// 如果已存在,返回已有模型
if ($XiaoShuoZhangJieModel) {
return [
'model' => $XiaoShuoZhangJieModel,
'isNew' => false, // 已存在
];
}
// 新增数据
$XiaoShuoZhangJieModel = new self;
$XiaoShuoZhangJieModel->save($arrData);
return [
'model' => $XiaoShuoZhangJieModel,
'isNew' => true, // 新增
];
} catch (\Throwable $T) {
var_dump($T);
throw $T;
}
}
// static public function add($arrData): self
// {
// try {
// $XiaoShuoZhangJieModel = self::where('vp_md5', $arrData['vp_md5'])->find();
// if ($XiaoShuoZhangJieModel) {
// return $XiaoShuoZhangJieModel;
// }
// $XiaoShuoZhangJieModel = new self;
// $XiaoShuoZhangJieModel->save($arrData);
// return $XiaoShuoZhangJieModel;
// } catch (\Throwable $T) {
// var_dump($T);
// // if ($T->getCode() == 10501) {
// // return self::add($arrData);
// // }
// throw $T;
// }
// }
/**
* 判断章节是否存在
*
* @param bool $strXszjSourceCode
* @return void
*/
static public function checkZhangJieExists($strXszjSourceCode)
{
$XiaoShuoZhangJieModel = self::where('vp_md5', $strXszjSourceCode)->find();
return !empty($XiaoShuoZhangJieModel);
}
}

View File

@@ -0,0 +1,90 @@
<?php
declare(strict_types=1);
namespace app\admin\model;
use think\facade\Cache;
use think\Model;
/**
* @mixin think\Model
*/
class VideoYearsModel extends BaseModel
{
protected $name = 'video_years';
protected $pk = 'vy_id';
static $arrVideoShuoFenLei = [];
static public function add($arrData): self
{
try {
if (empty($arrData['vy_name'])) {
throw new \Exception("年份信息不全!");
}else{
$VideoFenLeiModel = self::where('vy_name', $arrData['vy_name'])->where('vc_id', $arrData['vc_id'])->find();
if ($VideoFenLeiModel) {
return $VideoFenLeiModel;
}
$VideoFenLeiModel = new self;
$VideoFenLeiModel->save($arrData);
return $VideoFenLeiModel;
}
} catch (\Throwable $T) {
if ($T->getCode() == 10501) {
//return self::add($arrData);
}
throw $T;
}
}
static public function getIdByName($strName)
{
$intId = 0;
foreach (self::$arrVideoShuoFenLei as $arrFenLei) {
if ($arrFenLei['vy_name'] == $strName) {
$intId = $arrFenLei['vy_id'];
}
}
return $intId;
}
static public function getNameById($intId)
{
static::getDataByCache();
$strName = '未知';
// var_dump(self::$arrXiaoShuoFenLei);
foreach (self::$arrVideoShuoFenLei as $arrFenLei) {
if ($arrFenLei['vy_id'] === $intId) {
$strName = $arrFenLei['vy_name'];
}
}
return $strName;
}
static public function getDataByCache()
{
if (self::$arrVideoShuoFenLei == []) {
self::$arrVideoShuoFenLei = Cache::get('VIDEO_YEARS');
if (empty(self::$arrVideoShuoFenLei)) {
self::flushCache();
self::$arrVideoShuoFenLei = Cache::get('VIDEO_YEARS');
}
}
return self::$arrVideoShuoFenLei;
}
static public function flushCache()
{
$arrVideoShuoFenLei = self::select()->toArray();
Cache::tag('VIDEO_YEARS')->set('VIDEO_YEARS', $arrVideoShuoFenLei);
}
}

View File

@@ -0,0 +1,90 @@
<?php
declare(strict_types=1);
namespace app\admin\model;
use think\facade\Cache;
use think\Model;
/**
* @mixin think\Model
*/
class VideoYuyanModel extends BaseModel
{
protected $name = 'video_language';
protected $pk = 'vla_id';
static $arrVideoShuoFenLei = [];
static public function add($arrData): self
{
try {
if (empty($arrData['vla_name'])) {
throw new \Exception("年份信息不全!");
}else{
$VideoFenLeiModel = self::where('vla_name', $arrData['vla_name'])->where('vc_id', $arrData['vc_id'])->find();
if ($VideoFenLeiModel) {
return $VideoFenLeiModel;
}
$VideoFenLeiModel = new self;
$VideoFenLeiModel->save($arrData);
return $VideoFenLeiModel;
}
} catch (\Throwable $T) {
if ($T->getCode() == 10501) {
//return self::add($arrData);
}
throw $T;
}
}
static public function getIdByName($strName)
{
$intId = 0;
foreach (self::$arrVideoShuoFenLei as $arrFenLei) {
if ($arrFenLei['vla_name'] == $strName) {
$intId = $arrFenLei['vla_id'];
}
}
return $intId;
}
static public function getNameById($intId)
{
static::getDataByCache();
$strName = '未知';
// var_dump(self::$arrXiaoShuoFenLei);
foreach (self::$arrVideoShuoFenLei as $arrFenLei) {
if ($arrFenLei['vla_id'] === $intId) {
$strName = $arrFenLei['vla_name'];
}
}
return $strName;
}
static public function getDataByCache()
{
if (self::$arrVideoShuoFenLei == []) {
self::$arrVideoShuoFenLei = Cache::get('VIDEO_YUYAN');
if (empty(self::$arrVideoShuoFenLei)) {
self::flushCache();
self::$arrVideoShuoFenLei = Cache::get('VIDEO_YUYAN');
}
}
return self::$arrVideoShuoFenLei;
}
static public function flushCache()
{
$arrVideoShuoFenLei = self::select()->toArray();
Cache::tag('VIDEO_YUYAN')->set('VIDEO_YUYAN', $arrVideoShuoFenLei);
}
}

View File

@@ -0,0 +1,63 @@
<?php
declare(strict_types=1);
namespace app\admin\model;
use think\Model;
/**
* @mixin think\Model
*/
class VisitLogsModel extends BaseModel
{
protected $name = 'visit_logs';
protected $pk = 'vil_id';
static public function add(array $arrData): self
{
// 处理数据,避免 undefined index 问题
$arrData = [
'vil_ip' => $arrData['vil_ip'] ?? '',
'vil_user_agent' => $arrData['vil_user_agent'] ?? '',
'vil_url' => $arrData['vil_url'] ?? '',
'vil_method' => $arrData['vil_method'] ?? '',
'vil_headers' => $arrData['vil_headers'] ?? '',
'vil_timestamp' => date('Y-m-d H:i:s'),
'vil_content_type' => $arrData['vil_content_type'] ?? '',
'vil_content_id' => $arrData['vil_content_id'] ?? '',
'vil_is_spider' => $arrData['vil_is_spider'] ?? '',
'vil_location' => $arrData['vil_location'] ?? '',
];
try {
// 必填字段校验
$requiredFields = ['vil_ip', 'vil_url', 'vil_content_id', 'vil_content_type', 'vil_method'];
foreach ($requiredFields as $field) {
if (empty($arrData[$field])) {
var_dump('字段缺失: {}'.$field);
// throw new \InvalidArgumentException("字段缺失: {$field}");
}
}
// 创建模型实例
$visitLogModel = new self;
// 保存数据
if (!$visitLogModel::create($arrData)) {
var_dump('保存失败: ');
// throw new \RuntimeException('访问日志保存失败: ' . json_encode($arrData, JSON_UNESCAPED_UNICODE));
}
return $visitLogModel;
} catch (\Throwable $T) {
// 如果错误码是 10501可以在这里做特殊处理
if ($T->getCode() == 10501) {
// 记录日志或进行备用操作
// logError($T->getMessage());
}
throw $T; // 继续抛出异常
}
}
}

View File

@@ -0,0 +1,92 @@
<?php
declare(strict_types=1);
namespace app\admin\model;
use think\facade\Cache;
use think\Model;
/**
* @mixin think\Model
*/
class XiaoShuoFenLeiModel extends BaseModel
{
protected $name = 'xiao_shuo_fen_lei';
protected $pk = 'xsfl_id';
static $arrXiaoShuoFenLei = [];
static public function addXiaoShuoFenLei($arrData): self
{
try {
if (empty($arrData['xsfl_name']) || empty($arrData['xsfl_source_id'])) {
throw new \Exception("小说分类信息不全!");
}
$XiaoShuoFenLeiModel = self::where('xsfl_name', $arrData['xsfl_name'])->find();
if ($XiaoShuoFenLeiModel) {
return $XiaoShuoFenLeiModel;
}
$XiaoShuoFenLeiModel = new self;
$XiaoShuoFenLeiModel->xsfl_source_id = $arrData['xsfl_source_id'];
$XiaoShuoFenLeiModel->xsfl_source_code = $arrData['xsfl_source_code'];
$XiaoShuoFenLeiModel->xsfl_name = $arrData['xsfl_name'];
$XiaoShuoFenLeiModel->save();
return $XiaoShuoFenLeiModel;
} catch (\Throwable $T) {
if ($T->getCode() == 10501) {
return self::addXiaoShuoFenLei($arrData);
}
throw $T;
}
}
static public function getIdByName($strName)
{
$intId = 0;
foreach (self::$arrXiaoShuoFenLei as $arrFenLei) {
if ($arrFenLei['xsfl_name'] == $strName && $arrFenLei['site_id'] == config('app.default_app_id')) {
$intId = $arrFenLei['xsfl_id'];
}
}
return $intId;
}
static public function getNameById($intId)
{
static::getDataByCache();
$strName = '全部';
foreach (self::$arrXiaoShuoFenLei as $arrFenLei) {
if ($arrFenLei['xsfl_source_id'] == $intId && $arrFenLei['site_id'] == config('app.default_app_id')) {
$strName = $arrFenLei['xsfl_name'];
}
}
return $strName;
}
static public function getDataByCache()
{
$strCacheKey = 'XIAO_SHUO_FEN_LEI';
if (self::$arrXiaoShuoFenLei == []) {
self::$arrXiaoShuoFenLei = Cache::get( $strCacheKey);
if (empty(self::$arrXiaoShuoFenLei)) {
self::flushCache();
self::$arrXiaoShuoFenLei = Cache::get( $strCacheKey);
}
}
return self::$arrXiaoShuoFenLei;
}
static public function flushCache()
{
$strCacheKey = 'XIAO_SHUO_FEN_LEI';
$arrXiaoShuoFenLei = self::where('site_id',config('app.default_app_id'))->select()->toArray();
Cache::tag( $strCacheKey)->set( $strCacheKey, $arrXiaoShuoFenLei);
}
}

View File

@@ -0,0 +1,86 @@
<?php
declare(strict_types=1);
namespace app\admin\model;
use think\facade\Cache;
use think\Model;
use app\admin\model\XiaoShuoFenLeiModel;
/**
* @mixin think\Model
*/
class XiaoShuoXiangQingModel extends BaseModel
{
protected $name = 'xiao_shuo_xiang_qing';
protected $pk = 'xsxq_id';
protected $append = [
'xsxq_feng_mian_url',
'xsfl_name',
];
public function getXsxqFengMianUrlAttr($value, $data)
{
if(!empty($data['xsxq_feng_mian'])){
$arrUrl = json_decode($data['xsxq_feng_mian'], true);
$strDomain = ZiYuanDuanDianModel::getDomainByCode($arrUrl['code']);
return $strDomain . $arrUrl['uri'];
}
return '';
}
public function getXsxqTagAttr($value, $data)
{
return json_decode($data['xsxq_tag'], true);
}
public function getXsflNameAttr($value, $data)
{
$strXiaoShuoFenLeiName = '';
if(!empty($data['xsfl_id'])){
$strXiaoShuoFenLeiName = XiaoShuoFenLeiModel::getNameById($data['xsfl_id']);
}
return $strXiaoShuoFenLeiName;
}
static public function addXiaoShuoXiangQing($arrData): self
{
try {
$XiaoShuoXiangQingModel = self::where('xsxq_source_code', $arrData['xsxq_source_code'])->find();
if ($XiaoShuoXiangQingModel) {
if ($XiaoShuoXiangQingModel->xsxq_geng_xin_shi_jian != $arrData['xsxq_geng_xin_shi_jian']) {
$XiaoShuoXiangQingModel->xsxq_geng_xin_shi_jian = $arrData['xsxq_geng_xin_shi_jian'];
$XiaoShuoXiangQingModel->xsxq_zuozhe = $arrData['xsxq_zuozhe'];
$XiaoShuoXiangQingModel->save();
}
return $XiaoShuoXiangQingModel;
}
$XiaoShuoXiangQingModel = new self;
$XiaoShuoXiangQingModel->save($arrData);
return $XiaoShuoXiangQingModel;
} catch (\Throwable $T) {
// if ($T->getCode() == 10501 && $retryCount < 3) { // 限制重试次数
// $retryCount++;
// return self::addXiaoShuoXiangQing($arrData, $retryCount);
// }
throw $T; // 超过重试次数后抛出异常
// if ($T->getCode() == 10501) {
// return self::addXiaoShuoXiangQing($arrData);
// }
// throw $T;
}
}
}

View File

@@ -0,0 +1,49 @@
<?php
declare(strict_types=1);
namespace app\admin\model;
use think\facade\Cache;
use think\Model;
/**
* @mixin think\Model
*/
class XiaoShuoZhangJieModel extends BaseModel
{
protected $name = 'xiao_shuo_zhang_jie';
protected $pk = 'xszj_id';
static public function addXiaoShuoZhangJie($arrData): self
{
try {
$XiaoShuoZhangJieModel = self::where('xszj_source_code', $arrData['xszj_source_code'])->find();
if ($XiaoShuoZhangJieModel) {
return $XiaoShuoZhangJieModel;
}
$XiaoShuoZhangJieModel = new self;
$XiaoShuoZhangJieModel->save($arrData);
return $XiaoShuoZhangJieModel;
} catch (\Throwable $T) {
if ($T->getCode() == 10501) {
return self::addXiaoShuoZhangJie($arrData);
}
throw $T;
}
}
/**
* 判断章节是否存在
*
* @param bool $strXszjSourceCode
* @return void
*/
static public function checkZhangJieExists($strXszjSourceCode)
{
$XiaoShuoZhangJieModel = self::where('xszj_source_code', $strXszjSourceCode)->find();
return !empty($XiaoShuoZhangJieModel);
}
}

View File

@@ -0,0 +1,40 @@
<?php
declare(strict_types=1);
namespace app\admin\model;
use think\facade\Cache;
use think\Model;
/**
* @mixin think\Model
*/
class ZiYuanDuanDianModel extends BaseModel
{
protected $name = 'zi_yuan_duan_dian';
protected $pk = 'zydd_id';
static $arrZiYuanDuanDian = [];
static public function getDomainByCode($strCode)
{
if (self::$arrZiYuanDuanDian == []) {
self::$arrZiYuanDuanDian = Cache::get('Zydd');
if (empty(self::$arrZiYuanDuanDian)) {
self::flushCache();
self::$arrZiYuanDuanDian = Cache::get('Zydd');
}
}
return self::$arrZiYuanDuanDian[$strCode] ?? "";
}
static public function flushCache()
{
$arrZiYuanDuanDian = self::column('zydd_domain', 'zydd_code');
Cache::tag('Zydd')->set('Zydd', $arrZiYuanDuanDian);
}
}

View File

@@ -0,0 +1,12 @@
<?php
use app\admin\AdminServiceProvider;
use app\admin\middleware\AdminException;
use app\ExceptionHandle;
use app\Request;
// 容器Provider定义文件
return [
'think\Request' => Request::class,
'think\exception\Handle' => AdminException::class,
];

View File

@@ -1,2 +1,765 @@
<?php
// 应用公共文件
namespace {
function getUriByUrl($strUrl)
{
$arrUrl = parse_url($strUrl);
return $arrUrl['path'];
}
function getArgByUrl($strUrl, $strKey)
{
$arrPaseUrl = parse_url($strUrl);
$strUri = $arrPaseUrl['query'];
parse_str($strUri, $arrUrl);
return $arrUrl[$strKey];
}
/**
* 搜索字符串中某个词语出现后所有的字符
*
* @param string $string 要搜索的字符串
* @param string $searchWord 要搜索的词语
* @return string 搜索词语后面的所有字符,如果没有找到词语则返回空字符串
*/
function getStringAfterWord($string, $searchWord)
{
// 查找词语的位置
$pos = mb_strpos($string, $searchWord);
// 如果找到了词语
if ($pos !== false) {
// 获取词语出现后所有的字符
return trim(mb_substr($string, $pos + mb_strlen($searchWord)));
} else {
// 如果没有找到词语,返回空字符串
return '';
}
}
// 将 Cookie 字符串解析为数组
function parseCookies($cookieString)
{
$cookies = [];
$pairs = explode('; ', $cookieString);
foreach ($pairs as $pair) {
list($name, $value) = explode('=', $pair, 2);
$cookies[$name] = $value;
}
return $cookies;
}
// 获取域名函数
function getDomainFromUrl($url)
{
$host = parse_url($url, PHP_URL_HOST);
return $host ?: 'default.com';
}
function replaceLastPageNumber($url, $number)
{
// 使用正则表达式替换最后一个 "1" 为 "number"
return preg_replace('/1(?=[^1]*$)/', $number, $url);
}
function removeScriptTags($rpText, $newText)
{
// 使用正则表达式替换所有 <script> 标签
return preg_replace($rpText, '', $newText);
}
function extractFilename($url)
{
// 检查 URL 是否以 .html 结尾
if (substr($url, -5) !== '.html') {
throw new InvalidArgumentException('URL must end with .html');
}
// 解析 URL 获取路径部分
$path = parse_url($url, PHP_URL_PATH);
// 获取文件名部分
$filename = basename($path, '.html');
return $filename;
}
function extractIdFromUrl($url)
{
if (preg_match('/-id-(\d+)\.html/', $url, $matches)) {
return $matches[1];
}
return null;
}
function extractNumbersFromUrl($url)
{
// 使用正则表达式匹配路径中的数字部分
if (preg_match('#/(\d+)/(\d+)/#', $url, $matches)) {
return [
'first' => $matches[1], // 提取到的第一个数字
'second' => $matches[2], // 提取到的第二个数字
];
}
return null; // 如果未匹配,返回 null
}
function extractNumberFromUrl($url)
{
// 使用正则表达式匹配 /class/{数字}_ 的格式
if (preg_match('/\/class\/(\d+)_/', $url, $matches)) {
return $matches[1]; // 返回匹配的数字部分
}
return null; // 如果未匹配,返回 null
}
function encImgBy17C($strInputImgPath, $strOutImgPath, $strKey = 0x88)
{
// 读取图像内容
$strImgData = file_get_contents($strInputImgPath);
if ($strImgData === false) {
throw new Exception("无法读取图片文件: $strInputImgPath");
}
// 将图像内容转换为字节数组
$bytes = array_values(unpack('C*', $strImgData));
$len = count($bytes);
// 对每个字节进行异或操作加密
for ($i = 0; $i < $len; $i++) {
$bytes[$i] ^= $strKey;
}
// 将字节数组转换回二进制数据
$strEncryptedData = pack('C*', ...$bytes);
// 将加密后的数据写入文件
if (file_put_contents($strOutImgPath, $strEncryptedData) === false) {
throw new Exception("无法写入加密文件: $strOutImgPath");
}
}
function customEntities($text)
{
$result = '';
$length = mb_strlen($text, 'UTF-8');
for ($i = 0; $i < $length; $i++) {
$char = mb_substr($text, $i, 1, 'UTF-8');
$code = ord($char);
if ($code < 128) {
// 对于 ASCII 字符,直接使用 ord
$result .= '&#' . $code . ';';
} else {
// 对于非 ASCII 字符,使用 mb_ord 获取 Unicode 码
$code = mb_ord($char, 'UTF-8');
$result .= '&#' . $code . ';';
}
}
return $result;
}
function isValidDateTime($dateTimeStr, $format = 'Y-m-d H:i:s')
{
$date = DateTime::createFromFormat($format, $dateTimeStr);
return $date && $date->format($format) === $dateTimeStr;
}
// 创建保存文件
function saveContentToFile($filePath, $content)
{
// 确保目标路径的目录存在
$directory = dirname($filePath);
if (!is_dir($directory)) {
// 尝试创建目录
if (!mkdir($directory, 0777, true)) {
return [
'status' => false,
'message' => "Failed to create directory: $directory"
];
}
}
// 写入内容到文件
if (file_put_contents($filePath, $content) !== false) {
return [
'status' => true,
'message' => "Content successfully written to $filePath"
];
} else {
return [
'status' => false,
'message' => "Failed to write content to $filePath"
];
}
}
function saveCompressAndDeleteTxt($filePath, $content)
{
// 确保目标路径的目录存在
$directory = dirname($filePath);
if (!is_dir($directory)) {
if (!mkdir($directory, 0777, true)) {
return [
'status' => false,
'message' => "Failed to create directory: $directory"
];
}
}
// 写入内容到文件
$fp = fopen($filePath, "c+"); // 使用 c+ 模式,支持加锁
if (!$fp) {
return [
'status' => false,
'message' => "Failed to open file for writing: $filePath"
];
}
// 加锁防止其他进程同时操作
if (!flock($fp, LOCK_EX)) { // 加排他锁
fclose($fp);
return [
'status' => false,
'message' => "Failed to acquire lock for file: $filePath"
];
}
// 写入内容
if (fwrite($fp, $content) === false) {
flock($fp, LOCK_UN); // 释放锁
fclose($fp);
return [
'status' => false,
'message' => "Failed to write content to $filePath"
];
}
// 解锁并关闭文件
flock($fp, LOCK_UN);
fclose($fp);
// 压缩文件为 ZIP
$zipPath = $filePath . ".zip"; // 压缩后的 ZIP 文件路径
$zip = new ZipArchive();
$zipOpenResult = $zip->open($zipPath, ZipArchive::CREATE | ZipArchive::OVERWRITE);
if ($zipOpenResult !== true) {
return [
'status' => false,
'message' => "Failed to open zip archive: $zipPath"
];
}
// 添加文件到 ZIP 中
if (!$zip->addFile($filePath, basename($filePath))) {
$zip->close();
return [
'status' => false,
'message' => "Failed to add file to zip archive: $filePath"
];
}
if (!$zip->close()) {
return [
'status' => false,
'message' => "Failed to close zip archive: $zipPath"
];
}
// 删除原始文件
$fp = fopen($filePath, "c+"); // 重新打开文件以加锁
if ($fp && flock($fp, LOCK_EX)) {
if (!file_exists($filePath)) {
unlink($filePath); // 删除文件
}
flock($fp, LOCK_UN);
fclose($fp);
}
return [
'status' => true,
'message' => "File success save to $zipPath and del",
'zip_path' => $zipPath
];
}
/**
* 提取 ZIP 文件中的所有 .txt 文件内容
*
* @param string $zipPath ZIP 文件的完整路径
* @return array 返回一个数组,键是文件名,值是对应的文件内容
* @throws Exception 如果文件不存在或解压失败则抛出异常
*/
function extractTxtFromZip($zipPath)
{
if (!file_exists($zipPath)) {
throw new Exception("ZIP 文件路径无效: " . $zipPath);
}
$zip = new ZipArchive();
if ($zip->open($zipPath) === true) {
$txtContents = "";
// 遍历所有文件
for ($i = 0; $i < $zip->numFiles; $i++) {
$fileName = $zip->getNameIndex($i);
// 检查文件是否是 .txt 结尾
if (pathinfo($fileName, PATHINFO_EXTENSION) === 'txt') {
// 读取文件内容
$content = $zip->getFromName($fileName);
$txtContents = $content;
}
}
$zip->close();
return $txtContents;
} else {
throw new Exception("无法打开 ZIP 文件。");
}
}
function readExtractAndDeleteTxt($zipPath, $extractTo = null)
{
// 检查 ZIP 文件是否存在
if (!file_exists($zipPath)) {
return [
'status' => false,
'message' => "ZIP file does not exist: $zipPath"
];
}
// 打开 ZIP 文件
$zip = new ZipArchive();
if ($zip->open($zipPath) !== true) {
return [
'status' => false,
'message' => "Failed to open ZIP file: $zipPath"
];
}
// 获取 ZIP 文件内容列表
$fileList = [];
for ($i = 0; $i < $zip->numFiles; $i++) {
$fileList[] = $zip->getNameIndex($i);
}
// 如果需要提取文件
if ($extractTo) {
// 确保目标目录存在
if (!is_dir($extractTo)) {
if (!mkdir($extractTo, 0777, true)) {
return [
'status' => false,
'message' => "Failed to create extraction directory: $extractTo"
];
}
}
// 提取文件到指定目录
if (!$zip->extractTo($extractTo)) {
$zip->close();
return [
'status' => false,
'message' => "Failed to extract ZIP file to $extractTo"
];
}
}
// 关闭 ZIP 文件
$zip->close();
// 删除提取的 .txt 文件
$deletedFiles = [];
foreach ($fileList as $file) {
if (pathinfo($file, PATHINFO_EXTENSION) === 'txt') { // 检查是否是 .txt 文件
$filePath = rtrim($extractTo, '/') . '/' . $file;
if (file_exists($filePath) && unlink($filePath)) {
$deletedFiles[] = $file;
}
}
}
// 返回提取和删除状态
return [
'status' => true,
'message' => "ZIP file successfully extracted to $extractTo and .txt files deleted.",
'files_extracted' => $fileList,
'files_deleted' => $deletedFiles
];
}
function getRandomUserAgent()
{
$userAgents = [
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.0.0 Safari/537.36',
// 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
// 'Mozilla/5.0 (Linux; Android 9; ASUS_X00TD) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/359.0.0.288 Mobile Safari/537.36',
// 'Mozilla/5.0 (iPhone; CPU iPhone OS 15_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.0 Mobile/15E148 Safari/604.1',
// 'Mozilla/5.0 (Linux; Android 9; ASUS_X00TD; Flow) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/359.0.0.288 Mobile Safari/537.36'
];
return $userAgents[array_rand($userAgents)];
}
function getRandomProxies()
{
$proxies = [
'http://114.115.130.225:3128',
// 'http://36.6.145.131:8089',
// 'http://202.101.213.205:19926',
// 'http://218.87.205.35:21874',
// 'http://203.19.38.114:1080',
// 'http://121.89.222.174:8090',
// 'http://223.113.80.158:9091',
// 'http://120.232.194.134:8998',
// 'http://120.46.197.14:8083',
// 'http://139.9.5.196:7890',
// 'http://203.89.8.107:80',
// 'http://117.186.143.130:8118',
// 'http://117.186.232.73:8080',
// 'http://203.12.200.179:8085',
// 'http://120.46.215.52:15280',
// 'http://111.230.243.170:80',
// 'http://223.113.101.150:8060',
// 'http://117.50.113.39:1337',
];
return $proxies[array_rand($proxies)];
}
function fetchContentWithCurl($url, $headers = [], $timeout = 10, $proxy = null, $proxyAuth = null)
{
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_TIMEOUT, $timeout);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
// 设置头部
if (!empty($headers)) {
curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
}
// 模拟浏览器
curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/5.0 (Linux; Android 9; ASUS_X00TD; Flow) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/359.0.0.288 Mobile Safari/537.36');
// 如果需要代理
if ($proxy) {
curl_setopt($ch, CURLOPT_PROXY, $proxy);
if ($proxyAuth) {
curl_setopt($ch, CURLOPT_PROXYUSERPWD, $proxyAuth); // 代理用户名和密码
}
}
// 执行请求
$response = curl_exec($ch);
// 检查错误
if (curl_errno($ch)) {
$error = 'cURL Error: ' . curl_error($ch);
curl_close($ch);
return ['success' => false, 'error' => $error];
}
$httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
curl_close($ch);
// 返回结果
if ($httpCode >= 200 && $httpCode < 300) {
return ['success' => true, 'data' => $response];
} else {
return ['success' => false, 'error' => "HTTP Error: $httpCode"];
}
}
/**
* // 调用函数生成随机 HTML
* generateRandomHTML(10, 30); // 随机生成 10~30 个标签
*
* @param integer $minTags
* @param integer $maxTags
* @return void
*/
function generateRandomHTML($minTags = 5, $maxTags = 20)
{
// 可随机生成的标签字符集
$tagCharset = 'abcdefghijklmnopqrstuvwxyz';
// 属性的随机字符集
$attrCharset = 'abcdefghijklmnopqrstuvwxyz';
// 可生成的属性名称
$possibleAttributes = ['draggable', 'date-time', 'lang', 'id', 'dir', 'class'];
// 标签的最大长度和属性值的最大长度
$tagMaxLength = 8; // 标签名的最大长度
$attrValueMaxLength = 10; // 属性值的最大长度
// 随机生成的标签数量
$numTags = rand($minTags, $maxTags);
$htmlResult = '';
for ($i = 0; $i < $numTags; $i++) {
// 随机生成标签名称
$tagName = randomString(rand(3, $tagMaxLength), $tagCharset);
// 随机生成属性数量
$numAttributes = rand(1, 3); // 每个标签随机生成 1~3 个属性
$attributes = '';
for ($j = 0; $j < $numAttributes; $j++) {
// 随机选择属性名称
$attrName = $possibleAttributes[array_rand($possibleAttributes)];
// 随机生成属性值
$attrValue = randomString(rand(3, $attrValueMaxLength), $attrCharset);
$attributes .= " {$attrName}=\"{$attrValue}\"";
}
// 拼接随机标签
$htmlResult .= "<{$tagName}{$attributes}></{$tagName}>\n";
}
return $htmlResult;
}
// 辅助函数:生成随机字符串
function randomString($length = 8, $charset = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789')
{
$charsetLength = strlen($charset);
$randomString = '';
for ($i = 0; $i < $length; $i++) {
$randomString .= $charset[rand(0, $charsetLength - 1)];
}
return $randomString;
}
// 随机颜色生成函数
function getRandomColor()
{
return 'color:rgb(' . mt_rand(0, 255) . ', ' . mt_rand(0, 255) . ', ' . mt_rand(0, 255) . ');';
}
/**
* 生成分页 HTML 的通用函数
*
* @param int $currentPage 当前页码
* @param int $lastPage 总页数
* @param string $baseUrl 基础 URL
* @param string $queryParams 查询参数(包含占位符 :page
* @param int $maxPagesToShow 最大显示页码数
* @return string 分页 HTML
* 示例调用
* $paginationHtml = generatePaginationHtml(
* 3, // 当前页
* 10, // 总页数
* '/vodshow/dy-cn-hot-action-en-', // 基础 URL
* ':page---2024.html', // 查询参数,包含占位符 :page
* 5 // 最多显示 5 个页码
* );
* // 输出分页 HTML
* echo $paginationHtml;
*
*/
function generatePaginationHtml($currentPage, $lastPage, $baseUrl, $queryParams, $maxPagesToShow = 5)
{
$paginationHtml = '<ul class="stui-page text-center clearfix">';
// 首页按钮
$paginationHtml .= '<li><a style="' . getRandomColor() . '" href="' . str_replace(':page', 1, $baseUrl . $queryParams) . '">首页</a></li>';
// 上一页按钮
if ($currentPage > 1) {
$paginationHtml .= '<li><a style="' . getRandomColor() . '" href="' . str_replace(':page', $currentPage - 1, $baseUrl . $queryParams) . '">上一页</a></li>';
} else {
$paginationHtml .= '<li class="disabled"><span>上一页</span></li>';
}
// 页码范围控制:最多显示 $maxPagesToShow 个页码
$start = max(1, $currentPage - floor($maxPagesToShow / 2));
$end = min($lastPage, $start + $maxPagesToShow - 1);
if ($end - $start + 1 < $maxPagesToShow) {
$start = max(1, $end - $maxPagesToShow + 1);
}
// 页码按钮
for ($i = $start; $i <= $end; $i++) {
if ($i == $currentPage) {
$paginationHtml .= '<li class="active"><a style="' . getRandomColor() . '" href="' . str_replace(':page', $i, $baseUrl . $queryParams) . '">' . $i . '</a></li>';
} else {
$paginationHtml .= '<li><a style="' . getRandomColor() . '" href="' . str_replace(':page', $i, $baseUrl . $queryParams) . '">' . $i . '</a></li>';
}
}
// 下一页按钮
if ($currentPage < $lastPage) {
$paginationHtml .= '<li><a style="' . getRandomColor() . '" href="' . str_replace(':page', $currentPage + 1, $baseUrl . $queryParams) . '">下一页</a></li>';
} else {
$paginationHtml .= '<li class="disabled"><span>下一页</span></li>';
}
// 尾页按钮
$paginationHtml .= '<li><a style="' . getRandomColor() . '" href="' . str_replace(':page', $lastPage, $baseUrl . $queryParams) . '">尾页</a></li>';
$paginationHtml .= '</ul>';
return $paginationHtml;
}
/**
* 生成分页 HTML 的通用函数
*
* @param int $currentPage 当前页码
* @param int $lastPage 总页数
* @param string $baseUrl 基础 URL
* @param string $queryParams 查询参数(包含占位符 :page
* @param int $maxPagesToShow 最大显示页码数
* @return string 分页 HTML
* 示例调用
* $paginationHtml = generatePaginationHtml(
* 3, // 当前页
* 10, // 总页数
* '/vodshow/dy-cn-hot-action-en-', // 基础 URL
* ':page---2024.html', // 查询参数,包含占位符 :page
* 5 // 最多显示 5 个页码
* );
* // 输出分页 HTML
* echo $paginationHtml;
*
*/
function generateNovelPaginationHtml($currentPage, $lastPage, $baseUrl, $queryParams, $maxPagesToShow = 10)
{
$paginationHtml = '<div class="row sort_page_num">';
// 首页按钮
$paginationHtml .= '<a class="prev_off" href="' . str_replace(':page', 1, $baseUrl . $queryParams) . '">首 页</a>';
// 计算分页起始和结束
$start = max(1, $currentPage - floor($maxPagesToShow / 2));
$end = min($lastPage, $start + $maxPagesToShow - 1);
if ($end - $start + 1 < $maxPagesToShow) {
$start = max(1, $end - $maxPagesToShow + 1);
}
// 页码按钮
for ($i = $start; $i <= $end; $i++) {
if ($i == $currentPage) {
$paginationHtml .= '<a class="page_on" href="' . str_replace(':page', $i, $baseUrl . $queryParams) . '">&nbsp;' . $i . '&nbsp;</a>';
} else {
$paginationHtml .= '<a href="' . str_replace(':page', $i, $baseUrl . $queryParams) . '">&nbsp;' . $i . '&nbsp;</a>';
}
}
// 下一页按钮
if ($currentPage < $lastPage) {
$paginationHtml .= '<a class="prev_on" href="' . str_replace(':page', $currentPage + 1, $baseUrl . $queryParams) . '">下一页</a>';
}
$paginationHtml .= '</div>';
return $paginationHtml;
}
// 分页模板-时空小说模板
function generateNovelSkPaginationHtml($currentPage, $lastPage, $baseUrl, $queryParams, $maxPagesToShow = 10)
{
$paginationHtml = '<ul class="pagination justify-content-center">';
// 上一页按钮
if ($currentPage > 1) {
$paginationHtml .= '<li class="page-item"><a class="page-link" href="' . str_replace(':page', $currentPage - 1, $baseUrl . $queryParams) . '">上一页</a></li>';
} else {
$paginationHtml .= '<li class="page-item disabled"><span class="page-link">上一页</span></li>';
}
// 计算分页起始和结束
$start = max(1, $currentPage - floor($maxPagesToShow / 2));
$end = min($lastPage, $start + $maxPagesToShow - 1);
if ($end - $start + 1 < $maxPagesToShow) {
$start = max(1, $end - $maxPagesToShow + 1);
}
// 页码按钮
for ($i = $start; $i <= $end; $i++) {
if ($i == $currentPage) {
$paginationHtml .= '<li class="page-item active"><span class="page-link">' . $i . '</span></li>';
} else {
$paginationHtml .= '<li class="page-item"><a class="page-link" href="' . str_replace(':page', $i, $baseUrl . $queryParams) . '">' . $i . '</a></li>';
}
}
// 下一页按钮
if ($currentPage < $lastPage) {
$paginationHtml .= '<li class="page-item"><a class="page-link" href="' . str_replace(':page', $currentPage + 1, $baseUrl . $queryParams) . '">下一页</a></li>';
} else {
$paginationHtml .= '<li class="page-item disabled"><span class="page-link">下一页</span></li>';
}
// 显示当前页码/总页数
$paginationHtml .= '<li class="page-item disabled"><span class="page-link">' . $currentPage . '/' . $lastPage . '</span></li>';
$paginationHtml .= '</ul>';
return $paginationHtml;
}
// 更据IP 获取地区
function getLocationByIp($ip)
{
try {
$response = file_get_contents("http://ip-api.com/json/{$ip}");
$locationData = json_decode($response, true);
if ($locationData['status'] === 'success') {
return [
'country' => $locationData['country'],
'region' => $locationData['regionName'],
'city' => $locationData['city'],
'lat' => $locationData['lat'],
'lon' => $locationData['lon']
];
}
} catch (\Exception $e) {
// 记录异常或忽略
}
return null; // 无法获取地理信息时返回 null
}
// 检查解密内容是否包含潜在的漏洞代码
function checkForVulnerabilities($content)
{
// 检查是否包含 'eval' 函数
if (stripos($content, 'eval') !== false) {
return true;
}
// 检查是否包含恶意的 http 或域名
$vulnerablePatterns = [
'/http(s)?:\/\/[^ ]+/', // 匹配 URL
'/<script.*>.*<\/script>/i' // 匹配 script 标签
];
foreach ($vulnerablePatterns as $pattern) {
if (preg_match($pattern, $content)) {
return true;
}
}
return false;
}
}

View File

@@ -7,6 +7,11 @@ class Index
{
public function index()
{
$arrConfig = config();
echo '<pre>';
print_r($arrConfig);
echo '</pre>';
// exit;
return '您好!这是一个[home]示例应用';
}
}

View File

@@ -1,9 +1,11 @@
<?php
use app\admin\AdminServiceProvider;
use app\AppService;
// 系统服务定义文件
// 服务在完成全局初始化之后执行
return [
AppService::class,
AdminServiceProvider::class,
];

View File

@@ -0,0 +1,124 @@
<?php
declare(strict_types=1);
namespace app\task\collection;
use app\admin\model\CaiJiPeiZhiModel;
use app\admin\model\SystemConfigModel;
use app\task\core\TaskCore;
use GuzzleHttp\Client;
/**
* @mixin think\Model
*/
class PullDataColBase
{
/**
* Undocumented variable
*
* @var \GuzzleHttp\Client
*/
protected $Client = NULL;
/**
* Undocumented variable
*
* @var \app\task\core\TaskCore
*/
protected $TaskCore = NULL;
protected $arrCaiJiPeiZhi = [];
public function __construct()
{
$this->Client = new Client();
$this->TaskCore = new TaskCore();
}
public function getAilisiDomain()
{
if (!key_exists('AILISI_DOMAIN', $this->arrCaiJiPeiZhi)) {
$this->arrCaiJiPeiZhi['AILISI_DOMAIN'] = CaiJiPeiZhiModel::getValByCode('AILISI_DOMAIN');
}
return $this->arrCaiJiPeiZhi['AILISI_DOMAIN'];
}
public function getAilisiCookie()
{
if (!key_exists('AILISI_COOKIE', $this->arrCaiJiPeiZhi)) {
$strCookie = CaiJiPeiZhiModel::getValByCode('AILISI_COOKIE');
$this->arrCaiJiPeiZhi['AILISI_COOKIE'] = parseCookies($strCookie);
}
return $this->arrCaiJiPeiZhi['AILISI_COOKIE'];
}
public function getTianLaiDomain()
{
if (!key_exists('TIANLAI_DOMAIN', $this->arrCaiJiPeiZhi)) {
$this->arrCaiJiPeiZhi['TIANLAI_DOMAIN'] = CaiJiPeiZhiModel::getValByCode('TIANLAI_DOMAIN');
}
return $this->arrCaiJiPeiZhi['TIANLAI_DOMAIN'];
}
public function getBiquxsDomain()
{
if (!key_exists('BIQUXS_DOMAIN', $this->arrCaiJiPeiZhi)) {
$this->arrCaiJiPeiZhi['BIQUXS_DOMAIN'] = CaiJiPeiZhiModel::getValByCode('BIQUXS_DOMAIN');
}
return $this->arrCaiJiPeiZhi['BIQUXS_DOMAIN'];
}
public function get5cccDomain()
{
if (!key_exists('5CCC_DOMAIN', $this->arrCaiJiPeiZhi)) {
$this->arrCaiJiPeiZhi['5CCC_DOMAIN'] = CaiJiPeiZhiModel::getValByCode('5CCC_DOMAIN');
}
return $this->arrCaiJiPeiZhi['5CCC_DOMAIN'];
}
public function get630zwDomain()
{
if (!key_exists('630zw_DOMAIN', $this->arrCaiJiPeiZhi)) {
$this->arrCaiJiPeiZhi['630zw_DOMAIN'] = CaiJiPeiZhiModel::getValByCode('630zw_DOMAIN');
}
return $this->arrCaiJiPeiZhi['630zw_DOMAIN'];
}
public function getDadiDomain()
{
if (!key_exists('DADI_DOMAIN', $this->arrCaiJiPeiZhi)) {
$this->arrCaiJiPeiZhi['DADI_DOMAIN'] = CaiJiPeiZhiModel::getValByCode('DADI_DOMAIN');
}
return $this->arrCaiJiPeiZhi['DADI_DOMAIN'];
}
public function getHeiMuErDomain()
{
if (!key_exists('HEIMUER_DOMAIN', $this->arrCaiJiPeiZhi)) {
$this->arrCaiJiPeiZhi['HEIMUER_DOMAIN'] = CaiJiPeiZhiModel::getValByCode('HEIMUER_DOMAIN');
}
return $this->arrCaiJiPeiZhi['HEIMUER_DOMAIN'];
}
public function YouZhiZiYuanDomain()
{
if (!key_exists('YZZY_DOMAIN', $this->arrCaiJiPeiZhi)) {
$this->arrCaiJiPeiZhi['YZZY_DOMAIN'] = CaiJiPeiZhiModel::getValByCode('YZZY_DOMAIN');
}
return $this->arrCaiJiPeiZhi['YZZY_DOMAIN'];
}
}

View File

@@ -0,0 +1,427 @@
<?php
declare(strict_types=1);
namespace app\task\collection\biquxs;
use app\admin\model\XiaoShuoFenLeiModel;
use app\admin\model\XiaoShuoXiangQingModel;
use app\admin\model\XiaoShuoZhangJieModel;
use app\task\collection\PullDataColBase;
use GuzzleHttp\Cookie\CookieJar;
use processor\ImageProcessor;
use think\facade\Cache;
use QL\QueryList;
/**
* @mixin think\Model
*/
class BiquxsXiaoShuoCol extends PullDataColBase
{
public function pullXiaoShuo()
{
$TaskCore = $this->TaskCore;
// 20054
for ($i = 1; $i <= 20054; $i++) {
echo "当前值: $i\n";
$arrTask = [
'callback' => [self::class, 'getNovelInfo'],
'data' => [
'xs_source_id' => $i,
'xs_source_seo_id' => 1,
]
];
$TaskCore->set($arrTask);
var_dump($arrTask);
}
}
public function getNovelInfo($arrData)
{
$TaskCore = $this->TaskCore;
$Client = $this->Client;
$strSiteUrl = $this->getBiquxsDomain();
$strUrl = 'http://www.biquxs.com/book/' . $arrData['xs_source_id'] ; //'http://www.biquxs.com/book/20053';
$intXiaoShuoSourceId = $arrData['xs_source_id']; //20053;
$intXiaoShuoSourceSeoId = $arrData['xs_source_seo_id']; //1 ;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
$QueryList = QueryList::html($strResult);
// var_dump($QueryList);
// 提取所有 <meta> 标签的属性
$result = $QueryList->find('meta[property^="og:"]')->map(function ($item) {
return [
'property' => $item->attr('property'),
'content' => $item->attr('content'),
];
});
// 打印结果
// print_r($result->all());
$strZuoZhe = '';
$strMingZi = '';
$strLastDate = '';
$strJieShao = '';
$strCover = '';
$strFenlei = '';
$strZiShu = 0;
$strZhuangTai = $QueryList->find('#info p')->eq(1)->text();
$strZhuangTai = getStringAfterWord($strZhuangTai, '状态:');
foreach ($result as $item) {
switch ($item['property']) {
case 'og:title':
$strMingZi = $item['content'];
break;
case 'og:description':
$strJieShao = $item['content'];
break;
case 'og:image':
$strCover = $item['content'];
break;
case 'og:book:category':
$strFenlei = $item['content'];
break;
case 'og:book:author':
$strZuoZhe = $item['content'];
break;
case 'og:book:update_time':
$strLastDate = $item['content'];
break;
}
}
$strLastDate = trim((string)$strLastDate);
if (!isValidDateTime($strLastDate)) {
$strLastDate = date('Y-m-d H:i:s');
}
$arrTag = [];
if (strpos($strCover, 'http') === false) {
$strCover = $strSiteUrl . $strCover;
}
$strCoverUri = $strCover;
try {
$strCoverUri = "";
$strCoverUri = ImageProcessor::getInstance()->downloadImage( $intXiaoShuoSourceId,$strCover, 'xs');
} catch (\Throwable $t) {
$strErr = sprintf("采集[%s]小说图片错误,异常:%s", $strCover, $t->getMessage());
echo $strErr;
}
$arrCover = [
'code' => 'BI_QU_XS_COVER',
// 'uri' => getUriByUrl($strCover),
'uri' => $strCoverUri,
];
$strFenleiIId = 1 ;
switch ($strFenlei) {
case '玄幻小说':
$strName = "玄幻";
$strFenleiIId = 1;
break;
case '修真小说':
$strName = "修真";
$strFenleiIId = 2;
break;
case '都市小说':
$strName = "都市";
$strFenleiIId = 3;
break;
case '历史小说':
$strName = "历史";
$strFenleiIId = 4;
break;
case '网游小说':
$strName = "网游";
$strFenleiIId = 5;
break;
case '科幻小说':
$strName = "科幻";
$strFenleiIId = 6;
break;
case '女频小说':
$strName = "女频";
$strFenleiIId = 7;
break;
case '其它小说':
$strName = "其它";
$strFenleiIId = 8;
break;
default:
$strName = "其它";
$strFenleiIId = 8;
break;
}
$arrXiaoShuoInfo = [
'xsxq_ming_zi' => $strMingZi,
'xsxq_zuozhe' => $strZuoZhe , //$arrData['xiaoshuo_zuozhe'],
'xsfl_id' => $strFenleiIId,
'xsxq_zhuang_tai' => $strZhuangTai,
'xsxq_zi_shu' => $strZiShu,
'xsxq_jie_shao' => $strJieShao,
'xsxq_feng_mian' => json_encode($arrCover),
'xsxq_geng_xin_shi_jian' => $strLastDate,
'xsxq_tag' => json_encode($arrTag, JSON_UNESCAPED_UNICODE),
'xsxq_source_id' => $intXiaoShuoSourceId,
'xsxq_source_seo_id' => $intXiaoShuoSourceId.'1',
'xsxq_source_code' => 'BIQU_XIAOSHUO_' . $intXiaoShuoSourceId . '_' . $intXiaoShuoSourceSeoId,
];
// print_r($arrXiaoShuoInfo);
var_dump($arrXiaoShuoInfo['xsxq_source_code']);
$XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
$arrZhangjie = [];
$intChapterIndex = 0; // 定义一个计数器变量
// 检查是否找到元素
if ($QueryList->find('.listmain a')->count() > 0) {
$QueryList->find('.listmain a')->map(function ($ZhangJie) use (&$arrZhangjie, &$intChapterIndex) {
echo "Processing index $intChapterIndex: " . $ZhangJie->text() . PHP_EOL;
// 从索引6开始
if ($intChapterIndex >= 5) {
$arrZhangjie[] = [
'mingzi' => $ZhangJie->text(),
'url' => $ZhangJie->attr('href'),
];
}
$intChapterIndex++;
});
// 输出最终结果
var_dump($arrZhangjie);
} else {
echo "No elements found for .listmain a" . PHP_EOL;
}
//unset($QueryList);
// 获取最大章节索引
$intMaxXiaoShuoZhangJiePaixu = XiaoShuoZhangJieModel::where('xsxq_id', $intXiaoShuoSourceId)
->max('xszj_pai_xu'); // 获取最大值
var_dump('获取小说章节最大值-'.$intMaxXiaoShuoZhangJiePaixu);
// 如果没有结果max 返回 null可以设置默认值
$intMaxXiaoShuoZhangJiePaixu = $intMaxXiaoShuoZhangJiePaixu ?? 0;
foreach ($arrZhangjie as $intZhangJieNum => $arrZhangjieOne) {
var_dump($intZhangJieNum);
// 如果数据库里的章节最大索引大于当前索引,说明已经入库,不需要入队列了
if($intMaxXiaoShuoZhangJiePaixu > $intZhangJieNum){
var_dump("小说章节已经入库,不再推送任务");
continue;// 如果任务已存在,直接跳过 break
}
$strZhangJieUrl = $strSiteUrl . $arrZhangjieOne['url'];
$strZhangJieId = extractFilename($strZhangJieUrl);
# 'AILISI_XIAOSHUO_ZHANGJIE_小说源id_小说源章节id'
$strXszjSourceCode = 'AILISI_XIAOSHUO_ZHANGJIE_'. $XiaoShuoXiangQingModel->xsxq_source_id . '_' . $strZhangJieId;
$arrTask = [
'callback' => [self::class, 'getNovelReader'],
'data' => [
'xszj_source_id' => $strZhangJieId,
'xszj_paixu' => $intZhangJieNum,
'xszj_name' => $arrZhangjieOne['mingzi'],
'xszj_url' => $strZhangJieUrl,
'xszj_source_code' => $strXszjSourceCode,
'xsxq_source_id' => $XiaoShuoXiangQingModel->xsxq_source_id,
'xsfl_id' => $strFenleiIId,
]
];
$TaskCore->set($arrTask);
}
var_dump("章节任务投递成功");
} catch (\Throwable $T) {
throw $T;
}
}
public function getNovelFengMian($arrData)
{
//var_dump('getNovelFengMian');
//var_dump($arrData);
$Client = $this->Client;
$TaskCore = $this->TaskCore;
// 获取 Redis 实例
$redis = Cache::store('redis')->handler();
$intXiaoShuoSourceId = $arrData['xiaoshuo_id'];
$intXiaoShuoSourceSeoId = $arrData['xiaoshuo_seo_id'];
$redisKey = "task:novel_source_id:" . $intXiaoShuoSourceId;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($arrData['xiaoshuo_url']);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$strSiteUrl = $this->getAilisiDomain();
$strCover = $QueryList->find('#fmimg img')->eq(0)->attr("src");
if (strpos($strCover, 'http') === false) {
$strCover = $strSiteUrl . $strCover;
}
$strCoverUri = $strCover;
try {
$strCoverUri = "";
$strCoverUri = ImageProcessor::getInstance()->downloadAndEncryptImage($strCover, 'xs');
$arrCover = [
'code' => 'AI_LI_SI_COVER',
// 'uri' => getUriByUrl($strCover),
'uri' => $strCoverUri,
];
$arrXiaoShuoInfo = [
'xsxq_feng_mian' => json_encode($arrCover),
];
$XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
} catch (\Throwable $t) {
$strErr = sprintf("采集[%s]小说图片错误,异常:%s", $strCover, $t->getMessage());
echo $strErr;
}
} catch (\Throwable $T) {
throw $T;
}
}
public function getNovelReader($arrData)
{
var_dump('getNovelReader');
var_dump($arrData['xszj_url']);
$Client = $this->Client;
$strXszjSourceCode = $arrData['xszj_source_code'];
$strZhangJieUrl = $arrData['xszj_url'];
$intZhangJiePaiXu = $arrData['xszj_paixu'];
$intFenleiId = $arrData['xsfl_id'];
$intXiaoShuoSourceID = $arrData['xsxq_source_id'];
$strZhangJieSourceId = $arrData['xszj_source_id'];
$strZhangJieName = $arrData['xszj_name'];
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strZhangJieUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
if (XiaoShuoZhangJieModel::checkZhangJieExists($strXszjSourceCode)) {
return true;
}
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strZhangJieUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
unset($Response);
$ZhangJieQueryList = QueryList::html($strResult);
// 获取原始 HTML
$strNeiRong = $ZhangJieQueryList->find('#content')->eq(0)->html();
// 解码 HTML 实体
$strNeiRong = html_entity_decode($strNeiRong, ENT_QUOTES, 'UTF-8');
// 去掉 <p> 标签中的 class="content_detail" 属性
$strNeiRong = preg_replace('/<p\s+class="content_detail"/i', '<p', $strNeiRong);
// 清理多余的 HTML 标签
$strNeiRong = strip_tags($strNeiRong, '<p>');
// 替换不间断空格 (\xc2\xa0) 和 &nbsp; 为普通空格
$strNeiRong = preg_replace('/\xc2\xa0/', ' ', $strNeiRong); // 替换 UTF-8 不间断空格
$strNeiRong = preg_replace('/&nbsp;/', ' ', $strNeiRong); // 替换 &nbsp;
// 清理多余空格和换行符
$strNeiRong = preg_replace('/\s+/', ' ', $strNeiRong);
$strNeiRong = preg_replace('/<p>\s+/', '<p>', $strNeiRong); // 清理 <p> 标签内部多余空格
$strNeiRong = preg_replace('/\s+<\/p>/', '</p>', $strNeiRong); // 清理 <p> 标签结尾多余空格
// 输出结果
// var_dump($strNeiRong);
# '/novel/123yqw/分类id/小说源id/小说源章节id.txt'
$strNeiRongTxtPath = config("filesystem.novel").'/novel/biquxs/'.$intFenleiId.'/'.$intXiaoShuoSourceID.'/'.$strZhangJieSourceId.'.txt';
//$strNeiRongTxtPath = "/www/novel/123yqw/1/11/111.txt";
// 调用函数
$result = saveCompressAndDeleteTxt($strNeiRongTxtPath, $strNeiRong);
// 输出状态
if ($result['status']) {
var_dump("Success: " . $result['message']) ;
} else {
var_dump("Error: " . $result['message']) ;
}
$arrXiaoShuoZhangJie = [
'xsxq_id' => $intXiaoShuoSourceID,
'xszj_pai_xu' => $intZhangJiePaiXu,
'xszj_ming_zi' => $strZhangJieName,
'xszj_nei_rong' => $strNeiRongTxtPath,
'xszj_source_id' => $strZhangJieSourceId,
'xszj_source_code' => $strXszjSourceCode,
];
XiaoShuoZhangJieModel::addXiaoShuoZhangJie($arrXiaoShuoZhangJie);
unset($ZhangJieQueryList);
// 延时 3 秒
//sleep(3);
} catch (\Throwable $T) {
throw $T;
}
}
}

View File

@@ -0,0 +1,545 @@
<?php
declare(strict_types=1);
namespace app\task\collection\ccc5;
use app\admin\model\XiaoShuoFenLeiModel;
use app\admin\model\XiaoShuoXiangQingModel;
use app\admin\model\XiaoShuoZhangJieModel;
use app\task\collection\PullDataColBase;
use GuzzleHttp\Cookie\CookieJar;
use processor\ImageProcessor;
use think\facade\Cache;
use QL\QueryList;
/**
* @mixin think\Model
*/
class XiaoShuo5cccCol extends PullDataColBase
{
public function pullXiaoShuo()
{
$TaskCore = $this->TaskCore;
// 20054
for ($i = 3; $i <= 3; $i++) {
echo "当前值: $i\n";
$arrTask = [
'callback' => [self::class, 'getNovelInfo'],
'data' => [
'xs_source_id' => $i,
'xs_source_seo_id' => 1,
]
];
$TaskCore->set($arrTask);
var_dump($arrTask);
}
}
public function getNovelInfo($arrData)
{
$TaskCore = $this->TaskCore;
$Client = $this->Client;
$strSiteUrl = $this->get5cccDomain();
$strUrl = 'https://www.5ccc.org/chapter/' . $arrData['xs_source_id'].'.html' ; //'http://www.biquxs.com/book/20053';
$intXiaoShuoSourceId = $arrData['xs_source_id']; //20053;
$intXiaoShuoSourceSeoId = $arrData['xs_source_seo_id']; //1 ;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
$QueryList = QueryList::html($strResult);
// var_dump($QueryList);
// 提取所有 <meta> 标签的属性
$result = $QueryList->find('meta[property^="og:"]')->map(function ($item) {
return [
'property' => $item->attr('property'),
'content' => $item->attr('content'),
];
});
// 打印结果
print_r($result->all());
$strZuoZhe = '';
$strMingZi = '';
$strLastDate = '';
$strJieShao = '';
$strCover = '';
$strFenlei = '';
$strZiShu = 0;
$strZhuangTai = "";
foreach ($result as $item) {
switch ($item['property']) {
case 'og:title':
$strMingZi = $item['content'];
break;
case 'og:description':
$strJieShao = $item['content'];
break;
case 'og:image':
$strCover = $item['content'];
break;
case 'og:novel:category':
$strFenlei = $item['content'];
break;
case 'og:novel:author':
$strZuoZhe = $item['content'];
break;
case 'og:novel:update_time':
$strLastDate = $item['content'];
break;
case 'og:novel:status':
$strZhuangTai = $item['content'];
break;
}
}
$strLastDate = trim((string)$strLastDate);
if (!isValidDateTime($strLastDate)) {
$strLastDate = date('Y-m-d H:i:s');
}
$arrTag = [];
$strCover = $QueryList->find('.imgbox img')->eq(0)->attr("src");
if (strpos($strCover, 'http') === false) {
$strCover = $strSiteUrl . $strCover;
}
$strCoverUri = $strCover;
try {
$strCoverUri = "";
$strCoverUri = ImageProcessor::getInstance()->downloadImage( $intXiaoShuoSourceId,$strCover, 'xs');
} catch (\Throwable $t) {
$strErr = sprintf("采集[%s]小说图片错误,异常:%s", $strCover, $t->getMessage());
echo $strErr;
}
$arrCover = [
'code' => '5CCC_XS_COVER',
// 'uri' => getUriByUrl($strCover),
'uri' => $strCoverUri,
];
$strFenleiIId = 1 ;
switch ($strFenlei) {
case '玄幻修真':
$strName = "玄幻修真";
$strFenleiIId = 1;
break;
case '重生穿越':
$strName = "重生穿越";
$strFenleiIId = 2;
break;
case '都市小说':
$strName = "都市小说";
$strFenleiIId = 3;
break;
case '军史小说':
$strName = "军史小说";
$strFenleiIId = 4;
break;
case '网游小说':
$strName = "网游小说";
$strFenleiIId = 5;
break;
case '科幻小说':
$strName = "科幻小说";
$strFenleiIId = 6;
break;
case '灵异小说':
$strName = "灵异小说";
$strFenleiIId = 7;
break;
case '言情小说':
$strName = "言情小说";
$strFenleiIId = 8;
break;
case '其他小说':
$strName = "其他小说";
$strFenleiIId = 9;
break;
}
$arrXiaoShuoInfo = [
'xsxq_ming_zi' => $strMingZi,
'xsxq_zuozhe' => $strZuoZhe , //$arrData['xiaoshuo_zuozhe'],
'xsfl_id' => $strFenleiIId,
'xsxq_zhuang_tai' => $strZhuangTai,
'xsxq_zi_shu' => $strZiShu,
'xsxq_jie_shao' => $strJieShao,
'xsxq_feng_mian' => json_encode($arrCover),
'xsxq_geng_xin_shi_jian' => $strLastDate,
'xsxq_tag' => json_encode($arrTag, JSON_UNESCAPED_UNICODE),
'xsxq_source_id' => $intXiaoShuoSourceId,
'xsxq_source_seo_id' => $intXiaoShuoSourceId.'1',
'xsxq_source_code' => 'BIQU_XIAOSHUO_' . $intXiaoShuoSourceId . '_' . $intXiaoShuoSourceSeoId,
];
// print_r($arrXiaoShuoInfo);
var_dump($arrXiaoShuoInfo['xsxq_source_code']);
$XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
$strMuluHref = $QueryList->find('.btn-mulu')->eq(0)->attr("href");
$arrTask = [
'callback' => [self::class, 'getNovelReaderAll'],
'data' => [
'xszj_href' => $strMuluHref,
]
];
$TaskCore->set($arrTask);
/*
$arrZhangjie = [];
$intChapterIndex = 0; // 定义一个计数器变量
// 检查是否找到元素
if ($QueryList->find('.section-list a')->count() > 0) {
$QueryList->find('.section-list a')->map(function ($ZhangJie) use (&$arrZhangjie, &$intChapterIndex) {
echo "Processing index $intChapterIndex: " . $ZhangJie->text() . PHP_EOL;
// 从索引6开始
//if ($intChapterIndex >= 5) {
$arrZhangjie[] = [
'mingzi' => $ZhangJie->text(),
'url' => $ZhangJie->attr('href'),
];
//}
$intChapterIndex++;
});
// 输出最终结果
var_dump($arrZhangjie);
} else {
echo "No elements found for .section-list a" . PHP_EOL;
}
//unset($QueryList);
var_dump($arrZhangjie);
// 获取最大章节索引
$intMaxXiaoShuoZhangJiePaixu = XiaoShuoZhangJieModel::where('xsxq_id', $intXiaoShuoSourceId)
->max('xszj_pai_xu'); // 获取最大值
var_dump($intMaxXiaoShuoZhangJiePaixu);
// 如果没有结果max 返回 null可以设置默认值
$intMaxXiaoShuoZhangJiePaixu = $intMaxXiaoShuoZhangJiePaixu ?? 0;
foreach ($arrZhangjie as $intZhangJieNum => $arrZhangjieOne) {
var_dump($intZhangJieNum);
// 如果数据库里的章节最大索引大于当前索引,说明已经入库,不需要入队列了
if($intMaxXiaoShuoZhangJiePaixu > $intZhangJieNum){
var_dump("小说章节已经入库,不再推送任务");
continue;// 如果任务已存在,直接跳过 break
}
$strZhangJieUrl = $strSiteUrl . $arrZhangjieOne['url'];
$strZhangJieId = extractFilename($strZhangJieUrl);
# 'AILISI_XIAOSHUO_ZHANGJIE_小说源id_小说源章节id'
$strXszjSourceCode = '5CCC_XIAOSHUO_ZHANGJIE_'. $XiaoShuoXiangQingModel->xsxq_source_id . '_' . $strZhangJieId;
$arrTask = [
'callback' => [self::class, 'getNovelReader'],
'data' => [
'xszj_source_id' => $strZhangJieId,
'xszj_paixu' => $intZhangJieNum,
'xszj_name' => $arrZhangjieOne['mingzi'],
'xszj_url' => $strZhangJieUrl,
'xszj_source_code' => $strXszjSourceCode,
'xsxq_source_id' => $XiaoShuoXiangQingModel->xsxq_source_id,
'xsfl_id' => $strFenleiIId,
]
];
$TaskCore->set($arrTask);
var_dump("章节任务投递成功".$strXszjSourceCode);
}
*/
} catch (\Throwable $T) {
throw $T;
}
}
public function getNovelFengMian($arrData)
{
//var_dump('getNovelFengMian');
//var_dump($arrData);
$Client = $this->Client;
$TaskCore = $this->TaskCore;
// 获取 Redis 实例
$redis = Cache::store('redis')->handler();
$intXiaoShuoSourceId = $arrData['xiaoshuo_id'];
$intXiaoShuoSourceSeoId = $arrData['xiaoshuo_seo_id'];
$redisKey = "task:novel_source_id:" . $intXiaoShuoSourceId;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($arrData['xiaoshuo_url']);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$strSiteUrl = $this->getAilisiDomain();
$strCover = $QueryList->find('#fmimg img')->eq(0)->attr("src");
if (strpos($strCover, 'http') === false) {
$strCover = $strSiteUrl . $strCover;
}
$strCoverUri = $strCover;
try {
$strCoverUri = "";
$strCoverUri = ImageProcessor::getInstance()->downloadAndEncryptImage($strCover, 'xs');
$arrCover = [
'code' => 'AI_LI_SI_COVER',
// 'uri' => getUriByUrl($strCover),
'uri' => $strCoverUri,
];
$arrXiaoShuoInfo = [
'xsxq_feng_mian' => json_encode($arrCover),
];
$XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
} catch (\Throwable $t) {
$strErr = sprintf("采集[%s]小说图片错误,异常:%s", $strCover, $t->getMessage());
echo $strErr;
}
} catch (\Throwable $T) {
throw $T;
}
}
public function getNovelReaderAll($arrData)
{
var_dump('getNovelReaderAll');
$strMuluHref = $arrData['xszj_href'];
var_dump($strMuluHref);
$Client = $this->Client;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strMuluHref);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strMuluHref, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
unset($Response);
$QueryList = QueryList::html($strResult);
$intChapterIndex = 0; // 定义一个计数器变量
var_dump($QueryList->find('.section-list a')->count());
// 检查是否找到元素
if ($QueryList->find('.section-list a')->count() > 0) {
$QueryList->find('.section-list a')->map(function ($ZhangJie) use (&$arrZhangjie, &$intChapterIndex) {
echo "Processing index $intChapterIndex: " . $ZhangJie->text() . PHP_EOL;
// 从索引6开始
//if ($intChapterIndex >= 5) {
$arrZhangjie[] = [
'mingzi' => $ZhangJie->text(),
'url' => $ZhangJie->attr('href'),
];
//}
$intChapterIndex++;
});
// 输出最终结果
var_dump($arrZhangjie);
} else {
echo "No elements found for .section-list a" . PHP_EOL;
}
//unset($QueryList);
var_dump($arrZhangjie);
/*
// 获取最大章节索引
$intMaxXiaoShuoZhangJiePaixu = XiaoShuoZhangJieModel::where('xsxq_id', $intXiaoShuoSourceId)
->max('xszj_pai_xu'); // 获取最大值
var_dump($intMaxXiaoShuoZhangJiePaixu);
// 如果没有结果max 返回 null可以设置默认值
$intMaxXiaoShuoZhangJiePaixu = $intMaxXiaoShuoZhangJiePaixu ?? 0;
foreach ($arrZhangjie as $intZhangJieNum => $arrZhangjieOne) {
var_dump($intZhangJieNum);
// 如果数据库里的章节最大索引大于当前索引,说明已经入库,不需要入队列了
if($intMaxXiaoShuoZhangJiePaixu > $intZhangJieNum){
var_dump("小说章节已经入库,不再推送任务");
continue;// 如果任务已存在,直接跳过 break
}
$strZhangJieUrl = $strSiteUrl . $arrZhangjieOne['url'];
$strZhangJieId = extractFilename($strZhangJieUrl);
# 'AILISI_XIAOSHUO_ZHANGJIE_小说源id_小说源章节id'
$strXszjSourceCode = '5CCC_XIAOSHUO_ZHANGJIE_'. $XiaoShuoXiangQingModel->xsxq_source_id . '_' . $strZhangJieId;
$arrTask = [
'callback' => [self::class, 'getNovelReader'],
'data' => [
'xszj_source_id' => $strZhangJieId,
'xszj_paixu' => $intZhangJieNum,
'xszj_name' => $arrZhangjieOne['mingzi'],
'xszj_url' => $strZhangJieUrl,
'xszj_source_code' => $strXszjSourceCode,
'xsxq_source_id' => $XiaoShuoXiangQingModel->xsxq_source_id,
'xsfl_id' => $strFenleiIId,
]
];
$TaskCore->set($arrTask);
var_dump("章节任务投递成功".$strXszjSourceCode);
}
*/
} catch (\Throwable $T) {
throw $T;
}
}
public function getNovelReader($arrData)
{
var_dump('getNovelReader');
var_dump($arrData['xszj_url']);
$Client = $this->Client;
$strXszjSourceCode = $arrData['xszj_source_code'];
$strZhangJieUrl = $arrData['xszj_url'];
$intZhangJiePaiXu = $arrData['xszj_paixu'];
$intFenleiId = $arrData['xsfl_id'];
$intXiaoShuoSourceID = $arrData['xsxq_source_id'];
$strZhangJieSourceId = $arrData['xszj_source_id'];
$strZhangJieName = $arrData['xszj_name'];
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strZhangJieUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
if (XiaoShuoZhangJieModel::checkZhangJieExists($strXszjSourceCode)) {
return true;
}
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strZhangJieUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
unset($Response);
$ZhangJieQueryList = QueryList::html($strResult);
// 获取原始 HTML
$strNeiRong = $ZhangJieQueryList->find('#content')->eq(0)->html();
// 解码 HTML 实体
$strNeiRong = html_entity_decode($strNeiRong, ENT_QUOTES, 'UTF-8');
// 去掉 <p> 标签中的 class="content_detail" 属性
$strNeiRong = preg_replace('/<p\s+class="content_detail"/i', '<p', $strNeiRong);
// 清理多余的 HTML 标签
$strNeiRong = strip_tags($strNeiRong, '<p>');
// 替换不间断空格 (\xc2\xa0) 和 &nbsp; 为普通空格
$strNeiRong = preg_replace('/\xc2\xa0/', ' ', $strNeiRong); // 替换 UTF-8 不间断空格
$strNeiRong = preg_replace('/&nbsp;/', ' ', $strNeiRong); // 替换 &nbsp;
// 清理多余空格和换行符
$strNeiRong = preg_replace('/\s+/', ' ', $strNeiRong);
$strNeiRong = preg_replace('/<p>\s+/', '<p>', $strNeiRong); // 清理 <p> 标签内部多余空格
$strNeiRong = preg_replace('/\s+<\/p>/', '</p>', $strNeiRong); // 清理 <p> 标签结尾多余空格
// 输出结果
// var_dump($strNeiRong);
# '/novel/123yqw/分类id/小说源id/小说源章节id.txt'
$strNeiRongTxtPath = config("filesystem.novel").'/novel/biquxs/'.$intFenleiId.'/'.$intXiaoShuoSourceID.'/'.$strZhangJieSourceId.'.txt';
//$strNeiRongTxtPath = "/www/novel/123yqw/1/11/111.txt";
// 调用函数
$result = saveCompressAndDeleteTxt($strNeiRongTxtPath, $strNeiRong);
// 输出状态
if ($result['status']) {
var_dump("Success: " . $result['message']) ;
} else {
var_dump("Error: " . $result['message']) ;
}
$arrXiaoShuoZhangJie = [
'xsxq_id' => $intXiaoShuoSourceID,
'xszj_pai_xu' => $intZhangJiePaiXu,
'xszj_ming_zi' => $strZhangJieName,
'xszj_nei_rong' => $strNeiRongTxtPath,
'xszj_source_id' => $strZhangJieSourceId,
'xszj_source_code' => $strXszjSourceCode,
];
XiaoShuoZhangJieModel::addXiaoShuoZhangJie($arrXiaoShuoZhangJie);
unset($ZhangJieQueryList);
// 延时 3 秒
//sleep(3);
} catch (\Throwable $T) {
throw $T;
}
}
}

View File

@@ -0,0 +1,498 @@
<?php
declare(strict_types=1);
namespace app\task\collection\dadi;
use app\admin\model\VideoPlayurlModel;
use app\admin\model\VideoLeixinModel;
use app\admin\model\VideoDiquModel;
use app\admin\model\VideoYearsModel;
use app\admin\model\VideoJuqingModel;
use app\admin\model\XiaoShuoFenLeiModel;
use app\admin\model\XiaoShuoXiangQingModel;
use app\admin\model\VideoInfoModel;
use app\admin\model\XiaoShuoZhangJieModel;
use app\task\collection\PullDataColBase;
use GuzzleHttp\Cookie\CookieJar;
use processor\ImageProcessor;
use think\facade\Cache;
use QL\QueryList;
/**
* @mixin think\Model
*/
class DaDiDianyingCol extends PullDataColBase
{
public function pullDadiYingshi()
{
$TaskCore = $this->TaskCore;
// 1-75
for ($i = 1; $i <= 75; $i++) {
echo "当前值: $i\n";
$arrTask = [
'callback' => [self::class, 'getVideoPage'],
'data' => [
'video_page_url' => 'https://www.czdadi.net/vodshow/dy--------'.$i.'---.html',
'video_class_id' => 1
]
];
$TaskCore->set($arrTask);
}
//5-25 https://www.czdadi.net/vodshow/duanju-----------.html
for ($i = 1; $i <= 25; $i++) {
echo "当前值: $i\n";
$arrTask = [
'callback' => [self::class, 'getVideoPage'],
'data' => [
'video_page_url' => 'https://www.czdadi.net/vodshow/duanju--------'.$i.'---.html',
'video_class_id' => 5
]
];
$TaskCore->set($arrTask);
}
// 2--18 https://www.czdadi.net/vodshow/dsj--------18---.html
for ($i = 1; $i <= 18; $i++) {
echo "当前值: $i\n";
$arrTask = [
'callback' => [self::class, 'getVideoPage'],
'data' => [
'video_page_url' => 'https://www.czdadi.net/vodshow/dsj--------'.$i.'---.html',
'video_class_id' => 2
]
];
$TaskCore->set($arrTask);
}
// 3--11 https://www.czdadi.net/vodshow/zy--------11---.html
for ($i = 1; $i <= 11; $i++) {
echo "当前值: $i\n";
$arrTask = [
'callback' => [self::class, 'getVideoPage'],
'data' => [
'video_page_url' => 'https://www.czdadi.net/vodshow/zy--------'.$i.'---.html',
'video_class_id' => 3
]
];
$TaskCore->set($arrTask);
}
// 4--29 https://www.czdadi.net/vodshow/dm-----------.html
for ($i = 1; $i <= 29; $i++) {
echo "当前值: $i\n";
$arrTask = [
'callback' => [self::class, 'getVideoPage'],
'data' => [
'video_page_url' => 'https://www.czdadi.net/vodshow/dm--------'.$i.'---.html',
'video_class_id' => 4
]
];
$TaskCore->set($arrTask);
}
}
# 获取分页里面的视频列表
public function getVideoPage($arrData)
{
$TaskCore = $this->TaskCore;
$Client = $this->Client;
$strUrl= $arrData['video_page_url'];
$intClassId = $arrData['video_class_id'];
$strSiteUrl = $this->get5cccDomain();
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
$QueryList = QueryList::html($strResult);
//$arrVideo = [];
// 检查是否找到元素
if ($QueryList->find('.stui-vodlist li .stui-vodlist__detail a')->count() > 0) {
$QueryList->find('.stui-vodlist li .stui-vodlist__detail a ')->map(function ($Video) use (&$arrVideo,$intClassId,&$TaskCore) {
// $arrVideo[] = [
// 'mingzi' => $Video->text(),
// 'url' => $Video->attr('href'),
// ];
if(!empty($Video->text()) && !empty($Video->attr('href'))){
$arrTask = [
'callback' => [self::class, 'getVideoInfo'],
'data' => [
'class_id' => $intClassId,
'video_href' => $Video->attr('href'),
'video_name' => $Video->text(),
]
];
$TaskCore->set($arrTask);
echo "视频投递成功 : " . $Video->text() . PHP_EOL;
}
});
// foreach ($arrVideo as $intVideoNum => $arrVideoOne) {
// $arrTask = [
// 'callback' => [self::class, 'getVideoInfo'],
// 'data' => [
// 'class_id' => $intClassId,
// 'video_href' => $arrVideoOne['url'],
// 'video_name' => $arrVideoOne['mingzi'],
// ]
// ];
// $TaskCore->set($arrTask);
// }
// 输出最终结果
// var_dump($arrVideo);
} else {
echo "No elements found for .stui-vodlist a" . PHP_EOL;
}
}
public function getVideoInfo($arrData)
{
$TaskCore = $this->TaskCore;
$Client = $this->Client;
$strSiteUrl = $this->getDadiDomain();
$strUrl = $strSiteUrl . $arrData['video_href'] ;
$intClassId = $arrData['class_id'];
$strVideoName = $arrData['video_name'];
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
$QueryList = QueryList::html($strResult);
$strCover = $QueryList->find('.stui-content__thumb img')->eq(0)->attr("data-original");
if (strpos($strCover, 'http') === false) {
$strCover = $strSiteUrl . $strCover;
}
$strCoverUri = $strCover;
// try {
// $strCoverUri = "";
// $strCoverUri = ImageProcessor::getInstance()->downloadImage( $intXiaoShuoSourceId,$strCover, 'dadi');
// } catch (\Throwable $t) {
// $strErr = sprintf("采集[%s]小说图片错误,异常:%s", $strCover, $t->getMessage());
// echo $strErr;
// }
$arrCover = [
'code' => 'DADI_YINGSHI_COVER',
// 'uri' => getUriByUrl($strCover),
'uri' => $strCoverUri,
];
$strLanguage = "";
// if (strlen($strLanguage) > 0) {
// VideoLeixinModel::addVideoLeixin(['vc_id' => $intClassId, 'vla_name' => $strLanguage]);
// }
// 提取评分
$strPinfen = $QueryList->find('h1.title .score')->text();
// 提取类型-入库
$strLeixin = $QueryList->find('.col-pd a')->eq(1)->text();
if(strlen($strLeixin) > 0){
VideoLeixinModel::addVideoLeixin(['vc_id'=>$intClassId,'vl_name'=>$strLeixin]);
}
// 提取 HTML 的第一段 <p class="data">
$dataNode = $QueryList->find('p.data')->eq(0);
// 提取“类型”内容:确保只匹配 `href` 中包含 `/vodsearch/----`
$leixinNodes = $dataNode->find('a[href^="/vodsearch/----"]');
$strJuqing = $leixinNodes->map(function ($node) {
return $node->text();
})->all();
// 提取“地区”内容:确保只匹配 `href` 中包含 `/vodsearch/--`,但不包含 `/vodsearch/----`
$diquNodes = $dataNode->find('a[href^="/vodsearch/--"]')->not('a[href^="/vodsearch/----"]');
$strDiqu = $diquNodes->map(function ($node) {
return $node->text();
})->all();
// 提取“年份”内容:确保只匹配 `href` 中包含 `/vodsearch/-------------`
$yearsNode = $dataNode->find('a[href^="/vodsearch/-------------"]');
$strYears = $yearsNode->text();
if (!empty(trim($strYears))) {
$strYears = [trim($strYears)];
} else {
$strYears = [];
}
// 过滤掉“地区”和“年份”中的数据
$strJuqing = array_filter($strJuqing, function ($value) use ($strDiqu, $strYears) {
return !in_array($value, array_merge($strDiqu, $strYears));
});
// **在过滤后将“类型”数据入库**
foreach ($strJuqing as $value) {
if(strlen($value) > 0){
VideoJuqingModel::add(['vc_id' => $intClassId, 'vj_name' => $value]);
}
}
// 将“地区”数据入库
foreach ($strDiqu as $value) {
if(strlen($value) > 0){
VideoDiquModel::add(['vc_id' => $intClassId, 'vd_name' => $value]);
}
}
// 将“年份”数据入库
foreach ($strYears as $value) {
if(strlen($value) > 0){
VideoYearsModel::add(['vc_id' => $intClassId, 'vy_name' => $value]);
}
}
// 提取主演
$strZhuyan = $QueryList->find('p.data:contains("主演") a')->map(function ($item) {
return $item->text();
})->implode(', '); // 使用逗号分隔主演
// 提取导演
$strDaoyan = $QueryList->find('p.data:contains("导演") a')->map(function ($item) {
return $item->text();
})->implode(', '); // 使用逗号分隔主演
// 提取更新时间
$strLastDate = $QueryList->find('p.data.hidden-sm:contains("更新")')->text();
// 去掉“更新:”前缀
$strLastDate = str_replace('更新:', '', $strLastDate);
// 提取简介
$strDescription = $QueryList->find('#desc>.stui-pannel-box>.stui-pannel_bd>.col-pd')->text();
// 角标
$strJiaoBiao = $QueryList->find('.stui-content__thumb .text-right')->eq(0)->text();
$strPinyinId = extractFilename($strUrl);
$arrVideoInfo = [
'v_name' => $strVideoName,
'v_cover' => json_encode($arrCover),
'vc_id' => $intClassId,
'v_pinyin' => $strPinyinId,
'v_leixin' => $strLeixin ,
'v_juqing' => implode(', ', $strJuqing),
'v_diqu' => implode(', ', $strDiqu),
'v_years' => implode(', ', $strYears),
'v_language' => $strLanguage,
'v_daoyan' => $strZhuyan,
'v_zhuyan' => $strDaoyan,
'update_time' => $strLastDate,
'v_description'=>$strDescription,
'v_pingfen' => $strPinfen,
'v_orurl' => $strUrl,
'v_jiaobiao'=> $strJiaoBiao,
'v_md5' => md5($intClassId.$strVideoName.$strSiteUrl.$strUrl),
];
$VideoInfoModel = VideoInfoModel::addVideoXiangQing($arrVideoInfo);
$intVid = $VideoInfoModel->v_id;
$intChapterIndex = 0 ;
$arrZhangjie = [];
// // 检查是否找到元素
if ($QueryList->find('.stui-content__playlist a')->count() > 0) {
$QueryList->find('.stui-content__playlist a')->map(function ($ZhangJie) use (&$arrZhangjie, &$intChapterIndex ,&$intVid) {
//echo "Processing index $intChapterIndex: " . $ZhangJie->text() . PHP_EOL;
$arrZhangjie[] = [
'v_id' => $intVid,
'index' => $intChapterIndex,
'mingzi' => $ZhangJie->text(),
'url' => $ZhangJie->attr('href'),
];
$intChapterIndex++;
});
//var_dump($arrZhangjie);
foreach ($arrZhangjie as $intZhangJieNum => $arrZhangjieOne) {
var_dump($intZhangJieNum);
// 如果数据库里的章节最大索引大于当前索引,说明已经入库,不需要入队列了
// if($intMaxXiaoShuoZhangJiePaixu > $intZhangJieNum){
// var_dump("小说章节已经入库,不再推送任务");
// continue;// 如果任务已存在,直接跳过 break
// }
//if($intZhangJieNum<=3){
$arrTask = [
'callback' => [self::class, 'getNovelReader'],
'data' => [
'v_id' => $intVid,
'vp_sort' => $intZhangJieNum,
'vp_name' => $arrZhangjieOne['mingzi'],
'vp_url' => $strSiteUrl . $arrZhangjieOne['url']
]
];
$TaskCore->set($arrTask);
//}
}
} else {
echo "No elements found for .listmain a" . PHP_EOL;
}
} catch (\Throwable $T) {
throw $T;
}
// sleep(10);
}
public function getNovelFengMian($arrData)
{
//var_dump('getNovelFengMian');
//var_dump($arrData);
$Client = $this->Client;
$TaskCore = $this->TaskCore;
// 获取 Redis 实例
$redis = Cache::store('redis')->handler();
$intXiaoShuoSourceId = $arrData['xiaoshuo_id'];
$intXiaoShuoSourceSeoId = $arrData['xiaoshuo_seo_id'];
$redisKey = "task:novel_source_id:" . $intXiaoShuoSourceId;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($arrData['xiaoshuo_url']);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$strSiteUrl = $this->getAilisiDomain();
$strCover = $QueryList->find('#fmimg img')->eq(0)->attr("src");
if (strpos($strCover, 'http') === false) {
$strCover = $strSiteUrl . $strCover;
}
$strCoverUri = $strCover;
try {
$strCoverUri = "";
$strCoverUri = ImageProcessor::getInstance()->downloadAndEncryptImage($strCover, 'xs');
$arrCover = [
'code' => 'AI_LI_SI_COVER',
// 'uri' => getUriByUrl($strCover),
'uri' => $strCoverUri,
];
$arrXiaoShuoInfo = [
'xsxq_feng_mian' => json_encode($arrCover),
];
$XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
} catch (\Throwable $t) {
$strErr = sprintf("采集[%s]小说图片错误,异常:%s", $strCover, $t->getMessage());
echo $strErr;
}
} catch (\Throwable $T) {
throw $T;
}
}
public function getNovelReader($arrData)
{
if (Cache::has('video_no_m3u8_' . $arrData['v_id'])) {
echo "跳过抓取视频ID已在缓存中: " . $arrData['v_id'] . "\n";
return; // 跳过抓取逻辑
}
$strMuluHref = $arrData['vp_url'];
var_dump($strMuluHref);
$Client = $this->Client;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strMuluHref);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strMuluHref, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
unset($Response);
$QueryList = QueryList::html($strResult);
//提取 <script> 标签内容
$scriptContent = $QueryList->find('.vidoe-data-show')->text();
$strM3u8Url = "";
// 使用正则表达式提取 "url" 的值
if (preg_match('/"url":"(https:[^"]+\.m3u8)"/', $scriptContent, $matches)) {
$strM3u8Url = str_replace('\/', '/', $matches[1]); // 替换转义斜杠
echo "提取的 M3U8 地址: $strM3u8Url\n";
$arrM3u8Url = [
'code' => 'DADI_YINGSHI_M3U8',
'uri' => $strM3u8Url,
];
$arrVideoPlayurl = [
'v_id' => $arrData['v_id'],
'vp_sort' => $arrData['vp_sort'],
'vp_name' => $arrData['vp_name'],
'vp_url' => json_encode($arrM3u8Url),
'vp_md5' => md5($arrData['vp_url'] . $arrData['vp_url']),
];
VideoPlayurlModel::add($arrVideoPlayurl);
unset($QueryList);
} else {
echo "未找到 M3U8 地址\n";
// 将视频ID保存到缓存设置一个过期时间例如 1 小时
Cache::set('video_no_m3u8_' . $arrData['v_id'], true, 3600);
}
} catch (\Throwable $T) {
throw $T;
}
}
}

View File

@@ -0,0 +1,560 @@
<?php
declare(strict_types=1);
namespace app\task\collection\heimuer;
use app\admin\model\VideoPlayurlModel;
use app\admin\model\VideoLeixinModel;
use app\admin\model\VideoDiquModel;
use app\admin\model\VideoYearsModel;
use app\admin\model\VideoJuqingModel;
use app\admin\model\XiaoShuoFenLeiModel;
use app\admin\model\XiaoShuoXiangQingModel;
use app\admin\model\VideoInfoModel;
use app\admin\model\XiaoShuoZhangJieModel;
use app\task\collection\PullDataColBase;
use GuzzleHttp\Cookie\CookieJar;
use processor\ImageProcessor;
use think\facade\Cache;
use QL\QueryList;
/**
* @mixin think\Model
*/
class HeiMuErYingShiCol extends PullDataColBase
{
public function pullDadiYingshi()
{
$strSiteUrl = $this->getHeiMuErDomain();
// 配置分类任务参数
$categories = [
['id' => 1, 'video_class_id' => 1, 'name' => '电影'],
['id' => 27, 'video_class_id' => 5, 'name' => '短剧'],
['id' => 2, 'video_class_id' => 2, 'name' => '电视剧'],
['id' => 4, 'video_class_id' => 3, 'name' => '综艺'],
['id' => 3, 'video_class_id' => 4, 'name' => '动漫'],
];
foreach ($categories as $category) {
$intClassId = $category['video_class_id'];
$intCategoryId = $category['id'];
// 先抓第一页
$strUrl = $strSiteUrl . '/index.php/vod/type/id/' . $intCategoryId . '/page/1.html';
$maxPage = $this->getMaxPage($strUrl,$intClassId);
// 首次抓取,全部抓取
if (config('task.task_is_first')) {
$this->createTasks($strSiteUrl, $category['id'], $category['video_class_id'], $maxPage);
}
}
}
/**
* 获取分类的最大页码且抓取第一页
*/
protected function getMaxPage($strUrl, $intClassId)
{
$TaskCore = $this->TaskCore;
$Client = $this->Client;
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$Response = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'Accept-Language' => 'en-US,en;q=0.5',
'cookie' => '__sk_Vm8nTqM7LqSCWmR6__=be3bd4f4e0483619e1a434ec1b18a938',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 300,
]);
$strResult = (string)$Response->getBody();
$QueryList = QueryList::html($strResult);
// 检查是否找到元素
if ($QueryList->find('.stui-vodlist li.clearfix h3 a')->count() > 0) {
$QueryList->find('.stui-vodlist li.clearfix h3 a ')->map(function ($Video) use ($intClassId, &$TaskCore) {
$strHref = $Video->attr('href');
$strName = $Video->text();
if (!empty($strName) && !empty($strHref) && strlen($strName) > 0 && strlen($strHref) > 0) {
$arrTask = [
'callback' => [self::class, 'getVideoInfo'],
'data' => [
'class_id' => $intClassId,
'video_href' => $strHref,
'video_name' => $strName,
]
];
// var_dump($arrTask);
$TaskCore->set($arrTask);
echo "视频投递成功 : " . $Video->text() . PHP_EOL;
}
});
} else {
echo "No elements found for .stui-vodlist a" . PHP_EOL;
}
// 获取分页部分最后一个页码链接
$lastPageHref = $QueryList->find('.stui-page li')->eq($QueryList->find('.stui-page li')->count() - 1)->find('a')->attr('href');
// 提取最大页码
preg_match('/page\/(\d+)\.html/', $lastPageHref, $matches);
$maxPage = isset($matches[1]) ? (int)$matches[1] : 1; // 默认返回 1
return $maxPage;
}
/**
* 创建任务
*/
protected function createTasks($strSiteUrl, $categoryId, $videoClassId, $maxPage)
{
$TaskCore = $this->TaskCore;
for ($i = $maxPage; $i >= 1; $i--) {
echo "当前分类: $categoryId, 当前页: $i\n";
$arrTask = [
'callback' => [self::class, 'getVideoPage'],
'data' => [
'video_page_url' => $strSiteUrl . '/index.php/vod/type/id/' . $categoryId . '/page/' . $i . '.html',
'video_class_id' => $videoClassId,
]
];
$TaskCore->set($arrTask);
}
}
# 获取分页里面的视频列表
public function getVideoPage($arrData)
{
$TaskCore = $this->TaskCore;
$Client = $this->Client;
$strUrl = $arrData['video_page_url'];
$intClassId = $arrData['video_class_id'];
$strSiteUrl = $this->getHeiMuErDomain();
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
var_dump($strUrl);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'Accept-Language' => 'en-US,en;q=0.5',
'cookie' => '__sk_Vm8nTqM7LqSCWmR6__=be3bd4f4e0483619e1a434ec1b18a938',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 300,
]);
$strResult = (string)$Response->getBody();
$QueryList = QueryList::html($strResult);
// 检查是否找到元素
if ($QueryList->find('.stui-vodlist li.clearfix h3 a')->count() > 0) {
$QueryList->find('.stui-vodlist li.clearfix h3 a ')->map(function ($Video) use (&$arrVideo, $intClassId, &$TaskCore, $strSiteUrl) {
$strHref = $Video->attr('href');
$strName = $Video->text();
if (!empty($strName) && !empty($strHref) && strlen($strName) > 0 && strlen($strHref) > 0) {
$arrTask = [
'callback' => [self::class, 'getVideoInfo'],
'data' => [
'class_id' => $intClassId,
'video_href' => $strHref,
'video_name' => $strName,
]
];
// var_dump($arrTask);
$TaskCore->set($arrTask);
echo "视频投递成功 : " . $Video->text() . PHP_EOL;
}
});
} else {
echo "No elements found for .stui-vodlist a" . PHP_EOL;
}
}
public function getVideoInfo($arrData)
{
$TaskCore = $this->TaskCore;
$Client = $this->Client;
$strSiteUrl = $this->getHeiMuErDomain();
$strUrl = $strSiteUrl . $arrData['video_href'];
$intClassId = $arrData['class_id'];
$strVideoName = $arrData['video_name'];
var_dump($strUrl);
if (!empty($strVideoName) && !empty($strUrl) && strlen($strVideoName) > 0 && strlen($strUrl) > 0) {
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
$QueryList = QueryList::html($strResult);
$strCover = $QueryList->find('.img-responsive')->eq(0)->attr("src");
if (strpos($strCover, 'http') === false) {
$strCover = $strSiteUrl . $strCover;
}
$strCoverUri = $strCover;
// try {
// $strCoverUri = "";
// $strCoverUri = ImageProcessor::getInstance()->downloadImage( $intXiaoShuoSourceId,$strCover, 'dadi');
// } catch (\Throwable $t) {
// $strErr = sprintf("采集[%s]小说图片错误,异常:%s", $strCover, $t->getMessage());
// echo $strErr;
// }
$arrCover = [
'code' => 'HEIMUER_SOURCE_COVER',
'uri' => $strCoverUri,
];
$strLanguage = "";
// if (strlen($strLanguage) > 0) {
// VideoLeixinModel::addVideoLeixin(['vc_id' => $intClassId, 'vla_name' => $strLanguage]);
// }
// 提取别名
$strPinyinId = $QueryList->find('.stui-content__detail p.data')->eq(0)->text();
// 提取评分
$strPinfen = $QueryList->find('.stui-content__detail h1 .text-red')->text();
// 提取地区
$strDiqu = $QueryList->find('.stui-content__detail p.data')->eq(1)->text();
// 角标-状态
$strJiaoBiao = $QueryList->find('.stui-content__detail p.data')->eq(2)->find('.text-red')->eq(0)->text();
// 更新时间
$strLastDate = $QueryList->find('.stui-content__detail p.data')->eq(2)->find('.text-red')->eq(1)->text();
// 提取主演
$strZhuyan = $QueryList->find('.stui-content__detail p.data')->eq(3)->text();
// 提取导演
$strDaoyan = $QueryList->find('.stui-content__detail p.data')->eq(4)->text();
// 提取
$arrLeixinData = $QueryList->find('.stui-content__detail p.data')->eq(5)->text();
// 提取
$arrYuyanData = $QueryList->find('.stui-content__detail p.data')->eq(6)->text();
$strLeixin = '';
$strDiqu = '';
$strYears = '';
$strKuozhang = '';
// 使用正则表达式提取内容
$patternLeixin = '/类型:(.*?)扩展:(.*?)地区:(.*?)年份:(.*)/';
if (preg_match($patternLeixin, $arrLeixinData, $matches)) {
$strLeixin = trim($matches[1]);
$strKuozhang = trim($matches[2]);
$strDiqu = trim($matches[3]);
$strYears = trim($matches[4]);
}
$strLanguage = '';
// 使用正则表达式提取内容
$patternYuyan = '/语言:(.*?)集数:(.*?)时长:(.*?)点击:(.*)/';
if (preg_match($patternYuyan, $arrYuyanData, $matches)) {
$strLanguage = trim($matches[1]);
// $strKuozhang = trim($matches[2]);
// $strDiqu = trim($matches[3]);
// $strYears = trim($matches[4]);
}
if (strlen($strLeixin) > 0) {
VideoLeixinModel::addVideoLeixin(['vc_id' => $intClassId, 'vl_name' => $strLeixin]);
}
if (strlen($strDiqu) > 0) {
VideoDiquModel::add(['vc_id' => $intClassId, 'vd_name' => $strDiqu]);
}
if (strlen($strYears) > 0) {
VideoYearsModel::add(['vc_id' => $intClassId, 'vy_name' => $strYears]);
}
// 提取简介
$strDescription = $QueryList->find('.stui-content__desc')->text();
$arrVideoInfo = [
'v_name' => $strVideoName,
'v_cover' => json_encode($arrCover),
'vc_id' => $intClassId,
'v_pinyin' => trim(str_replace('别名:', '', $strPinyinId)),
'v_leixin' => $strLeixin,
'v_juqing' => '',
'v_diqu' => $strDiqu,
'v_years' => $strYears,
'v_language' => $strLanguage,
'v_daoyan' => trim(str_replace('主演:', '', $strZhuyan)),
'v_zhuyan' => trim(str_replace('导演:', '', $strDaoyan)),
'update_time' => $strLastDate,
'v_description' => $strDescription,
'v_pingfen' => $strPinfen,
'v_orurl' => $strUrl,
'v_jiaobiao' => $strJiaoBiao,
'v_md5' => md5($intClassId . $strVideoName . getUriByUrl($strUrl)),
];
//var_dump($arrVideoInfo);
// // 检查是否找到元素
if ($QueryList->find('.stui-content__playlist li')->count() > 0) {
// var_dump("视频入库成功--" . $strVideoName);
// $VideoInfoModel = VideoInfoModel::addVideoXiangQing($arrVideoInfo);
$resultVideoInfoModel = VideoInfoModel::addVideoXiangQing($arrVideoInfo);;
if ($resultVideoInfoModel['isNew']) {
var_dump("数据已新增:", $resultVideoInfoModel['model']->v_id."视频入库成功--" . $strVideoName);
} else {
echo "数据已存在:", $resultVideoInfoModel['model']->v_id;
}
$VideoInfoModel = $resultVideoInfoModel['model'];
$intVid = $VideoInfoModel->v_id;
$intChapterIndex = 0;
$QueryList->find('.stui-content__playlist li')->map(function ($ZhangJie) use (&$arrZhangjie, &$intChapterIndex, &$intVid) {
//echo "Processing index $intChapterIndex: " . $ZhangJie->text() . PHP_EOL;
$strM3u8 = $ZhangJie->find('a')->eq(1)->text();
$arrM3u8 = explode('$', $strM3u8);
$strM3u8Url = $arrM3u8[1];
$strM3u8Name = $arrM3u8[0];
$arrM3u8Url = [
'code' => 'HEIMUER_SOURCE_PLAY',
'uri' => $strM3u8Url,
];
$arrVideoPlayurl = [
'v_id' => $intVid,
'vp_sort' => $intChapterIndex,
'vp_name' => $strM3u8Name,
'vp_url' => json_encode($arrM3u8Url),
'vp_md5' => md5($intVid . getUriByUrl($strM3u8Url)),
];
$result = VideoPlayurlModel::add($arrVideoPlayurl);;
if ($result['isNew']) {
var_dump("数据已新增:", $result['model']->vp_id."播放线路入库成功--" . $strM3u8Url) ;
} else {
echo "数据已存在:", $result['model']->vp_id ;
}
$intChapterIndex++;
});
var_dump("--") ;
var_dump("一个视频任务完成------------------------------------------------------------------------------------") ;
} else {
echo "No elements found for .listmain a" . PHP_EOL;
}
} catch (\Throwable $T) {
throw $T;
}
}
// sleep(10);
}
public function getNovelFengMian($arrData)
{
//var_dump('getNovelFengMian');
//var_dump($arrData);
$Client = $this->Client;
$TaskCore = $this->TaskCore;
// 获取 Redis 实例
$redis = Cache::store('redis')->handler();
$intXiaoShuoSourceId = $arrData['xiaoshuo_id'];
$intXiaoShuoSourceSeoId = $arrData['xiaoshuo_seo_id'];
$redisKey = "task:novel_source_id:" . $intXiaoShuoSourceId;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($arrData['xiaoshuo_url']);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$strSiteUrl = $this->getAilisiDomain();
$strCover = $QueryList->find('#fmimg img')->eq(0)->attr("src");
if (strpos($strCover, 'http') === false) {
$strCover = $strSiteUrl . $strCover;
}
$strCoverUri = $strCover;
try {
$strCoverUri = "";
$strCoverUri = ImageProcessor::getInstance()->downloadAndEncryptImage($strCover, 'xs');
$arrCover = [
'code' => 'AI_LI_SI_COVER',
// 'uri' => getUriByUrl($strCover),
'uri' => $strCoverUri,
];
$arrXiaoShuoInfo = [
'xsxq_feng_mian' => json_encode($arrCover),
];
$XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
} catch (\Throwable $t) {
$strErr = sprintf("采集[%s]小说图片错误,异常:%s", $strCover, $t->getMessage());
echo $strErr;
}
} catch (\Throwable $T) {
throw $T;
}
}
public function getNovelReader($arrData)
{
$strMuluHref = $arrData['vp_url'];
var_dump($strMuluHref);
$Client = $this->Client;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strMuluHref);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strMuluHref, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
unset($Response);
$QueryList = QueryList::html($strResult);
//提取 <script> 标签内容
$scriptContent = $QueryList->find('.vidoe-data-show')->text();
$strM3u8Url = "";
// 使用正则表达式提取 "url" 的值
if (preg_match('/"url":"(https:[^"]+\.m3u8)"/', $scriptContent, $matches)) {
$strM3u8Url = str_replace('\/', '/', $matches[1]); // 替换转义斜杠
echo "提取的 M3U8 地址: $strM3u8Url\n";
$arrM3u8Url = [
'code' => 'DADI_YINGSHI_M3U8',
'uri' => $strM3u8Url,
];
$arrVideoPlayurl = [
'v_id' => $arrData['v_id'],
'vp_sort' => $arrData['vp_sort'],
'vp_name' => $arrData['vp_name'],
'vp_url' => json_encode($arrM3u8Url),
'vp_md5' => md5($arrData['vp_url'] . $arrData['vp_url']),
];
VideoPlayurlModel::add($arrVideoPlayurl);
unset($QueryList);
} else {
echo "未找到 M3U8 地址\n";
}
} catch (\Throwable $T) {
throw $T;
}
}
public function pullDadiYingshi2()
{
$strSiteUrl = $this->getHeiMuErDomain();
$TaskCore = $this->TaskCore;
//电影 307 https://heimuer.tv/index.php/vod/type/id/1/page/307.html
for ($i = 307; $i >= 1; $i--) {
if (!config('task.task_is_first') && $i > 1) {
continue;
}
echo "当前值: $i\n";
$arrTask = [
'callback' => [self::class, 'getVideoPage'],
'data' => [
'video_page_url' => $strSiteUrl . '/index.php/vod/type/id/1/page/' . $i . '.html',
'video_class_id' => 1
]
];
$TaskCore->set($arrTask);
}
//5-87 短剧 https://heimuer.tv/index.php/vod/type/id/27/page/87.html
for ($i = 87; $i >= 1; $i--) {
if (!config('task.task_is_first') && $i > 1) {
continue;
}
echo "当前值: $i\n";
$arrTask = [
'callback' => [self::class, 'getVideoPage'],
'data' => [
'video_page_url' => $strSiteUrl . '/index.php/vod/type/id/27/page/' . $i . '.html',
'video_class_id' => 5
]
];
$TaskCore->set($arrTask);
}
//电视剧 2--225 https://heimuer.tv/index.php/vod/type/id/2/page/225.html
for ($i = 225; $i >= 1; $i--) {
if (!config('task.task_is_first') && $i > 1) {
continue;
}
echo "当前值: $i\n";
$arrTask = [
'callback' => [self::class, 'getVideoPage'],
'data' => [
'video_page_url' => $strSiteUrl . '/index.php/vod/type/id/2/page/' . $i . '.html',
'video_class_id' => 2
]
];
$TaskCore->set($arrTask);
}
//综艺 78 https://heimuer.tv/index.php/vod/type/id/4/page/78.html
for ($i = 78; $i >= 1; $i--) {
if (!config('task.task_is_first') && $i > 1) {
continue;
}
echo "当前值: $i\n";
$arrTask = [
'callback' => [self::class, 'getVideoPage'],
'data' => [
'video_page_url' => $strSiteUrl . '/index.php/vod/type/id/4/page/' . $i . '.html',
'video_class_id' => 3
]
];
$TaskCore->set($arrTask);
}
// 动漫--100 https://heimuer.tv/index.php/vod/type/id/3/page/1.html
for ($i = 100; $i >= 1; $i--) {
if (!config('task.task_is_first') && $i > 1) {
continue;
}
echo "当前值: $i\n";
$arrTask = [
'callback' => [self::class, 'getVideoPage'],
'data' => [
'video_page_url' => $strSiteUrl . '/index.php/vod/type/id/3/page/' . $i . '.html',
'video_class_id' => 4
]
];
$TaskCore->set($arrTask);
}
}
}

View File

@@ -0,0 +1,850 @@
<?php
declare(strict_types=1);
namespace app\task\collection\novel;
use app\admin\model\XiaoShuoFenLeiModel;
use app\admin\model\XiaoShuoXiangQingModel;
use app\admin\model\XiaoShuoZhangJieModel;
use app\admin\model\NovelInfoSeoModel;
use app\task\collection\PullDataColBase;
use GuzzleHttp\Cookie\CookieJar;
use processor\ImageProcessor;
use think\facade\Cache;
use QL\QueryList;
use app\common\Base64Helper as qsbs;
/**
* @mixin think\Model https://www.630zw.org/
*/
class Zw630XiaoShuoCol extends PullDataColBase
{
public function pullXiaoShuo()
{
// $decoded = '美女';
// $encoded = qsbs::encode($decoded);
// $decodedString = qsbs::decode('5oC75LmL77yM57uT5bGA5piv5aW955qE77yM5amJ5YS/5bqU6K+l5LiN5Lya5Zyo5LmO6L+Z5Lqb57uG6IqC44CCPC9wPg==');
// var_dump($encoded);
// var_dump($decodedString);
// 首次抓取,全部抓取
if (config('task.task_is_first')) {
$TaskCore = $this->TaskCore;
// 527797
for ($i = config('task.task_start_number'); $i <= config('task.task_end_number'); $i++) {
//echo "当前值: $i\n";
$arrTask = [
'callback' => [self::class, 'getNovelInfo'],
'data' => [
'xs_source_id' => $i, //528049
'xs_source_seo_id' => 1,
]
];
$TaskCore->set($arrTask);
// var_dump($arrTask);
}
}else{
$this->pullFenlei();
}
}
public function pullFenlei()
{
$TaskCore = $this->TaskCore;
for ($intFenleiId = 1; $intFenleiId <= 9; $intFenleiId++) {
for ($intPage = 1; $intPage <= 10; $intPage++) {
// https://www.630zw.org/list/1/10.html
$arrTask = [
'callback' => [self::class, 'getFenLei'],
'data' => [
'xs_fenlei_url' => 'https://www.630zw.org/list/'.$intFenleiId.'/'.$intPage.'.html',
]
];
$TaskCore->set($arrTask);
// var_dump($arrTask);
}
}
}
public function getFenLei($arrData)
{
$TaskCore = $this->TaskCore;
$Client = $this->Client;
$strSiteUrl = $this->get630zwDomain();
$strUrl = $arrData['xs_fenlei_url']; //1 ;
var_dump($strUrl);
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
$QueryList = QueryList::html($strResult);
$arrNovel = [];
// 检查是否找到元素
if ($QueryList->find('.layout2 .txt-list-row5')->eq(0)->find('li')->count() > 0) {
$QueryList->find('.layout2 .txt-list-row5')->eq(0)->find('li')->map(function ($Novel) use (&$arrNovel) {
$strNovelHref = $Novel->find('.s2 a')->attr('href') ;
$intNovelId = extractFilename($strNovelHref);
var_dump($strNovelHref);
$arrNovel[] = [
'xs_source_id' => $intNovelId, //528049
'xs_source_seo_id' => 1,
];
});
} else {
echo 'qie h5' . PHP_EOL;
if ($QueryList->find('.sort_list')->eq(0)->find('li')->count() > 0) {
$QueryList->find('.sort_list')->eq(0)->find('li')->map(function ($Novel) use (&$arrNovel) {
$strNovelHref = $Novel->find('.s2 a')->attr('href') ;
$intNovelId = extractFilename($strNovelHref);
var_dump($strNovelHref);
$arrNovel[] = [
'xs_source_id' => $intNovelId, //528049
'xs_source_seo_id' => 1,
];
});
}
}
foreach ($arrNovel as $intNum => $arrNovelOne) {
$arrTask = [
'callback' => [self::class, 'getNovelInfo'],
'data' => $arrNovelOne,
];
$TaskCore->set($arrTask);
}
} catch (\Throwable $T) {
$arrTask = [
'callback' => [self::class, 'getFenLei'],
'data' => [
'xs_fenlei_url' => $strUrl,
]
];
$TaskCore->set($arrTask);
throw $T;
}
}
public function getNovelInfo($arrData)
{
$TaskCore = $this->TaskCore;
$Client = $this->Client;
$strSiteUrl = $this->get630zwDomain();
$strUrl = 'https://630zw.org/shu/' . $arrData['xs_source_id'] . '.html'; //'https://630zw.org/shu/93716.html';
$intXiaoShuoSourceId = $arrData['xs_source_id']; //20053;
$intXiaoShuoSourceSeoId = $arrData['xs_source_seo_id']; //1 ;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
$QueryList = QueryList::html($strResult);
// var_dump($QueryList);
// 提取所有 <meta> 标签的属性
$result = $QueryList->find('meta[property^="og:"]')->map(function ($item) {
return [
'property' => $item->attr('property'),
'content' => $item->attr('content'),
];
});
// 打印结果
// print_r($result->all());
$strZuoZhe = '';
$strMingZi = '';
$strLastDate = '';
$strJieShao = '';
$strCover = '';
$strFenlei = '';
$strZiShu = 0;
$strZhuangTai = '';
foreach ($result as $item) {
switch ($item['property']) {
case 'og:title':
$strMingZi = $item['content'];
break;
case 'og:description':
$strJieShao = preg_split('/<br\s*\/?>/i', $item['content'])[0];
break;
case 'og:image':
$strCover = $item['content'];
break;
case 'og:novel:category':
$strFenlei = $item['content'];
break;
case 'og:novel:author':
$strZuoZhe = $item['content'];
break;
case 'og:novel:status':
$strZhuangTai = $item['content'];
break;
case 'og:novel:update_time':
$strLastDate = $item['content'];
break;
}
}
$strLastDate = trim((string)$strLastDate);
if (!isValidDateTime($strLastDate)) {
$strLastDate = date('Y-m-d H:i:s');
}
$arrTag = [];
if (strpos($strCover, 'http') === false) {
$strCover = $strSiteUrl . $strCover;
}
$strCoverUri = $strCover;
try {
$strCoverUri = "";
$strCoverUri = ImageProcessor::getInstance()->downloadImage($intXiaoShuoSourceId, $strCover, 'xs/630zw');
} catch (\Throwable $t) {
$strErr = sprintf("采集[%s]小说图片错误,异常:%s", $strCover, $t->getMessage());
echo $strErr;
}
$arrCover = [
'code' => '630_ZW_XS_COVER',
// 'uri' => getUriByUrl($strCover),
'uri' => $strCoverUri,
];
$strFenleiIId = 1;
switch ($strFenlei) {
case '玄幻修真':
$strName = "玄幻";
$strFenleiIId = 1;
break;
case '重生穿越':
$strName = "修真";
$strFenleiIId = 2;
break;
case '都市小说':
$strName = "都市";
$strFenleiIId = 3;
break;
case '军史小说':
$strName = "历史";
$strFenleiIId = 4;
break;
case '网游小说':
$strName = "网游";
$strFenleiIId = 5;
break;
case '科幻小说':
$strName = "科幻";
$strFenleiIId = 6;
break;
case '灵异小说':
$strName = "女频";
$strFenleiIId = 7;
break;
case '言情小说':
$strName = "其它";
$strFenleiIId = 8;
break;
case '其他小说':
$strName = "其它";
$strFenleiIId = 9;
break;
default:
$strName = "其它";
$strFenleiIId = 9;
break;
}
$arrXiaoShuoInfo = [
'xsxq_ming_zi' => $strMingZi,
'xsxq_zuozhe' => $strZuoZhe, //$arrData['xiaoshuo_zuozhe'],
'xsfl_id' => $strFenleiIId,
'xsxq_zhuang_tai' => $strZhuangTai,
'xsxq_zi_shu' => $strZiShu,
'xsxq_jie_shao' => $strJieShao,
'xsxq_feng_mian' => json_encode($arrCover),
'xsxq_geng_xin_shi_jian' => $strLastDate,
'xsxq_tag' => json_encode($arrTag, JSON_UNESCAPED_UNICODE),
'xsxq_source_id' => $intXiaoShuoSourceId,
'xsxq_source_seo_id' => $intXiaoShuoSourceId . '1',
'xsxq_source_code' => 'ZW630_XIAOSHUO_' . $intXiaoShuoSourceId . '_' . $intXiaoShuoSourceSeoId,
];
// print_r($arrXiaoShuoInfo);
var_dump($arrXiaoShuoInfo['xsxq_source_code']);
$XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::where('xsxq_source_code', $arrXiaoShuoInfo['xsxq_source_code'])->find();
$arrZhangjie = [];
# seo 小说入库 - 只有新小说才会走这个步骤
if (!$XiaoShuoXiangQingModel) {
$XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
$intXiaoShuolId = $XiaoShuoXiangQingModel->xsxq_id;
// 检查是否找到元素-seo 小说入库
if ($QueryList->find('.first_txt a')->count() > 0 && $intXiaoShuolId) {
$QueryList->find('.first_txt a')->map(function ($ZhangJie) use ($intXiaoShuolId) {
$url = $ZhangJie->attr('href');
// 检查是否以 "/kan/" 开头
if (strpos($url, '/kan/') === 0) { // strpos 返回位置0 表示是以 "/kan/" 开头
// 处理符合条件的链接
// echo "Processing index : " . $ZhangJie->text() . PHP_EOL;
$arrXiaoShuoSeoInfo = [
'nis_name' => $ZhangJie->text(),
'xsxq_id' => $intXiaoShuolId,
'nis_md5' => md5($intXiaoShuolId . $ZhangJie->text()),
];
NovelInfoSeoModel::addXiaoShuoXiangQingSeo($arrXiaoShuoSeoInfo);
}
});
} else {
echo "No elements found for .first_txt a" . PHP_EOL;
var_dump($strResult);
}
// # 新小说全部章节入库
// $intChapterIndex = 1; // 定义一个计数器变量
// // 检查是否找到元素
// if ($QueryList->find('.section-list')->eq(1)->find('li')->count() > 0) {
// $QueryList->find('.section-list')->eq(1)->find('li')->map(function ($ZhangJie) use (&$arrZhangjie, &$intChapterIndex) {
// echo "Processing index $intChapterIndex: " . $ZhangJie->text() . PHP_EOL;
// $arrZhangjie[] = [
// 'paixu' => $intChapterIndex,
// 'mingzi' => $ZhangJie->find('a')->text(),
// 'url' => $ZhangJie->find('a')->attr('href'),
// ];
// $intChapterIndex++;
// });
// } else {
// echo "No elements found for .listmain a" . PHP_EOL;
// }
//unset($QueryList);
}else{
$intXiaoShuolId = $XiaoShuoXiangQingModel->xsxq_id;
// 获取最大章节索引
$intMaxXiaoShuoZhangJiePaixu = XiaoShuoZhangJieModel::where('xsxq_id', $intXiaoShuolId)
->max('xszj_pai_xu'); // 获取最大值
// var_dump('获取小说章节最大值-'.$intMaxXiaoShuoZhangJiePaixu);
}
# 新小说全部章节入库
$intChapterIndex = 1; // 定义一个计数器变量
// 检查是否找到元素
if ($QueryList->find('.section-list')->eq(1)->find('li')->count() > 0) {
$QueryList->find('.section-list')->eq(1)->find('li')->map(function ($ZhangJie) use (&$arrZhangjie, &$intChapterIndex) {
// echo "Processing index $intChapterIndex: " . $ZhangJie->text() . PHP_EOL;
$arrZhangjie[] = [
'paixu' => $intChapterIndex,
'mingzi' => $ZhangJie->find('a')->text(),
'url' => $ZhangJie->find('a')->attr('href'),
];
$intChapterIndex++;
});
$this->eachNovelReader($arrZhangjie,$strSiteUrl,$intXiaoShuolId,$strFenleiIId);
} else {
echo "No elements found for .listmain a" . PHP_EOL;
}
# 章节处理
# 新小说全部章节入库
# 旧小说,只更新最新章节,获取数据库最大章节索引,对比源总章节
// // 如果没有结果max 返回 null可以设置默认值
$intMaxXiaoShuoZhangJiePaixu = $intMaxXiaoShuoZhangJiePaixu ?? 0;
# 目录页面处理
$strMuluHref = $QueryList->find('.btn-mulu')->eq(0)->attr('href');
$strMuluHref = $strSiteUrl . $strMuluHref ;
$ResponseMulu = $Client->request('GET', $strMuluHref, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResultMulu = (string)$ResponseMulu->getBody();
$QueryListMulu = QueryList::html($strResultMulu);
$intMuluTotal = $QueryListMulu->find('.page_num select')->eq(0)->find('option')->count();
$strMuluEndHref = $QueryListMulu->find('.page_num select')->eq(0)->find('option')->eq($intMuluTotal-1)->attr('value');
$strMuluEndText = $QueryListMulu->find('.page_num select')->eq(0)->find('option')->eq($intMuluTotal-1)->text();
// 使用正则表达式匹配
preg_match_all('/\.section-list\.ycxsid>li:nth-child\(/', $strResultMulu, $matches);
preg_match_all('/\.section-list\.ycxsid>li:nth-last-child\(/', $strResultMulu, $matchesLast);
$intProLiNum = count($matches[0]);
$intLastLiNum = count($matchesLast[0]);
$arrZhangjieMulu = [];
//if($intMaxXiaoShuoZhangJiePaixu < 200){
$intChapterPageIndex = 101; // 定义一个计数器变量
// 检查是否找到元素
if ($QueryListMulu->find('.chapter-list')->eq(0)->find('li')->count() > 0) {
$QueryListMulu->find('.chapter-list')->eq(0)->find('li')->map(function ($ZhangJie) use (&$arrZhangjieMulu, &$intChapterPageIndex) {
$arrZhangjieMulu[] = [
'paixu' => $intChapterPageIndex,
'mingzi' => $ZhangJie->find('a')->text(),
'url' => $ZhangJie->find('a')->attr('href'),
];
$intChapterPageIndex ++ ;
});
$arrZhangjieMulu = array_slice($arrZhangjieMulu, $intProLiNum);
// 删除后 11 条数据
$arrZhangjieMulu = array_slice($arrZhangjieMulu, 0, count($arrZhangjieMulu) - $intLastLiNum);
$this->eachNovelReader($arrZhangjieMulu,$strSiteUrl,$intXiaoShuolId,$strFenleiIId);
var_dump("chapter章节任务投递成功 $strMuluHref");
} else {
// var_dump($QueryListMulu->find('.section-list')->eq(0)->find('li')->count());
//echo "No elements found for .chapter-list $strMuluHref" . PHP_EOL;
// 检查是否找到元素
$intListIndex = 1;
if ($QueryListMulu->find('.section-list')->eq(0)->find('li')->count() > 0) {
$QueryListMulu->find('.section-list')->eq(0)->find('li')->map(function ($ZhangJie) use (&$arrZhangjieMulu, &$intChapterPageIndex,&$intListIndex) {
if($intListIndex >14){
$arrZhangjieMulu[] = [
'paixu' => $intChapterPageIndex,
'mingzi' => $ZhangJie->find('a')->text(),
'url' => $ZhangJie->find('a')->attr('href'),
];
$intChapterPageIndex ++ ;
};
$intListIndex++ ;
});
$arrZhangjieMulu = array_slice($arrZhangjieMulu, $intProLiNum);
// 删除后 11 条数据
$arrZhangjieMulu = array_slice($arrZhangjieMulu, 0, count($arrZhangjieMulu) - $intLastLiNum);
$this->eachNovelReader($arrZhangjieMulu,$strSiteUrl,$intXiaoShuolId,$strFenleiIId);
var_dump("section章节任务投递成功 $strMuluHref");
} else {
var_dump("No elements found for .section-list $strMuluHref") ;
}
}
//}
// 输出最终结果
// var_dump(count($arrZhangjie));
// var_dump($QueryListMulu->find('.chapter-list li')->count());
// var_dump($intMuluTotal);
// var_dump($strMuluEndHref);
// var_dump($strMuluEndText);
// var_dump($intMaxXiaoShuoZhangJiePaixu); // 1010
preg_match('/(\d+)-(\d+)章/', $strMuluEndText, $matches);
$intMaxPaixu = $matches[2]; // 最大章节
$intMaxMuluPage = floor($intMaxPaixu / 100); // 最大目录页面
// $intMaxMuluPage = 2;
// 所有目录分页处理
for ($i = 2; $i <= $intMaxMuluPage; $i++) {
//echo "当前值: $i\n";
$arrTask = [
'callback' => [self::class, 'getNovelMulu'],
'data' => [
'strMuluHref' => $strSiteUrl . '/shu/' . $XiaoShuoXiangQingModel->xsxq_source_id . '_' . $i . '.html' ,
'xsxq_id' => $XiaoShuoXiangQingModel->xsxq_id,
'xsfl_id' => $strFenleiIId,
'intStartIndex' => $i*100 + 1,
]
];
$TaskCore->set($arrTask);
// var_dump($arrTask);
}
unset($QueryList);
unset($QueryListMulu);
} catch (\Throwable $T) {
$arrTask = [
'callback' => [self::class, 'getNovelInfo'],
'data' => [
'xs_source_id' => $arrData['xs_source_id'],
'xs_source_seo_id' => $intXiaoShuoSourceSeoId,
]
];
$TaskCore->set($arrTask);
throw $T;
}
}
public function getNovelMulu($arrData)
{
$TaskCore = $this->TaskCore;
$Client = $this->Client;
$strSiteUrl = $this->get630zwDomain();
$strUrl = $arrData['strMuluHref'];
$intXiaoShuoId = $arrData['xsxq_id'];
$intXiaoShuoFenleiId = $arrData['xsfl_id'];
$intStartIndex = $arrData['intStartIndex'];
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
$QueryList = QueryList::html($strResult);
// 使用正则表达式匹配
preg_match_all('/\.section-list\.ycxsid>li:nth-child\(/', $strResult, $matches);
preg_match_all('/\.section-list\.ycxsid>li:nth-last-child\(/', $strResult, $matchesLast);
$intProLiNum = count($matches[0]);
$intLastLiNum = count($matchesLast[0]);
$arrZhangjie = [];
if ($QueryList->find('.chapter-list')->eq(0)->find('li')->count() > 0) {
$QueryList->find('.chapter-list')->eq(0)->find('li')->map(function ($ZhangJie) use (&$arrZhangjie, &$intStartIndex) {
$arrZhangjie[] = [
'paixu' => $intStartIndex,
'mingzi' => $ZhangJie->find('a')->text(),
'url' => $ZhangJie->find('a')->attr('href'),
];
$intStartIndex ++ ;
// var_dump($intStartIndex);
});
$arrZhangjie = array_slice($arrZhangjie, $intProLiNum);
// 删除后 11 条数据
$arrZhangjie = array_slice($arrZhangjie, 0, count($arrZhangjie) - $intLastLiNum);
$this->eachNovelReader($arrZhangjie,$strSiteUrl,$intXiaoShuoId,$intXiaoShuoFenleiId);
var_dump("chapter章节任务投递成功 $strUrl");
} else {
// 检查是否找到元素
if ($QueryList->find('.section-list')->eq(0)->find('li')->count() > 0) {
$QueryList->find('.section-list')->eq(0)->find('li')->map(function ($ZhangJie) use (&$arrZhangjie, &$intStartIndex) {
$arrZhangjie[] = [
'paixu' => $intStartIndex,
'mingzi' => $ZhangJie->find('a')->text(),
'url' => $ZhangJie->find('a')->attr('href'),
];
$intStartIndex ++ ;
// var_dump($intStartIndex);
});
$arrZhangjie = array_slice($arrZhangjie, $intProLiNum);
// 删除后 11 条数据
$arrZhangjie = array_slice($arrZhangjie, 0, count($arrZhangjie) - $intLastLiNum);
$this->eachNovelReader($arrZhangjie,$strSiteUrl,$intXiaoShuoId,$intXiaoShuoFenleiId);
var_dump("section章节任务投递成功 $strUrl");
} else {
var_dump("No elements found for .section-list $strUrl") ;
}
}
unset($QueryList);
} catch (\Throwable $T) {
$arrTask = [
'callback' => [self::class, 'getNovelMulu'],
'data' => [
'strMuluHref' => $strUrl ,
'xsxq_id' => $intXiaoShuoId,
'xsfl_id' => $intXiaoShuoFenleiId,
'intStartIndex' => $intStartIndex,
]
];
$TaskCore->set($arrTask);
throw $T;
}
}
public function eachNovelReader($arrZhangjie,$strSiteUrl,$intXiaoShuoId,$intXiaoShuoFenleiId)
{
$TaskCore = $this->TaskCore;
// var_dump($arrZhangjie);
foreach ($arrZhangjie as $intZhangJieNum => $arrZhangjieOne) {
// 如果数据库里的章节最大索引大于当前索引,说明已经入库,不需要入队列了
// if($intMaxXiaoShuoZhangJiePaixu > $intZhangJieNum){
// var_dump("小说章节已经入库,不再推送任务");
// continue;// 如果任务已存在,直接跳过 break
// }
$strZhangJieUrl = 'https://www.630zw.org' . $arrZhangjieOne['url'];
$strZhangJieSourceId = extractFilename($strZhangJieUrl);
# 'AILISI_XIAOSHUO_ZHANGJIE_小说源id_小说源章节id'
$strXszjSourceCode = '630ZW_XIAOSHUO_ZHANGJIE_'. $intXiaoShuoId . '_' . $strZhangJieSourceId . '_1' ;
$arrTask = [
'callback' => [self::class, 'getNovelReader'],
'data' => [
'xszj_source_id' => $strZhangJieSourceId,
'xszj_paixu' => $arrZhangjieOne['paixu'],
'xszj_name' => $arrZhangjieOne['mingzi'],
'xszj_url' => $strZhangJieUrl,
'xszj_source_code' => $strXszjSourceCode,
'xsxq_id' => $intXiaoShuoId,
'xsfl_id' => $intXiaoShuoFenleiId,
'intZhangJieSort' => 1,
]
];
$TaskCore->set($arrTask);
}
}
public function getNovelFengMian($arrData)
{
//var_dump('getNovelFengMian');
//var_dump($arrData);
$Client = $this->Client;
$TaskCore = $this->TaskCore;
// 获取 Redis 实例
$redis = Cache::store('redis')->handler();
$intXiaoShuoSourceId = $arrData['xiaoshuo_id'];
$intXiaoShuoSourceSeoId = $arrData['xiaoshuo_seo_id'];
$redisKey = "task:novel_source_id:" . $intXiaoShuoSourceId;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($arrData['xiaoshuo_url']);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$strSiteUrl = $this->getAilisiDomain();
$strCover = $QueryList->find('#fmimg img')->eq(0)->attr("src");
if (strpos($strCover, 'http') === false) {
$strCover = $strSiteUrl . $strCover;
}
$strCoverUri = $strCover;
try {
$strCoverUri = "";
$strCoverUri = ImageProcessor::getInstance()->downloadAndEncryptImage($strCover, 'xs');
$arrCover = [
'code' => 'AI_LI_SI_COVER',
// 'uri' => getUriByUrl($strCover),
'uri' => $strCoverUri,
];
$arrXiaoShuoInfo = [
'xsxq_feng_mian' => json_encode($arrCover),
];
$XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
} catch (\Throwable $t) {
$strErr = sprintf("采集[%s]小说图片错误,异常:%s", $strCover, $t->getMessage());
echo $strErr;
}
} catch (\Throwable $T) {
throw $T;
}
}
//https://www.630zw.org/shu/438374/156339465.html
public function getNovelReader($arrData)
{
$TaskCore = $this->TaskCore;
$strSiteUrl = 'https://www.630zw.org';
var_dump('getNovelReader------------------------------------------------------------------------------------');
$Client = $this->Client;
$intZhangJieSort = $arrData['intZhangJieSort'];
$strXszjSourceCode = $arrData['xszj_source_code'];
$strZhangJieUrl = $arrData['xszj_url'];
$intZhangJiePaiXu = $arrData['xszj_paixu'];
$intFenleiId = $arrData['xsfl_id'];
$intXiaoShuoId = $arrData['xsxq_id'];
$strZhangJieSourceId = $arrData['xszj_source_id'];
$strZhangJieName = $arrData['xszj_name'];
$XiaoShuoZhangJieModel = XiaoShuoZhangJieModel::where('xszj_source_code', $strXszjSourceCode)->find();
if ($XiaoShuoZhangJieModel) {
var_dump($XiaoShuoZhangJieModel->xszj_source_code);
var_dump('章节已经存在 paixu: '.$intZhangJiePaiXu);
var_dump('章节已经存在strXszjSourceCode: '.$strXszjSourceCode);
var_dump('章节已经存在strZhangJieUrl: '.$strZhangJieUrl);
// return true;
}
var_dump($strZhangJieUrl);
var_dump('章节排序:'.$intZhangJiePaiXu);
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strZhangJieUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
if (XiaoShuoZhangJieModel::checkZhangJieExists($strXszjSourceCode)) {
return true;
}
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strZhangJieUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
unset($Response);
$ZhangJieQueryList = QueryList::html($strResult);
// 正则表达式,匹配 `_数字.html`
$pattern = '/_(\d+)\.html$/';
$patternHref = '/(\/shu\/\d+\/\d+_\d+\.html)/';
$strNeiRong = '';
$strNextHref = '';
// 使用 preg_match() 进行匹配
if (preg_match($patternHref, $strResult, $matches)) {
// 提取匹配到的部分
echo "提取到的 URL 是: " . $matches[1];
$strNextHref = $matches[1];
} else {
echo "没有匹配到 URL";
}
preg_match_all("/qsbs\.bb\('([^']+)'\)/", $strResult, $matches);
// 输出提取到的加密字符串
if (isset($matches[1])) {
$strNeiRong = implode(';;;', $matches[1]);
// var_dump($strNeiRong);
}
// 判断字符串是否匹配
if (preg_match($pattern, $strNextHref, $matches) ){
$intNextPage = 2;
// 确保匹配成功后再访问索引
if (isset($matches[1])) {
$intNextPage = (int)$matches[1];
$intNextPage +=1;
var_dump('提取到的数字是:' . $intNextPage);
}
var_dump('提取到的数字是-' . $intNextPage);
$arrTask = [
'callback' => [self::class, 'getNovelReader'],
'data' => [
'xszj_source_id' => $strZhangJieSourceId,
'xszj_paixu' => $intZhangJiePaiXu,
'xszj_name' => $strZhangJieName,
'xszj_url' => $strSiteUrl . $strNextHref,
'xszj_source_code' => '630ZW_XIAOSHUO_ZHANGJIE_'. $intXiaoShuoId . '_' . $strZhangJieSourceId . '_' . $intNextPage,
'xsxq_id' => $intXiaoShuoId,
'xsfl_id' => $intFenleiId,
'intZhangJieSort' => $intNextPage,
]
];
$TaskCore->set($arrTask);
}
# '/novel/123yqw/分类id/小说源id/小说源章节id.txt'
$strNeiRongTxtPath = config("filesystem.novel") . '/novel/630zwxs/' . $intFenleiId . '/' . $intXiaoShuoId .'/' . $intZhangJiePaiXu .'/' . $intZhangJieSort . '.txt';
//$strNeiRongTxtPath = "/www/novel/123yqw/1/11/111.txt";
// 调用函数
$result = saveCompressAndDeleteTxt($strNeiRongTxtPath, $strNeiRong);
// 输出状态
if ($result['status']) {
var_dump("Success: " . $result['message']);
} else {
var_dump("Error: " . $result['message']);
}
$arrXiaoShuoZhangJie = [
'xsxq_id' => $intXiaoShuoId,
'xszj_pai_xu' => $intZhangJiePaiXu,
'xszj_ming_zi' => $strZhangJieName,
'xszj_nei_rong' => $strNeiRongTxtPath,
'xszj_source_id' => $strZhangJieSourceId,
'xszj_source_code' => $strXszjSourceCode,
'xszj_sort' => $intZhangJieSort,
];
// var_dump($arrXiaoShuoZhangJie);
if($strZhangJieName){
XiaoShuoZhangJieModel::addXiaoShuoZhangJie($arrXiaoShuoZhangJie);
}
unset($ZhangJieQueryList);
// // 延时 3 秒
//sleep(3);
} catch (\Throwable $T) {
$arrTask = [
'callback' => [self::class, 'getNovelReader'],
'data' => [
'xszj_source_id' => $strZhangJieSourceId,
'xszj_paixu' => $intZhangJiePaiXu,
'xszj_name' => $strZhangJieName,
'xszj_url' => $strZhangJieUrl,
'xszj_source_code' => $strXszjSourceCode,
'xsxq_id' => $intXiaoShuoId,
'xsfl_id' => $intFenleiId,
'intZhangJieSort' => $intZhangJieSort,
]
];
$TaskCore->set($arrTask);
throw $T;
}
}
}

View File

@@ -0,0 +1,426 @@
<?php
declare(strict_types=1);
namespace app\task\collection\novelone;
use app\admin\model\XiaoShuoXiangQingModel;
use app\admin\model\XiaoShuoZhangJieModel;
use app\task\collection\PullDataColBase;
use GuzzleHttp\Cookie\CookieJar;
use processor\ImageProcessor;
use QL\QueryList;
use app\common\Base64Helper;
/**
* @ 抓取 gdbzkz.info 站点相关小说
*/
class NovelJiuYuFanXianCol extends PullDataColBase
{
public function pullXiaoShuo()
{
$TaskCore = $this->TaskCore;
// http://www.gdbzkz.info/jiuyufanxian/
$arrNovel = [
['xs_source_id' => 'jiuyufanxian'],
];
foreach ($arrNovel as $Novel) {
$arrTask = [
'callback' => [self::class, 'getNovelInfo'],
'data' => [
'xs_source_id' => $Novel['xs_source_id'],
'xs_source_seo_id' => 1,
]
];
$TaskCore->set($arrTask);
var_dump($arrTask);
}
}
public function getNovelInfo($arrData)
{
$TaskCore = $this->TaskCore;
$Client = $this->Client;
$strSiteUrl = 'http://www.gdbzkz.info';
$intXiaoShuoSourceId = $arrData['xs_source_id'];
$intXiaoShuoSourceSeoId = $arrData['xs_source_seo_id'];
$strUrl = $strSiteUrl . '/' . $intXiaoShuoSourceId;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
$QueryList = QueryList::html($strResult);
// var_dump($QueryList);
// 提取所有 <meta> 标签的属性
$result = $QueryList->find('meta[property^="og:"]')->map(function ($item) {
return [
'property' => $item->attr('property'),
'content' => $item->attr('content'),
];
});
// 打印结果
// print_r($result->all());
$strZuoZhe = '';
$strMingZi = '';
$strLastDate = '';
$strJieShao = '';
$strCover = '';
$strFenlei = '';
$strZiShu = 0;
$strZhuangTai = '';
foreach ($result as $item) {
switch ($item['property']) {
case 'og:title':
$strMingZi = $item['content'];
break;
case 'og:description':
$strJieShao = $item['content'];
break;
case 'og:image':
$strCover = $item['content'];
break;
case 'og:novel:category':
$strFenlei = $item['content'];
break;
case 'og:novel:author':
$strZuoZhe = $item['content'];
break;
case 'og:novel:status':
$strZhuangTai = $item['content'];
break;
case 'og:novel:update_time':
$strLastDate = $item['content'];
break;
}
}
$strLastDate = trim((string)$strLastDate);
if (!isValidDateTime($strLastDate)) {
$strLastDate = date('Y-m-d H:i:s');
}
$arrTag = [];
if (strpos($strCover, 'http') === false) {
$strCover = $strSiteUrl . $strCover;
}
$strCoverUri = $strCover;
try {
$strCoverUri = "";
$strCoverUri = ImageProcessor::getInstance()->downloadImage($intXiaoShuoSourceId, $strCover, 'xs');
} catch (\Throwable $t) {
$strErr = sprintf("采集[%s]小说图片错误,异常:%s", $strCover, $t->getMessage());
echo $strErr;
}
$arrCover = [
'code' => 'BI_QU_XS_COVER',
// 'uri' => getUriByUrl($strCover),
'uri' => $strCoverUri,
];
$strFenleiIId = 1;
switch ($strFenlei) {
case '玄幻小说':
$strName = "玄幻";
$strFenleiIId = 1;
break;
case '修真小说':
$strName = "修真";
$strFenleiIId = 2;
break;
case '都市小说':
$strName = "都市";
$strFenleiIId = 3;
break;
case '历史小说':
$strName = "历史";
$strFenleiIId = 4;
break;
case '网游小说':
$strName = "网游";
$strFenleiIId = 5;
break;
case '科幻小说':
$strName = "科幻";
$strFenleiIId = 6;
break;
case '女频小说':
$strName = "女频";
$strFenleiIId = 7;
break;
case '其它小说':
$strName = "其它";
$strFenleiIId = 8;
break;
default:
$strName = "其它";
$strFenleiIId = 8;
break;
}
$arrXiaoShuoInfo = [
'xsxq_ming_zi' => $strMingZi,
'xsxq_zuozhe' => $strZuoZhe, //$arrData['xiaoshuo_zuozhe'],
'xsfl_id' => $strFenleiIId,
'xsxq_zhuang_tai' => $strZhuangTai,
'xsxq_zi_shu' => $strZiShu,
'xsxq_jie_shao' => $strJieShao,
'xsxq_feng_mian' => json_encode($arrCover),
'xsxq_geng_xin_shi_jian' => $strLastDate,
'xsxq_tag' => json_encode($arrTag, JSON_UNESCAPED_UNICODE),
'xsxq_source_id' => $intXiaoShuoSourceId,
'xsxq_source_seo_id' => $intXiaoShuoSourceId . '1',
'xsxq_source_code' => 'BIQU_XIAOSHUO_' . $intXiaoShuoSourceId . '_' . $intXiaoShuoSourceSeoId,
];
// print_r($arrXiaoShuoInfo);
var_dump($arrXiaoShuoInfo['xsxq_source_code']);
$XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
$arrZhangjie = [];
$intChapterIndex = 0; // 定义一个计数器变量
// 检查是否找到元素
if ($QueryList->find('.listmain a')->count() > 0) {
$QueryList->find('.listmain a')->map(function ($ZhangJie) use (&$arrZhangjie, &$intChapterIndex) {
echo "Processing index $intChapterIndex: " . $ZhangJie->text() . PHP_EOL;
// 从索引6开始
if ($intChapterIndex >= 12) {
$arrZhangjie[] = [
'mingzi' => $ZhangJie->text(),
'url' => $ZhangJie->attr('href'),
];
}
$intChapterIndex++;
});
// 输出最终结果
// var_dump($arrZhangjie);
} else {
echo "No elements found for .listmain a" . PHP_EOL;
}
//unset($QueryList);
// 获取最大章节索引
$intMaxXiaoShuoZhangJiePaixu = XiaoShuoZhangJieModel::where('xsxq_id', $XiaoShuoXiangQingModel->xsxq_id)
->max('xszj_pai_xu'); // 获取最大值
var_dump('获取小说章节最大值-' . $intMaxXiaoShuoZhangJiePaixu);
// 如果没有结果max 返回 null可以设置默认值
$intMaxXiaoShuoZhangJiePaixu = $intMaxXiaoShuoZhangJiePaixu ?? 0;
foreach ($arrZhangjie as $intZhangJieNum => $arrZhangjieOne) {
var_dump($intZhangJieNum);
// 如果数据库里的章节最大索引大于当前索引,说明已经入库,不需要入队列了
// if ($intMaxXiaoShuoZhangJiePaixu > $intZhangJieNum) {
// var_dump("小说章节已经入库,不再推送任务");
// continue; // 如果任务已存在,直接跳过 break
// }
$strZhangJieUrl = $strSiteUrl . $arrZhangjieOne['url'];
$strZhangJieSourceId = extractFilename($strZhangJieUrl);
# 'AILISI_XIAOSHUO_ZHANGJIE_小说源id_小说源章节id'
$strXszjSourceCode = 'gdbzkz_XIAOSHUO_ZHANGJIE_' . $XiaoShuoXiangQingModel->xsxq_source_id . '_' . $strZhangJieSourceId . '_1';
$arrTask = [
'callback' => [self::class, 'getNovelReader'],
'data' => [
'xszj_source_id' => $strZhangJieSourceId,
'xszj_paixu' => $intZhangJieNum += 1,
'xszj_name' => $arrZhangjieOne['mingzi'],
'xszj_url' => $strZhangJieUrl,
'xszj_source_code' => $strXszjSourceCode,
'xsxq_id' => $XiaoShuoXiangQingModel->xsxq_id,
'xsfl_id' => $XiaoShuoXiangQingModel->xsfl_id,
'intZhangJieSort' => 1,
]
];
$TaskCore->set($arrTask);
}
var_dump("章节任务投递成功");
} catch (\Throwable $T) {
throw $T;
}
}
public function getNovelReader($arrData)
{
$TaskCore = $this->TaskCore;
$strSiteUrl = 'http://www.gdbzkz.info';
var_dump('getNovelReader------------------------------------------------------------------------------------');
$Client = $this->Client;
$intZhangJieSort = $arrData['intZhangJieSort'];
$strXszjSourceCode = $arrData['xszj_source_code'];
$strZhangJieUrl = $arrData['xszj_url'];
$intZhangJiePaiXu = $arrData['xszj_paixu'];
$intFenleiId = $arrData['xsfl_id'];
$intXiaoShuoId = $arrData['xsxq_id'];
$strZhangJieSourceId = $arrData['xszj_source_id'];
$strZhangJieName = $arrData['xszj_name'];
$XiaoShuoZhangJieModel = XiaoShuoZhangJieModel::where('xszj_source_code', $strXszjSourceCode)->find();
if ($XiaoShuoZhangJieModel) {
// var_dump($XiaoShuoZhangJieModel->xszj_source_code);
// var_dump('章节已经存在 paixu: ' . $intZhangJiePaiXu);
// var_dump('章节已经存在strXszjSourceCode: ' . $strXszjSourceCode);
// var_dump('章节已经存在strZhangJieUrl: ' . $strZhangJieUrl);
return true;
}
var_dump($strZhangJieUrl);
var_dump('章节排序:' . $intZhangJiePaiXu);
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strZhangJieUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
if (XiaoShuoZhangJieModel::checkZhangJieExists($strXszjSourceCode)) {
return true;
}
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strZhangJieUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
unset($Response);
$ZhangJieQueryList = QueryList::html($strResult);
$strNeiRong = '';
// 获取原始 HTML
$strNeiRong = $ZhangJieQueryList->find('#content')->html();
// 解码 HTML 实体
$strNeiRong = html_entity_decode($strNeiRong, ENT_QUOTES, 'UTF-8');
// 替换不间断空格 (\xc2\xa0) 和 &nbsp; 为普通空格
$strNeiRong = preg_replace('/\xc2\xa0/', ' ', $strNeiRong); // 替换 UTF-8 不间断空格
$strNeiRong = preg_replace('/&nbsp;/', ' ', $strNeiRong); // 替换 &nbsp;
// 清理多余空格和换行符
$strNeiRong = preg_replace('/\s+/', ' ', $strNeiRong);
// 1. 找到所有的 <br> 位置
$br_positions = [];
$offset = 0;
while (($pos = strpos($strNeiRong, "<br", $offset)) !== false) {
$br_positions[] = $pos;
$offset = $pos + 1;
}
// 2. 找到倒数第三个 <br> 的位置
if (count($br_positions) >= 3) {
$cut_position = $br_positions[count($br_positions) - 3]; // 倒数第三个 <br> 的索引
$strNeiRong = substr($strNeiRong, 0, $cut_position); // 截取前面部分,删除后面的内容
}
$arrNeiRong = explode('<br> <br>', $strNeiRong);
$arrEnCode = [];
foreach ($arrNeiRong as $intNeiRongNum => $strNeiRongOne) {
$strEnCode = Base64Helper::encode('<p>' . $strNeiRongOne . '</p>');
$arrEnCode[] = $strEnCode;
}
$strNeiRong = implode(';;;', $arrEnCode);
# '/novel/123yqw/分类id/小说源id/小说源章节id.txt'
$strNeiRongTxtPath = config("filesystem.novel") . '/novel/630zwxs/' . $intFenleiId . '/' . $intXiaoShuoId . '/' . $intZhangJiePaiXu . '/' . $intZhangJieSort . '.txt';
//$strNeiRongTxtPath = "/www/novel/123yqw/1/11/111.txt";
// 调用函数
$result = saveCompressAndDeleteTxt($strNeiRongTxtPath, $strNeiRong);
// 输出状态
if ($result['status']) {
var_dump("Success: " . $result['message']);
} else {
var_dump("Error: " . $result['message']);
}
$arrXiaoShuoZhangJie = [
'xsxq_id' => $intXiaoShuoId,
'xszj_pai_xu' => $intZhangJiePaiXu,
'xszj_ming_zi' => $strZhangJieName,
'xszj_nei_rong' => $strNeiRongTxtPath,
'xszj_source_id' => $strZhangJieSourceId,
'xszj_source_code' => $strXszjSourceCode,
'xszj_sort' => $intZhangJieSort,
];
var_dump($arrXiaoShuoZhangJie);
if ($strZhangJieName) {
XiaoShuoZhangJieModel::addXiaoShuoZhangJie($arrXiaoShuoZhangJie);
}
unset($ZhangJieQueryList);
// // 延时 3 秒
//sleep(3);
} catch (\Throwable $T) {
$arrTask = [
'callback' => [self::class, 'getNovelReader'],
'data' => [
'xszj_source_id' => $strZhangJieSourceId,
'xszj_paixu' => $intZhangJiePaiXu,
'xszj_name' => $strZhangJieName,
'xszj_url' => $strZhangJieUrl,
'xszj_source_code' => $strXszjSourceCode,
'xsxq_id' => $intXiaoShuoId,
'xsfl_id' => $intFenleiId,
'intZhangJieSort' => $intZhangJieSort,
]
];
$TaskCore->set($arrTask);
throw $T;
}
}
}

View File

@@ -0,0 +1,441 @@
<?php
declare(strict_types=1);
namespace app\task\collection\novelone;
use app\admin\model\XiaoShuoXiangQingModel;
use app\admin\model\XiaoShuoZhangJieModel;
use app\task\collection\PullDataColBase;
use GuzzleHttp\Cookie\CookieJar;
use processor\ImageProcessor;
use QL\QueryList;
use app\common\Base64Helper;
/**
* @ 抓取 bqvvxg8.cc 站点相关小说
*/
class NovelWenXueGuanCol extends PullDataColBase
{
public function pullXiaoShuo()
{
$TaskCore = $this->TaskCore;
// https://www.bqvvxg8.cc/wenzhang/86/86114/
$arrNovel = [
['xs_source_id' => 'wenxueguan86114', 'xs_url' => 'https://www.bqvvxg8.cc/wenzhang/86/86114/'],
['xs_source_id' => 'wenxueguan90205', 'xs_url' => 'https://www.bqvvxg8.cc/wenzhang/90/90205/'],
];
foreach ($arrNovel as $Novel) {
$arrTask = [
'callback' => [self::class, 'getNovelInfo'],
'data' => [
'xs_url' => $Novel['xs_url'],
'xs_source_id' => $Novel['xs_source_id'],
'xs_source_seo_id' => 1,
]
];
$TaskCore->set($arrTask);
var_dump($arrTask);
}
}
public function getNovelInfo($arrData)
{
$TaskCore = $this->TaskCore;
$Client = $this->Client;
$strSiteUrl = 'https://www.bqvvxg8.cc';
$intXiaoShuoSourceId = $arrData['xs_source_id'];
$intXiaoShuoSourceSeoId = $arrData['xs_source_seo_id'];
$strUrl = $arrData['xs_url'];
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
$QueryList = QueryList::html($strResult);
// var_dump($QueryList);
// 提取所有 <meta> 标签的属性
$result = $QueryList->find('meta[property^="og:"]')->map(function ($item) {
return [
'property' => $item->attr('property'),
'content' => $item->attr('content'),
];
});
// 打印结果
// print_r($result->all());
$strZuoZhe = '';
$strMingZi = '';
$strLastDate = '';
$strJieShao = '';
$strCover = '';
$strFenlei = '';
$strZiShu = 0;
$strZhuangTai = '';
foreach ($result as $item) {
switch ($item['property']) {
case 'og:title':
$strMingZi = $item['content'];
break;
case 'og:description':
$strJieShao = $item['content'];
break;
case 'og:image':
$strCover = $item['content'];
break;
case 'og:novel:category':
$strFenlei = $item['content'];
break;
case 'og:novel:author':
$strZuoZhe = $item['content'];
break;
case 'og:novel:status':
$strZhuangTai = $item['content'] . '中';
break;
case 'og:novel:update_time':
$strLastDate = $item['content'];
break;
}
}
$strLastDate = trim((string)$strLastDate);
if (!isValidDateTime($strLastDate)) {
$strLastDate = date('Y-m-d H:i:s');
}
$arrTag = [];
if (strpos($strCover, 'http') === false) {
$strCover = $strSiteUrl . $strCover;
}
$strCoverUri = $strCover;
try {
$strCoverUri = "";
$strCoverUri = ImageProcessor::getInstance()->downloadImage($intXiaoShuoSourceId, $strCover, 'xs');
} catch (\Throwable $t) {
$strErr = sprintf("采集[%s]小说图片错误,异常:%s", $strCover, $t->getMessage());
echo $strErr;
}
$arrCover = [
'code' => 'BI_QU_XS_COVER',
// 'uri' => getUriByUrl($strCover),
'uri' => $strCoverUri,
];
$strFenleiIId = 1;
switch ($strFenlei) {
case '玄幻小说':
$strName = "玄幻";
$strFenleiIId = 1;
break;
case '修真小说':
$strName = "修真";
$strFenleiIId = 2;
break;
case '言情小说':
$strName = "都市";
$strFenleiIId = 3;
break;
case '历史小说':
$strName = "历史";
$strFenleiIId = 4;
break;
case '网游小说':
$strName = "网游";
$strFenleiIId = 5;
break;
case '科幻小说':
$strName = "科幻";
$strFenleiIId = 6;
break;
case '女频小说':
$strName = "女频";
$strFenleiIId = 7;
break;
case '其它小说':
$strName = "其它";
$strFenleiIId = 8;
break;
default:
$strName = "其它";
$strFenleiIId = 8;
break;
}
$arrXiaoShuoInfo = [
'xsxq_ming_zi' => $strMingZi,
'xsxq_zuozhe' => $strZuoZhe, //$arrData['xiaoshuo_zuozhe'],
'xsfl_id' => $strFenleiIId,
'xsxq_zhuang_tai' => $strZhuangTai,
'xsxq_zi_shu' => $strZiShu,
'xsxq_jie_shao' => $strJieShao,
'xsxq_feng_mian' => json_encode($arrCover),
'xsxq_geng_xin_shi_jian' => $strLastDate,
'xsxq_tag' => json_encode($arrTag, JSON_UNESCAPED_UNICODE),
'xsxq_source_id' => $intXiaoShuoSourceId,
'xsxq_source_seo_id' => $intXiaoShuoSourceId . '1',
'xsxq_source_code' => 'WENXUEGUANG_XIAOSHUO_' . $intXiaoShuoSourceId . '_' . $intXiaoShuoSourceSeoId,
];
// print_r($arrXiaoShuoInfo);
var_dump($arrXiaoShuoInfo['xsxq_source_code']);
$XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
$arrZhangjie = [];
$intChapterIndex = 0; // 定义一个计数器变量
// 检查是否找到元素
if ($QueryList->find('.listmain a')->count() > 0) {
$QueryList->find('.listmain a')->map(function ($ZhangJie) use (&$arrZhangjie, &$intChapterIndex) {
echo "Processing index $intChapterIndex: " . $ZhangJie->text() . PHP_EOL;
// 从索引6开始
if ($intChapterIndex >= 12) {
$arrZhangjie[] = [
'mingzi' => $ZhangJie->text(),
'url' => $ZhangJie->attr('href'),
];
}
$intChapterIndex++;
});
// 输出最终结果
var_dump($arrZhangjie);
} else {
echo "No elements found for .listmain a" . PHP_EOL;
}
//unset($QueryList);
// 获取最大章节索引
$intMaxXiaoShuoZhangJiePaixu = XiaoShuoZhangJieModel::where('xsxq_id', $XiaoShuoXiangQingModel->xsxq_id)
->max('xszj_pai_xu'); // 获取最大值
var_dump('获取小说章节最大值-' . $intMaxXiaoShuoZhangJiePaixu);
// 如果没有结果max 返回 null可以设置默认值
$intMaxXiaoShuoZhangJiePaixu = $intMaxXiaoShuoZhangJiePaixu ?? 0;
foreach ($arrZhangjie as $intZhangJieNum => $arrZhangjieOne) {
var_dump($intZhangJieNum);
// 如果数据库里的章节最大索引大于当前索引,说明已经入库,不需要入队列了
// if ($intMaxXiaoShuoZhangJiePaixu > $intZhangJieNum) {
// var_dump("小说章节已经入库,不再推送任务");
// continue; // 如果任务已存在,直接跳过 break
// }
$strZhangJieUrl = $strSiteUrl . $arrZhangjieOne['url'];
$strZhangJieSourceId = extractFilename($strZhangJieUrl);
# 'AILISI_XIAOSHUO_ZHANGJIE_小说源id_小说源章节id'
$strXszjSourceCode = 'gdbzkz_XIAOSHUO_ZHANGJIE_' . $XiaoShuoXiangQingModel->xsxq_source_id . '_' . $strZhangJieSourceId . '_1';
$arrTask = [
'callback' => [self::class, 'getNovelReader'],
'data' => [
'xszj_source_id' => $strZhangJieSourceId,
'xszj_paixu' => $intZhangJieNum += 1,
'xszj_name' => $arrZhangjieOne['mingzi'],
'xszj_url' => $strZhangJieUrl,
'xszj_source_code' => $strXszjSourceCode,
'xsxq_id' => $XiaoShuoXiangQingModel->xsxq_id,
'xsfl_id' => $XiaoShuoXiangQingModel->xsfl_id,
'intZhangJieSort' => 1,
]
];
$TaskCore->set($arrTask);
}
var_dump("章节任务投递成功");
} catch (\Throwable $T) {
throw $T;
}
}
public function getNovelReader($arrData)
{
$TaskCore = $this->TaskCore;
$strSiteUrl = 'https://www.bqvvxg8.cc';
var_dump('getNovelReader------------------------------------------------------------------------------------');
$Client = $this->Client;
$intZhangJieSort = $arrData['intZhangJieSort'];
$strXszjSourceCode = $arrData['xszj_source_code'];
$strZhangJieUrl = $arrData['xszj_url'];
$intZhangJiePaiXu = $arrData['xszj_paixu'];
$intFenleiId = $arrData['xsfl_id'];
$intXiaoShuoId = $arrData['xsxq_id'];
$strZhangJieSourceId = $arrData['xszj_source_id'];
$strZhangJieName = $arrData['xszj_name'];
$XiaoShuoZhangJieModel = XiaoShuoZhangJieModel::where('xszj_source_code', $strXszjSourceCode)->find();
if ($XiaoShuoZhangJieModel) {
// var_dump($XiaoShuoZhangJieModel->xszj_source_code);
// var_dump('章节已经存在 paixu: ' . $intZhangJiePaiXu);
// var_dump('章节已经存在strXszjSourceCode: ' . $strXszjSourceCode);
// var_dump('章节已经存在strZhangJieUrl: ' . $strZhangJieUrl);
return true;
}
var_dump($strZhangJieUrl);
var_dump('章节排序:' . $intZhangJiePaiXu);
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strZhangJieUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
if (XiaoShuoZhangJieModel::checkZhangJieExists($strXszjSourceCode)) {
return true;
}
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strZhangJieUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
'Accept-Charset' => 'GBK, UTF-8, *;q=0.8',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
// 先尝试自动检测编码
$strEncoding = mb_detect_encoding($strResult, ['GBK', 'GB2312', 'BIG5', 'UTF-8'], true);
// 如果不是 UTF-8转换为 UTF-8
if ($strEncoding && $strEncoding !== 'UTF-8') {
$strResult = mb_convert_encoding($strResult, 'UTF-8', $strEncoding);
} else {
// 备用方案,强制转换
$strResult = @iconv('GBK', 'UTF-8//IGNORE', $strResult);
}
// **去掉可能影响解析的 meta 标签**
$strResult = preg_replace('/<meta[^>]+charset=[^>]+>/i', '', $strResult);
// **第一步:使用 QueryList**
$ZhangJieQueryList = QueryList::html($strResult);
$strNeiRong = '';
// 获取原始 HTML
$strNeiRong = $ZhangJieQueryList->find('#content')->html();
// 解码 HTML 实体
$strNeiRong = html_entity_decode($strNeiRong, ENT_QUOTES, 'UTF-8');
// 替换不间断空格 (\xc2\xa0) 和 &nbsp; 为普通空格
$strNeiRong = preg_replace('/\xc2\xa0/', ' ', $strNeiRong); // 替换 UTF-8 不间断空格
$strNeiRong = preg_replace('/&nbsp;/', ' ', $strNeiRong); // 替换 &nbsp;
// 清理多余空格和换行符
$strNeiRong = preg_replace('/\s+/', ' ', $strNeiRong);
// 1. 找到所有的 <br> 位置
$br_positions = [];
$offset = 0;
while (($pos = strpos($strNeiRong, "<br", $offset)) !== false) {
$br_positions[] = $pos;
$offset = $pos + 1;
}
// 2. 找到倒数第三个 <br> 的位置
if (count($br_positions) >= 3) {
$cut_position = $br_positions[count($br_positions) - 3]; // 倒数第三个 <br> 的索引
$strNeiRong = substr($strNeiRong, 0, $cut_position); // 截取前面部分,删除后面的内容
}
$arrNeiRong = explode('<script>app2();</script><br>', $strNeiRong);
$strNeiRong = $arrNeiRong[1];
$arrNeiRong = explode('<br><br>', $strNeiRong);
$arrEnCode = [];
foreach ($arrNeiRong as $intNeiRongNum => $strNeiRongOne) {
$strEnCode = Base64Helper::encode('<p>' . $strNeiRongOne . '</p>');
$arrEnCode[] = $strEnCode;
}
$strNeiRong = implode(';;;', $arrEnCode);
# '/novel/123yqw/分类id/小说源id/小说源章节id.txt'
$strNeiRongTxtPath = config("filesystem.novel") . '/novel/630zwxs/' . $intFenleiId . '/' . $intXiaoShuoId . '/' . $intZhangJiePaiXu . '/' . $intZhangJieSort . '.txt';
// 调用函数
$result = saveCompressAndDeleteTxt($strNeiRongTxtPath, $strNeiRong);
// 输出状态
if ($result['status']) {
var_dump("Success: " . $result['message']);
} else {
var_dump("Error: " . $result['message']);
}
$arrXiaoShuoZhangJie = [
'xsxq_id' => $intXiaoShuoId,
'xszj_pai_xu' => $intZhangJiePaiXu,
'xszj_ming_zi' => $strZhangJieName,
'xszj_nei_rong' => $strNeiRongTxtPath,
'xszj_source_id' => $strZhangJieSourceId,
'xszj_source_code' => $strXszjSourceCode,
'xszj_sort' => $intZhangJieSort,
];
if ($strZhangJieName) {
XiaoShuoZhangJieModel::addXiaoShuoZhangJie($arrXiaoShuoZhangJie);
}
unset($ZhangJieQueryList);
//sleep(3);
} catch (\Throwable $T) {
$arrTask = [
'callback' => [self::class, 'getNovelReader'],
'data' => [
'xszj_source_id' => $strZhangJieSourceId,
'xszj_paixu' => $intZhangJiePaiXu,
'xszj_name' => $strZhangJieName,
'xszj_url' => $strZhangJieUrl,
'xszj_source_code' => $strXszjSourceCode,
'xsxq_id' => $intXiaoShuoId,
'xsfl_id' => $intFenleiId,
'intZhangJieSort' => $intZhangJieSort,
]
];
$TaskCore->set($arrTask);
throw $T;
}
}
}

View File

@@ -0,0 +1,137 @@
<?php
declare(strict_types=1);
namespace app\task\collection\seo;
use app\task\collection\PullDataColBase;
use app\admin\model\VideoInfoModel;
use app\admin\model\SystemConfigModel;
use app\admin\model\VideoClassModel;
class GenerateSiteMapCol extends PullDataColBase
{
public function generate()
{
$strSiteDomain = SystemConfigModel::getValByCode('YONG_JIU_DOMAIN');
// 每个 Sitemap 包含的最大 URL 数量
$chunkSize = 40000;
// 从数据库获取所有视频总数
$totalVideoCount = VideoInfoModel::count();
// 获取分类信息
$categories = VideoClassModel::getDataByCache();
// 生成视频详情的 Sitemap
$videoSitemapFiles = $this->generateVideoSitemaps($chunkSize, $totalVideoCount, $strSiteDomain);
// 生成分类的 Sitemap
$categorySitemapFiles = $this->generateCategorySitemaps($categories, $strSiteDomain);
// 合并所有 Sitemap 文件路径
$allSitemapFiles = array_merge($videoSitemapFiles, $categorySitemapFiles);
// 生成 Sitemap Index 文件
$this->buildSitemapIndex($allSitemapFiles, $strSiteDomain);
var_dump(['message' => '所有 Sitemap 生成成功', 'files' => $allSitemapFiles]);
}
private function generateVideoSitemaps(int $chunkSize, int $totalCount, string $strSiteDomain): array
{
$totalFiles = ceil($totalCount / $chunkSize);
$sitemapFiles = [];
for ($i = 0; $i < $totalFiles; $i++) {
$videos = VideoInfoModel::field('v_pinyin')
->limit($i * $chunkSize, $chunkSize)
->select()
->toArray();
$sitemapXml = $this->buildSitemap($videos, $strSiteDomain);
$filePath = public_path() . "sitemap_video_{$i}.xml";
file_put_contents($filePath, $sitemapXml);
$sitemapFiles[] = $filePath;
}
return $sitemapFiles;
}
private function generateCategorySitemaps(array $categories, string $strSiteDomain): array
{
$sitemapFiles = [];
foreach ($categories as $category) {
$vcId = $category['vc_id'];
$vcNicheng = $category['vc_nicheng'];
$vcName = $category['vc_name'];
$videoCount = VideoInfoModel::where('vc_id', $vcId)->count();
$totalPages = ceil($videoCount / 30);
$categoryUrls = [];
$categoryUrls[] = [
'url' => "/vodtype/{$vcNicheng}.html",
'title' => "{$vcName} - 首页",
'is_home' => true,
];
for ($page = 2; $page <= $totalPages; $page++) {
$categoryUrls[] = [
'url' => "/vodshow/{$vcNicheng}--------{$page}---.html",
'title' => "{$vcName} - 第{$page}",
'is_home' => false,
];
}
$sitemapXml = $this->buildSitemap($categoryUrls, $strSiteDomain, true);
$filePath = public_path() . "sitemap_category_{$vcNicheng}.xml";
file_put_contents($filePath, $sitemapXml);
$sitemapFiles[] = $filePath;
}
return $sitemapFiles;
}
private function buildSitemap(array $items, string $strSiteDomain, bool $isCategory = false): string
{
$xmlHeader = '<?xml version="1.0" encoding="UTF-8"?>';
$urlsetStart = '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">';
$urlsetEnd = '</urlset>';
$urlEntries = '';
foreach ($items as $item) {
$urlEntries .= '<url>';
$urlEntries .= '<loc>https://' . $strSiteDomain . ($isCategory ? $item['url'] : '/voddetail/' . htmlspecialchars($item['v_pinyin'] . '.html', ENT_QUOTES)) . '</loc>';
$urlEntries .= '<changefreq>daily</changefreq>';
$urlEntries .= '<priority>0.8</priority>';
$urlEntries .= '</url>';
}
return $xmlHeader . $urlsetStart . $urlEntries . $urlsetEnd;
}
private function buildSitemapIndex(array $sitemapFiles, string $strSiteDomain): void
{
$xmlHeader = '<?xml version="1.0" encoding="UTF-8"?>';
$sitemapIndexStart = '<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">';
$sitemapIndexEnd = '</sitemapindex>';
$sitemapEntries = '';
foreach ($sitemapFiles as $file) {
$url = 'https://' . $strSiteDomain . '/' . basename($file);
$sitemapEntries .= '<sitemap>';
$sitemapEntries .= '<loc>' . htmlspecialchars($url, ENT_QUOTES) . '</loc>';
$sitemapEntries .= '<lastmod>' . date('Y-m-d') . '</lastmod>';
$sitemapEntries .= '</sitemap>';
}
$sitemapIndexXml = $xmlHeader . $sitemapIndexStart . $sitemapEntries . $sitemapIndexEnd;
file_put_contents(public_path() . 'sitemap_index.xml', $sitemapIndexXml);
var_dump(['message' => 'Sitemap Index 生成成功', 'file' => 'sitemap_index.xml']);
}
}

View File

@@ -0,0 +1,188 @@
<?php
declare(strict_types=1);
namespace app\task\collection\seo;
use app\task\collection\PullDataColBase;
use app\admin\model\XiaoShuoXiangQingModel;
use app\admin\model\XiaoShuoZhangJieModel;
use app\admin\model\SystemConfigModel;
use app\admin\model\SiteModel;
use app\admin\model\XiaoShuoFenLeiModel;
class GenerateSiteMapNovel extends PullDataColBase
{
public function generate()
{
$SiteModel = SiteModel::getValById(config('app.default_app_id'));
$strSiteDomain = $SiteModel['site_domain'];
$strFenleiPath = $SiteModel['site_fenlei_path'];
$strXqPath = $SiteModel['site_xiangqing_path'];
$strRankPath = $SiteModel['site_rank_path'];
$strTjPath = $SiteModel['site_tui_jian_path'];
$strSeoPath = $SiteModel['site_seo_path'];
$strZdyPath = $SiteModel['site_zdy_path'];
$strZjPath = $SiteModel['site_zhangjie_path'];
// 每个 Sitemap 包含的最大 URL 数量
$chunkSize = 40000;
// 从数据库获取所有总数
$totalNovelCount = XiaoShuoXiangQingModel::count();
// 从数据库获取所有总数
$totalNovelZjCount = XiaoShuoZhangJieModel::count();
// 获取分类信息
$categories = XiaoShuoFenLeiModel::getDataByCache();
// 生成详情的 Sitemap
$novelSitemapFiles = $this->generateNovelSitemaps($chunkSize, $totalNovelCount, $strSiteDomain);
// 生成章节的 Sitemap
$novelZjSitemapFiles = $this->generateNovelZjSitemaps($chunkSize, $totalNovelZjCount, $strSiteDomain);
// 生成分类的 Sitemap
$categorySitemapFiles = $this->generateCategorySitemaps($categories, $strSiteDomain);
// 合并所有 Sitemap 文件路径
$allSitemapFiles = array_merge($novelSitemapFiles, $categorySitemapFiles,$novelZjSitemapFiles);
// 生成 Sitemap Index 文件
$this->buildSitemapIndex($allSitemapFiles, $strSiteDomain);
var_dump(['message' => '所有 Sitemap 生成成功', 'files' => $allSitemapFiles]);
}
private function generateNovelZjSitemaps(int $chunkSize, int $totalCount, string $strSiteDomain): array
{
$totalFiles = ceil($totalCount / $chunkSize);
$sitemapFiles = [];
for ($i = 0; $i < $totalFiles; $i++) {
$Novels = XiaoShuoZhangJieModel::field(['xsxq_id','xszj_id']) //intNovelId-:intNovelSourceId-:intNovelSourceSeoId
->limit($i * $chunkSize, $chunkSize)
->select()
->toArray();
$sitemapXml = $this->buildZjSitemap($Novels, $strSiteDomain);
$filePath = public_path() . "sitemap_novel_zj_{$i}.xml";
file_put_contents($filePath, $sitemapXml);
$sitemapFiles[] = $filePath;
}
return $sitemapFiles;
}
private function generateNovelSitemaps(int $chunkSize, int $totalCount, string $strSiteDomain): array
{
$totalFiles = ceil($totalCount / $chunkSize);
$sitemapFiles = [];
for ($i = 0; $i < $totalFiles; $i++) {
$Novels = XiaoShuoXiangQingModel::field(['xsxq_id','xsxq_source_id','xsxq_source_seo_id']) //intNovelId-:intNovelSourceId-:intNovelSourceSeoId
->limit($i * $chunkSize, $chunkSize)
->select()
->toArray();
$sitemapXml = $this->buildSitemap($Novels, $strSiteDomain);
$filePath = public_path() . "sitemap_novel_{$i}.xml";
file_put_contents($filePath, $sitemapXml);
$sitemapFiles[] = $filePath;
}
return $sitemapFiles;
}
private function generateCategorySitemaps(array $categories, string $strSiteDomain): array
{
$sitemapFiles = [];
foreach ($categories as $category) {
$vcId = $category['xsfl_id'];
$vcName = $category['xsfl_name'];
$intNovelCount = XiaoShuoFenLeiModel::where('xsfl_id', $vcId)->count();
$totalPages = ceil($intNovelCount / 30);
$categoryUrls = [];
for ($type = 0; $type <= 3; $type++) {
for ($page = 1; $page <= $totalPages; $page++) {
$categoryUrls[] = [
// /fenlei/1-0-1.html
'url' => "/fenlei/{$vcId}-{$type}-{$page}.html",
'title' => "{$vcName} - 第{$page}",
'is_home' => false,
];
}
}
$sitemapXml = $this->buildSitemap($categoryUrls, $strSiteDomain, true);
$filePath = public_path() . "sitemap_category_{$vcId}.xml";
file_put_contents($filePath, $sitemapXml);
$sitemapFiles[] = $filePath;
}
return $sitemapFiles;
}
private function buildZjSitemap(array $items, string $strSiteDomain, bool $isCategory = false): string
{
$xmlHeader = '<?xml version="1.0" encoding="UTF-8"?>';
$urlsetStart = '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">';
$urlsetEnd = '</urlset>';
// zj/18648-206.html
$urlEntries = '';
foreach ($items as $item) {
$urlEntries .= '<url>';
$urlEntries .= '<loc>https://' . $strSiteDomain . ($isCategory ? $item['url'] : '/zj/' . htmlspecialchars($item['xsxq_id'].'-'.$item['xszj_id']. '.html', ENT_QUOTES)) . '</loc>';
$urlEntries .= '<changefreq>daily</changefreq>';
$urlEntries .= '<priority>0.8</priority>';
$urlEntries .= '</url>';
}
return $xmlHeader . $urlsetStart . $urlEntries . $urlsetEnd;
}
private function buildSitemap(array $items, string $strSiteDomain, bool $isCategory = false): string
{
$xmlHeader = '<?xml version="1.0" encoding="UTF-8"?>';
$urlsetStart = '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">';
$urlsetEnd = '</urlset>';
// /xq/20052-16107-161071.html
$urlEntries = '';
foreach ($items as $item) {
$urlEntries .= '<url>';
$urlEntries .= '<loc>https://' . $strSiteDomain . ($isCategory ? $item['url'] : '/xq/' . htmlspecialchars($item['xsxq_id'].'-'.$item['xsxq_source_id'].'-'.$item['xsxq_source_seo_id']. '.html', ENT_QUOTES)) . '</loc>';
$urlEntries .= '<changefreq>daily</changefreq>';
$urlEntries .= '<priority>1.0</priority>';
$urlEntries .= '</url>';
}
return $xmlHeader . $urlsetStart . $urlEntries . $urlsetEnd;
}
private function buildSitemapIndex(array $sitemapFiles, string $strSiteDomain): void
{
$xmlHeader = '<?xml version="1.0" encoding="UTF-8"?>';
$sitemapIndexStart = '<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">';
$sitemapIndexEnd = '</sitemapindex>';
$sitemapEntries = '';
foreach ($sitemapFiles as $file) {
$url = 'https://' . $strSiteDomain . '/' . basename($file);
$sitemapEntries .= '<sitemap>';
$sitemapEntries .= '<loc>' . htmlspecialchars($url, ENT_QUOTES) . '</loc>';
$sitemapEntries .= '<lastmod>' . date('Y-m-d') . '</lastmod>';
$sitemapEntries .= '</sitemap>';
}
$sitemapIndexXml = $xmlHeader . $sitemapIndexStart . $sitemapEntries . $sitemapIndexEnd;
file_put_contents(public_path() . 'sitemap_index.xml', $sitemapIndexXml);
var_dump(['message' => 'Sitemap Index 生成成功', 'file' => 'sitemap_index.xml']);
}
}

View File

@@ -0,0 +1,242 @@
<?php
declare(strict_types=1);
namespace app\task\collection\seo;
use app\task\collection\PullDataColBase;
use app\admin\model\XiaoShuoXiangQingModel;
use app\admin\model\NovelInfoSeoModel;
use app\admin\model\XiaoShuoZhangJieModel;
use app\admin\model\SiteModel;
use app\admin\model\XiaoShuoFenLeiModel;
class GenerateSiteMapNovelSk extends PullDataColBase
{
public function generate()
{
$SiteModel = SiteModel::getValById(config('app.default_app_id'));
// $strSiteDomain = SiteModel::getValBySiteIdByValCode('site_domain');
$strSiteDomain = $SiteModel['site_domain'];
$strFenleiPath = $SiteModel['site_fenlei_path'];
$strXqPath = $SiteModel['site_xiangqing_path'];
$strRankPath = $SiteModel['site_rank_path'];
$strTjPath = $SiteModel['site_tui_jian_path'];
$strSeoPath = $SiteModel['site_seo_path'];
$strZdyPath = $SiteModel['site_zdy_path'];
$strZjPath = $SiteModel['site_zhangjie_path'];
// 每个 Sitemap 包含的最大 URL 数量
$chunkSize = 40000;
// 从数据库获取所有总数
$totalNovelCount = XiaoShuoXiangQingModel::count();
// 从数据库获取所有总数
$totalNovelSeoCount = NovelInfoSeoModel::count();
// 从数据库获取所有总数
$totalNovelZjCount = XiaoShuoZhangJieModel::count();
// 获取分类信息
$categories = XiaoShuoFenLeiModel::getDataByCache();
// 生成详情的 Sitemap 小说详情
$novelSitemapFiles = $this->generateNovelSitemaps($chunkSize, $totalNovelCount, $strSiteDomain, $strXqPath);
// 生成详情的 Sitemap 小说seo
$novelSeoSitemapFiles = $this->generateNovelSeoSitemaps($chunkSize, $totalNovelSeoCount, $strSiteDomain, $strSeoPath);
// 生成章节的 Sitemap 小说章节
$novelZjSitemapFiles = $this->generateNovelZjSitemaps($chunkSize, $totalNovelZjCount, $strSiteDomain, $strXqPath);
// 生成分类的 Sitemap 小说分类
$categorySitemapFiles = $this->generateCategorySitemaps($categories, $strSiteDomain, $strFenleiPath, $strXqPath);
// 合并所有 Sitemap 文件路径
$allSitemapFiles = array_merge($novelSitemapFiles, $novelSeoSitemapFiles, $categorySitemapFiles, $novelZjSitemapFiles);
// 生成 Sitemap Index 文件
$this->buildSitemapIndex($allSitemapFiles, $strSiteDomain);
var_dump(['message' => '所有 Sitemap 生成成功', 'files' => $allSitemapFiles]);
}
# 生成小说章节 xml文件
private function generateNovelZjSitemaps(int $chunkSize, int $totalCount, string $strSiteDomain, string $strXqPath): array
{
$totalFiles = ceil($totalCount / $chunkSize);
$sitemapFiles = [];
for ($i = 0; $i < $totalFiles; $i++) {
$Novels = XiaoShuoZhangJieModel::field(['xsxq_id', 'xszj_id']) //intNovelId-:intNovelSourceId-:intNovelSourceSeoId
->limit($i * $chunkSize, $chunkSize)
->select()
->toArray();
$sitemapXml = $this->buildZjSitemap($Novels, $strSiteDomain, $strXqPath);
$filePath = public_path() . "sitemap_novel_zj_{$i}.xml";
file_put_contents($filePath, $sitemapXml);
$sitemapFiles[] = $filePath;
}
return $sitemapFiles;
}
# 生成小说详情 xml文件
private function generateNovelSitemaps(int $chunkSize, int $totalCount, string $strSiteDomain, string $strXqPath): array
{
$totalFiles = ceil($totalCount / $chunkSize);
$sitemapFiles = [];
for ($i = 0; $i < $totalFiles; $i++) {
$Novels = XiaoShuoXiangQingModel::field(['xsxq_id', 'xsxq_source_id', 'xsxq_source_seo_id']) //intNovelId-:intNovelSourceId-:intNovelSourceSeoId
->limit($i * $chunkSize, $chunkSize)
->select()
->toArray();
$sitemapXml = $this->buildSitemap($Novels, $strSiteDomain, $strXqPath);
$filePath = public_path() . "sitemap_novel_{$i}.xml";
file_put_contents($filePath, $sitemapXml);
$sitemapFiles[] = $filePath;
}
return $sitemapFiles;
}
# 生成小说详情 xml文件
private function generateNovelSeoSitemaps(int $chunkSize, int $totalCount, string $strSiteDomain, string $strSeoPath): array
{
$totalFiles = ceil($totalCount / $chunkSize);
$sitemapFiles = [];
for ($i = 0; $i < $totalFiles; $i++) {
$Novels = NovelInfoSeoModel::field(['xsxq_id', 'nis_id']) //show/nis_id
->limit($i * $chunkSize, $chunkSize)
->select()
->toArray();
$sitemapXml = $this->buildXiaoShuoSeoSitemap($Novels, $strSiteDomain, $strSeoPath);
$filePath = public_path() . "sitemap_novel_seo_{$i}.xml";
file_put_contents($filePath, $sitemapXml);
$sitemapFiles[] = $filePath;
}
return $sitemapFiles;
}
// 生成分类链接
private function generateCategorySitemaps(array $categories, string $strSiteDomain, string $strFenleiPath, string $strXqPath): array
{
$sitemapFiles = [];
foreach ($categories as $category) {
$vcId = $category['xsfl_id'];
$vcName = $category['xsfl_name'];
$intNovelCount = XiaoShuoFenLeiModel::where('xsfl_id', $vcId)->count();
$totalPages = ceil($intNovelCount / 30);
$categoryUrls = [];
for ($status = 0; $status <= 2; $status++) {
for ($sort = 0; $sort <= 4; $sort++) {
for ($page = 1; $page <= $totalPages; $page++) {
$categoryUrls[] = [
// /fl/:intCId-:intSort-[:intPage]
'url' => "/{$strFenleiPath}/{$vcId}-{$sort}-{$page}.html",
'title' => "{$vcName} - 第{$page}",
'is_home' => false,
];
}
}
}
$sitemapXml = $this->buildSitemap($categoryUrls, $strSiteDomain, $strXqPath, true);
$filePath = public_path() . "sitemap_category_{$vcId}.xml";
file_put_contents($filePath, $sitemapXml);
$sitemapFiles[] = $filePath;
}
return $sitemapFiles;
}
//生成章节链接
private function buildZjSitemap(array $items, string $strSiteDomain, string $strXqPath, bool $isCategory = false): string
{
$xmlHeader = '<?xml version="1.0" encoding="UTF-8"?>';
$urlsetStart = '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">';
$urlsetEnd = '</urlset>';
// /xq/:intNovelId/:intZhangJiePaiXu-[:strSort]
$urlEntries = '';
foreach ($items as $item) {
$urlEntries .= '<url>';
$urlEntries .= '<loc>https://' . $strSiteDomain . ($isCategory ? $item['url'] : '/' . $strXqPath . '/' . htmlspecialchars($item['xsxq_id'] . '-' . $item['xszj_pai_xu']. '-' . $item['xszj_sort'] . '.html', ENT_QUOTES)) . '</loc>';
$urlEntries .= '<changefreq>daily</changefreq>';
$urlEntries .= '<priority>0.8</priority>';
$urlEntries .= '</url>';
}
return $xmlHeader . $urlsetStart . $urlEntries . $urlsetEnd;
}
// 生成 详情链接
private function buildSitemap(array $items, string $strSiteDomain, string $strXqPath, bool $isCategory = false): string
{
$xmlHeader = '<?xml version="1.0" encoding="UTF-8"?>';
$urlsetStart = '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">';
$urlsetEnd = '</urlset>';
// /xq/:intNovelId
$urlEntries = '';
foreach ($items as $item) {
$urlEntries .= '<url>';
$urlEntries .= '<loc>https://' . $strSiteDomain . ($isCategory ? $item['url'] : '/' . $strXqPath . '/' . htmlspecialchars($item['xsxq_id'] . '.html', ENT_QUOTES)) . '</loc>';
$urlEntries .= '<changefreq>daily</changefreq>';
$urlEntries .= '<priority>1.0</priority>';
$urlEntries .= '</url>';
}
return $xmlHeader . $urlsetStart . $urlEntries . $urlsetEnd;
}
// 生成详情 seo 链接
private function buildXiaoShuoSeoSitemap(array $items, string $strSiteDomain, string $strSeoPath, bool $isCategory = false): string
{
$xmlHeader = '<?xml version="1.0" encoding="UTF-8"?>';
$urlsetStart = '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">';
$urlsetEnd = '</urlset>';
// /show/:intNovelId
$urlEntries = '';
foreach ($items as $item) {
$urlEntries .= '<url>';
$urlEntries .= '<loc>https://' . $strSiteDomain . ($isCategory ? $item['url'] : '/' . $strSeoPath . '/' . htmlspecialchars($item['nis_id'] . '.html', ENT_QUOTES)) . '</loc>';
$urlEntries .= '<changefreq>daily</changefreq>';
$urlEntries .= '<priority>1.0</priority>';
$urlEntries .= '</url>';
}
return $xmlHeader . $urlsetStart . $urlEntries . $urlsetEnd;
}
// 生成 xml 文件
private function buildSitemapIndex(array $sitemapFiles, string $strSiteDomain): void
{
$xmlHeader = '<?xml version="1.0" encoding="UTF-8"?>';
$sitemapIndexStart = '<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">';
$sitemapIndexEnd = '</sitemapindex>';
$sitemapEntries = '';
foreach ($sitemapFiles as $file) {
$url = 'https://' . $strSiteDomain . '/' . basename($file);
$sitemapEntries .= '<sitemap>';
$sitemapEntries .= '<loc>' . htmlspecialchars($url, ENT_QUOTES) . '</loc>';
$sitemapEntries .= '<lastmod>' . date('Y-m-d') . '</lastmod>';
$sitemapEntries .= '</sitemap>';
}
$sitemapIndexXml = $xmlHeader . $sitemapIndexStart . $sitemapEntries . $sitemapIndexEnd;
file_put_contents(public_path() . 'sitemap_index.xml', $sitemapIndexXml);
var_dump(['message' => 'Sitemap Index 生成成功', 'file' => 'sitemap_index.xml']);
}
}

View File

@@ -0,0 +1,242 @@
<?php
declare(strict_types=1);
namespace app\task\collection\seo;
use app\task\collection\PullDataColBase;
use app\admin\model\XiaoShuoXiangQingModel;
use app\admin\model\NovelInfoSeoModel;
use app\admin\model\XiaoShuoZhangJieModel;
use app\admin\model\SiteModel;
use app\admin\model\XiaoShuoFenLeiModel;
class GenerateSiteMapNovelZw extends PullDataColBase
{
public function generate()
{
$SiteModel = SiteModel::getValById(config('app.default_app_id'));
// $strSiteDomain = SiteModel::getValBySiteIdByValCode('site_domain');
$strSiteDomain = $SiteModel['site_domain'];
$strFenleiPath = $SiteModel['site_fenlei_path'];
$strXqPath = $SiteModel['site_xiangqing_path'];
$strRankPath = $SiteModel['site_rank_path'];
$strTjPath = $SiteModel['site_tui_jian_path'];
$strSeoPath = $SiteModel['site_seo_path'];
$strZdyPath = $SiteModel['site_zdy_path'];
$strZjPath = $SiteModel['site_zhangjie_path'];
// 每个 Sitemap 包含的最大 URL 数量
$chunkSize = 40000;
// 从数据库获取所有总数
$totalNovelCount = XiaoShuoXiangQingModel::count();
// 从数据库获取所有总数
$totalNovelSeoCount = NovelInfoSeoModel::count();
// 从数据库获取所有总数
$totalNovelZjCount = XiaoShuoZhangJieModel::count();
// 获取分类信息
$categories = XiaoShuoFenLeiModel::getDataByCache();
// 生成详情的 Sitemap 小说详情
$novelSitemapFiles = $this->generateNovelSitemaps($chunkSize, $totalNovelCount, $strSiteDomain, $strXqPath);
// 生成详情的 Sitemap 小说seo
$novelSeoSitemapFiles = $this->generateNovelSeoSitemaps($chunkSize, $totalNovelSeoCount, $strSiteDomain, $strSeoPath);
// 生成章节的 Sitemap 小说章节
$novelZjSitemapFiles = $this->generateNovelZjSitemaps($chunkSize, $totalNovelZjCount, $strSiteDomain, $strXqPath);
// 生成分类的 Sitemap 小说分类
$categorySitemapFiles = $this->generateCategorySitemaps($categories, $strSiteDomain, $strFenleiPath, $strXqPath);
// 合并所有 Sitemap 文件路径
$allSitemapFiles = array_merge($novelSitemapFiles, $novelSeoSitemapFiles, $categorySitemapFiles, $novelZjSitemapFiles);
// 生成 Sitemap Index 文件
$this->buildSitemapIndex($allSitemapFiles, $strSiteDomain);
var_dump(['message' => '所有 Sitemap 生成成功', 'files' => $allSitemapFiles]);
}
# 生成小说章节 xml文件
private function generateNovelZjSitemaps(int $chunkSize, int $totalCount, string $strSiteDomain, string $strXqPath): array
{
$totalFiles = ceil($totalCount / $chunkSize);
$sitemapFiles = [];
for ($i = 0; $i < $totalFiles; $i++) {
$Novels = XiaoShuoZhangJieModel::field(['xsxq_id', 'xszj_id']) //intNovelId-:intNovelSourceId-:intNovelSourceSeoId
->limit($i * $chunkSize, $chunkSize)
->select()
->toArray();
$sitemapXml = $this->buildZjSitemap($Novels, $strSiteDomain, $strXqPath);
$filePath = public_path() . "sitemap_novel_zj_{$i}.xml";
file_put_contents($filePath, $sitemapXml);
$sitemapFiles[] = $filePath;
}
return $sitemapFiles;
}
# 生成小说详情 xml文件
private function generateNovelSitemaps(int $chunkSize, int $totalCount, string $strSiteDomain, string $strXqPath): array
{
$totalFiles = ceil($totalCount / $chunkSize);
$sitemapFiles = [];
for ($i = 0; $i < $totalFiles; $i++) {
$Novels = XiaoShuoXiangQingModel::field(['xsxq_id', 'xsxq_source_id', 'xsxq_source_seo_id']) //intNovelId-:intNovelSourceId-:intNovelSourceSeoId
->limit($i * $chunkSize, $chunkSize)
->select()
->toArray();
$sitemapXml = $this->buildSitemap($Novels, $strSiteDomain, $strXqPath);
$filePath = public_path() . "sitemap_novel_{$i}.xml";
file_put_contents($filePath, $sitemapXml);
$sitemapFiles[] = $filePath;
}
return $sitemapFiles;
}
# 生成小说详情 xml文件
private function generateNovelSeoSitemaps(int $chunkSize, int $totalCount, string $strSiteDomain, string $strSeoPath): array
{
$totalFiles = ceil($totalCount / $chunkSize);
$sitemapFiles = [];
for ($i = 0; $i < $totalFiles; $i++) {
$Novels = NovelInfoSeoModel::field(['xsxq_id', 'nis_id']) //show/nis_id
->limit($i * $chunkSize, $chunkSize)
->select()
->toArray();
$sitemapXml = $this->buildXiaoShuoSeoSitemap($Novels, $strSiteDomain, $strSeoPath);
$filePath = public_path() . "sitemap_novel_seo_{$i}.xml";
file_put_contents($filePath, $sitemapXml);
$sitemapFiles[] = $filePath;
}
return $sitemapFiles;
}
// 生成分类链接
private function generateCategorySitemaps(array $categories, string $strSiteDomain, string $strFenleiPath, string $strXqPath): array
{
$sitemapFiles = [];
foreach ($categories as $category) {
$vcId = $category['xsfl_id'];
$vcName = $category['xsfl_name'];
$intNovelCount = XiaoShuoFenLeiModel::where('xsfl_id', $vcId)->count();
$totalPages = ceil($intNovelCount / 30);
$categoryUrls = [];
for ($status = 0; $status <= 2; $status++) {
for ($sort = 0; $sort <= 4; $sort++) {
for ($page = 1; $page <= $totalPages; $page++) {
$categoryUrls[] = [
// /fl/:intCId-:intSort-[:intPage]
'url' => "/{$strFenleiPath}/{$vcId}-{$sort}-{$page}.html",
'title' => "{$vcName} - 第{$page}",
'is_home' => false,
];
}
}
}
$sitemapXml = $this->buildSitemap($categoryUrls, $strSiteDomain, $strXqPath, true);
$filePath = public_path() . "sitemap_category_{$vcId}.xml";
file_put_contents($filePath, $sitemapXml);
$sitemapFiles[] = $filePath;
}
return $sitemapFiles;
}
//生成章节链接
private function buildZjSitemap(array $items, string $strSiteDomain, string $strXqPath, bool $isCategory = false): string
{
$xmlHeader = '<?xml version="1.0" encoding="UTF-8"?>';
$urlsetStart = '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">';
$urlsetEnd = '</urlset>';
// /xq/:intNovelId/:intZhangJiePaiXu-[:strSort]
$urlEntries = '';
foreach ($items as $item) {
$urlEntries .= '<url>';
$urlEntries .= '<loc>https://' . $strSiteDomain . ($isCategory ? $item['url'] : '/' . $strXqPath . '/' . htmlspecialchars($item['xsxq_id'] . '-' . $item['xszj_pai_xu']. '-' . $item['xszj_sort'] . '.html', ENT_QUOTES)) . '</loc>';
$urlEntries .= '<changefreq>daily</changefreq>';
$urlEntries .= '<priority>0.8</priority>';
$urlEntries .= '</url>';
}
return $xmlHeader . $urlsetStart . $urlEntries . $urlsetEnd;
}
// 生成 详情链接
private function buildSitemap(array $items, string $strSiteDomain, string $strXqPath, bool $isCategory = false): string
{
$xmlHeader = '<?xml version="1.0" encoding="UTF-8"?>';
$urlsetStart = '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">';
$urlsetEnd = '</urlset>';
// /xq/:intNovelId
$urlEntries = '';
foreach ($items as $item) {
$urlEntries .= '<url>';
$urlEntries .= '<loc>https://' . $strSiteDomain . ($isCategory ? $item['url'] : '/' . $strXqPath . '/' . htmlspecialchars($item['xsxq_id'] . '.html', ENT_QUOTES)) . '</loc>';
$urlEntries .= '<changefreq>daily</changefreq>';
$urlEntries .= '<priority>1.0</priority>';
$urlEntries .= '</url>';
}
return $xmlHeader . $urlsetStart . $urlEntries . $urlsetEnd;
}
// 生成详情 seo 链接
private function buildXiaoShuoSeoSitemap(array $items, string $strSiteDomain, string $strSeoPath, bool $isCategory = false): string
{
$xmlHeader = '<?xml version="1.0" encoding="UTF-8"?>';
$urlsetStart = '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">';
$urlsetEnd = '</urlset>';
// /show/:intNovelId
$urlEntries = '';
foreach ($items as $item) {
$urlEntries .= '<url>';
$urlEntries .= '<loc>https://' . $strSiteDomain . ($isCategory ? $item['url'] : '/' . $strSeoPath . '/' . htmlspecialchars($item['nis_id'] . '.html', ENT_QUOTES)) . '</loc>';
$urlEntries .= '<changefreq>daily</changefreq>';
$urlEntries .= '<priority>1.0</priority>';
$urlEntries .= '</url>';
}
return $xmlHeader . $urlsetStart . $urlEntries . $urlsetEnd;
}
// 生成 xml 文件
private function buildSitemapIndex(array $sitemapFiles, string $strSiteDomain): void
{
$xmlHeader = '<?xml version="1.0" encoding="UTF-8"?>';
$sitemapIndexStart = '<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">';
$sitemapIndexEnd = '</sitemapindex>';
$sitemapEntries = '';
foreach ($sitemapFiles as $file) {
$url = 'https://' . $strSiteDomain . '/' . basename($file);
$sitemapEntries .= '<sitemap>';
$sitemapEntries .= '<loc>' . htmlspecialchars($url, ENT_QUOTES) . '</loc>';
$sitemapEntries .= '<lastmod>' . date('Y-m-d') . '</lastmod>';
$sitemapEntries .= '</sitemap>';
}
$sitemapIndexXml = $xmlHeader . $sitemapIndexStart . $sitemapEntries . $sitemapIndexEnd;
file_put_contents(public_path() . 'sitemap_index.xml', $sitemapIndexXml);
var_dump(['message' => 'Sitemap Index 生成成功', 'file' => 'sitemap_index.xml']);
}
}

View File

@@ -0,0 +1,538 @@
<?php
declare(strict_types=1);
namespace app\task\collection\tianlai;
use app\admin\model\XiaoShuoFenLeiModel;
use app\admin\model\XiaoShuoXiangQingModel;
use app\admin\model\XiaoShuoZhangJieModel;
use app\task\collection\PullDataColBase;
use GuzzleHttp\Cookie\CookieJar;
use processor\ImageProcessor;
use think\facade\Cache;
use QL\QueryList;
/**
* @mixin think\Model
*/
class TianLaiXiaoShuoCol extends PullDataColBase
{
public function pullXiaoShuo()
{
$TaskCore = $this->TaskCore;
$strUrl = $this->getTianLaiDomain();
var_dump($strUrl);
$Client = $this->Client;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
// $proxy = '127.0.0.1:8080'; // 代理地址
// $response = fetchContentWithCurl($strUrl, [], 10, $proxy);
$strResult = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$strResult->getBody();
var_dump($strResult);
// exit;
// $strResult = $response['data'];
$QueryList = QueryList::html($strResult);
$QueryList->find('.nav a')->map(function ($FenLeiQueryList) use ($strUrl, $TaskCore) {
$strName = trim($FenLeiQueryList->text());
switch ($strName) {
case '都市言情':
$strName = "都市言情";
break;
case '历史军事':
$strName = "历史军事";
break;
case '玄幻奇幻':
$strName = "玄幻奇幻";
break;
case '武侠修真':
$strName = "武侠修真";
break;
case '女生纯爱':
$strName = "女频言情";
break;
case '科幻网游':
$strName = "科幻网游";
break;
case '悬疑灵异':
$strName = "悬疑灵异";
break;
default:
$strName = "其它";
break;
}
$strFenLeiUrl = $strUrl . $FenLeiQueryList->attr('href');
// $intFenLeiId = extractFilename($strFenLeiUrl."index.html");
$intFenLeiId = extractNumberFromUrl($strFenLeiUrl);
if($intFenLeiId !== null){
$arrClass = [
'xsfl_source_id' => $intFenLeiId,
'xsfl_source_code' => 'TIANLAI_FEN_LEI_ID_' . $intFenLeiId,
'xsfl_name' => $strName,
];
var_dump($arrClass);
$XiaoShuoFenLeiModel = XiaoShuoFenLeiModel::addXiaoShuoFenLei($arrClass);
$arrTask = [
'callback' => [self::class, 'getNovelPage'],
'data' => [
'url' => $strFenLeiUrl,
'xsfl_source_id' => $intFenLeiId,
'xsfl_name' => $strName,
'xsfl_id' => $XiaoShuoFenLeiModel->xsfl_id,
'xsfl_page' => 1,
'xsfl_max_page' => 3000,
]
];
$TaskCore->set($arrTask);
}
});
} catch (\Throwable $T) {
//var_dump($T);
throw $T;
}
}
public function getNovelPage($arrData)
{
var_dump('getNovelPage');
var_dump($arrData);
$strBasePage = $arrData['url'];
$Client = $this->Client;
$TaskCore = $this->TaskCore;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strBasePage);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$strSiteUrl = $this->getTianLaiDomain();
$intPage = $arrData['xsfl_page'];
$intMaxPage = $arrData['xsfl_max_page'];
$strUrl = $strBasePage;
var_dump($strUrl);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$strResult = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => $userAgent
],
'cookies' => $CookieJar,
#'proxy' => $proxy,
'timeout' => 10,
]);
$strResult = (string)$strResult->getBody();
$QueryList = QueryList::html($strResult);
$intMaxPage = $QueryList->find('#newscontent h2 span')->eq(1)->text();
$intMaxPage = ceil($intMaxPage / 100);
var_dump($intMaxPage);
$intMaxPage = 2;
var_dump('$strUrl');
// 如果是首页 则根据最大页面吧所有分页 追加到任务
if($intPage == 1){
for ($intCurrentPage = 2; $intCurrentPage <= $intMaxPage; $intCurrentPage++) {
$strNewsFenleiUrl = replaceLastPageNumber($strBasePage, $intCurrentPage);
var_dump($strNewsFenleiUrl);
$arrTask = [
'callback' => [self::class, 'getNovelPage'],
'data' => [
'url' => $strNewsFenleiUrl,
'xsfl_source_id' => $arrData['xsfl_source_id'],
'xsfl_name' => $arrData['xsfl_name'],
'xsfl_id' => $arrData['xsfl_id'],
'xsfl_page' => $intCurrentPage,
'xsfl_max_page' => $intMaxPage,
]
];
$TaskCore->set($arrTask);
}
}
var_dump('提取小说');
$XiaoShuoListQueryList = $QueryList->find('#newscontent li');
if (!$XiaoShuoListQueryList || $XiaoShuoListQueryList->count() <= 0) {
var_dump('这个分类已经市最后一页');
return true;
}
var_dump($XiaoShuoListQueryList->count());
var_dump('提取小说2');
// var_dump($XiaoShuoListQueryList);
$XiaoShuoListQueryList->map(function ($XiaoShuoQueryList) use ($strSiteUrl, $arrData, $TaskCore, $strUrl) {
// 提取小说链接
// var_dump($XiaoShuoQueryList->html());exit;
var_dump('提取小说链接');
$A = $XiaoShuoQueryList->find('a')->eq(0);
$strXiaoShuoUrl = $A->attr('href');
$strXiaoShuoUrl = strpos($strXiaoShuoUrl, 'https:') === 0 ? $strXiaoShuoUrl : 'https:' . $strXiaoShuoUrl;
var_dump('$A');
var_dump($A);
var_dump($strXiaoShuoUrl);
exit;
// var_dump($A);
// 提取作者信息
$strZuoZhe = trim($XiaoShuoQueryList->find('.s4')->eq(0)->text());
// 提取小说 ID
$arrXiaoShuoId = extractNumbersFromUrl($strXiaoShuoUrl);
if (!isset($arrXiaoShuoId['first'], $arrXiaoShuoId['second'])) {
return; // 如果 ID 提取失败,直接跳过
}
// 准备任务数据
$arrXiaoShuo = array_merge($arrData, [
'xiaoshuo_url' => $strXiaoShuoUrl,
'xiaoshuo_id' => $arrXiaoShuoId['first'],
'xiaoshuo_seo_id' => $arrXiaoShuoId['second'],
'xiaoshuo_zuozhe' => $strZuoZhe,
'url' => $strUrl
]);
var_dump($arrXiaoShuo);
$arrTask = [
'callback' => [self::class, 'getNovelInfo'],
'data' => $arrXiaoShuo,
];
// 将任务加入队列
$TaskCore->set($arrTask);
var_dump("将任务加入队列getNovelInfo-{$strXiaoShuoUrl}");
});
var_dump('提取小说end');
} catch (\Throwable $T) {
throw $T;
}
}
public function getNovelInfo($arrData)
{
var_dump('getNovelInfo');
var_dump($arrData);
$Client = $this->Client;
$TaskCore = $this->TaskCore;
// 获取 Redis 实例
$redis = Cache::store('redis')->handler();
$intXiaoShuoSourceId = $arrData['xiaoshuo_id'];
$intXiaoShuoSourceSeoId = $arrData['xiaoshuo_seo_id'];
$redisKey = "task:novel_source_id:" . $intXiaoShuoSourceId;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($arrData['xiaoshuo_url']);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$strSiteUrl = $this->getTianLaiDomain();
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $arrData['xiaoshuo_url'], [
'headers' => [
'User-Agent' => $userAgent
],
'cookies' => $CookieJar,
#'proxy' => $proxy,
'timeout' => 10,
]);
$strResult = (string)$Response->getBody();
unset($Response);
$QueryList = QueryList::html($strResult);
$strMingZi = $QueryList->find('#info h1')->eq(0)->text();
$strZhuangTai = "状态:连载中";
$strZiShu = 0;
$strZhuangTai = getStringAfterWord($strZhuangTai, '状态:');
// 提取作者信息
$strZuoZhe = trim($QueryList->find('p a')->eq(0)->text());
// $strLastDate = $QueryList->find('meta[property="og:novel:update_time"]')->attr('content');
$strLastDate = $QueryList->find('#info p')->eq(2)->text();
$strLastDate = getStringAfterWord($strLastDate, '最后更新:');
$strLastDate = trim((string)$strLastDate);
if (!isValidDateTime($strLastDate)) {
$strLastDate = date('Y-m-d H:i:s');
}
$arrTag = [];
// $QueryList->find('.tg span')->map(function ($TagQuery) use (&$arrTag) {
// $arrTag[] = trim($TagQuery->text());
// });
$strJieShao = $QueryList->find('#intro')->eq(0)->text();
$strJieShao = removeScriptTags('/<script>.*?<\/script>/is',$strJieShao) ;
$strJieShao = removeScriptTags('/myJs\.bookJs\(\);/',$strJieShao) ;
$strCover = $QueryList->find('#fmimg img')->eq(0)->attr("src");
// if (strpos($strCover, 'http') === false) {
// $strCover = $strSiteUrl . $strCover;
// }
$strCoverUri = $strCover;
// try {
// $strCoverUri = "";
// $strCoverUri = ImageProcessor::getInstance()->downloadAndEncryptImage($strCover, 'xs');
// } catch (\Throwable $t) {
// $strErr = sprintf("采集[%s]小说图片错误,异常:%s", $strCover, $t->getMessage());
// echo $strErr;
// }
$arrCover = [
'code' => 'AI_LI_SI_COVER',
// 'uri' => getUriByUrl($strCover),
'uri' => $strCoverUri,
];
$arrXiaoShuoInfo = [
'xsxq_ming_zi' => $strMingZi,
'xsxq_zuozhe' => $strZuoZhe , //$arrData['xiaoshuo_zuozhe'],
'xsfl_id' => $arrData['xsfl_id'],
'xsxq_zhuang_tai' => $strZhuangTai,
'xsxq_zi_shu' => $strZiShu,
'xsxq_jie_shao' => $strJieShao,
'xsxq_feng_mian' => json_encode($arrCover),
'xsxq_geng_xin_shi_jian' => $strLastDate,
'xsxq_tag' => json_encode($arrTag, JSON_UNESCAPED_UNICODE),
'xsxq_source_id' => $intXiaoShuoSourceId,
'xsxq_source_seo_id' => $intXiaoShuoSourceSeoId,
'xsxq_source_code' => 'AILISI_XIAOSHUO_' . $intXiaoShuoSourceId . '_' . $intXiaoShuoSourceSeoId,
];
// print_r($arrXiaoShuoInfo);
var_dump($arrXiaoShuoInfo['xsxq_source_code']);
$XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
/*
if ($redis->exists($redisKey)) {
var_dump('小说任务已存在,直接跳过');
return true; // 如果任务已存在,直接跳过
}
// 在 Redis 中标记任务为已存在
$redis->set($redisKey, 1);
$arrZhangjie = [];
$QueryList->find('#list a')->map(function ($ZhangJie) use (&$arrZhangjie) {
$arrZhangjie[] = [
'mingzi' => $ZhangJie->text(),
'url' => $ZhangJie->attr('href'),
];
});
unset($QueryList);
// 获取最大章节索引
$intMaxXiaoShuoZhangJiePaixu = XiaoShuoZhangJieModel::where('xsxq_id', $intXiaoShuoSourceId)
->max('xszj_pai_xu'); // 获取最大值
// 如果没有结果max 返回 null可以设置默认值
$intMaxXiaoShuoZhangJiePaixu = $intMaxXiaoShuoZhangJiePaixu ?? 0;
foreach ($arrZhangjie as $intZhangJieNum => $arrZhangjieOne) {
// 如果数据库里的章节最大索引大于当前索引,说明已经入库,不需要入队列了
if($intMaxXiaoShuoZhangJiePaixu > $intZhangJieNum){
var_dump("小说章节已经入库,不再推送任务");
continue;// 如果任务已存在,直接跳过 break
}
$strZhangJieUrl = "https:" . $arrZhangjieOne['url'];
$strZhangJieId = extractFilename($strZhangJieUrl);
# 'AILISI_XIAOSHUO_ZHANGJIE_小说源id_小说源章节id'
$strXszjSourceCode = 'AILISI_XIAOSHUO_ZHANGJIE_'. $XiaoShuoXiangQingModel->xsxq_source_id . '_' . $strZhangJieId;
$arrTask = [
'callback' => [self::class, 'getNovelReader'],
'data' => [
'xszj_source_id' => $strZhangJieId,
'xszj_paixu' => $intZhangJieNum,
'xszj_url' => $strZhangJieUrl,
'xszj_source_code' => $strXszjSourceCode,
'xsxq_source_id' => $XiaoShuoXiangQingModel->xsxq_source_id,
'xsfl_id' => $arrData['xsfl_id'],
]
];
$TaskCore->set($arrTask);
//var_dump("章节任务投递成功".$strXszjSourceCode);
}
// 任务成功后删除 Redis 标记
$redis->del($redisKey);
*/
} catch (\Throwable $T) {
throw $T;
}
}
public function getNovelFengMian($arrData)
{
//var_dump('getNovelFengMian');
//var_dump($arrData);
$Client = $this->Client;
$TaskCore = $this->TaskCore;
// 获取 Redis 实例
$redis = Cache::store('redis')->handler();
$intXiaoShuoSourceId = $arrData['xiaoshuo_id'];
$intXiaoShuoSourceSeoId = $arrData['xiaoshuo_seo_id'];
$redisKey = "task:novel_source_id:" . $intXiaoShuoSourceId;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($arrData['xiaoshuo_url']);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$strSiteUrl = $this->getTianLaiDomain();
$strCover = $QueryList->find('#fmimg img')->eq(0)->attr("src");
if (strpos($strCover, 'http') === false) {
$strCover = $strSiteUrl . $strCover;
}
$strCoverUri = $strCover;
try {
$strCoverUri = "";
$strCoverUri = ImageProcessor::getInstance()->downloadAndEncryptImage($strCover, 'xs');
$arrCover = [
'code' => 'AI_LI_SI_COVER',
// 'uri' => getUriByUrl($strCover),
'uri' => $strCoverUri,
];
$arrXiaoShuoInfo = [
'xsxq_feng_mian' => json_encode($arrCover),
];
$XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
} catch (\Throwable $t) {
$strErr = sprintf("采集[%s]小说图片错误,异常:%s", $strCover, $t->getMessage());
echo $strErr;
}
} catch (\Throwable $T) {
throw $T;
}
}
public function getNovelReader($arrData)
{
//var_dump('getNovelReader');
//var_dump($arrData);
$Client = $this->Client;
$strXszjSourceCode = $arrData['xszj_source_code'];
$strZhangJieUrl = $arrData['xszj_url'];
$intZhangJiePaiXu = $arrData['xszj_paixu'];
$intFenleiId = $arrData['xsfl_id'];
$intXiaoShuoSourceID = $arrData['xsxq_source_id'];
$strZhangJieSourceId = $arrData['xszj_source_id'];
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strZhangJieUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
if (XiaoShuoZhangJieModel::checkZhangJieExists($strXszjSourceCode)) {
return true;
}
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strZhangJieUrl, [
'headers' => [
'User-Agent' => $userAgent
],
'cookies' => $CookieJar,
#'proxy' => $proxy,
'timeout' => 10,
]);
$strResult = (string)$Response->getBody();
unset($Response);
$ZhangJieQueryList = QueryList::html($strResult);
$strZhangJieName = $ZhangJieQueryList->find('h1')->text();
$strNeiRong = $ZhangJieQueryList->find('#content')->eq(0)->text();
# '/novel/123yqw/分类id/小说源id/小说源章节id.txt'
$strNeiRongTxtPath = config("filesystem.novel").'/novel/123yqw/'.$intFenleiId.'/'.$intXiaoShuoSourceID.'/'.$strZhangJieSourceId.'.txt';
//$strNeiRongTxtPath = "/www/novel/123yqw/1/11/111.txt";
// 调用函数
$result = saveCompressAndDeleteTxt($strNeiRongTxtPath, $strNeiRong);
// 输出状态
if ($result['status']) {
var_dump("Success: " . $result['message']) ;
} else {
var_dump("Error: " . $result['message']) ;
}
$arrXiaoShuoZhangJie = [
'xsxq_id' => $intXiaoShuoSourceID,
'xszj_pai_xu' => $intZhangJiePaiXu,
'xszj_ming_zi' => $strZhangJieName,
'xszj_nei_rong' => $strNeiRongTxtPath,
'xszj_source_id' => $strZhangJieSourceId,
'xszj_source_code' => $strXszjSourceCode,
];
XiaoShuoZhangJieModel::addXiaoShuoZhangJie($arrXiaoShuoZhangJie);
unset($ZhangJieQueryList);
} catch (\Throwable $T) {
throw $T;
}
}
}

View File

@@ -0,0 +1,90 @@
<?php
declare(strict_types=1);
namespace app\task\collection\visitlogs;
use app\admin\model\XiaoShuoFenLeiModel;
use app\admin\model\XiaoShuoXiangQingModel;
use app\admin\model\XiaoShuoZhangJieModel;
use app\task\collection\PullDataColBase;
use GuzzleHttp\Cookie\CookieJar;
use processor\ImageProcessor;
use think\facade\Cache;
use QL\QueryList;
use app\admin\model\VisitLogsModel;
use app\admin\model\ContentStatsModel;
/**
* @mixin think\Model
*/
class ContentStatsCol extends PullDataColBase
{
public function run()
{
var_dump('ContentStatsCol');
try {
// 定义时间范围统计的周期类型和 SQL 表达式
$arrPeriods = [
'day' => "DATE(vil_timestamp)", // 按日统计
'week' => "YEARWEEK(vil_timestamp)", // 按周统计
'month' => "DATE_FORMAT(vil_timestamp, '%Y-%m')", // 按月统计
'quarter' => "CONCAT(YEAR(vil_timestamp), '-', QUARTER(vil_timestamp))", // 按季度统计
'year' => "YEAR(vil_timestamp)" // 按年统计
];
foreach ($arrPeriods as $periodName => $groupExpression) {
var_dump($groupExpression);
$results = VisitLogsModel::where('vil_is_spider', 0) // 排除爬虫访问
->field([
'vil_content_type',
'vil_content_id',
"$groupExpression" => 'stat_date', // 根据周期分组
'COUNT(*)' => 'pv', // PV
'COUNT(DISTINCT vil_ip)' => 'uv' // UV
])
->group('vil_content_type, vil_content_id, stat_date')
->select()->toArray();
// var_dump($results);
foreach ($results as $result) {
$ContentStatsModel = [
'cs_content_type' => $result['vil_content_type'],
'cs_content_id' => $result['vil_content_id'],
'cs_stat_date' => $result['stat_date'],
'cs_stat_period' => $periodName, // 周期类型day, week, month, quarter, year
'cs_pv' => $result['pv'],
'cs_uv' => $result['uv'],
'cs_md5' => md5($result['vil_content_type'].$result['vil_content_id'].$result['stat_date'].$periodName),
'updated_at' => date('Y-m-d H:i:s'),
];
var_dump($ContentStatsModel);
ContentStatsModel::add($ContentStatsModel);
// var_dump($r);
// 插入或更新到统计表
// ContentStatsModel::insertOrUpdate([
// 'cs_content_type' => $result['vil_content_type'],
// 'cs_content_id' => $result['vil_content_id'],
// 'cs_stat_date' => $result['stat_date'],
// 'cs_stat_period' => $periodName, // 周期类型day, week, month, quarter, year
// 'cs_pv' => $result['pv'],
// 'cs_uv' => $result['uv'],
// 'updated_at' => date('Y-m-d H:i:s'),
// ], [
// 'cs_content_type' => $result['vil_content_type'],
// 'cs_content_id' => $result['vil_content_id'],
// 'cs_stat_date' => $result['stat_date'],
// 'cs_stat_period' => $periodName,
// ]);
}
}
} catch (\Exception $e) {
// 记录异常日志
//\think\facade\Log::error('ContentStatsTask error: ' . $e->getMessage());
}
}
}

View File

@@ -0,0 +1,47 @@
<?php
declare(strict_types=1);
namespace app\task\collection\visitlogs;
use app\admin\model\XiaoShuoFenLeiModel;
use app\admin\model\XiaoShuoXiangQingModel;
use app\admin\model\XiaoShuoZhangJieModel;
use app\task\collection\PullDataColBase;
use GuzzleHttp\Cookie\CookieJar;
use processor\ImageProcessor;
use think\facade\Cache;
use QL\QueryList;
use app\admin\model\VisitLogsModel;
/**
* @mixin think\Model
*/
class VisitLogsCol extends PullDataColBase
{
public function add($arrData)
{
try {
// 'vil_ip', 'vil_url', 'vil_content_id', 'vil_content_type', 'vil_method'
if (empty($arrData['vil_ip']) || empty($arrData['vil_url']) || empty($arrData['vil_content_id']) || empty($arrData['vil_content_type']) || empty($arrData['vil_method'])) {
var_dump("信息不全!");
}else{
$strLocation = '';
// if(!empty($arrData['vil_ip'])){
// $strLocation = getLocationByIp($arrData['vil_ip']);
// }
$arrData['vil_location'] = $strLocation;
VisitLogsModel::add($arrData);
}
} catch (\Throwable $T) {
//var_dump($T);
throw $T;
}
}
}

View File

@@ -0,0 +1,375 @@
<?php
declare(strict_types=1);
namespace app\task\collection\yzzy;
use app\admin\model\VideoYuyanModel;
use app\admin\model\VideoPlayurlModel;
use app\admin\model\VideoLeixinModel;
use app\admin\model\VideoDiquModel;
use app\admin\model\VideoYearsModel;
use app\admin\model\VideoJuqingModel;
use app\admin\model\XiaoShuoFenLeiModel;
use app\admin\model\XiaoShuoXiangQingModel;
use app\admin\model\VideoInfoModel;
use app\admin\model\XiaoShuoZhangJieModel;
use app\task\collection\PullDataColBase;
use GuzzleHttp\Cookie\CookieJar;
use processor\ImageProcessor;
use think\facade\Cache;
use QL\QueryList;
/**
* @mixin think\Model 优质资源
*/
class YouZhiYingShiCol extends PullDataColBase
{
public function pullYouZhiYingshi()
{
// https://1080zyk1.com/?m=vod-type-id-1-pg-1.html
// https://1080zyk1.com/?m=vod-type-id-2-pg-2.html
// https://1080zyk1.com/?m=vod-type-id-3-pg-2.html
// https://1080zyk1.com/?m=vod-type-id-4-pg-2.html
// https://1080zyk1.com/?m=vod-type-id-83-pg-2.html
// https://1080zyk1.com/?m=vod-type-id-94-pg-2.html
$strSiteUrl = $this->YouZhiZiYuanDomain();
// 配置分类任务参数
$categories = [
['id' => 1, 'video_class_id' => 1, 'name' => '电影'],
['id' => 83, 'video_class_id' => 5, 'name' => '短剧'],
['id' => 2, 'video_class_id' => 2, 'name' => '电视剧'],
['id' => 3, 'video_class_id' => 3, 'name' => '综艺'],
['id' => 4, 'video_class_id' => 4, 'name' => '动漫'],
['id' => 94, 'video_class_id' => 6, 'name' => '体育'],
];
foreach ($categories as $category) {
$intClassId = $category['video_class_id'];
$intCategoryId = $category['id'];
// 先抓第一页
$strUrl = $strSiteUrl . '/?m=vod-type-id-' . $intCategoryId . '-pg-1.html';
var_dump($strUrl);
$maxPage = $this->getMaxPage($strUrl,$intClassId);
// 首次抓取,全部抓取
if (config('task.task_is_first')) {
$this->createTasks($strSiteUrl, $intCategoryId, $intClassId, $maxPage);
}
}
}
/**
* 获取分类的最大页码且抓取第一页
*/
protected function getMaxPage($strUrl, $intClassId)
{
$TaskCore = $this->TaskCore;
$Client = $this->Client;
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$Response = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'Accept-Language' => 'en-US,en;q=0.5',
'cookie' => '__sk_Vm8nTqM7LqSCWmR6__=be3bd4f4e0483619e1a434ec1b18a938',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 300,
]);
$strResult = (string)$Response->getBody();
$QueryList = QueryList::html($strResult);
// 检查是否找到元素
$intVideoListIndex = 0;
if ($QueryList->find('.xing_vb ul')->count() > 1) {
$QueryList->find('.xing_vb ul')->map(function ($Video) use ($intClassId, &$TaskCore,&$intVideoListIndex) {
if($intVideoListIndex >=1){
$strHref = $Video->find('.xing_vb4 a')->attr('href');
$strName = '';
if (!empty($strHref) && strlen($strHref) > 0) {
$arrTask = [
'callback' => [self::class, 'getVideoInfo'],
'data' => [
'class_id' => $intClassId,
'video_href' => $strHref,
'video_name' => $strName,
]
];
// var_dump($arrTask);
$TaskCore->set($arrTask);
echo "视频投递成功 : " . $Video->text() . PHP_EOL;
}
}
$intVideoListIndex ++ ;
});
} else {
echo "No elements found for .xing_vb ul" . PHP_EOL;
}
// 获取分页部分最后一个页码链接
$lastPageHref = $QueryList->find('.pages a')->eq($QueryList->find('.pages a')->count() - 1)->attr('href');
// 提取最大页码
preg_match('/-pg-(\d+)\.html/', $lastPageHref, $matches);
$maxPage = isset($matches[1]) ? (int)$matches[1] : 1; // 默认返回 1
return $maxPage;
}
/**
* 创建任务
*/
protected function createTasks($strSiteUrl, $intCategoryId, $videoClassId, $maxPage)
{
$TaskCore = $this->TaskCore;
for ($i = $maxPage; $i >= 1; $i--) {
echo "当前分类: $intCategoryId, 当前页: $i\n";
$arrTask = [
'callback' => [self::class, 'getVideoPage'],
'data' => [
'video_page_url' => $strSiteUrl . '/?m=vod-type-id-' . $intCategoryId . '-pg-' . $i . '.html',
'video_class_id' => $videoClassId,
]
];
$TaskCore->set($arrTask);
}
}
# 获取分页里面的视频列表
public function getVideoPage($arrData)
{
$TaskCore = $this->TaskCore;
$Client = $this->Client;
$strUrl = $arrData['video_page_url'];
$intClassId = $arrData['video_class_id'];
$strSiteUrl = $this->YouZhiZiYuanDomain();
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
var_dump($strUrl);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'Accept-Language' => 'en-US,en;q=0.5',
'cookie' => '__sk_Vm8nTqM7LqSCWmR6__=be3bd4f4e0483619e1a434ec1b18a938',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 300,
]);
$strResult = (string)$Response->getBody();
$QueryList = QueryList::html($strResult);
$intVideoListIndex = 0;
if ($QueryList->find('.xing_vb ul')->count() > 1) {
$QueryList->find('.xing_vb ul')->map(function ($Video) use ($intClassId, &$TaskCore,&$intVideoListIndex) {
if($intVideoListIndex >=1){
$strHref = $Video->find('.xing_vb4 a')->attr('href');
$strName = '';
if (!empty($strHref) && strlen($strHref) > 0) {
$arrTask = [
'callback' => [self::class, 'getVideoInfo'],
'data' => [
'class_id' => $intClassId,
'video_href' => $strHref,
'video_name' => $strName,
]
];
// var_dump($arrTask);
$TaskCore->set($arrTask);
echo "视频投递成功 : " . $strHref . PHP_EOL;
}
}
$intVideoListIndex ++ ;
});
} else {
echo "No elements found for .xing_vb ul" . PHP_EOL;
}
}
public function getVideoInfo($arrData)
{
$TaskCore = $this->TaskCore;
$Client = $this->Client;
$strSiteUrl = $this->YouZhiZiYuanDomain();
$strUrl = $strSiteUrl . $arrData['video_href'];
$intClassId = $arrData['class_id'];
$strVideoName = $arrData['video_name'] ?? '';
var_dump($strUrl);
if (!empty($strUrl) && strlen($strUrl) > 0) {
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
$QueryList = QueryList::html($strResult);
// 提取别名
$strPinyinId = 'yzzy'.extractIdFromUrl($strUrl);
$strVideoName = $QueryList->find('.vodh h2')->text();
$strJiaoBiao = $QueryList->find('.vodh span')->text();
$strPinfen = $QueryList->find('.vodh label')->text();
$strLeixin = $QueryList->find('.nvc dd a')->eq(1)->text();
$strCover = $QueryList->find('.vodImg img')->eq(0)->attr("src");
if (strpos($strCover, 'http') === false) {
$strCover = $strSiteUrl . $strCover;
}
$strCoverUri = $strCover;
$arrCover = [
'code' => 'YOUZHI_SOURCE_COVER',
'uri' => $strCoverUri,
];
// 提取导演
$strDaoyan = $QueryList->find('.vodInfo .vodinfobox ul li')->eq(1)->find('span')->text();
// 提取主演 多个逗号分割
$strZhuyan = $QueryList->find('.vodInfo .vodinfobox li')->eq(2)->find('span')->text();
// 提取类型
$arrLeixinData = $QueryList->find('.vodInfo .vodinfobox li')->eq(3)->find('span')->text();
// 切割后移除空元素
$arrJuqing = preg_split('/\s+/u', trim($arrLeixinData));
array_shift($arrJuqing);
// **在过滤后将“类型”数据入库**
foreach ($arrJuqing as $value) {
if(strlen($value) > 0){
VideoJuqingModel::add(['vc_id' => $intClassId, 'vj_name' => $value]);
}
}
// 提取地区
$strDiqu = $QueryList->find('.vodinfobox li span')->eq(4)->text();
// 提取语言
$strLanguage = $QueryList->find('.vodinfobox li span')->eq(5)->text();
// 提取年份
$strYears = $QueryList->find('.vodinfobox li span')->eq(6)->text();
// 提取更新时间
$strLastDate = $QueryList->find('.vodinfobox li span')->eq(8)->text();
if (strlen($strLeixin) > 0) {
VideoLeixinModel::addVideoLeixin(['vc_id' => $intClassId, 'vl_name' => $strLeixin]);
}
if (strlen($strDiqu) > 0) {
VideoDiquModel::add(['vc_id' => $intClassId, 'vd_name' => $strDiqu]);
}
if (strlen($strYears) > 0) {
VideoYearsModel::add(['vc_id' => $intClassId, 'vy_name' => $strYears]);
}
if (strlen($strLanguage) > 0) {
VideoYuyanModel::add(['vc_id' => $intClassId, 'vla_name' => $strLanguage]);
}
// 提取简介
$strDescription = $QueryList->find('.vodplayinfo')->eq(0)->text();
if(empty($strVideoName)){
var_dump('$strVideoName------');
var_dump($strVideoName);
return true;
}
$arrVideoInfo = [
'v_name' => $strVideoName,
'v_cover' => json_encode($arrCover),
'vc_id' => $intClassId,
'v_pinyin' => $strPinyinId,
'v_leixin' => $strLeixin,
'v_juqing' => implode(',', $arrJuqing),
'v_diqu' => $strDiqu,
'v_years' => $strYears,
'v_language' => $strLanguage,
'v_daoyan' => trim(str_replace('主演:', '', $strZhuyan)),
'v_zhuyan' => trim(str_replace('导演:', '', $strDaoyan)),
'update_time' => $strLastDate,
'v_description' => $strDescription,
'v_pingfen' => $strPinfen,
'v_orurl' => $strUrl,
'v_jiaobiao' => $strJiaoBiao,
'v_md5' => md5($intClassId .$strPinyinId. $strVideoName . getUriByUrl($strUrl)),
];
// var_dump($arrVideoInfo);
// // 检查是否找到元素
if ($QueryList->find('#play_2 ul li')->count() > 0) {
// var_dump("视频入库成功--" . $strVideoName);
$resultVideoInfoModel = VideoInfoModel::addVideoXiangQing($arrVideoInfo);;
if ($resultVideoInfoModel['isNew']) {
var_dump("视频详情数据已新增:", $resultVideoInfoModel['model']->v_id."视频入库成功--" . $strVideoName);
} else {
echo "视频详情数据已存在:", $resultVideoInfoModel['model']->v_id;
}
$VideoInfoModel = $resultVideoInfoModel['model'];
$intVid = $VideoInfoModel->v_id;
$intChapterIndex = 0;
$QueryList->find('#play_2 ul li')->map(function ($ZhangJie) use (&$arrZhangjie, &$intChapterIndex, &$intVid,&$intClassId) {
//echo "Processing index $intChapterIndex: " . $ZhangJie->text() . PHP_EOL;
$strM3u8 = $ZhangJie->find('a')->text();
$arrM3u8 = explode('$', $strM3u8);
$strM3u8Url = $arrM3u8[1];
$strM3u8Name = $arrM3u8[0];
$arrM3u8Url = [
'code' => 'YOUZHI_SOURCE_PLAY',
'uri' => $strM3u8Url,
];
$arrVideoPlayurl = [
'v_id' => $intVid,
'vp_sort' => $intChapterIndex,
'vp_name' => $strM3u8Name,
'vp_url' => json_encode($arrM3u8Url),
'vp_md5' => md5($intClassId.$intVid . getUriByUrl($strM3u8Url)),
];
$result = VideoPlayurlModel::add($arrVideoPlayurl);
if ($result['isNew']) {
var_dump("播放地址已新增:", $result['model']->vp_id."播放线路入库成功--" . $strM3u8Url) ;
} else {
echo "播放地址已存在:", $result['model']->vp_id ;
}
$intChapterIndex++;
});
var_dump("--") ;
var_dump("一个视频任务完成------------------------------------------------------------------------------------") ;
} else {
echo "No elements found for .listmain a" . PHP_EOL;
}
} catch (\Throwable $T) {
throw $T;
}
}
// sleep(10);
}
}

View File

@@ -0,0 +1,115 @@
<?php
declare(strict_types=1);
namespace app\task\command;
use database\seeders\SeedManager;
use think\console\Command;
use think\console\Input;
use think\console\input\Argument;
use think\console\Output;
use think\console\input\Option;
use think\facade\Db;
class DatabaseCommand extends Command
{
protected function configure()
{
$this->setName('db:manage')
->addArgument('action', \think\console\input\Argument::OPTIONAL, 'The action to perform: backup or init or seed')
->setDescription('Manage the database: backup or initialize');
}
protected function execute(Input $input, Output $Output)
{
$strAction = $input->getArgument('action');
if ($strAction === 'backup') {
$this->backupDatabase($Output);
} elseif ($strAction === 'init') {
$this->initializeDatabase($Output);
} elseif ($strAction === 'seed') {
$this->seed($Output);
} else {
$Output->writeln("Invalid action. Use 'backup' or 'init'. or 'seed'");
}
}
private function seed(Output $Output)
{
$SeedManager = new SeedManager;
$Output->writeln("开始数据播种" . PHP_EOL);
$SeedManager->handle();
$Output->writeln("数据播种结束");
}
private function backupDatabase(Output $Output)
{
$arrConfig = config('database.connections.mysql');
$strDb = $arrConfig['database'];
$strUserName = $arrConfig['username'];
$strPwd = $arrConfig['password'];
$strHost = $arrConfig['hostname'];
$strBackupFile = app()->getRootPath() . 'database/schema/mysql-schema.dump';
$strTempFile = tempnam(sys_get_temp_dir(), 'mysql-schema-') . '.dump';
$strCmd = "mysqldump -h{$strHost} -u{$strUserName} -p{$strPwd} --no-data {$strDb} > {$strTempFile}";
system($strCmd, $intResultCode);
if ($intResultCode === 0) {
if (file_exists($strBackupFile)) {
unlink($strBackupFile);
}
if (rename($strTempFile, $strBackupFile)) {
$Output->writeln("Database backup successful: {$strBackupFile}");
} else {
$Output->writeln("Failed to move backup file to destination.");
}
} else {
$Output->writeln("Database backup failed.");
unlink($strTempFile);
}
}
private function initializeDatabase(Output $Output)
{
$arrConfig = config('database.connections.mysql');
$strDb = $arrConfig['database'];
$strUserName = $arrConfig['username'];
$strPwd = $arrConfig['password'];
$strHost = $arrConfig['hostname'];
$strPort = $arrConfig['hostport'];
$strBackupFile = app()->getRootPath() . 'database/schema/mysql-schema.dump';
$checkTableQuery = "SHOW TABLES LIKE 'migrations'";
$boolMigrateExists = Db::query($checkTableQuery);
if (!empty($boolMigrateExists)) {
$Output->writeln("数据库已存在,跳过初始化导入");
return;
}
$strTempFile = tempnam(sys_get_temp_dir(), 'mysql-schema-') . '.dump';
if (!copy($strBackupFile, $strTempFile)) {
$Output->writeln("创建临时导入文件失败.");
return;
}
$strCmd = "mysql -h{$strHost} -P{$strPort} -u{$strUserName} -p{$strPwd} {$strDb} < {$strTempFile}";
system($strCmd, $intResultCode);
if ($intResultCode === 0) {
$Output->writeln("成功导入初始化文件");
} else {
$Output->writeln("初始化文件导入失败");
}
unlink($strTempFile);
}
}

View File

@@ -0,0 +1,50 @@
<?php
namespace app\task\command;
use think\console\Command;
use think\console\Input;
use think\console\Output;
use think\console\input\Argument;
use think\facade\Db;
class InsertSeoWordsCommand extends Command
{
protected function configure()
{
// 配置命令的名称和参数
$this->setName('insert:seowords')
->setDescription('Insert SEO words from a text file into the seo_keywords table')
->addArgument('file', Argument::REQUIRED, 'The path of the text file');
}
protected function execute(Input $input, Output $output)
{
// 获取传递的文件路径参数
$file = $input->getArgument('file');
// 检查文件是否存在
if (!file_exists($file)) {
$output->writeln("The file does not exist.");
return;
}
// 读取文件内容
$lines = file($file, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES);
// 遍历每一行,插入数据库
foreach ($lines as $line) {
// 假设每一行是以逗号分隔的字段值,例如:关键词标题,状态,HTML路径
// list($title, $status, $htmlPath) = explode(',', $line);
$title = $line;
// 插入数据到 seo_keywords 表
Db::name('seo_keywords')->insert([
'sw_title' => trim($title),
]);
$output->writeln("Inserted: $title");
}
$output->writeln("Data insertion completed.");
}
}

View File

@@ -0,0 +1,89 @@
<?php
declare(strict_types=1);
namespace app\task\command;
use think\console\Command;
use think\console\Input;
use think\console\input\Argument;
use think\console\Output;
use think\console\input\Option;
class Task extends Command
{
private $arrArguments = [
'action' => ['start', 'stop', 'restart', 'status',],
'daemonize' => ['d', "false",],
'forcibly' => ['f', "false",],
];
private $arrReflection = [
\app\task\core\Core::class,
];
protected function configure()
{
$this->setName('php think task')
->addArgument('action', Argument::REQUIRED, "action: (" . implode('|', $this->arrArguments['action']) . ')')
->addOption('daemonize', 'd', Option::VALUE_NONE, "daemonize start service")
->addOption('forcibly', 'f', Option::VALUE_NONE, "ignore user forcibly start service")
->setDescription('swoole task server')
->setHelp('php think task <action> [-d] [-f] run server');
}
protected function checkUser(Input $Input)
{
if (!$Input->getOption('forcibly')) {
$strUser = get_current_user();
if ($strUser == 'root') {
$strError = sprintf("Sorry , Can't start service by [%s] user!,\n but you can add option '-f' forcibly start on you debug ,\n Do not use in production environment !!!!!!!!!",$strUser);
throw new \InvalidArgumentException($strError);
}
if ($strUser != 'www') {
throw new \InvalidArgumentException("Sorry , Can't start service by [{$strUser}] user!, please use [www] user !");
}
}
}
protected function execute(Input $Input, Output $Output)
{
$this->checkArguments(['action'], $Input, $Output);
$strAction = trim($Input->getArgument('action'));
if ($strAction == 'start' || $strAction == 'restart') {
$this->checkUser($Input);
}
$arrArgs = array_merge($Input->getArguments(), $Input->getOptions());
$this->{$strAction}($arrArgs);
}
private function checkArguments($arrArguments, Input $Input)
{
foreach ($arrArguments as $strArguments) {
$strVal = trim($Input->getArgument($strArguments));
if (!in_array($strVal, $this->arrArguments[$strArguments])) {
throw new \InvalidArgumentException("Arguments [{$strVal}] is not defined! Optional parameters [" . implode(' ', $this->arrArguments[$strArguments]) . "] !");
}
}
}
public function __call($strAction, $arrArguments)
{
foreach ($this->arrReflection as $strClass) {
if (method_exists($strClass, $strAction)) {
return call_user_func([new $strClass, $strAction], $arrArguments);
}
}
throw new \InvalidArgumentException("Action [{$strAction}] no method available! ");
}
}

129
code/app/task/core/Core.php Normal file
View File

@@ -0,0 +1,129 @@
<?php
declare(strict_types=1);
namespace app\task\core;
class Core
{
static $HttpService;
static $arrDynamicConfig = [];
public function __init($arrArgvs = [])
{
$this->setConfigDaemonize($arrArgvs[0]['daemonize'] ?? 'false');
}
private function setConfigDaemonize($strMode)
{
self::$arrDynamicConfig['daemonize'] = (bool)$strMode;
}
private function getPid()
{
return file_exists(config('task.service.pid_file')) ? file_get_contents(config('task.service.pid_file')) : NULL;
}
public function start($arrArgvs = [])
{
$this->__init($arrArgvs);
$intPid = (int)$this->getPid();
if ($intPid > 0 && \Swoole\Process::kill($intPid, SIG_DFL)) {
echo " Service is \e[0;32mRunning\e[0m ! " . PHP_EOL;
exit;
}
echo " Execute the \e[0;32m start\e[0m command ...... " . PHP_EOL;
$this->createService();
$this->addProcessList();
$this->addScheduledTasks();
self::$HttpService->getServer()->start();
}
public function stop($arrArgvs = [])
{
$this->__init($arrArgvs);
$intPid = (int)$this->getPid();
if ($intPid <= 0) {
echo " Service is \e[0;31mStop\e[0m ! " . PHP_EOL;
} else {
echo " Execute the \e[0;32m stop\e[0m command ...... " . PHP_EOL;
\Swoole\Process::kill($intPid, SIGTERM);
echo " Service is \e[0;31mStop\e[0m ! " . PHP_EOL;
}
}
public function status($arrArgvs = [])
{
$this->__init($arrArgvs);
$intPid = (int)$this->getPid();
if ($intPid <= 0) {
echo " Service is \e[0;31mStop\e[0m ! " . PHP_EOL;
} else if (\Swoole\Process::kill($intPid, SIG_DFL)) {
echo " Service is \e[0;32mRunning\e[0m ! " . PHP_EOL;
} else {
echo " Service is \e[0;31mStop\e[0m ! " . PHP_EOL;
}
}
public function restart($arrArgvs = [])
{
$this->__init($arrArgvs);
$intPid = (int)$this->getPid();
if ($intPid <= 0) {
echo " \e[0;31m Get Pid Error\e[0m !" . PHP_EOL;
echo " Execute the \e[0;32m start\e[0m command ...... " . PHP_EOL;
$this->start();
} else {
echo " Execute the \e[0;32m restart\e[0m command ...... " . PHP_EOL;
\Swoole\Process::kill($intPid, SIGUSR1);
}
}
private function addScheduledTasks()
{
if (config('task.scheduled_tasks_pool')) {
foreach (config('task.scheduled_tasks_pool') as $arrProcessTask) {
if ($arrProcessTask['status']) {
$Process = new \Swoole\Process(function () use ($arrProcessTask) {
call_user_func([\app\task\core\ScheduledTasks::class, 'hander'], $arrProcessTask);
});
self::$HttpService->getServer()->addProcess($Process);
}
}
}
}
private function addProcessList()
{
if (config('task.process_pool')) {
foreach (config('task.process_pool') as $v) {
for ($i = 0; $i < $v['Num']; $i++) {
$Process = new \Swoole\Process($v['callback']);
self::$HttpService->getServer()->addProcess($Process);
}
}
}
}
private function createService()
{
$arrHttpServiceConfig = array_merge(config('task.service'), self::$arrDynamicConfig);
self::$HttpService = Service::instance($arrHttpServiceConfig);
}
}

View File

@@ -0,0 +1,90 @@
<?php
declare(strict_types=1);
namespace app\task\core;
use app\task\logic\SystemLogic;
class Listen
{
/**
* 监听队列1代理
*
* @param \Swoole\Process $Process
* @return void
*/
static public function innerConsume(\Swoole\Process $Process)
{
return self::innerConsumeCore($Process, 1);
}
/**
* 监听队列2代理
*
* @param \Swoole\Process $Process
* @return void
*/
static public function innerConsume02(\Swoole\Process $Process)
{
return self::innerConsumeCore($Process, 2);
}
/**
* 监听队列
*
* @param \Swoole\Process $Process
* @return void
*/
static private function innerConsumeCore(\Swoole\Process $Process, int $intIndex = 1)
{
Service::$Process = $Process;
SystemLogic::destructConnectSource();
$arrConfig = config('task.queue');
$intLimit = $arrConfig[$intIndex]['exec_num'];
$TaskCore = new TaskCore($intIndex);
return self::excuteTask($TaskCore, $intLimit);
}
/**
* 队列任务消费
*
* @param \Swoole\Process $Process
* @return void
*/
static private function excuteTask(TaskCore $TaskCore, $intLimit)
{
while ($intLimit > 0) {
$intLimit--;
try {
$strData = $TaskCore->get();
if (empty($strData)) {
sleep(1);
continue;
}
$arrData = json_decode($strData, true);
if (!is_array($arrData)) {
echo '监听Redis队列读取到异常无法解析的数据[' . $strData . ']' . PHP_EOL;
continue;
}
$Class = new $arrData['callback'][0];
$Class->{$arrData['callback'][1]}($arrData['data']);
unset($Class);
} catch (\Throwable $t) {
$strData = $strData ?? '';
echo date('Y-m-d H:i:s') . ':出现致命错误需要处理' . PHP_EOL .
' 队列数据:' . $strData . PHP_EOL .
' 文件:' . $t->getFile() . PHP_EOL .
' 行数:' . $t->getLine() . PHP_EOL .
' 错误描述:' . $t->getMessage() . PHP_EOL .
' 堆栈跟踪:' . $t->getTraceAsString() . PHP_EOL;
}
}
}
}

View File

@@ -0,0 +1,31 @@
<?php
declare (strict_types = 1);
namespace app\task\core;
class Rout
{
static public function httpDispense(\Swoole\Http\Request $Request, \Swoole\Http\Response $Response)
{
// print_r($Request->getData());
// print_r($Request->server);
# 简单的HTTP服务
$Response->status(999,'Hei Guys ~');
$Response->header("Content-Type", "text/html; charset=utf-8");
$Response->end("<h1>Hello reptile~. #".rand(1000, 9999)."</h1>");
}
static public function tcpDispense(\Swoole\Server $server, $fd, $reactor_id, $mixedData)
{
# 简单的HTTP服务
print_r($mixedData);
}
static public function test()
{
var_dump(date('Y-m-d H:i:s'));
return;
}
}

View File

@@ -0,0 +1,24 @@
<?php
declare(strict_types=1);
namespace app\task\core;
use app\task\logic\SystemLogic;
class ScheduledTasks
{
static public function hander($arrProcessTask = [])
{
SystemLogic::destructConnectSource();
// while(true){
call_user_func($arrProcessTask['callback'], $arrProcessTask['param']);
sleep($arrProcessTask['execution_interval']);
// }
// \Swoole\Timer::tick($arrProcessTask['execution_interval']*1000, $arrProcessTask['callback'], $arrProcessTask['param']);
// \Swoole\Event::wait();
}
}

View File

@@ -0,0 +1,87 @@
<?php
declare(strict_types=1);
namespace app\task\core;
class Service
{
private $arrConfig;
private static $obj;
private $Server;
static $Process = NULL;
static function instance($arrConfig)
{
if (self::$obj == null) {
self::$obj = new self($arrConfig);
}
return self::$obj;
}
public function __construct($arrConfig)
{
$this->arrConfig = $arrConfig;
$this->createServer();
}
private function serHttpConfig()
{
$this->Server->set($this->filterConfig());
$this->Server->on('request', [\app\task\core\Rout::class, 'httpDispense']);
}
private function filterConfig()
{
$arrConfig = $this->arrConfig;
unset($arrConfig['host']);
unset($arrConfig['port']);
unset($arrConfig['mode']);
unset($arrConfig['sockType']);
unset($arrConfig['server_type']);
return $arrConfig;
}
private function serTcpConfig()
{
$this->Server->set($this->filterConfig());
$this->Server->on('receive', [\app\task\core\Rout::class, 'tcpDispense']);
}
private function createServer()
{
switch ($this->arrConfig['server_type']) {
case 'HTTP':
$this->Server = new \Swoole\Http\Server($this->arrConfig['host'], $this->arrConfig['port']);
$this->serHttpConfig();
break;
default:
$this->Server = new \Swoole\Server($this->arrConfig['host'], $this->arrConfig['port'], $this->arrConfig['mode'], $this->arrConfig['sockType']);
$this->serTcpConfig();
break;
}
$this->Server->on('task', [$this, 'onTask']);
$this->Server->on('finish', [$this, 'onFinish']);
}
public function getServer()
{
return $this->Server;
}
public function onTask($Http, $task_id, $from_id, $arrData)
{
$mixedResult = call_user_func_array($arrData[0], $arrData[1]);
$Http->finish($mixedResult);
}
public function onFinish($Http, $task_id, $mixedData)
{
}
}

View File

@@ -0,0 +1,35 @@
<?php
declare(strict_types=1);
namespace app\task\core;
use think\facade\Cache;
class TaskCore
{
public $strQueueName = '';
public function __construct($intIndex = 1, $strQueueName = '')
{
if ($strQueueName == '') {
$arrConfig = config('task.queue');
$strQueueName = $arrConfig[$intIndex]['name'];
}
$this->strQueueName = $strQueueName;
}
public function get()
{
return Cache::store('redis')->lpop($this->strQueueName);
}
public function set($arrData)
{
$strData = json_encode($arrData);
// 打印调试信息
//\think\facade\Log::info('TaskCore set: '.$strData );
return Cache::store('redis')->rpush($this->strQueueName, $strData);
}
}

View File

@@ -0,0 +1,70 @@
<?php
declare(strict_types=1);
namespace app\task\logic;
use app\admin\model\GuangGaoFenZuModel;
use app\admin\model\GuangGaoModel;
use app\admin\model\GuangGaoPeiZhiModel;
use GuzzleHttp\Client;
/**
* @mixin think\Model
*/
class AdTask
{
/**
* Undocumented variable
*
* @var \GuzzleHttp\Client
*/
protected $Client = NULL;
/**
* Undocumented variable
*
* @var \app\task\core\TaskCore
*/
protected $TaskCore = NULL;
protected $arrCaiJiPeiZhi = [];
public function __construct()
{
$this->Client = new Client();
}
public function pullAdBySourceSite()
{
$intOpen = GuangGaoPeiZhiModel::getValByCode('GUANG_GAO_TONG_BU_KAI_GUAN');
if ($intOpen != 1) {
return false;
}
$strUrl = GuangGaoPeiZhiModel::getValByCode('GUANG_GAO_TONG_BU_DI_ZHI');
$strResult = $this->Client->request('GET', $strUrl);
$strResult = (string)$strResult->getBody();
$arrData = json_decode($strResult, true);
if (!is_array($arrData) || empty($arrData)) {
return false;
}
if ($arrData['code'] != 1000) {
return false;
}
GuangGaoModel::where('gg_id', '>', 0)->delete();
GuangGaoFenZuModel::where('ggfz_id', '>', 0)->delete();
GuangGaoFenZuModel::insertAll($arrData['data']['ggfz']);
GuangGaoModel::insertAll($arrData['data']['gg']);
GuangGaoModel::flushCache();
}
}

View File

@@ -0,0 +1,30 @@
<?php
declare (strict_types = 1);
namespace app\task\logic;
use think\Container;
use think\facade\Db;
/**
* @mixin think\Model
*/
class SystemLogic
{
static public function reloadRedis()
{
Container::getInstance()->delete('cache');
}
static public function reloadDb()
{
Db::close();
Container::getInstance()->delete('think\DbManager');
}
static public function destructConnectSource()
{
self::reloadDb();
self::reloadRedis();
}
}

View File

@@ -0,0 +1,358 @@
<?php
declare(strict_types=1);
namespace app\task\module;
use app\admin\model\PlanTaskModel;
use app\task\collection\ailisi\XiaoShuoCol;
use app\task\collection\biquxs\BiquxsXiaoShuoCol;
use app\task\collection\ccc5\XiaoShuo5cccCol;
use app\task\collection\novel\Zw630XiaoShuoCol;
use app\task\collection\novelone\NovelJiuYuFanXianCol;
use app\task\collection\novelone\NovelWenXueGuanCol;
use app\task\collection\tianlai\TianLaiXiaoShuoCol;
use app\task\collection\seo\GenerateSiteMapCol;
use app\task\collection\seo\GenerateSiteMapNovel;
use app\task\collection\seo\GenerateSiteMapNovelZw;
use app\task\collection\seo\GenerateSiteMapNovelSk;
use app\task\collection\visitlogs\ContentStatsCol;
use app\task\collection\dadi\DaDiDianyingCol;
use app\task\collection\heimuer\HeiMuErYingShiCol;
use app\task\collection\yzzy\YouZhiYingShiCol;
use app\task\core\TaskCore;
use app\task\logic\SystemLogic;
class PlanTask
{
public static function scanPlanTask($arrData)
{
SystemLogic::destructConnectSource();
$PlanTaskAll = PlanTaskModel::where('pt_enable', 1)->where('site_id', config('app.default_app_id'))->select();
foreach ($PlanTaskAll as $PlanTask) {
if ($PlanTask->pt_last_exec != 0 && ($PlanTask->pt_last_exec + $PlanTask->pt_limit) > time()) continue;
$PlanTask->pt_last_exec = time();
$PlanTask->save();
switch ($PlanTask->pt_code) {
case 'PULL_XIAOSHUO':
self::pullXiaoShuo();
break;
case 'PULL_TIANLAIXIAOSHUO':
self::pullTianLaiXiaoShuo();
break;
case 'PULL_BIQUXSXIAOSHUO':
self::pullBiquxsXiaoShuo();
break;
case 'PULL_5CCCXIAOSHUO':
self::pull5cccXiaoShuo();
break;
case 'PULL_DADI_DIANYHHING':
self::pullDadiYingshi();
break;
case 'PULL_HEI_MUER':
self::pullHeiMuEr();
break;
case 'PULL_YOUZHI':
self::pullYouZhiZiYuan();
break;
case 'PULL_TONJI_HUIZON':
self::AddTongJiHuizon();
break;
case 'GENERATE_SITEMAP':
self::generateSiteMap();
break;
case 'NOVEL_GENERATE_SITEMAP':
self::generateNovelSiteMap();
break;
case 'NOVELZW_GENERATE_SITEMAP':
self::generateSiteMapNovelZw();
break;
case 'NOVEL_SK_GENERATE_SITEMAP':
self::generateSiteMapNovelSk();
break;
case 'PULL_630ZWXIAOSHUO':
self::pull630ZwXiaoShuo();
break;
case 'PULL_GUICHUIDENG_XIAOSHUO':
self::pullGuiChuiDengJiuYuFanXianXiaoShuo();
break;
case 'PULL_WENXUEGUAN_XIAOSHUO':
self::pullWenXueGuangXianXiaoShuo();
break;
default:
break;
}
}
}
/**
* 拉取文学馆小说-都市极品医神
*
* @return void
*/
public static function pullWenXueGuangXianXiaoShuo()
{
$arrTask = [
'callback' => [NovelWenXueGuanCol::class, 'pullXiaoShuo'],
'data' => []
];
$TaskCore = new TaskCore();
$TaskCore->set($arrTask);
}
/**
* 拉取鬼吹灯小说-九域凡仙
*
* @return void
*/
public static function pullGuiChuiDengJiuYuFanXianXiaoShuo()
{
$arrTask = [
'callback' => [NovelJiuYuFanXianCol::class, 'pullXiaoShuo'],
'data' => []
];
$TaskCore = new TaskCore();
$TaskCore->set($arrTask);
}
/**
* 拉取小说
*
* @return void
*/
public static function pull630ZwXiaoShuo()
{
$arrTask = [
'callback' => [Zw630XiaoShuoCol::class, 'pullXiaoShuo'],
'data' => []
];
$TaskCore = new TaskCore();
$TaskCore->set($arrTask);
}
/**
* 生成地图-小说
*
* @return void
*/
public static function generateNovelSiteMap()
{
$arrTask = [
'callback' => [GenerateSiteMapNovel::class, 'generate'],
'data' => []
];
$TaskCore = new TaskCore();
$TaskCore->set($arrTask);
}
/**
* 生成地图-小说 novelzw 模板
*
* @return void
*/
public static function generateSiteMapNovelZw()
{
$arrTask = [
'callback' => [GenerateSiteMapNovelZw::class, 'generate'],
'data' => []
];
$TaskCore = new TaskCore();
$TaskCore->set($arrTask);
}
/**
* 生成地图-小说 novelzw 模板
*
* @return void
*/
public static function generateSiteMapNovelSk()
{
$arrTask = [
'callback' => [GenerateSiteMapNovelSk::class, 'generate'],
'data' => []
];
$TaskCore = new TaskCore();
$TaskCore->set($arrTask);
}
/**
* 生成地图-视频
*
* @return void
*/
public static function generateSiteMap()
{
$arrTask = [
'callback' => [GenerateSiteMapCol::class, 'generate'],
'data' => []
];
$TaskCore = new TaskCore();
$TaskCore->set($arrTask);
}
/**
* 拉取优质资源
*
* @return void
*/
public static function AddTongJiHuizon()
{
$arrTask = [
'callback' => [ContentStatsCol::class, 'run'],
'data' => []
];
$TaskCore = new TaskCore();
$TaskCore->set($arrTask);
}
/**
* 拉取优质资源
*
* @return void
*/
public static function AddVisitLogs()
{
$arrTask = [
'callback' => [YouZhiYingShiCol::class, 'add'],
'data' => []
];
$TaskCore = new TaskCore();
$TaskCore->set($arrTask);
}
/**
* 拉取优质资源
*
* @return void
*/
public static function pullYouZhiZiYuan()
{
$arrTask = [
'callback' => [YouZhiYingShiCol::class, 'pullYouZhiYingshi'],
'data' => []
];
$TaskCore = new TaskCore();
$TaskCore->set($arrTask);
}
/**
* 拉取黑木耳
*
* @return void
*/
public static function pullHeiMuEr()
{
$arrTask = [
'callback' => [HeiMuErYingShiCol::class, 'pullDadiYingshi'],
'data' => []
];
$TaskCore = new TaskCore();
$TaskCore->set($arrTask);
}
/**
* 拉取大地影视
*
* @return void
*/
public static function pullDadiYingshi()
{
$arrTask = [
'callback' => [DaDiDianyingCol::class, 'pullDadiYingshi'],
'data' => []
];
$TaskCore = new TaskCore();
$TaskCore->set($arrTask);
}
/**
* 拉取小说
*
* @return void
*/
public static function pull5cccXiaoShuo()
{
$arrTask = [
'callback' => [XiaoShuo5cccCol::class, 'pullXiaoShuo'],
'data' => []
];
$TaskCore = new TaskCore();
$TaskCore->set($arrTask);
}
/**
* 拉取小说
*
* @return void
*/
public static function pullXiaoShuo()
{
$arrTask = [
'callback' => [XiaoShuoCol::class, 'pullXiaoShuo'],
'data' => []
];
$TaskCore = new TaskCore();
$TaskCore->set($arrTask);
}
/**
* 拉取小说
*
* @return void
*/
public static function pullBiquxsXiaoShuo()
{
$arrTask = [
'callback' => [BiquxsXiaoShuoCol::class, 'pullXiaoShuo'],
'data' => []
];
$TaskCore = new TaskCore();
$TaskCore->set($arrTask);
}
/**
* 拉取小说
*
* @return void
*/
public static function pullTianLaiXiaoShuo()
{
$arrTask = [
'callback' => [TianLaiXiaoShuoCol::class, 'pullXiaoShuo'],
'data' => []
];
$TaskCore = new TaskCore();
$TaskCore->set($arrTask);
}
}