This commit is contained in:
make
2025-06-23 23:42:57 +08:00
parent 64b26f7484
commit 60c965f0c0
2 changed files with 254 additions and 26 deletions

View File

@@ -57,6 +57,43 @@ class Site
*/ */
public function initClient() public function initClient()
{ {
$arrUserAgents = [
// 华为浏览器
'Mozilla/5.0 (Linux; Android 11; HUAWEI Mate 40 Pro) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 HuaweiBrowser/12.1.4.301 Mobile Safari/537.36',
'Mozilla/5.0 (Linux; Android 13; HUAWEI P60) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 HuaweiBrowser/13.1.2.301 Mobile Safari/537.36',
// 小米/Redmi - MIUI浏览器
'Mozilla/5.0 (Linux; Android 12; Redmi K50) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 MQQBrowser/11.9 Mobile Safari/537.36 XiaoMi/MiuiBrowser/13.25.0',
'Mozilla/5.0 (Linux; Android 11; Mi 10) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 XiaoMi/MiuiBrowser/12.5.22 Mobile Safari/537.36',
// OPPO 浏览器
'Mozilla/5.0 (Linux; Android 13; OPPO Find X6 Pro) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 OPRU/14.3.8 Mobile Safari/537.36 HeyTapBrowser/14.3.8.0',
// vivo 浏览器
'Mozilla/5.0 (Linux; Android 12; VIVO X90) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 V1_AND_SQ_9.9.0_302 Mobile Safari/537.36 VivoBrowser/9.9.0.0',
// QQ 浏览器
'Mozilla/5.0 (Linux; Android 11; M2007J17C) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 MQQBrowser/11.7 Mobile Safari/537.36',
'Mozilla/5.0 (Linux; Android 10; OPPO Reno 4 Pro) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 MQQBrowser/11.3 Mobile Safari/537.36',
// UC 浏览器(优视)
'Mozilla/5.0 (Linux; U; Android 12; zh-CN; V2055A) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 UCBrowser/13.5.5.1303 Mobile Safari/537.36',
'Mozilla/5.0 (Linux; Android 11; 2201123C) AppleWebKit/537.36 (KHTML, like Gecko) UCBrowser/13.4.0.1306 Mobile Safari/537.36',
// 360 浏览器
'Mozilla/5.0 (Linux; Android 10; M2006J10C) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 360Browser/12.1.1.106 Mobile Safari/537.36',
// 搜狗浏览器
'Mozilla/5.0 (Linux; Android 11; V2073A) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 SogouMobileBrowser/6.0.0 Mobile Safari/537.36',
// 百度 App 内置浏览器
'Mozilla/5.0 (Linux; Android 12; V2123A) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 baiduboxapp/13.19.0.10 (Baidu; P1 12) Mobile Safari/537.36',
// 夸克浏览器Quark
'Mozilla/5.0 (Linux; Android 11; 2201122C) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Quark/6.5.2.234 Mobile Safari/537.36',
];
$arrConfig = [ $arrConfig = [
'http_version' => '1.1', // 强制用 HTTP/1.1,已有,保持 'http_version' => '1.1', // 强制用 HTTP/1.1,已有,保持
'base_uri' => $this->strDomain, // 基础域名,已有,保持 'base_uri' => $this->strDomain, // 基础域名,已有,保持
@@ -67,10 +104,11 @@ class Site
CURLOPT_TCP_KEEPALIVE => 1, // 启用 TCP Keep-Alive已有保持 CURLOPT_TCP_KEEPALIVE => 1, // 启用 TCP Keep-Alive已有保持
CURLOPT_TCP_KEEPIDLE => 120, // Keep-Alive 空闲时间 CURLOPT_TCP_KEEPIDLE => 120, // Keep-Alive 空闲时间
CURLOPT_TCP_KEEPINTVL => 60, // Keep-Alive 探测间隔 CURLOPT_TCP_KEEPINTVL => 60, // Keep-Alive 探测间隔
CURLOPT_SSLVERSION => CURL_SSLVERSION_TLSv1_2, // 强制使用 TLS 1.2
// CURLOPT_SSL_VERIFYPEER => true, // 验证 SSL已有保持 // CURLOPT_SSL_VERIFYPEER => true, // 验证 SSL已有保持
], ],
'headers' => [ 'headers' => [
'User-Agent' => 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36', 'User-Agent' => $arrUserAgents[array_rand($arrUserAgents)],
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7', 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'Accept-Language' => 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6', 'Accept-Language' => 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
'Referer' => $this->strDomain, 'Referer' => $this->strDomain,
@@ -133,7 +171,6 @@ class Site
$arrConfig['curl'][CURLOPT_PROXYAUTH] = CURLAUTH_BASIC; $arrConfig['curl'][CURLOPT_PROXYAUTH] = CURLAUTH_BASIC;
// $arrConfig['curl'][CURLOPT_PROXYUSERPWD] = "$strKey:$strSecret"; // $arrConfig['curl'][CURLOPT_PROXYUSERPWD] = "$strKey:$strSecret";
break; break;
} }
} }
@@ -257,7 +294,6 @@ class Site
$arrNovelPage[] = $NovelPage; $arrNovelPage[] = $NovelPage;
$this->redisDailyInc(config('task.app_name') . '_success_getNovelPageInfo:' . date('Y-m-d')); $this->redisDailyInc(config('task.app_name') . '_success_getNovelPageInfo:' . date('Y-m-d'));
} elseif ($arrResult['state'] == 'rejected') { } elseif ($arrResult['state'] == 'rejected') {
// 重新推送任务 // 重新推送任务
@@ -279,7 +315,6 @@ class Site
} }
} }
return $arrNovelPage; return $arrNovelPage;
} }
/** /**
@@ -376,6 +411,163 @@ class Site
return $arrChapterPage; return $arrChapterPage;
} }
// 调试使用
// public function getChapterPageList(array $arrFilter, int $intNid = 0): array
// {
// $QueueManage = QueueManage::getInstance();
// /** @var ChapterPage[] */
// $arrChapterPage = [];
// $arrPromises = [];
// $arrTimings = []; // 记录每个请求的耗时
// $arrRequestResults = []; // 记录每个请求的返回结果
// foreach ($arrFilter as $intKey => $arrItem) {
// $startTime = microtime(true); // 记录开始时间(微秒)
// $arrTimings[$intKey] = ['start_time' => $startTime];
// $arrPromises[$intKey] = $this->getClient()->getAsync($arrItem['uri'])->then(
// // 成功回调
// function (\Psr\Http\Message\ResponseInterface $response) use ($intKey, &$arrTimings, &$arrRequestResults, $arrItem) {
// $endTime = microtime(true); // 记录结束时间
// $arrTimings[$intKey]['end_time'] = $endTime;
// $arrTimings[$intKey]['duration'] = $endTime - $arrTimings[$intKey]['start_time']; // 计算耗时
// // 记录响应结果
// $arrRequestResults[$intKey] = [
// 'status' => 'fulfilled',
// 'status_code' => $response->getStatusCode(),
// 'body' => substr((string) $response->getBody(), 0, 1000), // 限制响应体长度
// 'uri' => $arrItem['uri'], // 添加 URI 记录
// ];
// return $response; // 保持 Promise 链
// },
// // 失败回调
// function (\Throwable $exception) use ($intKey, &$arrTimings, &$arrRequestResults, $arrItem) {
// $endTime = microtime(true); // 记录结束时间
// $arrTimings[$intKey]['end_time'] = $endTime;
// $arrTimings[$intKey]['duration'] = $endTime - $arrTimings[$intKey]['start_time']; // 计算耗时
// // 初始化错误结果
// $statusCode = null;
// $body = null;
// // 仅对非 ConnectException 的 RequestException 检查响应
// if ($exception instanceof \GuzzleHttp\Exception\RequestException && !($exception instanceof \GuzzleHttp\Exception\ConnectException)) {
// if (method_exists($exception, 'hasResponse') && $exception->hasResponse()) {
// $statusCode = $exception->getResponse()->getStatusCode();
// $body = substr((string) $exception->getResponse()->getBody(), 0, 1000);
// }
// }
// // 记录错误结果
// $arrRequestResults[$intKey] = [
// 'status' => 'rejected',
// 'error' => $exception->getMessage(),
// 'status_code' => $statusCode,
// 'body' => $body,
// 'uri' => $arrItem['uri'], // 添加 URI 记录
// ];
// throw $exception; // 保持 Promise 链
// }
// );
// }
// // 等待所有请求完成
// $arrResults = Promise\Utils::settle($arrPromises)->wait();
// // 处理结果并记录日志
// foreach ($arrResults as $intKey => $arrResult) {
// // 记录请求日志
// $logData = [
// 'uri' => $arrFilter[$intKey]['uri'] ?? 'unknown',
// 'duration' => $arrTimings[$intKey]['duration'] ?? 0,
// 'status' => $arrRequestResults[$intKey]['status'] ?? 'unknown',
// 'status_code' => $arrRequestResults[$intKey]['status_code'] ?? null,
// 'body' => $arrRequestResults[$intKey]['body'] ?? null,
// ];
// if (isset($arrRequestResults[$intKey]['error'])) {
// $logData['error'] = $arrRequestResults[$intKey]['error'];
// }
// if ($arrResult['state'] === 'fulfilled') {
// $Response = $arrResult['value'];
// $strContent = $Response->getBody()->getContents();
// // 跳过空内容
// if (empty($strContent) || $strContent == '索引文件不存在!') {
// continue;
// }
// // 如果不匹配有这个 说明没有请求到小说章节 或者 章节为空
// $strPattern = "/lastread\.set\((.*?)\);/";
// if (!preg_match($strPattern, $strContent, $arrMatches)) {
// if (config('log.task_debug_log_open_status')) {
// \think\facade\Log::warning("No match for pattern in response, URI: {$arrFilter[$intKey]['uri']}, Content: " . mb_substr($strContent, 0, 300));
// }
// if ($intNid > 0) {
// if (
// str_contains($strContent, '超过并发限制') ||
// str_contains($strContent, 'Timeout') ||
// str_contains($strContent, '403 Forbidden')
// ) {
// // 重新推送任务
// $arrTask = [
// 'callback' => [Zw74Scheduler::class, 'getChapterInfo'],
// 'data' => [
// 'n_id' => $intNid,
// 'filter' => $arrFilter,
// ],
// ];
// $QueueManage->set($arrTask);
// }
// }
// continue;
// }
// $ChapterPage = $this->getChapterPage($this, $arrFilter[$intKey]['uri']);
// $ChapterPage->setContent($strContent);
// $ChapterPage->setData($arrFilter[$intKey]);
// $arrChapterPage[] = $ChapterPage;
// // 成功指标
// $this->redisDailyInc(config('task.app_name') . '_success_getChapterPageList:' . date('Y-m-d'));
// } elseif ($arrResult['state'] === 'rejected') {
// \think\facade\Log::debug("Request log for getChapterPageList: " . json_encode($logData));
// if ($intNid > 0) {
// // 重新推送任务
// $arrTask = [
// 'callback' => [Zw74Scheduler::class, 'getChapterInfo'],
// 'data' => [
// 'n_id' => $intNid,
// 'filter' => $arrFilter,
// ],
// ];
// $QueueManage->set($arrTask);
// }
// $this->redisDailyInc(config('task.app_name') . '_errer_getChapterPageList:' . date('Y-m-d'));
// if (config('log.task_debug_log_open_status')) {
// $reason = $arrResult['reason'] instanceof \Exception
// ? $arrResult['reason']->getMessage()
// : 'Unknown error';
// $strUri = $arrFilter[$intKey]['uri'] ?? '';
// \think\facade\Log::error("Request failed for getNovelChapterInfo, URI: {$strUri}, Reason: {$reason}");
// }
// }
// }
// return $arrChapterPage;
// }
/** /**
@@ -414,7 +606,6 @@ class Site
$incKey = config('task.app_name') . '_success_getLatestListPageList:' . date('Y-m-d'); $incKey = config('task.app_name') . '_success_getLatestListPageList:' . date('Y-m-d');
$this->redisDailyInc($incKey); $this->redisDailyInc($incKey);
} elseif ($arrResult['state'] == 'rejected') { } elseif ($arrResult['state'] == 'rejected') {
// 重新推送任务 // 重新推送任务
@@ -480,7 +671,6 @@ class Site
$incKey = config('task.app_name') . '_success_downRemoteImgToLocal:' . date('Y-m-d'); $incKey = config('task.app_name') . '_success_downRemoteImgToLocal:' . date('Y-m-d');
$this->redisDailyInc($incKey); $this->redisDailyInc($incKey);
} elseif ($arrResult['state'] == 'rejected') { } elseif ($arrResult['state'] == 'rejected') {
// 重新推送任务 // 重新推送任务
@@ -552,7 +742,8 @@ class Site
* @param integer $expire * @param integer $expire
* @return void * @return void
*/ */
public function redisDailyInc(string $key, int $expire = 604800) { public function redisDailyInc(string $key, int $expire = 604800)
{
$cache = Cache::store('redis'); $cache = Cache::store('redis');
if (!$cache->has($key)) { if (!$cache->has($key)) {
$cache->set($key, 0, $expire); $cache->set($key, 0, $expire);

View File

@@ -39,20 +39,57 @@ class Site extends ZwSite
*/ */
public function initClient() public function initClient()
{ {
$arrUserAgents = [
// 华为浏览器
'Mozilla/5.0 (Linux; Android 11; HUAWEI Mate 40 Pro) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 HuaweiBrowser/12.1.4.301 Mobile Safari/537.36',
'Mozilla/5.0 (Linux; Android 13; HUAWEI P60) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 HuaweiBrowser/13.1.2.301 Mobile Safari/537.36',
// 小米/Redmi - MIUI浏览器
'Mozilla/5.0 (Linux; Android 12; Redmi K50) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 MQQBrowser/11.9 Mobile Safari/537.36 XiaoMi/MiuiBrowser/13.25.0',
'Mozilla/5.0 (Linux; Android 11; Mi 10) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 XiaoMi/MiuiBrowser/12.5.22 Mobile Safari/537.36',
// OPPO 浏览器
'Mozilla/5.0 (Linux; Android 13; OPPO Find X6 Pro) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 OPRU/14.3.8 Mobile Safari/537.36 HeyTapBrowser/14.3.8.0',
// vivo 浏览器
'Mozilla/5.0 (Linux; Android 12; VIVO X90) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 V1_AND_SQ_9.9.0_302 Mobile Safari/537.36 VivoBrowser/9.9.0.0',
// QQ 浏览器
'Mozilla/5.0 (Linux; Android 11; M2007J17C) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 MQQBrowser/11.7 Mobile Safari/537.36',
'Mozilla/5.0 (Linux; Android 10; OPPO Reno 4 Pro) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 MQQBrowser/11.3 Mobile Safari/537.36',
// UC 浏览器(优视)
'Mozilla/5.0 (Linux; U; Android 12; zh-CN; V2055A) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 UCBrowser/13.5.5.1303 Mobile Safari/537.36',
'Mozilla/5.0 (Linux; Android 11; 2201123C) AppleWebKit/537.36 (KHTML, like Gecko) UCBrowser/13.4.0.1306 Mobile Safari/537.36',
// 360 浏览器
'Mozilla/5.0 (Linux; Android 10; M2006J10C) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 360Browser/12.1.1.106 Mobile Safari/537.36',
// 搜狗浏览器
'Mozilla/5.0 (Linux; Android 11; V2073A) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 SogouMobileBrowser/6.0.0 Mobile Safari/537.36',
// 百度 App 内置浏览器
'Mozilla/5.0 (Linux; Android 12; V2123A) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 baiduboxapp/13.19.0.10 (Baidu; P1 12) Mobile Safari/537.36',
// 夸克浏览器Quark
'Mozilla/5.0 (Linux; Android 11; 2201122C) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Quark/6.5.2.234 Mobile Safari/537.36',
];
$arrConfig = [ $arrConfig = [
'http_version' => '1.1', // 强制用 HTTP/1.1,已有,保持 'http_version' => '1.1', // 强制用 HTTP/1.1,已有,保持
'base_uri' => $this->strDomain, // 基础域名,已有,保持 'base_uri' => $this->strDomain, // 基础域名,已有,保持
'http_errors' => false, // 不抛 HTTP 错误,已有,保持 'http_errors' => false, // 不抛 HTTP 错误,已有,保持
'timeout' => 60, // 从 160 秒改为 60 秒 'timeout' => 60, // 从 160 秒改为 60 秒
'connect_timeout' => 10, // 从 30 秒改为 10 秒 'connect_timeout' => 30, // 从 30 秒改为 10 秒
'curl' => [ 'curl' => [
CURLOPT_TCP_KEEPALIVE => 1, // 启用 TCP Keep-Alive已有保持 CURLOPT_TCP_KEEPALIVE => 1, // 启用 TCP Keep-Alive已有保持
CURLOPT_TCP_KEEPIDLE => 120, // Keep-Alive 空闲时间 CURLOPT_TCP_KEEPIDLE => 120, // Keep-Alive 空闲时间
CURLOPT_TCP_KEEPINTVL => 60, // Keep-Alive 探测间隔 CURLOPT_TCP_KEEPINTVL => 60, // Keep-Alive 探测间隔
CURLOPT_SSLVERSION => CURL_SSLVERSION_TLSv1_2, // 强制使用 TLS 1.2
// CURLOPT_SSL_VERIFYPEER => true, // 验证 SSL已有保持 // CURLOPT_SSL_VERIFYPEER => true, // 验证 SSL已有保持
], ],
'headers' => [ 'headers' => [
'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36 Edg/135.0.0.0', 'User-Agent' => $arrUserAgents[array_rand($arrUserAgents)],
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7', 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'Accept-Language' => 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6', 'Accept-Language' => 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
'Referer' => $this->strDomain, 'Referer' => $this->strDomain,