admin/SEONexus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

debug

Browse Source
This commit is contained in:
make
2025-06-23 23:42:57 +08:00
parent 64b26f7484
commit 60c965f0c0
2 changed files with 254 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

239
code/app/task/crawler/zw/page/Site.php
View File

@@ -57,6 +57,43 @@ class Site
*/
public function initClient()
{
$arrUserAgents = [
// 华为浏览器
'Mozilla/5.0 (Linux; Android 11; HUAWEI Mate 40 Pro) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 HuaweiBrowser/12.1.4.301 Mobile Safari/537.36',
'Mozilla/5.0 (Linux; Android 13; HUAWEI P60) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 HuaweiBrowser/13.1.2.301 Mobile Safari/537.36',
// 小米/Redmi - MIUI浏览器
'Mozilla/5.0 (Linux; Android 12; Redmi K50) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 MQQBrowser/11.9 Mobile Safari/537.36 XiaoMi/MiuiBrowser/13.25.0',
'Mozilla/5.0 (Linux; Android 11; Mi 10) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 XiaoMi/MiuiBrowser/12.5.22 Mobile Safari/537.36',
// OPPO 浏览器
'Mozilla/5.0 (Linux; Android 13; OPPO Find X6 Pro) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 OPRU/14.3.8 Mobile Safari/537.36 HeyTapBrowser/14.3.8.0',
// vivo 浏览器
'Mozilla/5.0 (Linux; Android 12; VIVO X90) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 V1_AND_SQ_9.9.0_302 Mobile Safari/537.36 VivoBrowser/9.9.0.0',
// QQ 浏览器
'Mozilla/5.0 (Linux; Android 11; M2007J17C) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 MQQBrowser/11.7 Mobile Safari/537.36',
'Mozilla/5.0 (Linux; Android 10; OPPO Reno 4 Pro) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 MQQBrowser/11.3 Mobile Safari/537.36',
// UC 浏览器（优视）
'Mozilla/5.0 (Linux; U; Android 12; zh-CN; V2055A) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 UCBrowser/13.5.5.1303 Mobile Safari/537.36',
'Mozilla/5.0 (Linux; Android 11; 2201123C) AppleWebKit/537.36 (KHTML, like Gecko) UCBrowser/13.4.0.1306 Mobile Safari/537.36',
// 360 浏览器
'Mozilla/5.0 (Linux; Android 10; M2006J10C) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 360Browser/12.1.1.106 Mobile Safari/537.36',
// 搜狗浏览器
'Mozilla/5.0 (Linux; Android 11; V2073A) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 SogouMobileBrowser/6.0.0 Mobile Safari/537.36',
// 百度 App 内置浏览器
'Mozilla/5.0 (Linux; Android 12; V2123A) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 baiduboxapp/13.19.0.10 (Baidu; P1 12) Mobile Safari/537.36',
// 夸克浏览器（Quark）
'Mozilla/5.0 (Linux; Android 11; 2201122C) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Quark/6.5.2.234 Mobile Safari/537.36',
];
$arrConfig = [
'http_version' => '1.1', // 强制用 HTTP/1.1，已有，保持
'base_uri' => $this->strDomain, // 基础域名，已有，保持
@@ -67,10 +104,11 @@ class Site
CURLOPT_TCP_KEEPALIVE => 1, // 启用 TCP Keep-Alive，已有，保持
CURLOPT_TCP_KEEPIDLE => 120, // Keep-Alive 空闲时间
CURLOPT_TCP_KEEPINTVL => 60, // Keep-Alive 探测间隔
CURLOPT_SSLVERSION => CURL_SSLVERSION_TLSv1_2, // 强制使用 TLS 1.2
// CURLOPT_SSL_VERIFYPEER => true, // 验证 SSL，已有，保持
],
'headers' => [
'User-Agent' => 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36',
'User-Agent' => $arrUserAgents[array_rand($arrUserAgents)],
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'Accept-Language' => 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
'Referer' => $this->strDomain,
@@ -119,7 +157,7 @@ class Site
case 'JULIANG':
// 隧道代理信息
// $strProxy = 'http://username:password@XXX.juliangip.com:23456';
$strProxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost;
$strProxy = "http://" . $strKey . ":" . $strSecret . "@" . $strProxyHost;
$arrConfig['proxy'] = $strProxy;
break;
case 'LUOTUO':
@@ -133,7 +171,6 @@ class Site
$arrConfig['curl'][CURLOPT_PROXYAUTH] = CURLAUTH_BASIC;
// $arrConfig['curl'][CURLOPT_PROXYUSERPWD] = "$strKey:$strSecret";
break;
}
}
@@ -233,8 +270,8 @@ class Site
// 判断是否包含 $strContent <body class="novel_info">，不包含跳过 Gateway Timeout
if (strpos($strContent, '<body class="novel_info">') === false) {
// 不包含该字符串，跳过
var_dump(mb_substr($strContent, 0, 300));
var_dump(mb_substr($strContent, 0, 300));
if (
str_contains($strContent, '超过并发限制') ||
str_contains($strContent, 'Timeout') ||
@@ -249,15 +286,14 @@ class Site
}
continue;
}
$NovelPage = $this->getNovelPage($this, $arrFilter[$intKey]['uri']);
$NovelPage->setContent($strContent);
$NovelPage->setData($arrFilter[$intKey]);
$arrNovelPage[] = $NovelPage;
$this->redisDailyInc(config('task.app_name').'_success_getNovelPageInfo:' . date('Y-m-d'));
$this->redisDailyInc(config('task.app_name') . '_success_getNovelPageInfo:' . date('Y-m-d'));
} elseif ($arrResult['state'] == 'rejected') {
// 重新推送任务
@@ -267,19 +303,18 @@ class Site
];
$QueueManage->set($arrTask);
$incKey = config('task.app_name').'_errer_getNovelPageInfo:' . date('Y-m-d');
$incKey = config('task.app_name') . '_errer_getNovelPageInfo:' . date('Y-m-d');
$this->redisDailyInc($incKey);
if(config('log.task_debug_log_open_status')){
if (config('log.task_debug_log_open_status')) {
// 记录失败原因
$reason = $arrResult['reason'] instanceof \Exception ? $arrResult['reason']->getMessage() : 'Unknown error';
$strUri = $arrFilter[$intKey]['uri'] ?? '';
\think\facade\Log::error("Request failed for getNovelPageInfo , URI: {$strUri }, Reason: {$reason}");
\think\facade\Log::error("Request failed for getNovelPageInfo , URI: {$strUri}, Reason: {$reason}");
}
}
}
return $arrNovelPage;
}
/**
@@ -376,6 +411,163 @@ class Site
return $arrChapterPage;
}
// 调试使用
// public function getChapterPageList(array $arrFilter, int $intNid = 0): array
// {
// $QueueManage = QueueManage::getInstance();
// /** @var ChapterPage[] */
// $arrChapterPage = [];
// $arrPromises = [];
// $arrTimings = []; // 记录每个请求的耗时
// $arrRequestResults = []; // 记录每个请求的返回结果
// foreach ($arrFilter as $intKey => $arrItem) {
// $startTime = microtime(true); // 记录开始时间（微秒）
// $arrTimings[$intKey] = ['start_time' => $startTime];
// $arrPromises[$intKey] = $this->getClient()->getAsync($arrItem['uri'])->then(
// // 成功回调
// function (\Psr\Http\Message\ResponseInterface $response) use ($intKey, &$arrTimings, &$arrRequestResults, $arrItem) {
// $endTime = microtime(true); // 记录结束时间
// $arrTimings[$intKey]['end_time'] = $endTime;
// $arrTimings[$intKey]['duration'] = $endTime - $arrTimings[$intKey]['start_time']; // 计算耗时
// // 记录响应结果
// $arrRequestResults[$intKey] = [
// 'status' => 'fulfilled',
// 'status_code' => $response->getStatusCode(),
// 'body' => substr((string) $response->getBody(), 0, 1000), // 限制响应体长度
// 'uri' => $arrItem['uri'], // 添加 URI 记录
// ];
// return $response; // 保持 Promise 链
// },
// // 失败回调
// function (\Throwable $exception) use ($intKey, &$arrTimings, &$arrRequestResults, $arrItem) {
// $endTime = microtime(true); // 记录结束时间
// $arrTimings[$intKey]['end_time'] = $endTime;
// $arrTimings[$intKey]['duration'] = $endTime - $arrTimings[$intKey]['start_time']; // 计算耗时
// // 初始化错误结果
// $statusCode = null;
// $body = null;
// // 仅对非 ConnectException 的 RequestException 检查响应
// if ($exception instanceof \GuzzleHttp\Exception\RequestException && !($exception instanceof \GuzzleHttp\Exception\ConnectException)) {
// if (method_exists($exception, 'hasResponse') && $exception->hasResponse()) {
// $statusCode = $exception->getResponse()->getStatusCode();
// $body = substr((string) $exception->getResponse()->getBody(), 0, 1000);
// }
// }
// // 记录错误结果
// $arrRequestResults[$intKey] = [
// 'status' => 'rejected',
// 'error' => $exception->getMessage(),
// 'status_code' => $statusCode,
// 'body' => $body,
// 'uri' => $arrItem['uri'], // 添加 URI 记录
// ];
// throw $exception; // 保持 Promise 链
// }
// );
// }
// // 等待所有请求完成
// $arrResults = Promise\Utils::settle($arrPromises)->wait();
// // 处理结果并记录日志
// foreach ($arrResults as $intKey => $arrResult) {
// // 记录请求日志
// $logData = [
// 'uri' => $arrFilter[$intKey]['uri'] ?? 'unknown',
// 'duration' => $arrTimings[$intKey]['duration'] ?? 0,
// 'status' => $arrRequestResults[$intKey]['status'] ?? 'unknown',
// 'status_code' => $arrRequestResults[$intKey]['status_code'] ?? null,
// 'body' => $arrRequestResults[$intKey]['body'] ?? null,
// ];
// if (isset($arrRequestResults[$intKey]['error'])) {
// $logData['error'] = $arrRequestResults[$intKey]['error'];
// }
// if ($arrResult['state'] === 'fulfilled') {
// $Response = $arrResult['value'];
// $strContent = $Response->getBody()->getContents();
// // 跳过空内容
// if (empty($strContent) || $strContent == '索引文件不存在！') {
// continue;
// }
// // 如果不匹配有这个 说明没有请求到小说章节 或者 章节为空
// $strPattern = "/lastread\.set\((.*?)\);/";
// if (!preg_match($strPattern, $strContent, $arrMatches)) {
// if (config('log.task_debug_log_open_status')) {
// \think\facade\Log::warning("No match for pattern in response, URI: {$arrFilter[$intKey]['uri']}, Content: " . mb_substr($strContent, 0, 300));
// }
// if ($intNid > 0) {
// if (
// str_contains($strContent, '超过并发限制') ||
// str_contains($strContent, 'Timeout') ||
// str_contains($strContent, '403 Forbidden')
// ) {
// // 重新推送任务
// $arrTask = [
// 'callback' => [Zw74Scheduler::class, 'getChapterInfo'],
// 'data' => [
// 'n_id' => $intNid,
// 'filter' => $arrFilter,
// ],
// ];
// $QueueManage->set($arrTask);
// }
// }
// continue;
// }
// $ChapterPage = $this->getChapterPage($this, $arrFilter[$intKey]['uri']);
// $ChapterPage->setContent($strContent);
// $ChapterPage->setData($arrFilter[$intKey]);
// $arrChapterPage[] = $ChapterPage;
// // 成功指标
// $this->redisDailyInc(config('task.app_name') . '_success_getChapterPageList:' . date('Y-m-d'));
// } elseif ($arrResult['state'] === 'rejected') {
// \think\facade\Log::debug("Request log for getChapterPageList: " . json_encode($logData));
// if ($intNid > 0) {
// // 重新推送任务
// $arrTask = [
// 'callback' => [Zw74Scheduler::class, 'getChapterInfo'],
// 'data' => [
// 'n_id' => $intNid,
// 'filter' => $arrFilter,
// ],
// ];
// $QueueManage->set($arrTask);
// }
// $this->redisDailyInc(config('task.app_name') . '_errer_getChapterPageList:' . date('Y-m-d'));
// if (config('log.task_debug_log_open_status')) {
// $reason = $arrResult['reason'] instanceof \Exception
// ? $arrResult['reason']->getMessage()
// : 'Unknown error';
// $strUri = $arrFilter[$intKey]['uri'] ?? '';
// \think\facade\Log::error("Request failed for getNovelChapterInfo, URI: {$strUri}, Reason: {$reason}");
// }
// }
// }
// return $arrChapterPage;
// }
/**
@@ -394,10 +586,10 @@ class Site
$arrPromises = [];
// $arrSuccessfulResults = [];
foreach ($arrFilter as $intKey => $strUri) {
$arrPromises[$intKey] = $this->getClient()->getAsync($strUri);
}
$arrResults = Promise\Utils::settle($arrPromises)->wait();
// $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
@@ -412,9 +604,8 @@ class Site
$LatestListPage->setContent($strContent);
$arrLatestListPage[] = $LatestListPage;
$incKey = config('task.app_name').'_success_getLatestListPageList:' . date('Y-m-d');
$incKey = config('task.app_name') . '_success_getLatestListPageList:' . date('Y-m-d');
$this->redisDailyInc($incKey);
} elseif ($arrResult['state'] == 'rejected') {
// 重新推送任务
@@ -425,14 +616,14 @@ class Site
$QueueManage = QueueManage::getInstance();
$QueueManage->set($arrTask);
$incKey = config('task.app_name').'_errer_getLatestListPageList:' . date('Y-m-d');
$incKey = config('task.app_name') . '_errer_getLatestListPageList:' . date('Y-m-d');
$this->redisDailyInc($incKey);
if(config('log.task_debug_log_open_status')){
if (config('log.task_debug_log_open_status')) {
// 记录失败原因
$reason = $arrResult['reason'] instanceof \Exception ? $arrResult['reason']->getMessage() : 'Unknown error';
$strUri = $arrFilter[$intKey] ?? '';
\think\facade\Log::error("Request failed for getLatestListPageList , URI: {$strUri }, Reason: {$reason}");
\think\facade\Log::error("Request failed for getLatestListPageList , URI: {$strUri}, Reason: {$reason}");
}
}
}
@@ -478,9 +669,8 @@ class Site
$Response = $arrResult['value'];
$StorageCore->writeStream($arrFilter[$intKey]['local_path'], $Response->getBody()->detach());
$incKey = config('task.app_name').'_success_downRemoteImgToLocal:' . date('Y-m-d');
$incKey = config('task.app_name') . '_success_downRemoteImgToLocal:' . date('Y-m-d');
$this->redisDailyInc($incKey);
} elseif ($arrResult['state'] == 'rejected') {
// 重新推送任务
@@ -491,10 +681,10 @@ class Site
$QueueManage = QueueManage::getInstance();
$QueueManage->set($arrTask);
$incKey = config('task.app_name').'_errer_downRemoteImgToLocal:' . date('Y-m-d');
$incKey = config('task.app_name') . '_errer_downRemoteImgToLocal:' . date('Y-m-d');
$this->redisDailyInc($incKey);
if(config('log.task_debug_log_open_status')){
if (config('log.task_debug_log_open_status')) {
// 记录失败原因
$reason = $arrResult['reason'] instanceof \Exception ? $arrResult['reason']->getMessage() : 'Unknown error';
\think\facade\Log::error("Request failed for downRemoteImgToLocal , Reason: {$reason}");
@@ -552,7 +742,8 @@ class Site
* @param integer $expire
* @return void
*/
public function redisDailyInc(string $key, int $expire = 604800) {
public function redisDailyInc(string $key, int $expire = 604800)
{
$cache = Cache::store('redis');
if (!$cache->has($key)) {
$cache->set($key, 0, $expire);

41
code/app/task/crawler/zw74/page/Site.php
View File

@@ -39,20 +39,57 @@ class Site extends ZwSite
*/
public function initClient()
{
$arrUserAgents = [
// 华为浏览器
'Mozilla/5.0 (Linux; Android 11; HUAWEI Mate 40 Pro) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 HuaweiBrowser/12.1.4.301 Mobile Safari/537.36',
'Mozilla/5.0 (Linux; Android 13; HUAWEI P60) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 HuaweiBrowser/13.1.2.301 Mobile Safari/537.36',
// 小米/Redmi - MIUI浏览器
'Mozilla/5.0 (Linux; Android 12; Redmi K50) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 MQQBrowser/11.9 Mobile Safari/537.36 XiaoMi/MiuiBrowser/13.25.0',
'Mozilla/5.0 (Linux; Android 11; Mi 10) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 XiaoMi/MiuiBrowser/12.5.22 Mobile Safari/537.36',
// OPPO 浏览器
'Mozilla/5.0 (Linux; Android 13; OPPO Find X6 Pro) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 OPRU/14.3.8 Mobile Safari/537.36 HeyTapBrowser/14.3.8.0',
// vivo 浏览器
'Mozilla/5.0 (Linux; Android 12; VIVO X90) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 V1_AND_SQ_9.9.0_302 Mobile Safari/537.36 VivoBrowser/9.9.0.0',
// QQ 浏览器
'Mozilla/5.0 (Linux; Android 11; M2007J17C) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 MQQBrowser/11.7 Mobile Safari/537.36',
'Mozilla/5.0 (Linux; Android 10; OPPO Reno 4 Pro) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 MQQBrowser/11.3 Mobile Safari/537.36',
// UC 浏览器（优视）
'Mozilla/5.0 (Linux; U; Android 12; zh-CN; V2055A) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 UCBrowser/13.5.5.1303 Mobile Safari/537.36',
'Mozilla/5.0 (Linux; Android 11; 2201123C) AppleWebKit/537.36 (KHTML, like Gecko) UCBrowser/13.4.0.1306 Mobile Safari/537.36',
// 360 浏览器
'Mozilla/5.0 (Linux; Android 10; M2006J10C) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 360Browser/12.1.1.106 Mobile Safari/537.36',
// 搜狗浏览器
'Mozilla/5.0 (Linux; Android 11; V2073A) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 SogouMobileBrowser/6.0.0 Mobile Safari/537.36',
// 百度 App 内置浏览器
'Mozilla/5.0 (Linux; Android 12; V2123A) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 baiduboxapp/13.19.0.10 (Baidu; P1 12) Mobile Safari/537.36',
// 夸克浏览器（Quark）
'Mozilla/5.0 (Linux; Android 11; 2201122C) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Quark/6.5.2.234 Mobile Safari/537.36',
];
$arrConfig = [
'http_version' => '1.1', // 强制用 HTTP/1.1，已有，保持
'base_uri' => $this->strDomain, // 基础域名，已有，保持
'http_errors' => false, // 不抛 HTTP 错误，已有，保持
'timeout' => 60, // 从 160 秒改为 60 秒
'connect_timeout' => 10, // 从 30 秒改为 10 秒
'connect_timeout' => 30, // 从 30 秒改为 10 秒
'curl' => [
CURLOPT_TCP_KEEPALIVE => 1, // 启用 TCP Keep-Alive，已有，保持
CURLOPT_TCP_KEEPIDLE => 120, // Keep-Alive 空闲时间
CURLOPT_TCP_KEEPINTVL => 60, // Keep-Alive 探测间隔
CURLOPT_SSLVERSION => CURL_SSLVERSION_TLSv1_2, // 强制使用 TLS 1.2
// CURLOPT_SSL_VERIFYPEER => true, // 验证 SSL，已有，保持
],
'headers' => [
'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36 Edg/135.0.0.0',
'User-Agent' => $arrUserAgents[array_rand($arrUserAgents)],
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'Accept-Language' => 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
'Referer' => $this->strDomain,