From 60c965f0c0bdb0dfca6a7c0c4991a615376d79f4 Mon Sep 17 00:00:00 2001 From: make Date: Mon, 23 Jun 2025 23:42:57 +0800 Subject: [PATCH] debug --- code/app/task/crawler/zw/page/Site.php | 239 ++++++++++++++++++++--- code/app/task/crawler/zw74/page/Site.php | 41 +++- 2 files changed, 254 insertions(+), 26 deletions(-) diff --git a/code/app/task/crawler/zw/page/Site.php b/code/app/task/crawler/zw/page/Site.php index c32e28f..b399aaa 100644 --- a/code/app/task/crawler/zw/page/Site.php +++ b/code/app/task/crawler/zw/page/Site.php @@ -57,6 +57,43 @@ class Site */ public function initClient() { + $arrUserAgents = [ + // 华为浏览器 + 'Mozilla/5.0 (Linux; Android 11; HUAWEI Mate 40 Pro) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 HuaweiBrowser/12.1.4.301 Mobile Safari/537.36', + 'Mozilla/5.0 (Linux; Android 13; HUAWEI P60) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 HuaweiBrowser/13.1.2.301 Mobile Safari/537.36', + + // 小米/Redmi - MIUI浏览器 + 'Mozilla/5.0 (Linux; Android 12; Redmi K50) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 MQQBrowser/11.9 Mobile Safari/537.36 XiaoMi/MiuiBrowser/13.25.0', + 'Mozilla/5.0 (Linux; Android 11; Mi 10) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 XiaoMi/MiuiBrowser/12.5.22 Mobile Safari/537.36', + + // OPPO 浏览器 + 'Mozilla/5.0 (Linux; Android 13; OPPO Find X6 Pro) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 OPRU/14.3.8 Mobile Safari/537.36 HeyTapBrowser/14.3.8.0', + + // vivo 浏览器 + 'Mozilla/5.0 (Linux; Android 12; VIVO X90) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 V1_AND_SQ_9.9.0_302 Mobile Safari/537.36 VivoBrowser/9.9.0.0', + + // QQ 浏览器 + 'Mozilla/5.0 (Linux; Android 11; M2007J17C) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 MQQBrowser/11.7 Mobile Safari/537.36', + 'Mozilla/5.0 (Linux; Android 10; OPPO Reno 4 Pro) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 MQQBrowser/11.3 Mobile Safari/537.36', + + // UC 浏览器（优视） + 'Mozilla/5.0 (Linux; U; Android 12; zh-CN; V2055A) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 UCBrowser/13.5.5.1303 Mobile Safari/537.36', + 'Mozilla/5.0 (Linux; Android 11; 2201123C) AppleWebKit/537.36 (KHTML, like Gecko) UCBrowser/13.4.0.1306 Mobile Safari/537.36', + + // 360 浏览器 + 'Mozilla/5.0 (Linux; Android 10; M2006J10C) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 360Browser/12.1.1.106 Mobile Safari/537.36', + + // 搜狗浏览器 + 'Mozilla/5.0 (Linux; Android 11; V2073A) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 SogouMobileBrowser/6.0.0 Mobile Safari/537.36', + + // 百度 App 内置浏览器 + 'Mozilla/5.0 (Linux; Android 12; V2123A) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 baiduboxapp/13.19.0.10 (Baidu; P1 12) Mobile Safari/537.36', + + // 夸克浏览器（Quark） + 'Mozilla/5.0 (Linux; Android 11; 2201122C) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Quark/6.5.2.234 Mobile Safari/537.36', + ]; + + $arrConfig = [ 'http_version' => '1.1', // 强制用 HTTP/1.1，已有，保持 'base_uri' => $this->strDomain, // 基础域名，已有，保持 @@ -67,10 +104,11 @@ class Site CURLOPT_TCP_KEEPALIVE => 1, // 启用 TCP Keep-Alive，已有，保持 CURLOPT_TCP_KEEPIDLE => 120, // Keep-Alive 空闲时间 CURLOPT_TCP_KEEPINTVL => 60, // Keep-Alive 探测间隔 + CURLOPT_SSLVERSION => CURL_SSLVERSION_TLSv1_2, // 强制使用 TLS 1.2 // CURLOPT_SSL_VERIFYPEER => true, // 验证 SSL，已有，保持 ], 'headers' => [ - 'User-Agent' => 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36', + 'User-Agent' => $arrUserAgents[array_rand($arrUserAgents)], 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7', 'Accept-Language' => 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6', 'Referer' => $this->strDomain, @@ -119,7 +157,7 @@ class Site case 'JULIANG': // 隧道代理信息 // $strProxy = 'http://username:password@XXX.juliangip.com:23456'; - $strProxy = "http://".$strKey . ":" . $strSecret. "@".$strProxyHost; + $strProxy = "http://" . $strKey . ":" . $strSecret . "@" . $strProxyHost; $arrConfig['proxy'] = $strProxy; break; case 'LUOTUO': @@ -133,7 +171,6 @@ class Site $arrConfig['curl'][CURLOPT_PROXYAUTH] = CURLAUTH_BASIC; // $arrConfig['curl'][CURLOPT_PROXYUSERPWD] = "$strKey:$strSecret"; break; - } } @@ -233,8 +270,8 @@ class Site // 判断是否包含 $strContent ，不包含跳过 Gateway Timeout if (strpos($strContent, '') === false) { // 不包含该字符串，跳过 - var_dump(mb_substr($strContent, 0, 300)); - + var_dump(mb_substr($strContent, 0, 300)); + if ( str_contains($strContent, '超过并发限制') || str_contains($strContent, 'Timeout') || @@ -249,15 +286,14 @@ class Site } continue; } - + $NovelPage = $this->getNovelPage($this, $arrFilter[$intKey]['uri']); $NovelPage->setContent($strContent); $NovelPage->setData($arrFilter[$intKey]); $arrNovelPage[] = $NovelPage; - $this->redisDailyInc(config('task.app_name').'_success_getNovelPageInfo:' . date('Y-m-d')); - + $this->redisDailyInc(config('task.app_name') . '_success_getNovelPageInfo:' . date('Y-m-d')); } elseif ($arrResult['state'] == 'rejected') { // 重新推送任务 @@ -267,19 +303,18 @@ class Site ]; $QueueManage->set($arrTask); - $incKey = config('task.app_name').'_errer_getNovelPageInfo:' . date('Y-m-d'); + $incKey = config('task.app_name') . '_errer_getNovelPageInfo:' . date('Y-m-d'); $this->redisDailyInc($incKey); - if(config('log.task_debug_log_open_status')){ + if (config('log.task_debug_log_open_status')) { // 记录失败原因 $reason = $arrResult['reason'] instanceof \Exception ? $arrResult['reason']->getMessage() : 'Unknown error'; $strUri = $arrFilter[$intKey]['uri'] ?? ''; - \think\facade\Log::error("Request failed for getNovelPageInfo , URI: {$strUri }, Reason: {$reason}"); + \think\facade\Log::error("Request failed for getNovelPageInfo , URI: {$strUri}, Reason: {$reason}"); } } } return $arrNovelPage; - } /** @@ -376,6 +411,163 @@ class Site return $arrChapterPage; } + // 调试使用 + // public function getChapterPageList(array $arrFilter, int $intNid = 0): array + // { + // $QueueManage = QueueManage::getInstance(); + + // /** @var ChapterPage[] */ + // $arrChapterPage = []; + + // $arrPromises = []; + // $arrTimings = []; // 记录每个请求的耗时 + // $arrRequestResults = []; // 记录每个请求的返回结果 + + // foreach ($arrFilter as $intKey => $arrItem) { + // $startTime = microtime(true); // 记录开始时间（微秒） + // $arrTimings[$intKey] = ['start_time' => $startTime]; + + // $arrPromises[$intKey] = $this->getClient()->getAsync($arrItem['uri'])->then( + // // 成功回调 + // function (\Psr\Http\Message\ResponseInterface $response) use ($intKey, &$arrTimings, &$arrRequestResults, $arrItem) { + // $endTime = microtime(true); // 记录结束时间 + // $arrTimings[$intKey]['end_time'] = $endTime; + // $arrTimings[$intKey]['duration'] = $endTime - $arrTimings[$intKey]['start_time']; // 计算耗时 + + // // 记录响应结果 + // $arrRequestResults[$intKey] = [ + // 'status' => 'fulfilled', + // 'status_code' => $response->getStatusCode(), + // 'body' => substr((string) $response->getBody(), 0, 1000), // 限制响应体长度 + // 'uri' => $arrItem['uri'], // 添加 URI 记录 + // ]; + + // return $response; // 保持 Promise 链 + // }, + // // 失败回调 + // function (\Throwable $exception) use ($intKey, &$arrTimings, &$arrRequestResults, $arrItem) { + // $endTime = microtime(true); // 记录结束时间 + // $arrTimings[$intKey]['end_time'] = $endTime; + // $arrTimings[$intKey]['duration'] = $endTime - $arrTimings[$intKey]['start_time']; // 计算耗时 + + // // 初始化错误结果 + // $statusCode = null; + // $body = null; + + // // 仅对非 ConnectException 的 RequestException 检查响应 + // if ($exception instanceof \GuzzleHttp\Exception\RequestException && !($exception instanceof \GuzzleHttp\Exception\ConnectException)) { + // if (method_exists($exception, 'hasResponse') && $exception->hasResponse()) { + // $statusCode = $exception->getResponse()->getStatusCode(); + // $body = substr((string) $exception->getResponse()->getBody(), 0, 1000); + // } + // } + + // // 记录错误结果 + // $arrRequestResults[$intKey] = [ + // 'status' => 'rejected', + // 'error' => $exception->getMessage(), + // 'status_code' => $statusCode, + // 'body' => $body, + // 'uri' => $arrItem['uri'], // 添加 URI 记录 + // ]; + + // throw $exception; // 保持 Promise 链 + // } + // ); + // } + // // 等待所有请求完成 + // $arrResults = Promise\Utils::settle($arrPromises)->wait(); + + // // 处理结果并记录日志 + // foreach ($arrResults as $intKey => $arrResult) { + // // 记录请求日志 + // $logData = [ + // 'uri' => $arrFilter[$intKey]['uri'] ?? 'unknown', + // 'duration' => $arrTimings[$intKey]['duration'] ?? 0, + // 'status' => $arrRequestResults[$intKey]['status'] ?? 'unknown', + // 'status_code' => $arrRequestResults[$intKey]['status_code'] ?? null, + // 'body' => $arrRequestResults[$intKey]['body'] ?? null, + // ]; + // if (isset($arrRequestResults[$intKey]['error'])) { + // $logData['error'] = $arrRequestResults[$intKey]['error']; + // } + + + // if ($arrResult['state'] === 'fulfilled') { + // $Response = $arrResult['value']; + // $strContent = $Response->getBody()->getContents(); + + // // 跳过空内容 + // if (empty($strContent) || $strContent == '索引文件不存在！') { + // continue; + // } + + // // 如果不匹配有这个 说明没有请求到小说章节 或者 章节为空 + // $strPattern = "/lastread\.set\((.*?)\);/"; + // if (!preg_match($strPattern, $strContent, $arrMatches)) { + // if (config('log.task_debug_log_open_status')) { + // \think\facade\Log::warning("No match for pattern in response, URI: {$arrFilter[$intKey]['uri']}, Content: " . mb_substr($strContent, 0, 300)); + // } + + // if ($intNid > 0) { + // if ( + // str_contains($strContent, '超过并发限制') || + // str_contains($strContent, 'Timeout') || + // str_contains($strContent, '403 Forbidden') + // ) { + // // 重新推送任务 + // $arrTask = [ + // 'callback' => [Zw74Scheduler::class, 'getChapterInfo'], + // 'data' => [ + // 'n_id' => $intNid, + // 'filter' => $arrFilter, + // ], + // ]; + // $QueueManage->set($arrTask); + // } + // } + // continue; + // } + + // $ChapterPage = $this->getChapterPage($this, $arrFilter[$intKey]['uri']); + // $ChapterPage->setContent($strContent); + // $ChapterPage->setData($arrFilter[$intKey]); + + // $arrChapterPage[] = $ChapterPage; + + // // 成功指标 + // $this->redisDailyInc(config('task.app_name') . '_success_getChapterPageList:' . date('Y-m-d')); + // } elseif ($arrResult['state'] === 'rejected') { + + // \think\facade\Log::debug("Request log for getChapterPageList: " . json_encode($logData)); + + // if ($intNid > 0) { + // // 重新推送任务 + // $arrTask = [ + // 'callback' => [Zw74Scheduler::class, 'getChapterInfo'], + // 'data' => [ + // 'n_id' => $intNid, + // 'filter' => $arrFilter, + // ], + // ]; + // $QueueManage->set($arrTask); + // } + + // $this->redisDailyInc(config('task.app_name') . '_errer_getChapterPageList:' . date('Y-m-d')); + + // if (config('log.task_debug_log_open_status')) { + // $reason = $arrResult['reason'] instanceof \Exception + // ? $arrResult['reason']->getMessage() + // : 'Unknown error'; + // $strUri = $arrFilter[$intKey]['uri'] ?? ''; + // \think\facade\Log::error("Request failed for getNovelChapterInfo, URI: {$strUri}, Reason: {$reason}"); + // } + // } + // } + + // return $arrChapterPage; + // } + /** @@ -394,10 +586,10 @@ class Site $arrPromises = []; // $arrSuccessfulResults = []; foreach ($arrFilter as $intKey => $strUri) { - + $arrPromises[$intKey] = $this->getClient()->getAsync($strUri); } - + $arrResults = Promise\Utils::settle($arrPromises)->wait(); // $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) { @@ -412,9 +604,8 @@ class Site $LatestListPage->setContent($strContent); $arrLatestListPage[] = $LatestListPage; - $incKey = config('task.app_name').'_success_getLatestListPageList:' . date('Y-m-d'); + $incKey = config('task.app_name') . '_success_getLatestListPageList:' . date('Y-m-d'); $this->redisDailyInc($incKey); - } elseif ($arrResult['state'] == 'rejected') { // 重新推送任务 @@ -425,14 +616,14 @@ class Site $QueueManage = QueueManage::getInstance(); $QueueManage->set($arrTask); - $incKey = config('task.app_name').'_errer_getLatestListPageList:' . date('Y-m-d'); + $incKey = config('task.app_name') . '_errer_getLatestListPageList:' . date('Y-m-d'); $this->redisDailyInc($incKey); - if(config('log.task_debug_log_open_status')){ + if (config('log.task_debug_log_open_status')) { // 记录失败原因 $reason = $arrResult['reason'] instanceof \Exception ? $arrResult['reason']->getMessage() : 'Unknown error'; $strUri = $arrFilter[$intKey] ?? ''; - \think\facade\Log::error("Request failed for getLatestListPageList , URI: {$strUri }, Reason: {$reason}"); + \think\facade\Log::error("Request failed for getLatestListPageList , URI: {$strUri}, Reason: {$reason}"); } } } @@ -478,9 +669,8 @@ class Site $Response = $arrResult['value']; $StorageCore->writeStream($arrFilter[$intKey]['local_path'], $Response->getBody()->detach()); - $incKey = config('task.app_name').'_success_downRemoteImgToLocal:' . date('Y-m-d'); + $incKey = config('task.app_name') . '_success_downRemoteImgToLocal:' . date('Y-m-d'); $this->redisDailyInc($incKey); - } elseif ($arrResult['state'] == 'rejected') { // 重新推送任务 @@ -491,10 +681,10 @@ class Site $QueueManage = QueueManage::getInstance(); $QueueManage->set($arrTask); - $incKey = config('task.app_name').'_errer_downRemoteImgToLocal:' . date('Y-m-d'); + $incKey = config('task.app_name') . '_errer_downRemoteImgToLocal:' . date('Y-m-d'); $this->redisDailyInc($incKey); - if(config('log.task_debug_log_open_status')){ + if (config('log.task_debug_log_open_status')) { // 记录失败原因 $reason = $arrResult['reason'] instanceof \Exception ? $arrResult['reason']->getMessage() : 'Unknown error'; \think\facade\Log::error("Request failed for downRemoteImgToLocal , Reason: {$reason}"); @@ -552,7 +742,8 @@ class Site * @param integer $expire * @return void */ - public function redisDailyInc(string $key, int $expire = 604800) { + public function redisDailyInc(string $key, int $expire = 604800) + { $cache = Cache::store('redis'); if (!$cache->has($key)) { $cache->set($key, 0, $expire); diff --git a/code/app/task/crawler/zw74/page/Site.php b/code/app/task/crawler/zw74/page/Site.php index d648b0e..16a900f 100644 --- a/code/app/task/crawler/zw74/page/Site.php +++ b/code/app/task/crawler/zw74/page/Site.php @@ -39,20 +39,57 @@ class Site extends ZwSite */ public function initClient() { + $arrUserAgents = [ + // 华为浏览器 + 'Mozilla/5.0 (Linux; Android 11; HUAWEI Mate 40 Pro) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 HuaweiBrowser/12.1.4.301 Mobile Safari/537.36', + 'Mozilla/5.0 (Linux; Android 13; HUAWEI P60) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 HuaweiBrowser/13.1.2.301 Mobile Safari/537.36', + + // 小米/Redmi - MIUI浏览器 + 'Mozilla/5.0 (Linux; Android 12; Redmi K50) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 MQQBrowser/11.9 Mobile Safari/537.36 XiaoMi/MiuiBrowser/13.25.0', + 'Mozilla/5.0 (Linux; Android 11; Mi 10) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 XiaoMi/MiuiBrowser/12.5.22 Mobile Safari/537.36', + + // OPPO 浏览器 + 'Mozilla/5.0 (Linux; Android 13; OPPO Find X6 Pro) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 OPRU/14.3.8 Mobile Safari/537.36 HeyTapBrowser/14.3.8.0', + + // vivo 浏览器 + 'Mozilla/5.0 (Linux; Android 12; VIVO X90) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 V1_AND_SQ_9.9.0_302 Mobile Safari/537.36 VivoBrowser/9.9.0.0', + + // QQ 浏览器 + 'Mozilla/5.0 (Linux; Android 11; M2007J17C) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 MQQBrowser/11.7 Mobile Safari/537.36', + 'Mozilla/5.0 (Linux; Android 10; OPPO Reno 4 Pro) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 MQQBrowser/11.3 Mobile Safari/537.36', + + // UC 浏览器（优视） + 'Mozilla/5.0 (Linux; U; Android 12; zh-CN; V2055A) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 UCBrowser/13.5.5.1303 Mobile Safari/537.36', + 'Mozilla/5.0 (Linux; Android 11; 2201123C) AppleWebKit/537.36 (KHTML, like Gecko) UCBrowser/13.4.0.1306 Mobile Safari/537.36', + + // 360 浏览器 + 'Mozilla/5.0 (Linux; Android 10; M2006J10C) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 360Browser/12.1.1.106 Mobile Safari/537.36', + + // 搜狗浏览器 + 'Mozilla/5.0 (Linux; Android 11; V2073A) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 SogouMobileBrowser/6.0.0 Mobile Safari/537.36', + + // 百度 App 内置浏览器 + 'Mozilla/5.0 (Linux; Android 12; V2123A) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 baiduboxapp/13.19.0.10 (Baidu; P1 12) Mobile Safari/537.36', + + // 夸克浏览器（Quark） + 'Mozilla/5.0 (Linux; Android 11; 2201122C) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Quark/6.5.2.234 Mobile Safari/537.36', + ]; + $arrConfig = [ 'http_version' => '1.1', // 强制用 HTTP/1.1，已有，保持 'base_uri' => $this->strDomain, // 基础域名，已有，保持 'http_errors' => false, // 不抛 HTTP 错误，已有，保持 'timeout' => 60, // 从 160 秒改为 60 秒 - 'connect_timeout' => 10, // 从 30 秒改为 10 秒 + 'connect_timeout' => 30, // 从 30 秒改为 10 秒 'curl' => [ CURLOPT_TCP_KEEPALIVE => 1, // 启用 TCP Keep-Alive，已有，保持 CURLOPT_TCP_KEEPIDLE => 120, // Keep-Alive 空闲时间 CURLOPT_TCP_KEEPINTVL => 60, // Keep-Alive 探测间隔 + CURLOPT_SSLVERSION => CURL_SSLVERSION_TLSv1_2, // 强制使用 TLS 1.2 // CURLOPT_SSL_VERIFYPEER => true, // 验证 SSL，已有，保持 ], 'headers' => [ - 'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36 Edg/135.0.0.0', + 'User-Agent' => $arrUserAgents[array_rand($arrUserAgents)], 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7', 'Accept-Language' => 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6', 'Referer' => $this->strDomain,