admin/SEONexus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

123

Browse Source
This commit is contained in:
Default
2025-04-13 21:35:38 +08:00
parent 2e48c61a37
commit ea2c6119f1
2 changed files with 26 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
code/app/task/crawler/zw630/Scheduler.php
View File

@@ -33,7 +33,7 @@ class Scheduler
*/ */
public function crawlFull() public function crawlFull()
{ {
$intSliceSize = 4; $intSliceSize = 3;
$arrFilter = []; $arrFilter = [];
for ($intNId = 1; $intNId <= 531208; $intNId++) { for ($intNId = 1; $intNId <= 531208; $intNId++) {
@@ -61,7 +61,7 @@ class Scheduler
*/ */
public function crawLatest() public function crawLatest()
{ {
$intSliceSize = 4; $intSliceSize = 3;
$arrUri = []; $arrUri = [];
$strUri = '/list/lastupdate_%s_0_0_%s.html'; $strUri = '/list/lastupdate_%s_0_0_%s.html';
@@ -108,7 +108,7 @@ class Scheduler
$arrNSourceId = array_unique($arrNSourceId); $arrNSourceId = array_unique($arrNSourceId);
$arrNSourceId = array_values($arrNSourceId); $arrNSourceId = array_values($arrNSourceId);
$intSliceSize = 4; $intSliceSize = 3;
$arrFilter = []; $arrFilter = [];
foreach ($arrNSourceId as $intKey => $intNSourceId) { foreach ($arrNSourceId as $intKey => $intNSourceId) {
@@ -152,7 +152,7 @@ class Scheduler
*/ */
public function fetchChapter(int $intNId, int $intPage, array $arrChapterList = []) public function fetchChapter(int $intNId, int $intPage, array $arrChapterList = [])
{ {
$intSliceSize = 4; $intSliceSize = 3;
$intPageSize = 100; $intPageSize = 100;
$arrTask = [ $arrTask = [

76
code/app/task/crawler/zw630/page/Site.php
View File

@@ -55,34 +55,7 @@ class Site
*/ */
public function __construct() public function __construct()
{ {
// $this->Client = new Client([ $arrConfig = [
// 'http_version' => '1.1', // 强制用 HTTP/1.1
// 'base_uri' => $this->strDomain,
// 'http_errors' => false,
// // 'version' => 2.0,
// 'timeout' => 160,
// 'connect_timeout' => 30,
// 'curl' => [
// // CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_2_0,
// CURLOPT_TCP_KEEPALIVE => 1,
// CURLOPT_SSL_VERIFYPEER => true,
// ],
// 'headers' => [
// 'User-Agent' => 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Mobile Safari/537.36 Edg/134.0.0.0',
// 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
// 'Accept-Language' => 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
// 'Referer' => $this->strDomain,
// 'sec-ch-ua-mobile' => '?1',
// 'sec-ch-ua-platform' => '"Android"',
// 'sec-fetch-dest' => 'document',
// 'sec-fetch-mode' => 'navigate',
// 'sec-fetch-site' => 'same-origin',
// 'sec-fetch-user' => '?1',
// 'upgrade-insecure-requests' => '1',
// ],
// ]);
$this->Client = new Client([
'http_version' => '1.1', // 强制用 HTTP/1.1，已有，保持 'http_version' => '1.1', // 强制用 HTTP/1.1，已有，保持
'base_uri' => $this->strDomain, // 基础域名，已有，保持 'base_uri' => $this->strDomain, // 基础域名，已有，保持
'http_errors' => false, // 不抛 HTTP 错误，已有，保持 'http_errors' => false, // 不抛 HTTP 错误，已有，保持
@@ -92,33 +65,15 @@ class Site
CURLOPT_TCP_KEEPALIVE => 1, // 启用 TCP Keep-Alive，已有，保持 CURLOPT_TCP_KEEPALIVE => 1, // 启用 TCP Keep-Alive，已有，保持
CURLOPT_TCP_KEEPIDLE => 120, // Keep-Alive 空闲时间 CURLOPT_TCP_KEEPIDLE => 120, // Keep-Alive 空闲时间
CURLOPT_TCP_KEEPINTVL => 60, // Keep-Alive 探测间隔 CURLOPT_TCP_KEEPINTVL => 60, // Keep-Alive 探测间隔
CURLOPT_SSL_VERIFYPEER => true, // 验证 SSL，已有，保持 // CURLOPT_SSL_VERIFYPEER => true, // 验证 SSL，已有，保持
CURLOPT_SSL_VERIFYHOST => 2, // 确保主机名匹配 // CURLOPT_SSL_VERIFYHOST => 2, // 确保主机名匹配
], ],
'headers' => [ 'headers' => [
// Mobile
// 'User-Agent' => 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Mobile Safari/537.36 Edg/134.0.0.0',
// 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
//PC
'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36 Edg/134.0.0.0', 'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36 Edg/134.0.0.0',
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7', 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'Accept-Language' => 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6', 'Accept-Language' => 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
'Referer' => $this->strDomain, 'Referer' => $this->strDomain,
'Connection' => 'keep-alive', // 显式启用 Keep-Alive 'Connection' => 'keep-alive',
// Mobile
// 'sec-ch-ua-mobile' => '?1',
// 'sec-ch-ua-platform' => '"Android"',
// 'sec-fetch-dest' => 'document',
// 'sec-fetch-mode' => 'navigate',
// 'sec-fetch-site' => 'same-origin',
// 'sec-fetch-user' => '?1',
// PC
'sec-ch-ua' => '"Chromium";v="134", "Not:A-Brand";v="24", "Microsoft Edge";v="134"', 'sec-ch-ua' => '"Chromium";v="134", "Not:A-Brand";v="24", "Microsoft Edge";v="134"',
'sec-ch-ua-mobile' => '?0', 'sec-ch-ua-mobile' => '?0',
'sec-ch-ua-platform' => '"Windows"', 'sec-ch-ua-platform' => '"Windows"',
@@ -126,11 +81,24 @@ class Site
'sec-fetch-mode' => 'navigate', 'sec-fetch-mode' => 'navigate',
'sec-fetch-site' => 'same-origin', 'sec-fetch-site' => 'same-origin',
'sec-fetch-user' => '?1', 'sec-fetch-user' => '?1',
'upgrade-insecure-requests' => '1', 'upgrade-insecure-requests' => '1',
], ],
'pool_size' => 40, // 并发连接池大小（视需求调整） 'pool_size' => 5,
]); 'verify' => false, // 忽略 SSL 证书验证
];
$time = time();
$orderno = "DT20250413204710pFcgjMR9";
$secret = "583b141edcf65ce94f83591c5cb592ed";
$txt = "orderno=" . $orderno . ",secret=" . $secret . ",timestamp=" . $time;
$sign = strtoupper(md5($txt));
$auth = 'sign=' . $sign . '&orderno=' . $orderno . '&timestamp=' . $time;
$arrConfig['proxy'] = 'http://dtbf.xiongmaodaili.com:8089';
$arrConfig['headers']['Proxy-Authorization'] = $auth;
$this->Client = new Client($arrConfig);
} }
/** /**
@@ -180,7 +148,7 @@ class Site
} }
$arrResults = Promise\Utils::settle($arrPromises)->wait(); $arrResults = Promise\Utils::settle($arrPromises)->wait();
$arrSuccessfulResults = array_filter($arrResults, function ($arrResult) { $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
return $arrResult['state'] === 'fulfilled'; return $arrResult['state'] === 'fulfilled';
}); });
@@ -245,7 +213,7 @@ class Site
return $arrChapterPage; return $arrChapterPage;
} }
/** /**
* retrieve multiple chapter page model * retrieve multiple chapter page model
* *