admin/SEONexus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

123

Browse Source
This commit is contained in:
Default
2025-03-27 23:10:03 +08:00
parent 81b573c296
commit dd0d52f693
45 changed files with 2502 additions and 1279 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
code/app/task/collection/PullDataColBase.php
View File

@@ -1,48 +0,0 @@
<?php
declare(strict_types=1);
namespace app\task\collection;
use app\model\CaiJiPeiZhiModel;
use app\model\SystemConfigModel;
use minserver\QueueManage;
use GuzzleHttp\Client;
/**
* @mixin think\Model
*/
class PullDataColBase
{
/**
* Undocumented variable
*
* @var \GuzzleHttp\Client
*/
protected $Client = NULL;
/**
* Undocumented variable
*
* @var \minserver\QueueManage
*/
protected $QueueManage = NULL;
protected $arrCaiJiPeiZhi = [];
public function __construct()
{
$this->Client = new Client();
$this->QueueManage = new QueueManage();
}
public function getAilisiDomain()
{
if (!key_exists('AILISI_DOMAIN', $this->arrCaiJiPeiZhi)) {
$this->arrCaiJiPeiZhi['AILISI_DOMAIN'] = CaiJiPeiZhiModel::getValByCode('AILISI_DOMAIN');
}
return $this->arrCaiJiPeiZhi['AILISI_DOMAIN'];
}
}

611
code/app/task/collection/zw630/Zw630Col.php
View File

@@ -1,611 +0,0 @@
<?php
declare(strict_types=1);
namespace app\task\collection\zw630;
use app\model\XiaoShuoXiangQingModel;
use app\task\collection\PullDataColBase;
use db\mongo\MongoBase;
use GuzzleHttp\Cookie\CookieJar;
use processor\ImageProcessor;
use think\facade\Cache;
use QL\QueryList;
use minserver\QueueManage;
use GuzzleHttp\Client;
/**
* @mixin think\Model
*/
class Zw630Col extends PullDataColBase
{
private $strDomain = 'https://www.630zw.org';
/**
* Undocumented variable
*
* @var MongoBase
*/
private $MongoBase;
public function __construct()
{
$this->MongoBase = MongoBase::getInstance();
$this->Client = new Client([
'base_uri' => $this->strDomain,
'http_errors' => false,
'version' => 2.0, // 强制使用 HTTP/2
'timeout' => 160,
'connect_timeout' => 30, // 连接超时 10 秒
'curl' => [
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_2_0, // 强制 HTTP/2
CURLOPT_TCP_KEEPALIVE => 1, // 启用 TCP Keep-Alive
CURLOPT_SSL_VERIFYPEER => true, // 验证 SSL
],
'headers' => [
'User-Agent' => 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Mobile Safari/537.36 Edg/134.0.0.0',
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'Accept-Language' => 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
'Referer' => $this->strDomain,
'sec-ch-ua-mobile' => '?1',
'sec-ch-ua-platform' => '"Android"',
'sec-fetch-dest' => 'document',
'sec-fetch-mode' => 'navigate',
'sec-fetch-site' => 'same-origin',
'sec-fetch-user' => '?1',
'upgrade-insecure-requests' => '1',
],
]);
$this->QueueManage = new QueueManage();
}
public function pull630ZwNovel()
{
// 全局扫描任务
$QueueManage = $this->QueueManage;
// for ($intNovelId = 1; $intNovelId <= 531208; $intNovelId++) {
// for ($intNovelId = 1; $intNovelId <= 10000; $intNovelId++) {
for ($intNovelId = 1; $intNovelId <= 1; $intNovelId++) {
$arrTask = [
'callback' => [self::class, 'getNovelInfo'],
'data' => [
'n_id' => $intNovelId,
'page' => 0,
]
];
// print_r($arrTask);
$QueueManage->set($arrTask);
}
}
public function getNovelInfo($arrData)
{
// content="mobile" 有这个的是手机，否则是PC
print_r($arrData);
if ($arrData['page'] == 0) {
$strUri = sprintf('/shu/%s.html', $arrData['n_id']);
} else {
$strUri = sprintf('/shu/%s_%s.html', $arrData['n_id']);
}
$ResponseBase = $this->Client->get($strUri);
$strNovelContent = $ResponseBase->getBody()->getContents();
$NovelQuery = QueryList::html($strNovelContent);
$arrNovel = $this->getNovelHead($NovelQuery, $arrData['n_id'], $arrData['page']);
if (empty($arrNovel)) {
return false;
}
print_r($arrNovel);
// $strResult = (string)$Response->getBody();
}
private function getNovelHead(QueryList $QueryList, $intSourceNovelId, $intPage = 0)
{
$arrResult = [];
$QueryList->find('meta[property]')->each(function ($Item) use (&$arrResult) {
$property = str_replace(':', '_', $Item->attr('property'));
$arrResult[$property] = $Item->attr('content');
});
if (empty($arrResult)) {
return false;
}
// print_r($arrResult);
$arrResult['novel_source_id'] = 'zw630-' . $intSourceNovelId;
try {
$Col = $this->MongoBase->getDb()->selectCollection('novel');
$arrExisting = $Col->findOne(['novel_source_id' => $arrResult['novel_source_id']]);
// use novel
// db.novel.createIndex({ "novel_source_id": 1 }, { unique: true })
if ($arrExisting) {
$arrExisting = (array) $arrExisting; // MongoDB\BSON\Document 转为数组
$arrResult = array_merge($arrExisting, $arrResult); // 合并，保留原有字段
$arrResult['updated_at'] = new \MongoDB\BSON\UTCDateTime(); // 添加更新时间
$Col->updateOne(
['novel_source_id' => $arrResult['novel_source_id']],
['$set' => $arrResult],
// ['upsert' => false]
);
} else {
$arrResult['novel_id'] = $this->MongoBase->getNextSequence('novel');
$this->MongoBase->insertOne('novel', $arrResult);
}
return $arrResult;
} catch (\MongoDB\Exception\Exception $e) {
throw new \Exception("Upsert failed: " . $e->getMessage());
}
}
public function getNovelInfo1($arrData)
{
//var_dump('getNovelFengMian');
//var_dump($arrData);
$Client = $this->Client;
$QueueManage = $this->QueueManage;
// 获取 Redis 实例
$redis = Cache::store('redis')->handler();
$intXiaoShuoSourceId = $arrData['xiaoshuo_id'];
$intXiaoShuoSourceSeoId = $arrData['xiaoshuo_seo_id'];
$redisKey = "task:novel_source_id:" . $intXiaoShuoSourceId;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($arrData['xiaoshuo_url']);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$strSiteUrl = $this->getAilisiDomain();
$strCover = $QueryList->find('#fmimg img')->eq(0)->attr("src");
if (strpos($strCover, 'http') === false) {
$strCover = $strSiteUrl . $strCover;
}
$strCoverUri = $strCover;
try {
$strCoverUri = "";
$strCoverUri = ImageProcessor::getInstance()->downloadAndEncryptImage($strCover, 'xs');
$arrCover = [
'code' => 'AI_LI_SI_COVER',
// 'uri' => getUriByUrl($strCover),
'uri' => $strCoverUri,
];
$arrXiaoShuoInfo = [
'xsxq_feng_mian' => json_encode($arrCover),
];
$XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
} catch (\Throwable $t) {
$strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
echo $strErr;
}
} catch (\Throwable $T) {
throw $T;
}
}
// public function getNovelInfo($arrData)
// {
// $QueueManage = $this->QueueManage;
// $Client = $this->Client;
// $strSiteUrl = $this->getBiquxsDomain();
// $strUrl = 'http://www.biquxs.com/book/' . $arrData['xs_source_id'] ; //'http://www.biquxs.com/book/20053';
// $intXiaoShuoSourceId = $arrData['xs_source_id']; //20053;
// $intXiaoShuoSourceSeoId = $arrData['xs_source_seo_id']; //1 ;
// try {
// $arrCookie = $this->getAilisiCookie();
// $strHost = getDomainFromUrl($strUrl);
// $CookieJar = CookieJar::fromArray($arrCookie, $strHost);
// $proxy = getRandomProxies();
// $userAgent = getRandomUserAgent();
// $Response = $Client->request('GET', $strUrl, [
// 'headers' => [
// 'User-Agent' => $userAgent,
// 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
// 'Accept-Language' => 'en-US,en;q=0.5',
// ],
// 'cookies' => $CookieJar,
// // 'proxy' => $proxy,
// 'timeout' => 30,
// ]);
// $strResult = (string)$Response->getBody();
// $QueryList = QueryList::html($strResult);
// // var_dump($QueryList);
// // 提取所有 <meta> 标签的属性
// $result = $QueryList->find('meta[property^="og:"]')->map(function ($item) {
// return [
// 'property' => $item->attr('property'),
// 'content' => $item->attr('content'),
// ];
// });
// // 打印结果
// // print_r($result->all());
// $strZuoZhe = '';
// $strMingZi = '';
// $strLastDate = '';
// $strJieShao = '';
// $strCover = '';
// $strFenlei = '';
// $strZiShu = 0;
// $strZhuangTai = $QueryList->find('#info p')->eq(1)->text();
// $strZhuangTai = getStringAfterWord($strZhuangTai, '状态：');
// foreach ($result as $item) {
// switch ($item['property']) {
// case 'og:title':
// $strMingZi = $item['content'];
// break;
// case 'og:description':
// $strJieShao = $item['content'];
// break;
// case 'og:image':
// $strCover = $item['content'];
// break;
// case 'og:book:category':
// $strFenlei = $item['content'];
// break;
// case 'og:book:author':
// $strZuoZhe = $item['content'];
// break;
// case 'og:book:update_time':
// $strLastDate = $item['content'];
// break;
// }
// }
// $strLastDate = trim((string)$strLastDate);
// if (!isValidDateTime($strLastDate)) {
// $strLastDate = date('Y-m-d H:i:s');
// }
// $arrTag = [];
// if (strpos($strCover, 'http') === false) {
// $strCover = $strSiteUrl . $strCover;
// }
// $strCoverUri = $strCover;
// try {
// $strCoverUri = "";
// $strCoverUri = ImageProcessor::getInstance()->downloadImage( $intXiaoShuoSourceId,$strCover, 'xs');
// } catch (\Throwable $t) {
// $strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
// echo $strErr;
// }
// $arrCover = [
// 'code' => 'BI_QU_XS_COVER',
// // 'uri' => getUriByUrl($strCover),
// 'uri' => $strCoverUri,
// ];
// $strFenleiIId = 1 ;
// switch ($strFenlei) {
// case '玄幻小说':
// $strName = "玄幻";
// $strFenleiIId = 1;
// break;
// case '修真小说':
// $strName = "修真";
// $strFenleiIId = 2;
// break;
// case '都市小说':
// $strName = "都市";
// $strFenleiIId = 3;
// break;
// case '历史小说':
// $strName = "历史";
// $strFenleiIId = 4;
// break;
// case '网游小说':
// $strName = "网游";
// $strFenleiIId = 5;
// break;
// case '科幻小说':
// $strName = "科幻";
// $strFenleiIId = 6;
// break;
// case '女频小说':
// $strName = "女频";
// $strFenleiIId = 7;
// break;
// case '其它小说':
// $strName = "其它";
// $strFenleiIId = 8;
// break;
// default:
// $strName = "其它";
// $strFenleiIId = 8;
// break;
// }
// $arrXiaoShuoInfo = [
// 'xsxq_ming_zi' => $strMingZi,
// 'xsxq_zuozhe' => $strZuoZhe , //$arrData['xiaoshuo_zuozhe'],
// 'xsfl_id' => $strFenleiIId,
// 'xsxq_zhuang_tai' => $strZhuangTai,
// 'xsxq_zi_shu' => $strZiShu,
// 'xsxq_jie_shao' => $strJieShao,
// 'xsxq_feng_mian' => json_encode($arrCover),
// 'xsxq_geng_xin_shi_jian' => $strLastDate,
// 'xsxq_tag' => json_encode($arrTag, JSON_UNESCAPED_UNICODE),
// 'xsxq_source_id' => $intXiaoShuoSourceId,
// 'xsxq_source_seo_id' => $intXiaoShuoSourceId.'1',
// 'xsxq_source_code' => 'BIQU_XIAOSHUO_' . $intXiaoShuoSourceId . '_' . $intXiaoShuoSourceSeoId,
// ];
// // print_r($arrXiaoShuoInfo);
// var_dump($arrXiaoShuoInfo['xsxq_source_code']);
// $XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
// $arrZhangjie = [];
// $intChapterIndex = 0; // 定义一个计数器变量
// // 检查是否找到元素
// if ($QueryList->find('.listmain a')->count() > 0) {
// $QueryList->find('.listmain a')->map(function ($ZhangJie) use (&$arrZhangjie, &$intChapterIndex) {
// echo "Processing index $intChapterIndex: " . $ZhangJie->text() . PHP_EOL;
// // 从索引6开始
// if ($intChapterIndex >= 5) {
// $arrZhangjie[] = [
// 'mingzi' => $ZhangJie->text(),
// 'url' => $ZhangJie->attr('href'),
// ];
// }
// $intChapterIndex++;
// });
// // 输出最终结果
// var_dump($arrZhangjie);
// } else {
// echo "No elements found for .listmain a" . PHP_EOL;
// }
// //unset($QueryList);
// // 获取最大章节索引
// $intMaxXiaoShuoZhangJiePaixu = XiaoShuoZhangJieModel::where('xsxq_id', $intXiaoShuoSourceId)
// ->max('xszj_pai_xu'); // 获取最大值
// var_dump('获取小说章节最大值-'.$intMaxXiaoShuoZhangJiePaixu);
// // 如果没有结果，max 返回 null，可以设置默认值
// $intMaxXiaoShuoZhangJiePaixu = $intMaxXiaoShuoZhangJiePaixu ?? 0;
// foreach ($arrZhangjie as $intZhangJieNum => $arrZhangjieOne) {
// var_dump($intZhangJieNum);
// // 如果数据库里的章节最大索引大于当前索引，说明已经入库，不需要入队列了
// if($intMaxXiaoShuoZhangJiePaixu > $intZhangJieNum){
// var_dump("小说章节已经入库，不再推送任务");
// continue;// 如果任务已存在，直接跳过 break
// }
// $strZhangJieUrl = $strSiteUrl . $arrZhangjieOne['url'];
// $strZhangJieId = extractFilename($strZhangJieUrl);
// # 'AILISI_XIAOSHUO_ZHANGJIE_小说源id_小说源章节id'
// $strXszjSourceCode = 'AILISI_XIAOSHUO_ZHANGJIE_'. $XiaoShuoXiangQingModel->xsxq_source_id . '_' . $strZhangJieId;
// $arrTask = [
// 'callback' => [self::class, 'getNovelReader'],
// 'data' => [
// 'xszj_source_id' => $strZhangJieId,
// 'xszj_paixu' => $intZhangJieNum,
// 'xszj_name' => $arrZhangjieOne['mingzi'],
// 'xszj_url' => $strZhangJieUrl,
// 'xszj_source_code' => $strXszjSourceCode,
// 'xsxq_source_id' => $XiaoShuoXiangQingModel->xsxq_source_id,
// 'xsfl_id' => $strFenleiIId,
// ]
// ];
// $QueueManage->set($arrTask);
// }
// var_dump("章节任务投递成功");
// } catch (\Throwable $T) {
// throw $T;
// }
// }
// public function getNovelFengMian($arrData)
// {
// //var_dump('getNovelFengMian');
// //var_dump($arrData);
// $Client = $this->Client;
// $QueueManage = $this->QueueManage;
// // 获取 Redis 实例
// $redis = Cache::store('redis')->handler();
// $intXiaoShuoSourceId = $arrData['xiaoshuo_id'];
// $intXiaoShuoSourceSeoId = $arrData['xiaoshuo_seo_id'];
// $redisKey = "task:novel_source_id:" . $intXiaoShuoSourceId;
// try {
// $arrCookie = $this->getAilisiCookie();
// $strHost = getDomainFromUrl($arrData['xiaoshuo_url']);
// $CookieJar = CookieJar::fromArray($arrCookie, $strHost);
// $strSiteUrl = $this->getAilisiDomain();
// $strCover = $QueryList->find('#fmimg img')->eq(0)->attr("src");
// if (strpos($strCover, 'http') === false) {
// $strCover = $strSiteUrl . $strCover;
// }
// $strCoverUri = $strCover;
// try {
// $strCoverUri = "";
// $strCoverUri = ImageProcessor::getInstance()->downloadAndEncryptImage($strCover, 'xs');
// $arrCover = [
// 'code' => 'AI_LI_SI_COVER',
// // 'uri' => getUriByUrl($strCover),
// 'uri' => $strCoverUri,
// ];
// $arrXiaoShuoInfo = [
// 'xsxq_feng_mian' => json_encode($arrCover),
// ];
// $XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
// } catch (\Throwable $t) {
// $strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
// echo $strErr;
// }
// } catch (\Throwable $T) {
// throw $T;
// }
// }
// public function getNovelReader($arrData)
// {
// var_dump('getNovelReader');
// var_dump($arrData['xszj_url']);
// $Client = $this->Client;
// $strXszjSourceCode = $arrData['xszj_source_code'];
// $strZhangJieUrl = $arrData['xszj_url'];
// $intZhangJiePaiXu = $arrData['xszj_paixu'];
// $intFenleiId = $arrData['xsfl_id'];
// $intXiaoShuoSourceID = $arrData['xsxq_source_id'];
// $strZhangJieSourceId = $arrData['xszj_source_id'];
// $strZhangJieName = $arrData['xszj_name'];
// try {
// $arrCookie = $this->getAilisiCookie();
// $strHost = getDomainFromUrl($strZhangJieUrl);
// $CookieJar = CookieJar::fromArray($arrCookie, $strHost);
// if (XiaoShuoZhangJieModel::checkZhangJieExists($strXszjSourceCode)) {
// return true;
// }
// $proxy = getRandomProxies();
// $userAgent = getRandomUserAgent();
// $Response = $Client->request('GET', $strZhangJieUrl, [
// 'headers' => [
// 'User-Agent' => $userAgent,
// 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
// 'Accept-Language' => 'en-US,en;q=0.5',
// ],
// 'cookies' => $CookieJar,
// // 'proxy' => $proxy,
// 'timeout' => 30,
// ]);
// $strResult = (string)$Response->getBody();
// unset($Response);
// $ZhangJieQueryList = QueryList::html($strResult);
// // 获取原始 HTML
// $strNeiRong = $ZhangJieQueryList->find('#content')->eq(0)->html();
// // 解码 HTML 实体
// $strNeiRong = html_entity_decode($strNeiRong, ENT_QUOTES, 'UTF-8');
// // 去掉 <p> 标签中的 class="content_detail" 属性
// $strNeiRong = preg_replace('/<p\s+class="content_detail"/i', '<p', $strNeiRong);
// // 清理多余的 HTML 标签
// $strNeiRong = strip_tags($strNeiRong, '<p>');
// // 替换不间断空格 (\xc2\xa0) 和 &nbsp; 为普通空格
// $strNeiRong = preg_replace('/\xc2\xa0/', ' ', $strNeiRong); // 替换 UTF-8 不间断空格
// $strNeiRong = preg_replace('/&nbsp;/', ' ', $strNeiRong); // 替换 &nbsp;
// // 清理多余空格和换行符
// $strNeiRong = preg_replace('/\s+/', ' ', $strNeiRong);
// $strNeiRong = preg_replace('/<p>\s+/', '<p>', $strNeiRong); // 清理 <p> 标签内部多余空格
// $strNeiRong = preg_replace('/\s+<\/p>/', '</p>', $strNeiRong); // 清理 <p> 标签结尾多余空格
// // 输出结果
// // var_dump($strNeiRong);
// # '/novel/123yqw/分类id/小说源id/小说源章节id.txt'
// $strNeiRongTxtPath = config("filesystem.novel").'/novel/biquxs/'.$intFenleiId.'/'.$intXiaoShuoSourceID.'/'.$strZhangJieSourceId.'.txt';
// //$strNeiRongTxtPath = "/www/novel/123yqw/1/11/111.txt";
// // 调用函数
// $result = saveCompressAndDeleteTxt($strNeiRongTxtPath, $strNeiRong);
// // 输出状态
// if ($result['status']) {
// var_dump("Success: " . $result['message']) ;
// } else {
// var_dump("Error: " . $result['message']) ;
// }
// $arrXiaoShuoZhangJie = [
// 'xsxq_id' => $intXiaoShuoSourceID,
// 'xszj_pai_xu' => $intZhangJiePaiXu,
// 'xszj_ming_zi' => $strZhangJieName,
// 'xszj_nei_rong' => $strNeiRongTxtPath,
// 'xszj_source_id' => $strZhangJieSourceId,
// 'xszj_source_code' => $strXszjSourceCode,
// ];
// XiaoShuoZhangJieModel::addXiaoShuoZhangJie($arrXiaoShuoZhangJie);
// unset($ZhangJieQueryList);
// // 延时 3 秒
// //sleep(3);
// } catch (\Throwable $T) {
// throw $T;
// }
// }
}

213
code/app/task/crawler/zw630/Scheduler.php Normal file
View File

@@ -0,0 +1,213 @@
<?php
declare(strict_types=1);
namespace app\task\crawler\zw630;
use app\task\crawler\zw630\page\NovelPage;
use app\task\crawler\zw630\page\Site;
use microserver\QueueManage;
/**
* @mixin think\Model
*/
class Scheduler
{
/**
* Undocumented variable
*
* @var \microserver\QueueManage
*/
protected $QueueManage = NULL;
public function __construct()
{
$this->QueueManage = new QueueManage();
}
/**
* 全局扫描任务
*
* @return void
*/
public function crawlFull()
{
$intSliceSize = 6;
$arrFilter = [];
// for ($intNId = 1; $intNId <= 531208; $intNId++) {
// for ($intNId = 1; $intNId <= 10000; $intNId++) {
for ($intNId = 1; $intNId <= 20; $intNId++) {
$arrFilter[] = [
'n_source_id' => $intNId,
'page' => 0,
];
if ($intNId % $intSliceSize == 0) {
$this->fetchNovel($arrFilter);
$arrFilter = [];
}
}
if (!empty($arrFilter)) {
$this->fetchNovel($arrFilter);
}
return;
// $arrTest = [
// 'n_id' => 9973,
// 'filter' => [
// [
// 'name' => '第十四章 神兽空青蛇妖',
// 'uri' => '/shu/1/33.html',
// 'c_sort_num' => 32,
// 'c_source_id' => 33,
// 'c_page' => 0,
// ],
// ]
// ];
// $arrTask = [
// 'callback' => [self::class, 'getChapterInfo'],
// 'data' => $arrTest
// ];
// $this->QueueManage->set($arrTask);
}
/**
* Undocumented function
*
* @param array $arrFilter
* @return void
*/
public function fetchNovel(array $arrFilter) //(int $intNSourceId, int $intPage = 0)
{
$arrTask = [
'callback' => [self::class, 'getNovelInfo'],
'data' => $arrFilter,
// 'data' => [
// 'n_source_id' => $intNSourceId,
// 'page' => $intPage,
// ]
];
$this->QueueManage->set($arrTask);
}
/**
* Undocumented function
*
* @param integer $intNId
* @param integer $intPage
* @param array $arrChapterList
* @return void
*/
public function fetchChapter(int $intNId, int $intPage, array $arrChapterList = [])
{
$intSliceSize = 6;
$intPageSize = 100;
$arrTask = [
'callback' => [self::class, 'getChapterInfo'],
'data' => [
'n_id' => $intNId,
// 'filter' => $arrFilter,
]
];
$arrFilter = [];
foreach ($arrChapterList as $intKey => $arrChapter) {
$arrChapter['c_sort_num'] = $intPage * $intPageSize + $intKey + 1;
$arrChapter['c_source_id'] = supperExtractFilename($arrChapter['uri']);
$arrChapter['c_page'] = 0;
$arrFilter[] = $arrChapter;
if ($intKey % $intSliceSize == 0) {
$arrTask['data']['filter'] = $arrFilter;
$this->QueueManage->set($arrTask);
$arrFilter = [];
}
}
if (!empty($arrFilter)) {
$arrTask['data']['filter'] = $arrFilter;
$this->QueueManage->set($arrTask);
}
}
/**
* Undocumented function
*
* @param array $arrFilter
* @return void
*/
public function getNovelInfo(array $arrFilter)
{
$Site = new Site;
$arrNovelPageList = $Site->getNovelPageList($arrFilter);
$arrCover = [];
foreach ($arrNovelPageList as $NovelPage) {
$arrNovel = $NovelPage->getNovelInfo();
if (empty($arrNovel)) {
continue;
}
$arrNextPageData = $NovelPage->getNextPageArgs();
if (!empty($arrNextPageData)) {
$this->fetchNovel([
[
'n_source_id' => $arrNextPageData['n_source_id'],
'page' => $arrNextPageData['page'],
]
]);
}
// $arrCover[] = $NovelPage->getDownCoverInfo();
// $arrChapterList = $NovelPage->getChapters();
// $this->fetchChapter($arrNovel['n_id'], $NovelPage->getData()['page'], $arrChapterList);
}
if ($arrCover) {
$Site->downRemoteImgToLocal($arrCover);
}
}
/**
* Undocumented function
*
* @param array $arrData
* @return void
*/
public function getChapterInfo(array $arrData)
{
$Site = new Site;
$arrChapterPageList = $Site->getChapterPageList($arrData['filter']);
$arrNextPageFilterList = [];
foreach ($arrChapterPageList as $ChapterPage) {
$ChapterPage->saveChapter($arrData['n_id']);
$arrNextPageFilter = $ChapterPage->getNextPage();
if (!empty($arrNextPageFilter)) {
$arrNextPageFilterList[] = $arrNextPageFilter;
}
}
if (!empty($arrNextPageFilterList)) {
$arrTask = $arrData;
$arrTask['filter'] = $arrNextPageFilterList;
$arrTask = [
'callback' => [self::class, 'getChapterInfo'],
'data' => $arrTask
];
$this->QueueManage->set($arrTask);
}
}
}

258
code/app/task/crawler/zw630/page/BasePage.php Normal file
View File

@@ -0,0 +1,258 @@
<?php
declare(strict_types=1);
namespace app\task\crawler\zw630\page;
use db\mongo\MongoBase;
use ddl\DDLManage;
use GuzzleHttp\Client;
use microserver\QueueManage;
use QL\QueryList;
use Psr\Http\Message\ResponseInterface;
/**
* @mixin think\Model
*/
abstract class BasePage
{
/**
* Undocumented variable
*
* @var string
*/
// protected $strDomain;
/**
* Undocumented variable
*
* @var string
*/
protected $strUri;
/**
* Undocumented variable
*
* @var MongoBase
*/
protected $MongoBase;
/**
* Undocumented variable
*
* @var \GuzzleHttp\Client
*/
protected $Client = NULL;
/**
* Undocumented variable
*
* @var \microserver\QueueManage
*/
protected $QueueManage = NULL;
/**
* Undocumented variable
*
* @var array
*/
protected $arrCaiJiPeiZhi = [];
/**
* Undocumented variable
*
* @var string
*/
protected $strContent;
/**
* Undocumented variable
*
* @var QueryList
*/
protected $QueryList;
/**
* Undocumented variable
*
* @var array
*/
protected $arrData = [];
/**
* Undocumented variable
*
* @var bool
*/
protected $boolIsMobile;
/**
* Undocumented variable
*
* @var ResponseInterface
*/
private $Response;
/**
* Undocumented function
*
* @param string $strUri
*/
public function __construct(Site $Site, string $strUri)
{
$this->setSite($Site);
$this->strUri = $strUri;
}
/**
* Undocumented function
*
* @param array $arrData
* @return void
*/
public function setData(array $arrData)
{
$this->arrData = $arrData;
}
/**
* Undocumented function
*
* @return array
*/
public function getData(): array
{
return $this->arrData;
}
/**
* Undocumented function
*
* @return QueueManage
*/
public function getQueueManage(): QueueManage
{
if ($this->QueueManage == NULL) {
$this->QueueManage = QueueManage::getInstance();
}
return $this->QueueManage;
}
/**
* Undocumented function
*
* @return MongoBase
*/
public function getMongoBase(): MongoBase
{
if ($this->MongoBase == NULL) {
DDLManage::load('MongoBase');
$this->MongoBase = MongoBase::getInstance();
}
return $this->MongoBase;
}
public function getQueryList(): QueryList
{
if ($this->QueryList == NULL) {
$this->QueryList = QueryList::html($this->getContent());
}
return $this->QueryList;
}
/**
* Undocumented function
*
* @return string
*/
public function getContent(): string
{
if ($this->strContent == NULL) {
$this->Response = $this->getClient()->get($this->strUri);
$this->strContent = $this->Response->getBody()->getContents();
}
return $this->strContent;
}
/**
* force set page content
*
* @param string $strContent
* @return void
*/
public function setContent($strContent)
{
$this->strContent = $strContent;
}
/**
* Undocumented function
*
* @return boolean
*/
public function isMobile($strMark = 'content="mobile"'): bool
{
if ($this->boolIsMobile === NULL) {
$this->boolIsMobile = is_numeric(strpos($this->getContent(), $strMark));
}
return $this->boolIsMobile;
}
/**
* Undocumented function
*
* @return Client
*/
protected function getClient(): Client
{
if ($this->Client == NULL) {
$this->Client = $this->getSite()->getClient();
}
return $this->Client;
}
/**
* Undocumented variable
*
* @var Site
*/
protected $Site;
public function setSite(Site $Site)
{
$this->Site = $Site;
}
public function getSite()
{
return $this->Site;
}
/**
* Undocumented variable
*
* @var boolean
*/
protected $boolIsFirstSave = false;
/**
* Undocumented function
*
* @return boolean
*/
public function getIsFirstSave(): bool
{
return $this->boolIsFirstSave;
}
}

139
code/app/task/crawler/zw630/page/ChapterPage.php Normal file
View File

@@ -0,0 +1,139 @@
<?php
declare(strict_types=1);
namespace app\task\crawler\zw630\page;
use storage\StorageCore;
class ChapterPage extends BasePage
{
/**
* Undocumented variable
*
* @var string
*/
protected $strChapterContent = '';
/**
* Undocumented function
*
* @return array
*/
public function getChapterInfo(): array
{
if ($this->arrChapterInfo === NULL) {
$this->arrChapterInfo = [];
$strPattern = "/qsbs\.bb\('([^']*)'\)/";
preg_match_all($strPattern, $this->getContent(), $arrMatches);
$strContent = '';
foreach ($arrMatches[1] as $str) {
$strContent .= base64_decode($str);
}
// $this->arrChapterInfo['c_content'] = $strContent;
$this->strChapterContent = $strContent;
$this->arrChapterInfo['c_page_title'] = $this->getQueryList()->find('title')->text();
$this->arrChapterInfo['c_page_keywords'] = $this->getQueryList()->find('meta[name="keywords"]')->attr('content');
$this->arrChapterInfo['c_page_description'] = $this->getQueryList()->find('meta[name="description"]')->attr('content');
}
return $this->arrChapterInfo;
}
/**
* Undocumented variable
*
* @var array
*/
protected $arrChapterInfo;
public function saveChapter(int $intNId): array
{
try {
$this->getChapterInfo();
$strUUId = 'zw630-chapter-' . $this->arrData['c_source_id'] . '-' . $this->arrData['c_page'];
$Col = $this->getMongoBase()->getDb()->selectCollection('chapter');
$arrExisting = $Col->findOne(['c_site_uuid' => $strUUId]);
if ($arrExisting) {
$arrExisting = (array) $arrExisting;
$this->arrChapterInfo = array_merge($arrExisting, $this->getChapterInfo());
$this->arrChapterInfo['updated_at'] = new \MongoDB\BSON\UTCDateTime(); // 添加更新时间
$Col->updateOne(
['c_site_uuid' => $strUUId],
['$set' => $this->arrChapterInfo],
// ['upsert' => false]
);
} else {
$this->arrChapterInfo['n_id'] = $intNId;
$this->arrChapterInfo['name'] = $this->arrData['name'];
$this->arrChapterInfo['c_sort_num'] = $this->arrData['c_sort_num'];
$this->arrChapterInfo['c_page'] = $this->arrData['c_page'];
$this->arrChapterInfo['c_source_id'] = $this->arrData['c_source_id'];
$this->arrChapterInfo['c_source_uri'] = $this->arrData['uri'];
$this->arrChapterInfo['c_site_name'] = '恋上你中文';
$this->arrChapterInfo['c_site_code'] = 'zw630';
$this->arrChapterInfo['c_site_uuid'] = $strUUId;
$this->arrChapterInfo['c_content_path'] = '/novel/' . ($intNId % 100) . '/' . $intNId . '/' . $this->arrChapterInfo['c_sort_num'] . '_' . $this->arrChapterInfo['c_page'] . '.txt';
$this->getMongoBase()->insertOne('chapter', $this->arrChapterInfo);
if (!StorageCore::getInstance()->has($this->arrChapterInfo['c_content_path'])) {
StorageCore::getInstance()->set($this->arrChapterInfo['c_content_path'], $this->strChapterContent);
}
$this->boolIsFirstSave = true;
}
return $this->arrChapterInfo;
} catch (\MongoDB\Exception\Exception $e) {
throw new \Exception("Upsert failed: " . $e->getMessage());
}
}
protected $arrNextPage;
/**
* Undocumented function
*
* @return array
*/
public function getNextPage(): array
{
if ($this->arrNextPage === NULL) {
$this->arrNextPage = [];
if ($this->isMobile()) {
$strPattern = "/var\s+hhekgsv\s*=\s*['\"](.*?)['\"]/";
preg_match_all($strPattern, $this->getContent(), $arrMatches);
$strNextPageUri = end($arrMatches[1]) ?: '';
} else {
$strHref = $this->getQueryList()->find('.read_btn a:contains("下一章")')->attr('href');
$strNextPageUri = $strHref ?: '';
}
if (is_numeric(strpos($strNextPageUri, '_'))) {
$this->arrNextPage = $this->arrData;
$this->arrNextPage['c_page'] += 1;
$this->arrNextPage['uri'] = $strNextPageUri;
}
}
return $this->arrNextPage;
}
}

304
code/app/task/crawler/zw630/page/NovelPage.php Normal file
View File

@@ -0,0 +1,304 @@
<?php
declare(strict_types=1);
namespace app\task\crawler\zw630\page;
use ddl\DDLManage;
use QL\QueryList;
class NovelPage extends BasePage
{
/**
* Undocumented variable
*
* @var array
*/
protected $arrNovelInfo = [];
/**
* Undocumented variable
*
* @var array
*/
protected $arrRelatedNovel = [];
/**
* Undocumented variable
*
* @var array
*/
protected $arrForgeNovel = [];
private function findOrCreateNovel(int $intNSourceId, string $strName): array
{
$strUUId = 'zw630-' . $intNSourceId;
$Col = $this->getMongoBase()->getDb()->selectCollection('novel');
$arrExisting = $Col->findOne(['n_site_uuid' => $strUUId]);
if ($arrExisting) {
$arrNovelInfo = (array) $arrExisting;
} else {
$arrNovelInfo['og_title'] = $strName;
$arrNovelInfo['og_novel_book_name'] = $strName;
$arrNovelInfo['og_novel_pin_yin'] = zhToPinYin($strName);
$arrNovelInfo['n_site_name'] = '恋上你中文';
$arrNovelInfo['n_site_code'] = 'zw630';
$arrNovelInfo['n_site_uuid'] = $strUUId;
$arrNovelInfo['n_source_id'] = $intNSourceId;
$arrNovelInfo['n_level'] = 1;
$arrNovelInfo['n_id'] = $this->getMongoBase()->getNextSequence('novel');
$this->getMongoBase()->insertOne('novel', $arrNovelInfo);
}
return $arrNovelInfo;
}
/**
* Find or create a category by name, handling concurrency safely.
*
* @param string $strCategory
* @return array|null The category document (existing or newly created)
*/
public function findOrCreateCategory(string $strCategory)
{
if (empty($strCategory)) {
return false;
}
$Col = $this->getMongoBase()->getDb()->selectCollection('category');
$arrCategory = ['ca_name' => $strCategory];
try {
$Result = $Col->findOneAndUpdate(
$arrCategory,
['$set' => $arrCategory],
[
'upsert' => true,
'returnDocument' => \MongoDB\Operation\FindOneAndUpdate::RETURN_DOCUMENT_AFTER
]
);
return $Result;
} catch (\MongoDB\Driver\Exception\Exception $e) {
return null;
}
}
/**
* Undocumented function
*
* @return array
*/
public function getNovelInfo($boolSave = true): array
{
if (empty($this->arrNovelInfo)) {
$this->getQueryList()->find('meta[property]')->each(function ($Item) {
$strKey = str_replace(':', '_', $Item->attr('property'));
$this->arrNovelInfo[$strKey] = $Item->attr('content');
});
$this->arrNovelInfo['n_page_title'] = $this->getQueryList()->find('title')->text();
$this->arrNovelInfo['n_page_keywords'] = $this->getQueryList()->find('meta[name="keywords"]')->attr('content');
$this->arrNovelInfo['n_page_description'] = $this->getQueryList()->find('meta[name="description"]')->attr('content');
$this->getQueryList()->find('.first_txt')->eq(0)->find("a")->map(function ($A) {
$strName = $A->text();
$strUri = $A->attr('href');
if (is_numeric(strpos($strUri, 'shu'))) {
$intNSourceId = (int)supperExtractFilename($strUri);
$arrNovelInfo = $this->findOrCreateNovel($intNSourceId, $strName);
$this->arrRelatedNovel[] = [
"n_id" => $arrNovelInfo["n_id"],
"n_source_id" => $arrNovelInfo["n_source_id"],
"og_title" => $arrNovelInfo["og_title"],
];
} elseif (is_numeric(strpos($strUri, 'kan'))) {
$intNForgeId = (int)supperExtractFilename($strUri);
$this->arrForgeNovel[] = [
"n_forge_id" => $intNForgeId,
"n_forge_title" => $strName,
];
}
});
$this->arrNovelInfo['n_related_novel'] = $this->arrRelatedNovel;
$this->arrNovelInfo['n_forge_novel'] = $this->arrForgeNovel;
}
if ($boolSave && !empty($this->arrNovelInfo)) {
$this->saveNovelInfo();
$this->findOrCreateCategory($this->arrNovelInfo['og_novel_category']);
}
return $this->arrNovelInfo;
}
/**
* Undocumented function
*
* @return void
*/
public function saveNovelInfo()
{
try {
$strUUId = 'zw630-' . $this->arrData['n_source_id'];
$Col = $this->getMongoBase()->getDb()->selectCollection('novel');
$arrExisting = $Col->findOne(['n_site_uuid' => $strUUId]);
if ($arrExisting) {
$arrExisting = (array) $arrExisting;
$this->arrNovelInfo = array_merge($arrExisting, $this->arrNovelInfo);
if (!isset($this->arrNovelInfo['og_novel_pin_yin']) || empty($this->arrNovelInfo['og_novel_pin_yin'])) {
$this->arrNovelInfo['og_novel_pin_yin'] = zhToPinYin($this->arrNovelInfo['og_novel_book_name'] ?? '');
}
if (!isset($this->arrNovelInfo['n_cover_path']) || empty($this->arrNovelInfo['n_cover_path'])) {
$this->arrNovelInfo['n_cover_path'] = 'img/' . date('Y/m/d', time() - 24 * 3600 * 14) . '/' . $this->arrNovelInfo['n_id'] . '.jpg';
}
$this->arrNovelInfo['updated_at'] = new \MongoDB\BSON\UTCDateTime();
$Col->updateOne(
['n_site_uuid' => $strUUId],
['$set' => $this->arrNovelInfo],
// ['upsert' => false]
);
} else {
$this->arrNovelInfo['og_novel_pin_yin'] = zhToPinYin($this->arrNovelInfo['og_novel_book_name'] ?? '');
$this->arrNovelInfo['n_site_name'] = '恋上你中文';
$this->arrNovelInfo['n_site_code'] = 'zw630';
$this->arrNovelInfo['n_site_uuid'] = $strUUId;
$this->arrNovelInfo['n_source_id'] = $this->arrData['n_source_id'];
$this->arrNovelInfo['n_id'] = $this->getMongoBase()->getNextSequence('novel');
$this->arrNovelInfo['n_level'] = 1;
$this->arrNovelInfo['n_cover_path'] = '/img/' . date('Y/m/d', time() - 24 * 3600 * 14) . '/' . $this->arrNovelInfo['n_id'] . '.jpg';
$this->getMongoBase()->insertOne('novel', $this->arrNovelInfo);
$this->boolIsFirstSave = true;
}
return $this->arrNovelInfo;
} catch (\MongoDB\Exception\Exception $e) {
throw new \Exception("Upsert failed: " . $e->getMessage());
}
}
/**
* Undocumented function
*
* @return array
*/
public function getDownCoverInfo(): array
{
return [
'uri' => $this->arrNovelInfo['og_image'],
'local_path' => $this->arrNovelInfo['n_cover_path'],
];
}
/**
* Undocumented variable
*
* @var array
*/
protected $arrLatestChapters = NULL;
/**
* Undocumented variable
*
* @var array
*/
protected $arrChapters = NULL;
public function getLatestChaptersList(): array
{
if ($this->arrLatestChapters === NULL) {
$this->arrLatestChapters = [];
if ($this->isMobile()) {
$strClass = '.chapter-list';
} else {
$strClass = '.section-list';
}
if ($this->arrData['page'] == 0) {
$this->getQueryList()->find($strClass)->eq(0)->find("a")->map(function ($A) {
$this->arrLatestChapters[] = [
'name' => $A->text(),
'uri' => $A->attr('href'),
];
});
}
}
return $this->arrLatestChapters;
}
public function getChapters(): array
{
if ($this->arrChapters === NULL) {
$this->arrChapters = [];
if ($this->isMobile()) {
$strClass = '.chapter-list';
} else {
$strClass = '.section-list';
}
$intDivIndex = 0;
if ($this->arrData['page'] == 0) {
$intDivIndex = 1;
}
$this->getQueryList()->find($strClass)->eq($intDivIndex)->find("a")->map(function ($A) {
$this->arrChapters[] = [
'name' => $A->text(),
'uri' => $A->attr('href'),
];
});
}
return $this->arrChapters;
}
protected $arrNextPageArgs = NULL;
/**
* Undocumented function
*
* @return array
*/
public function getNextPageArgs(): array
{
$this->arrNextPageArgs = [];
if ($this->arrData['page'] == 0) {
$this->getLatestChaptersList();
$this->getChapters();
$arrLatestChapterFirstBlock = count($this->arrLatestChapters) > 0 ? $this->arrLatestChapters[0] : null;
$arrLatestChapterSecondBlock = count($this->arrChapters) > 0 ? $this->arrChapters[count($this->arrChapters) - 1] : null;
if (
!empty($arrLatestChapterFirstBlock) && !empty($arrLatestChapterSecondBlock)
&& ($arrLatestChapterFirstBlock['uri'] != $arrLatestChapterSecondBlock['uri'])
) {
$this->arrNextPageArgs = [
'n_source_id' => $this->arrData['n_source_id'],
'page' => $this->arrData['page'] + 1,
];
}
} else {
$strLastUri = $this->getQueryList()->find('select:first option:last')->attr('value');
if ($strLastUri != '' && $strLastUri != $this->strUri) {
$this->arrNextPageArgs = [
'n_source_id' => $this->arrData['n_source_id'],
'page' => $this->arrData['page'] + 1,
];
}
}
return $this->arrNextPageArgs;
}
}

257
code/app/task/crawler/zw630/page/Site.php Normal file
View File

@@ -0,0 +1,257 @@
<?php
declare(strict_types=1);
namespace app\task\crawler\zw630\page;
use GuzzleHttp\Client;
use GuzzleHttp\Promise;
use storage\StorageCore;
/**
* @mixin think\Model
*/
class Site
{
/**
* Undocumented variable
*
* @var Client
*/
private $Client;
/**
* Undocumented variable
*
* @var string
*/
protected $strDomain = 'https://www.630zw.org';
/**
* Undocumented function
*/
public function __construct()
{
// $this->Client = new Client([
// 'http_version' => '1.1', // 强制用 HTTP/1.1
// 'base_uri' => $this->strDomain,
// 'http_errors' => false,
// // 'version' => 2.0,
// 'timeout' => 160,
// 'connect_timeout' => 30,
// 'curl' => [
// // CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_2_0,
// CURLOPT_TCP_KEEPALIVE => 1,
// CURLOPT_SSL_VERIFYPEER => true,
// ],
// 'headers' => [
// 'User-Agent' => 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Mobile Safari/537.36 Edg/134.0.0.0',
// 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
// 'Accept-Language' => 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
// 'Referer' => $this->strDomain,
// 'sec-ch-ua-mobile' => '?1',
// 'sec-ch-ua-platform' => '"Android"',
// 'sec-fetch-dest' => 'document',
// 'sec-fetch-mode' => 'navigate',
// 'sec-fetch-site' => 'same-origin',
// 'sec-fetch-user' => '?1',
// 'upgrade-insecure-requests' => '1',
// ],
// ]);
$this->Client = new Client([
'http_version' => '1.1', // 强制用 HTTP/1.1，已有，保持
'base_uri' => $this->strDomain, // 基础域名，已有，保持
'http_errors' => false, // 不抛 HTTP 错误，已有，保持
'timeout' => 60, // 从 160 秒改为 60 秒
'connect_timeout' => 10, // 从 30 秒改为 10 秒
'curl' => [
CURLOPT_TCP_KEEPALIVE => 1, // 启用 TCP Keep-Alive，已有，保持
CURLOPT_TCP_KEEPIDLE => 120, // Keep-Alive 空闲时间
CURLOPT_TCP_KEEPINTVL => 60, // Keep-Alive 探测间隔
CURLOPT_SSL_VERIFYPEER => true, // 验证 SSL，已有，保持
CURLOPT_SSL_VERIFYHOST => 2, // 确保主机名匹配
],
'headers' => [
// Mobile
// 'User-Agent' => 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Mobile Safari/537.36 Edg/134.0.0.0',
// 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
//PC
'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36 Edg/134.0.0.0',
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'Accept-Language' => 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
'Referer' => $this->strDomain,
'Connection' => 'keep-alive', // 显式启用 Keep-Alive
// Mobile
// 'sec-ch-ua-mobile' => '?1',
// 'sec-ch-ua-platform' => '"Android"',
// 'sec-fetch-dest' => 'document',
// 'sec-fetch-mode' => 'navigate',
// 'sec-fetch-site' => 'same-origin',
// 'sec-fetch-user' => '?1',
// PC
'sec-ch-ua' => '"Chromium";v="134", "Not:A-Brand";v="24", "Microsoft Edge";v="134"',
'sec-ch-ua-mobile' => '?0',
'sec-ch-ua-platform' => '"Windows"',
'sec-fetch-dest' => 'document',
'sec-fetch-mode' => 'navigate',
'sec-fetch-site' => 'same-origin',
'sec-fetch-user' => '?1',
'upgrade-insecure-requests' => '1',
],
'pool_size' => 40, // 并发连接池大小（视需求调整）
]);
}
/**
* Undocumented function
*
* @return Client
*/
public function getClient(): Client
{
return $this->Client;
}
/**
* Undocumented function
*
* @param string $strUri
* @return NovelPage
*/
public function getNovelPage(string $strUri): NovelPage
{
return new NovelPage($this, $strUri);
}
/**
* Undocumented function
*
* @param string $strUri
* @return NovelPage[]
*/
public function getNovelPageList(array $arrFilter): array
{
/** @var NovelPage[] */
$arrNovelPage = [];
foreach ($arrFilter as $intKey => $arrItem) {
if ($arrItem['page'] == 0) {
$arrItem['uri'] = sprintf('/shu/%s.html', $arrItem['n_source_id']);
} else {
$arrItem['uri'] = sprintf('/shu/%s_%s.html', $arrItem['n_source_id'], $arrItem['page']);
}
$arrFilter[$intKey]['uri'] = $arrItem['uri'];
$arrPromises[$intKey] = $this->getClient()->getAsync($arrItem['uri']);
}
$arrResults = Promise\Utils::settle($arrPromises)->wait();
$arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
return $arrResult['state'] === 'fulfilled';
});
foreach ($arrSuccessfulResults as $intKey => $arrResult) {
$Response = $arrResult['value'];
$strContent = $Response->getBody()->getContents();
if ($strContent == '索引文件不存在！') {
continue;
}
$NovelPage = new NovelPage($this, $arrItem['uri']);
$NovelPage->setContent($strContent);
$NovelPage->setData($arrFilter[$intKey]);
$arrNovelPage[] = $NovelPage;
}
return $arrNovelPage;
}
/**
* Undocumented function
*
* @param string $strUri
* @return ChapterPage[]
*/
public function getChapterPageList(array $arrFilter): array
{
/** @var ChapterPage[] */
$arrChapterPage = [];
$arrPromises = [];
foreach ($arrFilter as $intKey => $arrItem) {
$arrPromises[$intKey] = $this->getClient()->getAsync($arrItem['uri']);
}
$arrResults = Promise\Utils::settle($arrPromises)->wait();
$arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
return $arrResult['state'] === 'fulfilled';
});
foreach ($arrSuccessfulResults as $intKey => $arrResult) {
$Response = $arrResult['value'];
$strContent = $Response->getBody()->getContents();;
$ChapterPage = new ChapterPage($this, $arrItem['uri']);
$ChapterPage->setContent($strContent);
$ChapterPage->setData($arrFilter[$intKey]);
$arrChapterPage[] = $ChapterPage;
}
return $arrChapterPage;
}
/**
* Undocumented function
*
* @param array $arrFilter
* @return void
*/
public function downRemoteImgToLocal(array $arrFilter)
{
$StorageCore = StorageCore::getInstance();
$arrPromises = [];
foreach ($arrFilter as $intKey => $arrItem) {
if (!$StorageCore->has($arrItem['local_path'])) {
$arrPromises[$intKey] = $this->getClient()->getAsync($arrItem['uri']);
}
}
$arrResults = Promise\Utils::settle($arrPromises)->wait();
$arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
return $arrResult['state'] === 'fulfilled';
});
foreach ($arrSuccessfulResults as $intKey => $arrResult) {
$Response = $arrResult['value'];
// $strContent = $Response->getBody()->getContents();
$StorageCore->writeStream($arrFilter[$intKey]['local_path'], $Response->getBody()->detach());;
}
}
}

4
code/app/task/module/Listen.php
View File

@@ -4,8 +4,8 @@ declare(strict_types=1);
namespace app\task\module;
use minserver\ProcessSanitizer;
use minserver\QueueManage;
use microserver\ProcessSanitizer;
use microserver\QueueManage;
class Listen
{

8
code/app/task/module/PlanTask.php
View File

@@ -5,9 +5,9 @@ declare(strict_types=1);
namespace app\task\module;
use app\model\PlanTaskModel;
use app\task\collection\zw630\Zw630Col;
use minserver\QueueManage;
use minserver\ProcessSanitizer;
use app\task\crawler\zw630\Scheduler as Zw630Scheduler;
use microserver\QueueManage;
use microserver\ProcessSanitizer;
class PlanTask
@@ -41,7 +41,7 @@ class PlanTask
public static function pull630ZwNovel()
{
$arrTask = [
'callback' => [Zw630Col::class, 'pull630ZwNovel'],
'callback' => [Zw630Scheduler::class, 'crawlFull'],
'data' => []
];