admin/SEONexus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

debug

Browse Source
This commit is contained in:
Default
2025-03-15 20:12:21 +08:00
parent 105ffe11c7
commit eb9035a7ae
134 changed files with 15450 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files

850
code/app/task/collection/novel/Zw630XiaoShuoCol.php Normal file
View File

@@ -0,0 +1,850 @@
<?php
declare(strict_types=1);
namespace app\task\collection\novel;
use app\admin\model\XiaoShuoFenLeiModel;
use app\admin\model\XiaoShuoXiangQingModel;
use app\admin\model\XiaoShuoZhangJieModel;
use app\admin\model\NovelInfoSeoModel;
use app\task\collection\PullDataColBase;
use GuzzleHttp\Cookie\CookieJar;
use processor\ImageProcessor;
use think\facade\Cache;
use QL\QueryList;
use app\common\Base64Helper as qsbs;
/**
* @mixin think\Model https://www.630zw.org/
*/
class Zw630XiaoShuoCol extends PullDataColBase
{
public function pullXiaoShuo()
{
// $decoded = '美女';
// $encoded = qsbs::encode($decoded);
// $decodedString = qsbs::decode('5oC75LmL77yM57uT5bGA5piv5aW955qE77yM5amJ5YS/5bqU6K+l5LiN5Lya5Zyo5LmO6L+Z5Lqb57uG6IqC44CCPC9wPg==');
// var_dump($encoded);
// var_dump($decodedString);
// 首次抓取，全部抓取
if (config('task.task_is_first')) {
$TaskCore = $this->TaskCore;
// 527797
for ($i = config('task.task_start_number'); $i <= config('task.task_end_number'); $i++) {
//echo "当前值: $i\n";
$arrTask = [
'callback' => [self::class, 'getNovelInfo'],
'data' => [
'xs_source_id' => $i, //528049
'xs_source_seo_id' => 1,
]
];
$TaskCore->set($arrTask);
// var_dump($arrTask);
}
}else{
$this->pullFenlei();
}
}
public function pullFenlei()
{
$TaskCore = $this->TaskCore;
for ($intFenleiId = 1; $intFenleiId <= 9; $intFenleiId++) {
for ($intPage = 1; $intPage <= 10; $intPage++) {
// https://www.630zw.org/list/1/10.html
$arrTask = [
'callback' => [self::class, 'getFenLei'],
'data' => [
'xs_fenlei_url' => 'https://www.630zw.org/list/'.$intFenleiId.'/'.$intPage.'.html',
]
];
$TaskCore->set($arrTask);
// var_dump($arrTask);
}
}
}
public function getFenLei($arrData)
{
$TaskCore = $this->TaskCore;
$Client = $this->Client;
$strSiteUrl = $this->get630zwDomain();
$strUrl = $arrData['xs_fenlei_url']; //1 ;
var_dump($strUrl);
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
$QueryList = QueryList::html($strResult);
$arrNovel = [];
// 检查是否找到元素
if ($QueryList->find('.layout2 .txt-list-row5')->eq(0)->find('li')->count() > 0) {
$QueryList->find('.layout2 .txt-list-row5')->eq(0)->find('li')->map(function ($Novel) use (&$arrNovel) {
$strNovelHref = $Novel->find('.s2 a')->attr('href') ;
$intNovelId = extractFilename($strNovelHref);
var_dump($strNovelHref);
$arrNovel[] = [
'xs_source_id' => $intNovelId, //528049
'xs_source_seo_id' => 1,
];
});
} else {
echo 'qie h5' . PHP_EOL;
if ($QueryList->find('.sort_list')->eq(0)->find('li')->count() > 0) {
$QueryList->find('.sort_list')->eq(0)->find('li')->map(function ($Novel) use (&$arrNovel) {
$strNovelHref = $Novel->find('.s2 a')->attr('href') ;
$intNovelId = extractFilename($strNovelHref);
var_dump($strNovelHref);
$arrNovel[] = [
'xs_source_id' => $intNovelId, //528049
'xs_source_seo_id' => 1,
];
});
}
}
foreach ($arrNovel as $intNum => $arrNovelOne) {
$arrTask = [
'callback' => [self::class, 'getNovelInfo'],
'data' => $arrNovelOne,
];
$TaskCore->set($arrTask);
}
} catch (\Throwable $T) {
$arrTask = [
'callback' => [self::class, 'getFenLei'],
'data' => [
'xs_fenlei_url' => $strUrl,
]
];
$TaskCore->set($arrTask);
throw $T;
}
}
public function getNovelInfo($arrData)
{
$TaskCore = $this->TaskCore;
$Client = $this->Client;
$strSiteUrl = $this->get630zwDomain();
$strUrl = 'https://630zw.org/shu/' . $arrData['xs_source_id'] . '.html'; //'https://630zw.org/shu/93716.html';
$intXiaoShuoSourceId = $arrData['xs_source_id']; //20053;
$intXiaoShuoSourceSeoId = $arrData['xs_source_seo_id']; //1 ;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
$QueryList = QueryList::html($strResult);
// var_dump($QueryList);
// 提取所有 <meta> 标签的属性
$result = $QueryList->find('meta[property^="og:"]')->map(function ($item) {
return [
'property' => $item->attr('property'),
'content' => $item->attr('content'),
];
});
// 打印结果
// print_r($result->all());
$strZuoZhe = '';
$strMingZi = '';
$strLastDate = '';
$strJieShao = '';
$strCover = '';
$strFenlei = '';
$strZiShu = 0;
$strZhuangTai = '';
foreach ($result as $item) {
switch ($item['property']) {
case 'og:title':
$strMingZi = $item['content'];
break;
case 'og:description':
$strJieShao = preg_split('/<br\s*\/?>/i', $item['content'])[0];
break;
case 'og:image':
$strCover = $item['content'];
break;
case 'og:novel:category':
$strFenlei = $item['content'];
break;
case 'og:novel:author':
$strZuoZhe = $item['content'];
break;
case 'og:novel:status':
$strZhuangTai = $item['content'];
break;
case 'og:novel:update_time':
$strLastDate = $item['content'];
break;
}
}
$strLastDate = trim((string)$strLastDate);
if (!isValidDateTime($strLastDate)) {
$strLastDate = date('Y-m-d H:i:s');
}
$arrTag = [];
if (strpos($strCover, 'http') === false) {
$strCover = $strSiteUrl . $strCover;
}
$strCoverUri = $strCover;
try {
$strCoverUri = "";
$strCoverUri = ImageProcessor::getInstance()->downloadImage($intXiaoShuoSourceId, $strCover, 'xs/630zw');
} catch (\Throwable $t) {
$strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
echo $strErr;
}
$arrCover = [
'code' => '630_ZW_XS_COVER',
// 'uri' => getUriByUrl($strCover),
'uri' => $strCoverUri,
];
$strFenleiIId = 1;
switch ($strFenlei) {
case '玄幻修真':
$strName = "玄幻";
$strFenleiIId = 1;
break;
case '重生穿越':
$strName = "修真";
$strFenleiIId = 2;
break;
case '都市小说':
$strName = "都市";
$strFenleiIId = 3;
break;
case '军史小说':
$strName = "历史";
$strFenleiIId = 4;
break;
case '网游小说':
$strName = "网游";
$strFenleiIId = 5;
break;
case '科幻小说':
$strName = "科幻";
$strFenleiIId = 6;
break;
case '灵异小说':
$strName = "女频";
$strFenleiIId = 7;
break;
case '言情小说':
$strName = "其它";
$strFenleiIId = 8;
break;
case '其他小说':
$strName = "其它";
$strFenleiIId = 9;
break;
default:
$strName = "其它";
$strFenleiIId = 9;
break;
}
$arrXiaoShuoInfo = [
'xsxq_ming_zi' => $strMingZi,
'xsxq_zuozhe' => $strZuoZhe, //$arrData['xiaoshuo_zuozhe'],
'xsfl_id' => $strFenleiIId,
'xsxq_zhuang_tai' => $strZhuangTai,
'xsxq_zi_shu' => $strZiShu,
'xsxq_jie_shao' => $strJieShao,
'xsxq_feng_mian' => json_encode($arrCover),
'xsxq_geng_xin_shi_jian' => $strLastDate,
'xsxq_tag' => json_encode($arrTag, JSON_UNESCAPED_UNICODE),
'xsxq_source_id' => $intXiaoShuoSourceId,
'xsxq_source_seo_id' => $intXiaoShuoSourceId . '1',
'xsxq_source_code' => 'ZW630_XIAOSHUO_' . $intXiaoShuoSourceId . '_' . $intXiaoShuoSourceSeoId,
];
// print_r($arrXiaoShuoInfo);
var_dump($arrXiaoShuoInfo['xsxq_source_code']);
$XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::where('xsxq_source_code', $arrXiaoShuoInfo['xsxq_source_code'])->find();
$arrZhangjie = [];
# seo 小说入库 - 只有新小说才会走这个步骤
if (!$XiaoShuoXiangQingModel) {
$XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
$intXiaoShuolId = $XiaoShuoXiangQingModel->xsxq_id;
// 检查是否找到元素-seo 小说入库
if ($QueryList->find('.first_txt a')->count() > 0 && $intXiaoShuolId) {
$QueryList->find('.first_txt a')->map(function ($ZhangJie) use ($intXiaoShuolId) {
$url = $ZhangJie->attr('href');
// 检查是否以 "/kan/" 开头
if (strpos($url, '/kan/') === 0) { // strpos 返回位置，0 表示是以 "/kan/" 开头
// 处理符合条件的链接
// echo "Processing index : " . $ZhangJie->text() . PHP_EOL;
$arrXiaoShuoSeoInfo = [
'nis_name' => $ZhangJie->text(),
'xsxq_id' => $intXiaoShuolId,
'nis_md5' => md5($intXiaoShuolId . $ZhangJie->text()),
];
NovelInfoSeoModel::addXiaoShuoXiangQingSeo($arrXiaoShuoSeoInfo);
}
});
} else {
echo "No elements found for .first_txt a" . PHP_EOL;
var_dump($strResult);
}
// # 新小说全部章节入库
// $intChapterIndex = 1; // 定义一个计数器变量
// // 检查是否找到元素
// if ($QueryList->find('.section-list')->eq(1)->find('li')->count() > 0) {
// $QueryList->find('.section-list')->eq(1)->find('li')->map(function ($ZhangJie) use (&$arrZhangjie, &$intChapterIndex) {
// echo "Processing index $intChapterIndex: " . $ZhangJie->text() . PHP_EOL;
// $arrZhangjie[] = [
// 'paixu' => $intChapterIndex,
// 'mingzi' => $ZhangJie->find('a')->text(),
// 'url' => $ZhangJie->find('a')->attr('href'),
// ];
// $intChapterIndex++;
// });
// } else {
// echo "No elements found for .listmain a" . PHP_EOL;
// }
//unset($QueryList);
}else{
$intXiaoShuolId = $XiaoShuoXiangQingModel->xsxq_id;
// 获取最大章节索引
$intMaxXiaoShuoZhangJiePaixu = XiaoShuoZhangJieModel::where('xsxq_id', $intXiaoShuolId)
->max('xszj_pai_xu'); // 获取最大值
// var_dump('获取小说章节最大值-'.$intMaxXiaoShuoZhangJiePaixu);
}
# 新小说全部章节入库
$intChapterIndex = 1; // 定义一个计数器变量
// 检查是否找到元素
if ($QueryList->find('.section-list')->eq(1)->find('li')->count() > 0) {
$QueryList->find('.section-list')->eq(1)->find('li')->map(function ($ZhangJie) use (&$arrZhangjie, &$intChapterIndex) {
// echo "Processing index $intChapterIndex: " . $ZhangJie->text() . PHP_EOL;
$arrZhangjie[] = [
'paixu' => $intChapterIndex,
'mingzi' => $ZhangJie->find('a')->text(),
'url' => $ZhangJie->find('a')->attr('href'),
];
$intChapterIndex++;
});
$this->eachNovelReader($arrZhangjie,$strSiteUrl,$intXiaoShuolId,$strFenleiIId);
} else {
echo "No elements found for .listmain a" . PHP_EOL;
}
# 章节处理
# 新小说全部章节入库
# 旧小说，只更新最新章节，获取数据库最大章节索引，对比源总章节
// // 如果没有结果，max 返回 null，可以设置默认值
$intMaxXiaoShuoZhangJiePaixu = $intMaxXiaoShuoZhangJiePaixu ?? 0;
# 目录页面处理
$strMuluHref = $QueryList->find('.btn-mulu')->eq(0)->attr('href');
$strMuluHref = $strSiteUrl . $strMuluHref ;
$ResponseMulu = $Client->request('GET', $strMuluHref, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResultMulu = (string)$ResponseMulu->getBody();
$QueryListMulu = QueryList::html($strResultMulu);
$intMuluTotal = $QueryListMulu->find('.page_num select')->eq(0)->find('option')->count();
$strMuluEndHref = $QueryListMulu->find('.page_num select')->eq(0)->find('option')->eq($intMuluTotal-1)->attr('value');
$strMuluEndText = $QueryListMulu->find('.page_num select')->eq(0)->find('option')->eq($intMuluTotal-1)->text();
// 使用正则表达式匹配
preg_match_all('/\.section-list\.ycxsid>li:nth-child\(/', $strResultMulu, $matches);
preg_match_all('/\.section-list\.ycxsid>li:nth-last-child\(/', $strResultMulu, $matchesLast);
$intProLiNum = count($matches[0]);
$intLastLiNum = count($matchesLast[0]);
$arrZhangjieMulu = [];
//if($intMaxXiaoShuoZhangJiePaixu < 200){
$intChapterPageIndex = 101; // 定义一个计数器变量
// 检查是否找到元素
if ($QueryListMulu->find('.chapter-list')->eq(0)->find('li')->count() > 0) {
$QueryListMulu->find('.chapter-list')->eq(0)->find('li')->map(function ($ZhangJie) use (&$arrZhangjieMulu, &$intChapterPageIndex) {
$arrZhangjieMulu[] = [
'paixu' => $intChapterPageIndex,
'mingzi' => $ZhangJie->find('a')->text(),
'url' => $ZhangJie->find('a')->attr('href'),
];
$intChapterPageIndex ++ ;
});
$arrZhangjieMulu = array_slice($arrZhangjieMulu, $intProLiNum);
// 删除后 11 条数据
$arrZhangjieMulu = array_slice($arrZhangjieMulu, 0, count($arrZhangjieMulu) - $intLastLiNum);
$this->eachNovelReader($arrZhangjieMulu,$strSiteUrl,$intXiaoShuolId,$strFenleiIId);
var_dump("chapter章节任务投递成功 $strMuluHref");
} else {
// var_dump($QueryListMulu->find('.section-list')->eq(0)->find('li')->count());
//echo "No elements found for .chapter-list $strMuluHref" . PHP_EOL;
// 检查是否找到元素
$intListIndex = 1;
if ($QueryListMulu->find('.section-list')->eq(0)->find('li')->count() > 0) {
$QueryListMulu->find('.section-list')->eq(0)->find('li')->map(function ($ZhangJie) use (&$arrZhangjieMulu, &$intChapterPageIndex,&$intListIndex) {
if($intListIndex >14){
$arrZhangjieMulu[] = [
'paixu' => $intChapterPageIndex,
'mingzi' => $ZhangJie->find('a')->text(),
'url' => $ZhangJie->find('a')->attr('href'),
];
$intChapterPageIndex ++ ;
};
$intListIndex++ ;
});
$arrZhangjieMulu = array_slice($arrZhangjieMulu, $intProLiNum);
// 删除后 11 条数据
$arrZhangjieMulu = array_slice($arrZhangjieMulu, 0, count($arrZhangjieMulu) - $intLastLiNum);
$this->eachNovelReader($arrZhangjieMulu,$strSiteUrl,$intXiaoShuolId,$strFenleiIId);
var_dump("section章节任务投递成功 $strMuluHref");
} else {
var_dump("No elements found for .section-list $strMuluHref") ;
}
}
//}
// 输出最终结果
// var_dump(count($arrZhangjie));
// var_dump($QueryListMulu->find('.chapter-list li')->count());
// var_dump($intMuluTotal);
// var_dump($strMuluEndHref);
// var_dump($strMuluEndText);
// var_dump($intMaxXiaoShuoZhangJiePaixu); // 1010
preg_match('/(\d+)-(\d+)章/', $strMuluEndText, $matches);
$intMaxPaixu = $matches[2]; // 最大章节
$intMaxMuluPage = floor($intMaxPaixu / 100); // 最大目录页面
// $intMaxMuluPage = 2;
// 所有目录分页处理
for ($i = 2; $i <= $intMaxMuluPage; $i++) {
//echo "当前值: $i\n";
$arrTask = [
'callback' => [self::class, 'getNovelMulu'],
'data' => [
'strMuluHref' => $strSiteUrl . '/shu/' . $XiaoShuoXiangQingModel->xsxq_source_id . '_' . $i . '.html' ,
'xsxq_id' => $XiaoShuoXiangQingModel->xsxq_id,
'xsfl_id' => $strFenleiIId,
'intStartIndex' => $i*100 + 1,
]
];
$TaskCore->set($arrTask);
// var_dump($arrTask);
}
unset($QueryList);
unset($QueryListMulu);
} catch (\Throwable $T) {
$arrTask = [
'callback' => [self::class, 'getNovelInfo'],
'data' => [
'xs_source_id' => $arrData['xs_source_id'],
'xs_source_seo_id' => $intXiaoShuoSourceSeoId,
]
];
$TaskCore->set($arrTask);
throw $T;
}
}
public function getNovelMulu($arrData)
{
$TaskCore = $this->TaskCore;
$Client = $this->Client;
$strSiteUrl = $this->get630zwDomain();
$strUrl = $arrData['strMuluHref'];
$intXiaoShuoId = $arrData['xsxq_id'];
$intXiaoShuoFenleiId = $arrData['xsfl_id'];
$intStartIndex = $arrData['intStartIndex'];
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
$QueryList = QueryList::html($strResult);
// 使用正则表达式匹配
preg_match_all('/\.section-list\.ycxsid>li:nth-child\(/', $strResult, $matches);
preg_match_all('/\.section-list\.ycxsid>li:nth-last-child\(/', $strResult, $matchesLast);
$intProLiNum = count($matches[0]);
$intLastLiNum = count($matchesLast[0]);
$arrZhangjie = [];
if ($QueryList->find('.chapter-list')->eq(0)->find('li')->count() > 0) {
$QueryList->find('.chapter-list')->eq(0)->find('li')->map(function ($ZhangJie) use (&$arrZhangjie, &$intStartIndex) {
$arrZhangjie[] = [
'paixu' => $intStartIndex,
'mingzi' => $ZhangJie->find('a')->text(),
'url' => $ZhangJie->find('a')->attr('href'),
];
$intStartIndex ++ ;
// var_dump($intStartIndex);
});
$arrZhangjie = array_slice($arrZhangjie, $intProLiNum);
// 删除后 11 条数据
$arrZhangjie = array_slice($arrZhangjie, 0, count($arrZhangjie) - $intLastLiNum);
$this->eachNovelReader($arrZhangjie,$strSiteUrl,$intXiaoShuoId,$intXiaoShuoFenleiId);
var_dump("chapter章节任务投递成功 $strUrl");
} else {
// 检查是否找到元素
if ($QueryList->find('.section-list')->eq(0)->find('li')->count() > 0) {
$QueryList->find('.section-list')->eq(0)->find('li')->map(function ($ZhangJie) use (&$arrZhangjie, &$intStartIndex) {
$arrZhangjie[] = [
'paixu' => $intStartIndex,
'mingzi' => $ZhangJie->find('a')->text(),
'url' => $ZhangJie->find('a')->attr('href'),
];
$intStartIndex ++ ;
// var_dump($intStartIndex);
});
$arrZhangjie = array_slice($arrZhangjie, $intProLiNum);
// 删除后 11 条数据
$arrZhangjie = array_slice($arrZhangjie, 0, count($arrZhangjie) - $intLastLiNum);
$this->eachNovelReader($arrZhangjie,$strSiteUrl,$intXiaoShuoId,$intXiaoShuoFenleiId);
var_dump("section章节任务投递成功 $strUrl");
} else {
var_dump("No elements found for .section-list $strUrl") ;
}
}
unset($QueryList);
} catch (\Throwable $T) {
$arrTask = [
'callback' => [self::class, 'getNovelMulu'],
'data' => [
'strMuluHref' => $strUrl ,
'xsxq_id' => $intXiaoShuoId,
'xsfl_id' => $intXiaoShuoFenleiId,
'intStartIndex' => $intStartIndex,
]
];
$TaskCore->set($arrTask);
throw $T;
}
}
public function eachNovelReader($arrZhangjie,$strSiteUrl,$intXiaoShuoId,$intXiaoShuoFenleiId)
{
$TaskCore = $this->TaskCore;
// var_dump($arrZhangjie);
foreach ($arrZhangjie as $intZhangJieNum => $arrZhangjieOne) {
// 如果数据库里的章节最大索引大于当前索引，说明已经入库，不需要入队列了
// if($intMaxXiaoShuoZhangJiePaixu > $intZhangJieNum){
// var_dump("小说章节已经入库，不再推送任务");
// continue;// 如果任务已存在，直接跳过 break
// }
$strZhangJieUrl = 'https://www.630zw.org' . $arrZhangjieOne['url'];
$strZhangJieSourceId = extractFilename($strZhangJieUrl);
# 'AILISI_XIAOSHUO_ZHANGJIE_小说源id_小说源章节id'
$strXszjSourceCode = '630ZW_XIAOSHUO_ZHANGJIE_'. $intXiaoShuoId . '_' . $strZhangJieSourceId . '_1' ;
$arrTask = [
'callback' => [self::class, 'getNovelReader'],
'data' => [
'xszj_source_id' => $strZhangJieSourceId,
'xszj_paixu' => $arrZhangjieOne['paixu'],
'xszj_name' => $arrZhangjieOne['mingzi'],
'xszj_url' => $strZhangJieUrl,
'xszj_source_code' => $strXszjSourceCode,
'xsxq_id' => $intXiaoShuoId,
'xsfl_id' => $intXiaoShuoFenleiId,
'intZhangJieSort' => 1,
]
];
$TaskCore->set($arrTask);
}
}
public function getNovelFengMian($arrData)
{
//var_dump('getNovelFengMian');
//var_dump($arrData);
$Client = $this->Client;
$TaskCore = $this->TaskCore;
// 获取 Redis 实例
$redis = Cache::store('redis')->handler();
$intXiaoShuoSourceId = $arrData['xiaoshuo_id'];
$intXiaoShuoSourceSeoId = $arrData['xiaoshuo_seo_id'];
$redisKey = "task:novel_source_id:" . $intXiaoShuoSourceId;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($arrData['xiaoshuo_url']);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$strSiteUrl = $this->getAilisiDomain();
$strCover = $QueryList->find('#fmimg img')->eq(0)->attr("src");
if (strpos($strCover, 'http') === false) {
$strCover = $strSiteUrl . $strCover;
}
$strCoverUri = $strCover;
try {
$strCoverUri = "";
$strCoverUri = ImageProcessor::getInstance()->downloadAndEncryptImage($strCover, 'xs');
$arrCover = [
'code' => 'AI_LI_SI_COVER',
// 'uri' => getUriByUrl($strCover),
'uri' => $strCoverUri,
];
$arrXiaoShuoInfo = [
'xsxq_feng_mian' => json_encode($arrCover),
];
$XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
} catch (\Throwable $t) {
$strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
echo $strErr;
}
} catch (\Throwable $T) {
throw $T;
}
}
//https://www.630zw.org/shu/438374/156339465.html
public function getNovelReader($arrData)
{
$TaskCore = $this->TaskCore;
$strSiteUrl = 'https://www.630zw.org';
var_dump('getNovelReader------------------------------------------------------------------------------------');
$Client = $this->Client;
$intZhangJieSort = $arrData['intZhangJieSort'];
$strXszjSourceCode = $arrData['xszj_source_code'];
$strZhangJieUrl = $arrData['xszj_url'];
$intZhangJiePaiXu = $arrData['xszj_paixu'];
$intFenleiId = $arrData['xsfl_id'];
$intXiaoShuoId = $arrData['xsxq_id'];
$strZhangJieSourceId = $arrData['xszj_source_id'];
$strZhangJieName = $arrData['xszj_name'];
$XiaoShuoZhangJieModel = XiaoShuoZhangJieModel::where('xszj_source_code', $strXszjSourceCode)->find();
if ($XiaoShuoZhangJieModel) {
var_dump($XiaoShuoZhangJieModel->xszj_source_code);
var_dump('章节已经存在 paixu: '.$intZhangJiePaiXu);
var_dump('章节已经存在strXszjSourceCode: '.$strXszjSourceCode);
var_dump('章节已经存在strZhangJieUrl: '.$strZhangJieUrl);
// return true;
}
var_dump($strZhangJieUrl);
var_dump('章节排序：'.$intZhangJiePaiXu);
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strZhangJieUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
if (XiaoShuoZhangJieModel::checkZhangJieExists($strXszjSourceCode)) {
return true;
}
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strZhangJieUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
unset($Response);
$ZhangJieQueryList = QueryList::html($strResult);
// 正则表达式，匹配 `_数字.html`
$pattern = '/_(\d+)\.html$/';
$patternHref = '/(\/shu\/\d+\/\d+_\d+\.html)/';
$strNeiRong = '';
$strNextHref = '';
// 使用 preg_match() 进行匹配
if (preg_match($patternHref, $strResult, $matches)) {
// 提取匹配到的部分
echo "提取到的 URL 是: " . $matches[1];
$strNextHref = $matches[1];
} else {
echo "没有匹配到 URL";
}
preg_match_all("/qsbs\.bb\('([^']+)'\)/", $strResult, $matches);
// 输出提取到的加密字符串
if (isset($matches[1])) {
$strNeiRong = implode(';;;', $matches[1]);
// var_dump($strNeiRong);
}
// 判断字符串是否匹配
if (preg_match($pattern, $strNextHref, $matches) ){
$intNextPage = 2;
// 确保匹配成功后再访问索引
if (isset($matches[1])) {
$intNextPage = (int)$matches[1];
$intNextPage +=1;
var_dump('提取到的数字是：' . $intNextPage);
}
var_dump('提取到的数字是-：' . $intNextPage);
$arrTask = [
'callback' => [self::class, 'getNovelReader'],
'data' => [
'xszj_source_id' => $strZhangJieSourceId,
'xszj_paixu' => $intZhangJiePaiXu,
'xszj_name' => $strZhangJieName,
'xszj_url' => $strSiteUrl . $strNextHref,
'xszj_source_code' => '630ZW_XIAOSHUO_ZHANGJIE_'. $intXiaoShuoId . '_' . $strZhangJieSourceId . '_' . $intNextPage,
'xsxq_id' => $intXiaoShuoId,
'xsfl_id' => $intFenleiId,
'intZhangJieSort' => $intNextPage,
]
];
$TaskCore->set($arrTask);
}
# '/novel/123yqw/分类id/小说源id/小说源章节id.txt'
$strNeiRongTxtPath = config("filesystem.novel") . '/novel/630zwxs/' . $intFenleiId . '/' . $intXiaoShuoId .'/' . $intZhangJiePaiXu .'/' . $intZhangJieSort . '.txt';
//$strNeiRongTxtPath = "/www/novel/123yqw/1/11/111.txt";
// 调用函数
$result = saveCompressAndDeleteTxt($strNeiRongTxtPath, $strNeiRong);
// 输出状态
if ($result['status']) {
var_dump("Success: " . $result['message']);
} else {
var_dump("Error: " . $result['message']);
}
$arrXiaoShuoZhangJie = [
'xsxq_id' => $intXiaoShuoId,
'xszj_pai_xu' => $intZhangJiePaiXu,
'xszj_ming_zi' => $strZhangJieName,
'xszj_nei_rong' => $strNeiRongTxtPath,
'xszj_source_id' => $strZhangJieSourceId,
'xszj_source_code' => $strXszjSourceCode,
'xszj_sort' => $intZhangJieSort,
];
// var_dump($arrXiaoShuoZhangJie);
if($strZhangJieName){
XiaoShuoZhangJieModel::addXiaoShuoZhangJie($arrXiaoShuoZhangJie);
}
unset($ZhangJieQueryList);
// // 延时 3 秒
//sleep(3);
} catch (\Throwable $T) {
$arrTask = [
'callback' => [self::class, 'getNovelReader'],
'data' => [
'xszj_source_id' => $strZhangJieSourceId,
'xszj_paixu' => $intZhangJiePaiXu,
'xszj_name' => $strZhangJieName,
'xszj_url' => $strZhangJieUrl,
'xszj_source_code' => $strXszjSourceCode,
'xsxq_id' => $intXiaoShuoId,
'xsfl_id' => $intFenleiId,
'intZhangJieSort' => $intZhangJieSort,
]
];
$TaskCore->set($arrTask);
throw $T;
}
}
}