admin/SEONexus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

debug

Browse Source
This commit is contained in:
Default
2025-03-20 21:10:55 +08:00
parent 62d2e78347
commit b09e4fca11
41 changed files with 1369 additions and 6258 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
code/app/task/collection/PullDataColBase.php
View File

@@ -6,7 +6,7 @@ namespace app\task\collection;
use app\model\CaiJiPeiZhiModel;
use app\model\SystemConfigModel;
use app\task\core\TaskCore;
use minserver\QueueManage;
use GuzzleHttp\Client;
/**
@@ -24,9 +24,9 @@ class PullDataColBase
/**
* Undocumented variable
*
* @var \app\task\core\TaskCore
* @var \minserver\QueueManage
*/
protected $TaskCore = NULL;
protected $QueueManage = NULL;
protected $arrCaiJiPeiZhi = [];
@@ -34,7 +34,7 @@ class PullDataColBase
{
$this->Client = new Client();
$this->TaskCore = new TaskCore();
$this->QueueManage = new QueueManage();
}

427
code/app/task/collection/biquxs/BiquxsXiaoShuoCol.php
View File

@@ -1,427 +0,0 @@
<?php
declare(strict_types=1);
namespace app\task\collection\biquxs;
use app\model\XiaoShuoFenLeiModel;
use app\model\XiaoShuoXiangQingModel;
use app\model\XiaoShuoZhangJieModel;
use app\task\collection\PullDataColBase;
use GuzzleHttp\Cookie\CookieJar;
use processor\ImageProcessor;
use think\facade\Cache;
use QL\QueryList;
/**
* @mixin think\Model
*/
class BiquxsXiaoShuoCol extends PullDataColBase
{
public function pullXiaoShuo()
{
$TaskCore = $this->TaskCore;
// 20054
for ($i = 1; $i <= 20054; $i++) {
echo "当前值: $i\n";
$arrTask = [
'callback' => [self::class, 'getNovelInfo'],
'data' => [
'xs_source_id' => $i,
'xs_source_seo_id' => 1,
]
];
$TaskCore->set($arrTask);
var_dump($arrTask);
}
}
public function getNovelInfo($arrData)
{
$TaskCore = $this->TaskCore;
$Client = $this->Client;
$strSiteUrl = $this->getBiquxsDomain();
$strUrl = 'http://www.biquxs.com/book/' . $arrData['xs_source_id'] ; //'http://www.biquxs.com/book/20053';
$intXiaoShuoSourceId = $arrData['xs_source_id']; //20053;
$intXiaoShuoSourceSeoId = $arrData['xs_source_seo_id']; //1 ;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
$QueryList = QueryList::html($strResult);
// var_dump($QueryList);
// 提取所有 <meta> 标签的属性
$result = $QueryList->find('meta[property^="og:"]')->map(function ($item) {
return [
'property' => $item->attr('property'),
'content' => $item->attr('content'),
];
});
// 打印结果
// print_r($result->all());
$strZuoZhe = '';
$strMingZi = '';
$strLastDate = '';
$strJieShao = '';
$strCover = '';
$strFenlei = '';
$strZiShu = 0;
$strZhuangTai = $QueryList->find('#info p')->eq(1)->text();
$strZhuangTai = getStringAfterWord($strZhuangTai, '状态：');
foreach ($result as $item) {
switch ($item['property']) {
case 'og:title':
$strMingZi = $item['content'];
break;
case 'og:description':
$strJieShao = $item['content'];
break;
case 'og:image':
$strCover = $item['content'];
break;
case 'og:book:category':
$strFenlei = $item['content'];
break;
case 'og:book:author':
$strZuoZhe = $item['content'];
break;
case 'og:book:update_time':
$strLastDate = $item['content'];
break;
}
}
$strLastDate = trim((string)$strLastDate);
if (!isValidDateTime($strLastDate)) {
$strLastDate = date('Y-m-d H:i:s');
}
$arrTag = [];
if (strpos($strCover, 'http') === false) {
$strCover = $strSiteUrl . $strCover;
}
$strCoverUri = $strCover;
try {
$strCoverUri = "";
$strCoverUri = ImageProcessor::getInstance()->downloadImage( $intXiaoShuoSourceId,$strCover, 'xs');
} catch (\Throwable $t) {
$strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
echo $strErr;
}
$arrCover = [
'code' => 'BI_QU_XS_COVER',
// 'uri' => getUriByUrl($strCover),
'uri' => $strCoverUri,
];
$strFenleiIId = 1 ;
switch ($strFenlei) {
case '玄幻小说':
$strName = "玄幻";
$strFenleiIId = 1;
break;
case '修真小说':
$strName = "修真";
$strFenleiIId = 2;
break;
case '都市小说':
$strName = "都市";
$strFenleiIId = 3;
break;
case '历史小说':
$strName = "历史";
$strFenleiIId = 4;
break;
case '网游小说':
$strName = "网游";
$strFenleiIId = 5;
break;
case '科幻小说':
$strName = "科幻";
$strFenleiIId = 6;
break;
case '女频小说':
$strName = "女频";
$strFenleiIId = 7;
break;
case '其它小说':
$strName = "其它";
$strFenleiIId = 8;
break;
default:
$strName = "其它";
$strFenleiIId = 8;
break;
}
$arrXiaoShuoInfo = [
'xsxq_ming_zi' => $strMingZi,
'xsxq_zuozhe' => $strZuoZhe , //$arrData['xiaoshuo_zuozhe'],
'xsfl_id' => $strFenleiIId,
'xsxq_zhuang_tai' => $strZhuangTai,
'xsxq_zi_shu' => $strZiShu,
'xsxq_jie_shao' => $strJieShao,
'xsxq_feng_mian' => json_encode($arrCover),
'xsxq_geng_xin_shi_jian' => $strLastDate,
'xsxq_tag' => json_encode($arrTag, JSON_UNESCAPED_UNICODE),
'xsxq_source_id' => $intXiaoShuoSourceId,
'xsxq_source_seo_id' => $intXiaoShuoSourceId.'1',
'xsxq_source_code' => 'BIQU_XIAOSHUO_' . $intXiaoShuoSourceId . '_' . $intXiaoShuoSourceSeoId,
];
// print_r($arrXiaoShuoInfo);
var_dump($arrXiaoShuoInfo['xsxq_source_code']);
$XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
$arrZhangjie = [];
$intChapterIndex = 0; // 定义一个计数器变量
// 检查是否找到元素
if ($QueryList->find('.listmain a')->count() > 0) {
$QueryList->find('.listmain a')->map(function ($ZhangJie) use (&$arrZhangjie, &$intChapterIndex) {
echo "Processing index $intChapterIndex: " . $ZhangJie->text() . PHP_EOL;
// 从索引6开始
if ($intChapterIndex >= 5) {
$arrZhangjie[] = [
'mingzi' => $ZhangJie->text(),
'url' => $ZhangJie->attr('href'),
];
}
$intChapterIndex++;
});
// 输出最终结果
var_dump($arrZhangjie);
} else {
echo "No elements found for .listmain a" . PHP_EOL;
}
//unset($QueryList);
// 获取最大章节索引
$intMaxXiaoShuoZhangJiePaixu = XiaoShuoZhangJieModel::where('xsxq_id', $intXiaoShuoSourceId)
->max('xszj_pai_xu'); // 获取最大值
var_dump('获取小说章节最大值-'.$intMaxXiaoShuoZhangJiePaixu);
// 如果没有结果，max 返回 null，可以设置默认值
$intMaxXiaoShuoZhangJiePaixu = $intMaxXiaoShuoZhangJiePaixu ?? 0;
foreach ($arrZhangjie as $intZhangJieNum => $arrZhangjieOne) {
var_dump($intZhangJieNum);
// 如果数据库里的章节最大索引大于当前索引，说明已经入库，不需要入队列了
if($intMaxXiaoShuoZhangJiePaixu > $intZhangJieNum){
var_dump("小说章节已经入库，不再推送任务");
continue;// 如果任务已存在，直接跳过 break
}
$strZhangJieUrl = $strSiteUrl . $arrZhangjieOne['url'];
$strZhangJieId = extractFilename($strZhangJieUrl);
# 'AILISI_XIAOSHUO_ZHANGJIE_小说源id_小说源章节id'
$strXszjSourceCode = 'AILISI_XIAOSHUO_ZHANGJIE_'. $XiaoShuoXiangQingModel->xsxq_source_id . '_' . $strZhangJieId;
$arrTask = [
'callback' => [self::class, 'getNovelReader'],
'data' => [
'xszj_source_id' => $strZhangJieId,
'xszj_paixu' => $intZhangJieNum,
'xszj_name' => $arrZhangjieOne['mingzi'],
'xszj_url' => $strZhangJieUrl,
'xszj_source_code' => $strXszjSourceCode,
'xsxq_source_id' => $XiaoShuoXiangQingModel->xsxq_source_id,
'xsfl_id' => $strFenleiIId,
]
];
$TaskCore->set($arrTask);
}
var_dump("章节任务投递成功");
} catch (\Throwable $T) {
throw $T;
}
}
public function getNovelFengMian($arrData)
{
//var_dump('getNovelFengMian');
//var_dump($arrData);
$Client = $this->Client;
$TaskCore = $this->TaskCore;
// 获取 Redis 实例
$redis = Cache::store('redis')->handler();
$intXiaoShuoSourceId = $arrData['xiaoshuo_id'];
$intXiaoShuoSourceSeoId = $arrData['xiaoshuo_seo_id'];
$redisKey = "task:novel_source_id:" . $intXiaoShuoSourceId;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($arrData['xiaoshuo_url']);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$strSiteUrl = $this->getAilisiDomain();
$strCover = $QueryList->find('#fmimg img')->eq(0)->attr("src");
if (strpos($strCover, 'http') === false) {
$strCover = $strSiteUrl . $strCover;
}
$strCoverUri = $strCover;
try {
$strCoverUri = "";
$strCoverUri = ImageProcessor::getInstance()->downloadAndEncryptImage($strCover, 'xs');
$arrCover = [
'code' => 'AI_LI_SI_COVER',
// 'uri' => getUriByUrl($strCover),
'uri' => $strCoverUri,
];
$arrXiaoShuoInfo = [
'xsxq_feng_mian' => json_encode($arrCover),
];
$XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
} catch (\Throwable $t) {
$strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
echo $strErr;
}
} catch (\Throwable $T) {
throw $T;
}
}
public function getNovelReader($arrData)
{
var_dump('getNovelReader');
var_dump($arrData['xszj_url']);
$Client = $this->Client;
$strXszjSourceCode = $arrData['xszj_source_code'];
$strZhangJieUrl = $arrData['xszj_url'];
$intZhangJiePaiXu = $arrData['xszj_paixu'];
$intFenleiId = $arrData['xsfl_id'];
$intXiaoShuoSourceID = $arrData['xsxq_source_id'];
$strZhangJieSourceId = $arrData['xszj_source_id'];
$strZhangJieName = $arrData['xszj_name'];
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strZhangJieUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
if (XiaoShuoZhangJieModel::checkZhangJieExists($strXszjSourceCode)) {
return true;
}
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strZhangJieUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
unset($Response);
$ZhangJieQueryList = QueryList::html($strResult);
// 获取原始 HTML
$strNeiRong = $ZhangJieQueryList->find('#content')->eq(0)->html();
// 解码 HTML 实体
$strNeiRong = html_entity_decode($strNeiRong, ENT_QUOTES, 'UTF-8');
// 去掉 <p> 标签中的 class="content_detail" 属性
$strNeiRong = preg_replace('/<p\s+class="content_detail"/i', '<p', $strNeiRong);
// 清理多余的 HTML 标签
$strNeiRong = strip_tags($strNeiRong, '<p>');
// 替换不间断空格 (\xc2\xa0) 和 &nbsp; 为普通空格
$strNeiRong = preg_replace('/\xc2\xa0/', ' ', $strNeiRong); // 替换 UTF-8 不间断空格
$strNeiRong = preg_replace('/&nbsp;/', ' ', $strNeiRong); // 替换 &nbsp;
// 清理多余空格和换行符
$strNeiRong = preg_replace('/\s+/', ' ', $strNeiRong);
$strNeiRong = preg_replace('/<p>\s+/', '<p>', $strNeiRong); // 清理 <p> 标签内部多余空格
$strNeiRong = preg_replace('/\s+<\/p>/', '</p>', $strNeiRong); // 清理 <p> 标签结尾多余空格
// 输出结果
// var_dump($strNeiRong);
# '/novel/123yqw/分类id/小说源id/小说源章节id.txt'
$strNeiRongTxtPath = config("filesystem.novel").'/novel/biquxs/'.$intFenleiId.'/'.$intXiaoShuoSourceID.'/'.$strZhangJieSourceId.'.txt';
//$strNeiRongTxtPath = "/www/novel/123yqw/1/11/111.txt";
// 调用函数
$result = saveCompressAndDeleteTxt($strNeiRongTxtPath, $strNeiRong);
// 输出状态
if ($result['status']) {
var_dump("Success: " . $result['message']) ;
} else {
var_dump("Error: " . $result['message']) ;
}
$arrXiaoShuoZhangJie = [
'xsxq_id' => $intXiaoShuoSourceID,
'xszj_pai_xu' => $intZhangJiePaiXu,
'xszj_ming_zi' => $strZhangJieName,
'xszj_nei_rong' => $strNeiRongTxtPath,
'xszj_source_id' => $strZhangJieSourceId,
'xszj_source_code' => $strXszjSourceCode,
];
XiaoShuoZhangJieModel::addXiaoShuoZhangJie($arrXiaoShuoZhangJie);
unset($ZhangJieQueryList);
// 延时 3 秒
//sleep(3);
} catch (\Throwable $T) {
throw $T;
}
}
}

545
code/app/task/collection/ccc5/XiaoShuo5cccCol.php
View File

@@ -1,545 +0,0 @@
<?php
declare(strict_types=1);
namespace app\task\collection\ccc5;
use app\model\XiaoShuoFenLeiModel;
use app\model\XiaoShuoXiangQingModel;
use app\model\XiaoShuoZhangJieModel;
use app\task\collection\PullDataColBase;
use GuzzleHttp\Cookie\CookieJar;
use processor\ImageProcessor;
use think\facade\Cache;
use QL\QueryList;
/**
* @mixin think\Model
*/
class XiaoShuo5cccCol extends PullDataColBase
{
public function pullXiaoShuo()
{
$TaskCore = $this->TaskCore;
// 20054
for ($i = 3; $i <= 3; $i++) {
echo "当前值: $i\n";
$arrTask = [
'callback' => [self::class, 'getNovelInfo'],
'data' => [
'xs_source_id' => $i,
'xs_source_seo_id' => 1,
]
];
$TaskCore->set($arrTask);
var_dump($arrTask);
}
}
public function getNovelInfo($arrData)
{
$TaskCore = $this->TaskCore;
$Client = $this->Client;
$strSiteUrl = $this->get5cccDomain();
$strUrl = 'https://www.5ccc.org/chapter/' . $arrData['xs_source_id'].'.html' ; //'http://www.biquxs.com/book/20053';
$intXiaoShuoSourceId = $arrData['xs_source_id']; //20053;
$intXiaoShuoSourceSeoId = $arrData['xs_source_seo_id']; //1 ;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
$QueryList = QueryList::html($strResult);
// var_dump($QueryList);
// 提取所有 <meta> 标签的属性
$result = $QueryList->find('meta[property^="og:"]')->map(function ($item) {
return [
'property' => $item->attr('property'),
'content' => $item->attr('content'),
];
});
// 打印结果
print_r($result->all());
$strZuoZhe = '';
$strMingZi = '';
$strLastDate = '';
$strJieShao = '';
$strCover = '';
$strFenlei = '';
$strZiShu = 0;
$strZhuangTai = "";
foreach ($result as $item) {
switch ($item['property']) {
case 'og:title':
$strMingZi = $item['content'];
break;
case 'og:description':
$strJieShao = $item['content'];
break;
case 'og:image':
$strCover = $item['content'];
break;
case 'og:novel:category':
$strFenlei = $item['content'];
break;
case 'og:novel:author':
$strZuoZhe = $item['content'];
break;
case 'og:novel:update_time':
$strLastDate = $item['content'];
break;
case 'og:novel:status':
$strZhuangTai = $item['content'];
break;
}
}
$strLastDate = trim((string)$strLastDate);
if (!isValidDateTime($strLastDate)) {
$strLastDate = date('Y-m-d H:i:s');
}
$arrTag = [];
$strCover = $QueryList->find('.imgbox img')->eq(0)->attr("src");
if (strpos($strCover, 'http') === false) {
$strCover = $strSiteUrl . $strCover;
}
$strCoverUri = $strCover;
try {
$strCoverUri = "";
$strCoverUri = ImageProcessor::getInstance()->downloadImage( $intXiaoShuoSourceId,$strCover, 'xs');
} catch (\Throwable $t) {
$strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
echo $strErr;
}
$arrCover = [
'code' => '5CCC_XS_COVER',
// 'uri' => getUriByUrl($strCover),
'uri' => $strCoverUri,
];
$strFenleiIId = 1 ;
switch ($strFenlei) {
case '玄幻修真':
$strName = "玄幻修真";
$strFenleiIId = 1;
break;
case '重生穿越':
$strName = "重生穿越";
$strFenleiIId = 2;
break;
case '都市小说':
$strName = "都市小说";
$strFenleiIId = 3;
break;
case '军史小说':
$strName = "军史小说";
$strFenleiIId = 4;
break;
case '网游小说':
$strName = "网游小说";
$strFenleiIId = 5;
break;
case '科幻小说':
$strName = "科幻小说";
$strFenleiIId = 6;
break;
case '灵异小说':
$strName = "灵异小说";
$strFenleiIId = 7;
break;
case '言情小说':
$strName = "言情小说";
$strFenleiIId = 8;
break;
case '其他小说':
$strName = "其他小说";
$strFenleiIId = 9;
break;
}
$arrXiaoShuoInfo = [
'xsxq_ming_zi' => $strMingZi,
'xsxq_zuozhe' => $strZuoZhe , //$arrData['xiaoshuo_zuozhe'],
'xsfl_id' => $strFenleiIId,
'xsxq_zhuang_tai' => $strZhuangTai,
'xsxq_zi_shu' => $strZiShu,
'xsxq_jie_shao' => $strJieShao,
'xsxq_feng_mian' => json_encode($arrCover),
'xsxq_geng_xin_shi_jian' => $strLastDate,
'xsxq_tag' => json_encode($arrTag, JSON_UNESCAPED_UNICODE),
'xsxq_source_id' => $intXiaoShuoSourceId,
'xsxq_source_seo_id' => $intXiaoShuoSourceId.'1',
'xsxq_source_code' => 'BIQU_XIAOSHUO_' . $intXiaoShuoSourceId . '_' . $intXiaoShuoSourceSeoId,
];
// print_r($arrXiaoShuoInfo);
var_dump($arrXiaoShuoInfo['xsxq_source_code']);
$XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
$strMuluHref = $QueryList->find('.btn-mulu')->eq(0)->attr("href");
$arrTask = [
'callback' => [self::class, 'getNovelReaderAll'],
'data' => [
'xszj_href' => $strMuluHref,
]
];
$TaskCore->set($arrTask);
/*
$arrZhangjie = [];
$intChapterIndex = 0; // 定义一个计数器变量
// 检查是否找到元素
if ($QueryList->find('.section-list a')->count() > 0) {
$QueryList->find('.section-list a')->map(function ($ZhangJie) use (&$arrZhangjie, &$intChapterIndex) {
echo "Processing index $intChapterIndex: " . $ZhangJie->text() . PHP_EOL;
// 从索引6开始
//if ($intChapterIndex >= 5) {
$arrZhangjie[] = [
'mingzi' => $ZhangJie->text(),
'url' => $ZhangJie->attr('href'),
];
//}
$intChapterIndex++;
});
// 输出最终结果
var_dump($arrZhangjie);
} else {
echo "No elements found for .section-list a" . PHP_EOL;
}
//unset($QueryList);
var_dump($arrZhangjie);
// 获取最大章节索引
$intMaxXiaoShuoZhangJiePaixu = XiaoShuoZhangJieModel::where('xsxq_id', $intXiaoShuoSourceId)
->max('xszj_pai_xu'); // 获取最大值
var_dump($intMaxXiaoShuoZhangJiePaixu);
// 如果没有结果，max 返回 null，可以设置默认值
$intMaxXiaoShuoZhangJiePaixu = $intMaxXiaoShuoZhangJiePaixu ?? 0;
foreach ($arrZhangjie as $intZhangJieNum => $arrZhangjieOne) {
var_dump($intZhangJieNum);
// 如果数据库里的章节最大索引大于当前索引，说明已经入库，不需要入队列了
if($intMaxXiaoShuoZhangJiePaixu > $intZhangJieNum){
var_dump("小说章节已经入库，不再推送任务");
continue;// 如果任务已存在，直接跳过 break
}
$strZhangJieUrl = $strSiteUrl . $arrZhangjieOne['url'];
$strZhangJieId = extractFilename($strZhangJieUrl);
# 'AILISI_XIAOSHUO_ZHANGJIE_小说源id_小说源章节id'
$strXszjSourceCode = '5CCC_XIAOSHUO_ZHANGJIE_'. $XiaoShuoXiangQingModel->xsxq_source_id . '_' . $strZhangJieId;
$arrTask = [
'callback' => [self::class, 'getNovelReader'],
'data' => [
'xszj_source_id' => $strZhangJieId,
'xszj_paixu' => $intZhangJieNum,
'xszj_name' => $arrZhangjieOne['mingzi'],
'xszj_url' => $strZhangJieUrl,
'xszj_source_code' => $strXszjSourceCode,
'xsxq_source_id' => $XiaoShuoXiangQingModel->xsxq_source_id,
'xsfl_id' => $strFenleiIId,
]
];
$TaskCore->set($arrTask);
var_dump("章节任务投递成功".$strXszjSourceCode);
}
*/
} catch (\Throwable $T) {
throw $T;
}
}
public function getNovelFengMian($arrData)
{
//var_dump('getNovelFengMian');
//var_dump($arrData);
$Client = $this->Client;
$TaskCore = $this->TaskCore;
// 获取 Redis 实例
$redis = Cache::store('redis')->handler();
$intXiaoShuoSourceId = $arrData['xiaoshuo_id'];
$intXiaoShuoSourceSeoId = $arrData['xiaoshuo_seo_id'];
$redisKey = "task:novel_source_id:" . $intXiaoShuoSourceId;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($arrData['xiaoshuo_url']);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$strSiteUrl = $this->getAilisiDomain();
$strCover = $QueryList->find('#fmimg img')->eq(0)->attr("src");
if (strpos($strCover, 'http') === false) {
$strCover = $strSiteUrl . $strCover;
}
$strCoverUri = $strCover;
try {
$strCoverUri = "";
$strCoverUri = ImageProcessor::getInstance()->downloadAndEncryptImage($strCover, 'xs');
$arrCover = [
'code' => 'AI_LI_SI_COVER',
// 'uri' => getUriByUrl($strCover),
'uri' => $strCoverUri,
];
$arrXiaoShuoInfo = [
'xsxq_feng_mian' => json_encode($arrCover),
];
$XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
} catch (\Throwable $t) {
$strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
echo $strErr;
}
} catch (\Throwable $T) {
throw $T;
}
}
public function getNovelReaderAll($arrData)
{
var_dump('getNovelReaderAll');
$strMuluHref = $arrData['xszj_href'];
var_dump($strMuluHref);
$Client = $this->Client;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strMuluHref);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strMuluHref, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
unset($Response);
$QueryList = QueryList::html($strResult);
$intChapterIndex = 0; // 定义一个计数器变量
var_dump($QueryList->find('.section-list a')->count());
// 检查是否找到元素
if ($QueryList->find('.section-list a')->count() > 0) {
$QueryList->find('.section-list a')->map(function ($ZhangJie) use (&$arrZhangjie, &$intChapterIndex) {
echo "Processing index $intChapterIndex: " . $ZhangJie->text() . PHP_EOL;
// 从索引6开始
//if ($intChapterIndex >= 5) {
$arrZhangjie[] = [
'mingzi' => $ZhangJie->text(),
'url' => $ZhangJie->attr('href'),
];
//}
$intChapterIndex++;
});
// 输出最终结果
var_dump($arrZhangjie);
} else {
echo "No elements found for .section-list a" . PHP_EOL;
}
//unset($QueryList);
var_dump($arrZhangjie);
/*
// 获取最大章节索引
$intMaxXiaoShuoZhangJiePaixu = XiaoShuoZhangJieModel::where('xsxq_id', $intXiaoShuoSourceId)
->max('xszj_pai_xu'); // 获取最大值
var_dump($intMaxXiaoShuoZhangJiePaixu);
// 如果没有结果，max 返回 null，可以设置默认值
$intMaxXiaoShuoZhangJiePaixu = $intMaxXiaoShuoZhangJiePaixu ?? 0;
foreach ($arrZhangjie as $intZhangJieNum => $arrZhangjieOne) {
var_dump($intZhangJieNum);
// 如果数据库里的章节最大索引大于当前索引，说明已经入库，不需要入队列了
if($intMaxXiaoShuoZhangJiePaixu > $intZhangJieNum){
var_dump("小说章节已经入库，不再推送任务");
continue;// 如果任务已存在，直接跳过 break
}
$strZhangJieUrl = $strSiteUrl . $arrZhangjieOne['url'];
$strZhangJieId = extractFilename($strZhangJieUrl);
# 'AILISI_XIAOSHUO_ZHANGJIE_小说源id_小说源章节id'
$strXszjSourceCode = '5CCC_XIAOSHUO_ZHANGJIE_'. $XiaoShuoXiangQingModel->xsxq_source_id . '_' . $strZhangJieId;
$arrTask = [
'callback' => [self::class, 'getNovelReader'],
'data' => [
'xszj_source_id' => $strZhangJieId,
'xszj_paixu' => $intZhangJieNum,
'xszj_name' => $arrZhangjieOne['mingzi'],
'xszj_url' => $strZhangJieUrl,
'xszj_source_code' => $strXszjSourceCode,
'xsxq_source_id' => $XiaoShuoXiangQingModel->xsxq_source_id,
'xsfl_id' => $strFenleiIId,
]
];
$TaskCore->set($arrTask);
var_dump("章节任务投递成功".$strXszjSourceCode);
}
*/
} catch (\Throwable $T) {
throw $T;
}
}
public function getNovelReader($arrData)
{
var_dump('getNovelReader');
var_dump($arrData['xszj_url']);
$Client = $this->Client;
$strXszjSourceCode = $arrData['xszj_source_code'];
$strZhangJieUrl = $arrData['xszj_url'];
$intZhangJiePaiXu = $arrData['xszj_paixu'];
$intFenleiId = $arrData['xsfl_id'];
$intXiaoShuoSourceID = $arrData['xsxq_source_id'];
$strZhangJieSourceId = $arrData['xszj_source_id'];
$strZhangJieName = $arrData['xszj_name'];
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strZhangJieUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
if (XiaoShuoZhangJieModel::checkZhangJieExists($strXszjSourceCode)) {
return true;
}
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strZhangJieUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
unset($Response);
$ZhangJieQueryList = QueryList::html($strResult);
// 获取原始 HTML
$strNeiRong = $ZhangJieQueryList->find('#content')->eq(0)->html();
// 解码 HTML 实体
$strNeiRong = html_entity_decode($strNeiRong, ENT_QUOTES, 'UTF-8');
// 去掉 <p> 标签中的 class="content_detail" 属性
$strNeiRong = preg_replace('/<p\s+class="content_detail"/i', '<p', $strNeiRong);
// 清理多余的 HTML 标签
$strNeiRong = strip_tags($strNeiRong, '<p>');
// 替换不间断空格 (\xc2\xa0) 和 &nbsp; 为普通空格
$strNeiRong = preg_replace('/\xc2\xa0/', ' ', $strNeiRong); // 替换 UTF-8 不间断空格
$strNeiRong = preg_replace('/&nbsp;/', ' ', $strNeiRong); // 替换 &nbsp;
// 清理多余空格和换行符
$strNeiRong = preg_replace('/\s+/', ' ', $strNeiRong);
$strNeiRong = preg_replace('/<p>\s+/', '<p>', $strNeiRong); // 清理 <p> 标签内部多余空格
$strNeiRong = preg_replace('/\s+<\/p>/', '</p>', $strNeiRong); // 清理 <p> 标签结尾多余空格
// 输出结果
// var_dump($strNeiRong);
# '/novel/123yqw/分类id/小说源id/小说源章节id.txt'
$strNeiRongTxtPath = config("filesystem.novel").'/novel/biquxs/'.$intFenleiId.'/'.$intXiaoShuoSourceID.'/'.$strZhangJieSourceId.'.txt';
//$strNeiRongTxtPath = "/www/novel/123yqw/1/11/111.txt";
// 调用函数
$result = saveCompressAndDeleteTxt($strNeiRongTxtPath, $strNeiRong);
// 输出状态
if ($result['status']) {
var_dump("Success: " . $result['message']) ;
} else {
var_dump("Error: " . $result['message']) ;
}
$arrXiaoShuoZhangJie = [
'xsxq_id' => $intXiaoShuoSourceID,
'xszj_pai_xu' => $intZhangJiePaiXu,
'xszj_ming_zi' => $strZhangJieName,
'xszj_nei_rong' => $strNeiRongTxtPath,
'xszj_source_id' => $strZhangJieSourceId,
'xszj_source_code' => $strXszjSourceCode,
];
XiaoShuoZhangJieModel::addXiaoShuoZhangJie($arrXiaoShuoZhangJie);
unset($ZhangJieQueryList);
// 延时 3 秒
//sleep(3);
} catch (\Throwable $T) {
throw $T;
}
}
}

498
code/app/task/collection/dadi/DaDiDianyingCol.php
View File

@@ -1,498 +0,0 @@
<?php
declare(strict_types=1);
namespace app\task\collection\dadi;
use app\model\VideoPlayurlModel;
use app\model\VideoLeixinModel;
use app\model\VideoDiquModel;
use app\model\VideoYearsModel;
use app\model\VideoJuqingModel;
use app\model\XiaoShuoFenLeiModel;
use app\model\XiaoShuoXiangQingModel;
use app\model\VideoInfoModel;
use app\model\XiaoShuoZhangJieModel;
use app\task\collection\PullDataColBase;
use GuzzleHttp\Cookie\CookieJar;
use processor\ImageProcessor;
use think\facade\Cache;
use QL\QueryList;
/**
* @mixin think\Model
*/
class DaDiDianyingCol extends PullDataColBase
{
public function pullDadiYingshi()
{
$TaskCore = $this->TaskCore;
// 1-75
for ($i = 1; $i <= 75; $i++) {
echo "当前值: $i\n";
$arrTask = [
'callback' => [self::class, 'getVideoPage'],
'data' => [
'video_page_url' => 'https://www.czdadi.net/vodshow/dy--------'.$i.'---.html',
'video_class_id' => 1
]
];
$TaskCore->set($arrTask);
}
//5-25 https://www.czdadi.net/vodshow/duanju-----------.html
for ($i = 1; $i <= 25; $i++) {
echo "当前值: $i\n";
$arrTask = [
'callback' => [self::class, 'getVideoPage'],
'data' => [
'video_page_url' => 'https://www.czdadi.net/vodshow/duanju--------'.$i.'---.html',
'video_class_id' => 5
]
];
$TaskCore->set($arrTask);
}
// 2--18 https://www.czdadi.net/vodshow/dsj--------18---.html
for ($i = 1; $i <= 18; $i++) {
echo "当前值: $i\n";
$arrTask = [
'callback' => [self::class, 'getVideoPage'],
'data' => [
'video_page_url' => 'https://www.czdadi.net/vodshow/dsj--------'.$i.'---.html',
'video_class_id' => 2
]
];
$TaskCore->set($arrTask);
}
// 3--11 https://www.czdadi.net/vodshow/zy--------11---.html
for ($i = 1; $i <= 11; $i++) {
echo "当前值: $i\n";
$arrTask = [
'callback' => [self::class, 'getVideoPage'],
'data' => [
'video_page_url' => 'https://www.czdadi.net/vodshow/zy--------'.$i.'---.html',
'video_class_id' => 3
]
];
$TaskCore->set($arrTask);
}
// 4--29 https://www.czdadi.net/vodshow/dm-----------.html
for ($i = 1; $i <= 29; $i++) {
echo "当前值: $i\n";
$arrTask = [
'callback' => [self::class, 'getVideoPage'],
'data' => [
'video_page_url' => 'https://www.czdadi.net/vodshow/dm--------'.$i.'---.html',
'video_class_id' => 4
]
];
$TaskCore->set($arrTask);
}
}
# 获取分页里面的视频列表
public function getVideoPage($arrData)
{
$TaskCore = $this->TaskCore;
$Client = $this->Client;
$strUrl= $arrData['video_page_url'];
$intClassId = $arrData['video_class_id'];
$strSiteUrl = $this->get5cccDomain();
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
$QueryList = QueryList::html($strResult);
//$arrVideo = [];
// 检查是否找到元素
if ($QueryList->find('.stui-vodlist li .stui-vodlist__detail a')->count() > 0) {
$QueryList->find('.stui-vodlist li .stui-vodlist__detail a ')->map(function ($Video) use (&$arrVideo,$intClassId,&$TaskCore) {
// $arrVideo[] = [
// 'mingzi' => $Video->text(),
// 'url' => $Video->attr('href'),
// ];
if(!empty($Video->text()) && !empty($Video->attr('href'))){
$arrTask = [
'callback' => [self::class, 'getVideoInfo'],
'data' => [
'class_id' => $intClassId,
'video_href' => $Video->attr('href'),
'video_name' => $Video->text(),
]
];
$TaskCore->set($arrTask);
echo "视频投递成功 : " . $Video->text() . PHP_EOL;
}
});
// foreach ($arrVideo as $intVideoNum => $arrVideoOne) {
// $arrTask = [
// 'callback' => [self::class, 'getVideoInfo'],
// 'data' => [
// 'class_id' => $intClassId,
// 'video_href' => $arrVideoOne['url'],
// 'video_name' => $arrVideoOne['mingzi'],
// ]
// ];
// $TaskCore->set($arrTask);
// }
// 输出最终结果
// var_dump($arrVideo);
} else {
echo "No elements found for .stui-vodlist a" . PHP_EOL;
}
}
public function getVideoInfo($arrData)
{
$TaskCore = $this->TaskCore;
$Client = $this->Client;
$strSiteUrl = $this->getDadiDomain();
$strUrl = $strSiteUrl . $arrData['video_href'] ;
$intClassId = $arrData['class_id'];
$strVideoName = $arrData['video_name'];
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
$QueryList = QueryList::html($strResult);
$strCover = $QueryList->find('.stui-content__thumb img')->eq(0)->attr("data-original");
if (strpos($strCover, 'http') === false) {
$strCover = $strSiteUrl . $strCover;
}
$strCoverUri = $strCover;
// try {
// $strCoverUri = "";
// $strCoverUri = ImageProcessor::getInstance()->downloadImage( $intXiaoShuoSourceId,$strCover, 'dadi');
// } catch (\Throwable $t) {
// $strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
// echo $strErr;
// }
$arrCover = [
'code' => 'DADI_YINGSHI_COVER',
// 'uri' => getUriByUrl($strCover),
'uri' => $strCoverUri,
];
$strLanguage = "";
// if (strlen($strLanguage) > 0) {
// VideoLeixinModel::addVideoLeixin(['vc_id' => $intClassId, 'vla_name' => $strLanguage]);
// }
// 提取评分
$strPinfen = $QueryList->find('h1.title .score')->text();
// 提取类型-入库
$strLeixin = $QueryList->find('.col-pd a')->eq(1)->text();
if(strlen($strLeixin) > 0){
VideoLeixinModel::addVideoLeixin(['vc_id'=>$intClassId,'vl_name'=>$strLeixin]);
}
// 提取 HTML 的第一段 <p class="data">
$dataNode = $QueryList->find('p.data')->eq(0);
// 提取“类型”内容：确保只匹配 `href` 中包含 `/vodsearch/----`
$leixinNodes = $dataNode->find('a[href^="/vodsearch/----"]');
$strJuqing = $leixinNodes->map(function ($node) {
return $node->text();
})->all();
// 提取“地区”内容：确保只匹配 `href` 中包含 `/vodsearch/--`，但不包含 `/vodsearch/----`
$diquNodes = $dataNode->find('a[href^="/vodsearch/--"]')->not('a[href^="/vodsearch/----"]');
$strDiqu = $diquNodes->map(function ($node) {
return $node->text();
})->all();
// 提取“年份”内容：确保只匹配 `href` 中包含 `/vodsearch/-------------`
$yearsNode = $dataNode->find('a[href^="/vodsearch/-------------"]');
$strYears = $yearsNode->text();
if (!empty(trim($strYears))) {
$strYears = [trim($strYears)];
} else {
$strYears = [];
}
// 过滤掉“地区”和“年份”中的数据
$strJuqing = array_filter($strJuqing, function ($value) use ($strDiqu, $strYears) {
return !in_array($value, array_merge($strDiqu, $strYears));
});
// **在过滤后将“类型”数据入库**
foreach ($strJuqing as $value) {
if(strlen($value) > 0){
VideoJuqingModel::add(['vc_id' => $intClassId, 'vj_name' => $value]);
}
}
// 将“地区”数据入库
foreach ($strDiqu as $value) {
if(strlen($value) > 0){
VideoDiquModel::add(['vc_id' => $intClassId, 'vd_name' => $value]);
}
}
// 将“年份”数据入库
foreach ($strYears as $value) {
if(strlen($value) > 0){
VideoYearsModel::add(['vc_id' => $intClassId, 'vy_name' => $value]);
}
}
// 提取主演
$strZhuyan = $QueryList->find('p.data:contains("主演") a')->map(function ($item) {
return $item->text();
})->implode(', '); // 使用逗号分隔主演
// 提取导演
$strDaoyan = $QueryList->find('p.data:contains("导演") a')->map(function ($item) {
return $item->text();
})->implode(', '); // 使用逗号分隔主演
// 提取更新时间
$strLastDate = $QueryList->find('p.data.hidden-sm:contains("更新")')->text();
// 去掉“更新：”前缀
$strLastDate = str_replace('更新：', '', $strLastDate);
// 提取简介
$strDescription = $QueryList->find('#desc>.stui-pannel-box>.stui-pannel_bd>.col-pd')->text();
// 角标
$strJiaoBiao = $QueryList->find('.stui-content__thumb .text-right')->eq(0)->text();
$strPinyinId = extractFilename($strUrl);
$arrVideoInfo = [
'v_name' => $strVideoName,
'v_cover' => json_encode($arrCover),
'vc_id' => $intClassId,
'v_pinyin' => $strPinyinId,
'v_leixin' => $strLeixin ,
'v_juqing' => implode(', ', $strJuqing),
'v_diqu' => implode(', ', $strDiqu),
'v_years' => implode(', ', $strYears),
'v_language' => $strLanguage,
'v_daoyan' => $strZhuyan,
'v_zhuyan' => $strDaoyan,
'update_time' => $strLastDate,
'v_description'=>$strDescription,
'v_pingfen' => $strPinfen,
'v_orurl' => $strUrl,
'v_jiaobiao'=> $strJiaoBiao,
'v_md5' => md5($intClassId.$strVideoName.$strSiteUrl.$strUrl),
];
$VideoInfoModel = VideoInfoModel::addVideoXiangQing($arrVideoInfo);
$intVid = $VideoInfoModel->v_id;
$intChapterIndex = 0 ;
$arrZhangjie = [];
// // 检查是否找到元素
if ($QueryList->find('.stui-content__playlist a')->count() > 0) {
$QueryList->find('.stui-content__playlist a')->map(function ($ZhangJie) use (&$arrZhangjie, &$intChapterIndex ,&$intVid) {
//echo "Processing index $intChapterIndex: " . $ZhangJie->text() . PHP_EOL;
$arrZhangjie[] = [
'v_id' => $intVid,
'index' => $intChapterIndex,
'mingzi' => $ZhangJie->text(),
'url' => $ZhangJie->attr('href'),
];
$intChapterIndex++;
});
//var_dump($arrZhangjie);
foreach ($arrZhangjie as $intZhangJieNum => $arrZhangjieOne) {
var_dump($intZhangJieNum);
// 如果数据库里的章节最大索引大于当前索引，说明已经入库，不需要入队列了
// if($intMaxXiaoShuoZhangJiePaixu > $intZhangJieNum){
// var_dump("小说章节已经入库，不再推送任务");
// continue;// 如果任务已存在，直接跳过 break
// }
//if($intZhangJieNum<=3){
$arrTask = [
'callback' => [self::class, 'getNovelReader'],
'data' => [
'v_id' => $intVid,
'vp_sort' => $intZhangJieNum,
'vp_name' => $arrZhangjieOne['mingzi'],
'vp_url' => $strSiteUrl . $arrZhangjieOne['url']
]
];
$TaskCore->set($arrTask);
//}
}
} else {
echo "No elements found for .listmain a" . PHP_EOL;
}
} catch (\Throwable $T) {
throw $T;
}
// sleep(10);
}
public function getNovelFengMian($arrData)
{
//var_dump('getNovelFengMian');
//var_dump($arrData);
$Client = $this->Client;
$TaskCore = $this->TaskCore;
// 获取 Redis 实例
$redis = Cache::store('redis')->handler();
$intXiaoShuoSourceId = $arrData['xiaoshuo_id'];
$intXiaoShuoSourceSeoId = $arrData['xiaoshuo_seo_id'];
$redisKey = "task:novel_source_id:" . $intXiaoShuoSourceId;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($arrData['xiaoshuo_url']);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$strSiteUrl = $this->getAilisiDomain();
$strCover = $QueryList->find('#fmimg img')->eq(0)->attr("src");
if (strpos($strCover, 'http') === false) {
$strCover = $strSiteUrl . $strCover;
}
$strCoverUri = $strCover;
try {
$strCoverUri = "";
$strCoverUri = ImageProcessor::getInstance()->downloadAndEncryptImage($strCover, 'xs');
$arrCover = [
'code' => 'AI_LI_SI_COVER',
// 'uri' => getUriByUrl($strCover),
'uri' => $strCoverUri,
];
$arrXiaoShuoInfo = [
'xsxq_feng_mian' => json_encode($arrCover),
];
$XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
} catch (\Throwable $t) {
$strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
echo $strErr;
}
} catch (\Throwable $T) {
throw $T;
}
}
public function getNovelReader($arrData)
{
if (Cache::has('video_no_m3u8_' . $arrData['v_id'])) {
echo "跳过抓取，视频ID已在缓存中: " . $arrData['v_id'] . "\n";
return; // 跳过抓取逻辑
}
$strMuluHref = $arrData['vp_url'];
var_dump($strMuluHref);
$Client = $this->Client;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strMuluHref);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strMuluHref, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
unset($Response);
$QueryList = QueryList::html($strResult);
//提取 <script> 标签内容
$scriptContent = $QueryList->find('.vidoe-data-show')->text();
$strM3u8Url = "";
// 使用正则表达式提取 "url" 的值
if (preg_match('/"url":"(https:[^"]+\.m3u8)"/', $scriptContent, $matches)) {
$strM3u8Url = str_replace('\/', '/', $matches[1]); // 替换转义斜杠
echo "提取的 M3U8 地址: $strM3u8Url\n";
$arrM3u8Url = [
'code' => 'DADI_YINGSHI_M3U8',
'uri' => $strM3u8Url,
];
$arrVideoPlayurl = [
'v_id' => $arrData['v_id'],
'vp_sort' => $arrData['vp_sort'],
'vp_name' => $arrData['vp_name'],
'vp_url' => json_encode($arrM3u8Url),
'vp_md5' => md5($arrData['vp_url'] . $arrData['vp_url']),
];
VideoPlayurlModel::add($arrVideoPlayurl);
unset($QueryList);
} else {
echo "未找到 M3U8 地址\n";
// 将视频ID保存到缓存，设置一个过期时间，例如 1 小时
Cache::set('video_no_m3u8_' . $arrData['v_id'], true, 3600);
}
} catch (\Throwable $T) {
throw $T;
}
}
}

560
code/app/task/collection/heimuer/HeiMuErYingShiCol.php
View File

@@ -1,560 +0,0 @@
<?php
declare(strict_types=1);
namespace app\task\collection\heimuer;
use app\model\VideoPlayurlModel;
use app\model\VideoLeixinModel;
use app\model\VideoDiquModel;
use app\model\VideoYearsModel;
use app\model\VideoJuqingModel;
use app\model\XiaoShuoFenLeiModel;
use app\model\XiaoShuoXiangQingModel;
use app\model\VideoInfoModel;
use app\model\XiaoShuoZhangJieModel;
use app\task\collection\PullDataColBase;
use GuzzleHttp\Cookie\CookieJar;
use processor\ImageProcessor;
use think\facade\Cache;
use QL\QueryList;
/**
* @mixin think\Model
*/
class HeiMuErYingShiCol extends PullDataColBase
{
public function pullDadiYingshi()
{
$strSiteUrl = $this->getHeiMuErDomain();
// 配置分类任务参数
$categories = [
['id' => 1, 'video_class_id' => 1, 'name' => '电影'],
['id' => 27, 'video_class_id' => 5, 'name' => '短剧'],
['id' => 2, 'video_class_id' => 2, 'name' => '电视剧'],
['id' => 4, 'video_class_id' => 3, 'name' => '综艺'],
['id' => 3, 'video_class_id' => 4, 'name' => '动漫'],
];
foreach ($categories as $category) {
$intClassId = $category['video_class_id'];
$intCategoryId = $category['id'];
// 先抓第一页
$strUrl = $strSiteUrl . '/index.php/vod/type/id/' . $intCategoryId . '/page/1.html';
$maxPage = $this->getMaxPage($strUrl,$intClassId);
// 首次抓取，全部抓取
if (config('task.task_is_first')) {
$this->createTasks($strSiteUrl, $category['id'], $category['video_class_id'], $maxPage);
}
}
}
/**
* 获取分类的最大页码且抓取第一页
*/
protected function getMaxPage($strUrl, $intClassId)
{
$TaskCore = $this->TaskCore;
$Client = $this->Client;
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$Response = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'Accept-Language' => 'en-US,en;q=0.5',
'cookie' => '__sk_Vm8nTqM7LqSCWmR6__=be3bd4f4e0483619e1a434ec1b18a938',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 300,
]);
$strResult = (string)$Response->getBody();
$QueryList = QueryList::html($strResult);
// 检查是否找到元素
if ($QueryList->find('.stui-vodlist li.clearfix h3 a')->count() > 0) {
$QueryList->find('.stui-vodlist li.clearfix h3 a ')->map(function ($Video) use ($intClassId, &$TaskCore) {
$strHref = $Video->attr('href');
$strName = $Video->text();
if (!empty($strName) && !empty($strHref) && strlen($strName) > 0 && strlen($strHref) > 0) {
$arrTask = [
'callback' => [self::class, 'getVideoInfo'],
'data' => [
'class_id' => $intClassId,
'video_href' => $strHref,
'video_name' => $strName,
]
];
// var_dump($arrTask);
$TaskCore->set($arrTask);
echo "视频投递成功 : " . $Video->text() . PHP_EOL;
}
});
} else {
echo "No elements found for .stui-vodlist a" . PHP_EOL;
}
// 获取分页部分最后一个页码链接
$lastPageHref = $QueryList->find('.stui-page li')->eq($QueryList->find('.stui-page li')->count() - 1)->find('a')->attr('href');
// 提取最大页码
preg_match('/page\/(\d+)\.html/', $lastPageHref, $matches);
$maxPage = isset($matches[1]) ? (int)$matches[1] : 1; // 默认返回 1
return $maxPage;
}
/**
* 创建任务
*/
protected function createTasks($strSiteUrl, $categoryId, $videoClassId, $maxPage)
{
$TaskCore = $this->TaskCore;
for ($i = $maxPage; $i >= 1; $i--) {
echo "当前分类: $categoryId, 当前页: $i\n";
$arrTask = [
'callback' => [self::class, 'getVideoPage'],
'data' => [
'video_page_url' => $strSiteUrl . '/index.php/vod/type/id/' . $categoryId . '/page/' . $i . '.html',
'video_class_id' => $videoClassId,
]
];
$TaskCore->set($arrTask);
}
}
# 获取分页里面的视频列表
public function getVideoPage($arrData)
{
$TaskCore = $this->TaskCore;
$Client = $this->Client;
$strUrl = $arrData['video_page_url'];
$intClassId = $arrData['video_class_id'];
$strSiteUrl = $this->getHeiMuErDomain();
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
var_dump($strUrl);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'Accept-Language' => 'en-US,en;q=0.5',
'cookie' => '__sk_Vm8nTqM7LqSCWmR6__=be3bd4f4e0483619e1a434ec1b18a938',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 300,
]);
$strResult = (string)$Response->getBody();
$QueryList = QueryList::html($strResult);
// 检查是否找到元素
if ($QueryList->find('.stui-vodlist li.clearfix h3 a')->count() > 0) {
$QueryList->find('.stui-vodlist li.clearfix h3 a ')->map(function ($Video) use (&$arrVideo, $intClassId, &$TaskCore, $strSiteUrl) {
$strHref = $Video->attr('href');
$strName = $Video->text();
if (!empty($strName) && !empty($strHref) && strlen($strName) > 0 && strlen($strHref) > 0) {
$arrTask = [
'callback' => [self::class, 'getVideoInfo'],
'data' => [
'class_id' => $intClassId,
'video_href' => $strHref,
'video_name' => $strName,
]
];
// var_dump($arrTask);
$TaskCore->set($arrTask);
echo "视频投递成功 : " . $Video->text() . PHP_EOL;
}
});
} else {
echo "No elements found for .stui-vodlist a" . PHP_EOL;
}
}
public function getVideoInfo($arrData)
{
$TaskCore = $this->TaskCore;
$Client = $this->Client;
$strSiteUrl = $this->getHeiMuErDomain();
$strUrl = $strSiteUrl . $arrData['video_href'];
$intClassId = $arrData['class_id'];
$strVideoName = $arrData['video_name'];
var_dump($strUrl);
if (!empty($strVideoName) && !empty($strUrl) && strlen($strVideoName) > 0 && strlen($strUrl) > 0) {
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
$QueryList = QueryList::html($strResult);
$strCover = $QueryList->find('.img-responsive')->eq(0)->attr("src");
if (strpos($strCover, 'http') === false) {
$strCover = $strSiteUrl . $strCover;
}
$strCoverUri = $strCover;
// try {
// $strCoverUri = "";
// $strCoverUri = ImageProcessor::getInstance()->downloadImage( $intXiaoShuoSourceId,$strCover, 'dadi');
// } catch (\Throwable $t) {
// $strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
// echo $strErr;
// }
$arrCover = [
'code' => 'HEIMUER_SOURCE_COVER',
'uri' => $strCoverUri,
];
$strLanguage = "";
// if (strlen($strLanguage) > 0) {
// VideoLeixinModel::addVideoLeixin(['vc_id' => $intClassId, 'vla_name' => $strLanguage]);
// }
// 提取别名
$strPinyinId = $QueryList->find('.stui-content__detail p.data')->eq(0)->text();
// 提取评分
$strPinfen = $QueryList->find('.stui-content__detail h1 .text-red')->text();
// 提取地区
$strDiqu = $QueryList->find('.stui-content__detail p.data')->eq(1)->text();
// 角标-状态
$strJiaoBiao = $QueryList->find('.stui-content__detail p.data')->eq(2)->find('.text-red')->eq(0)->text();
// 更新时间
$strLastDate = $QueryList->find('.stui-content__detail p.data')->eq(2)->find('.text-red')->eq(1)->text();
// 提取主演
$strZhuyan = $QueryList->find('.stui-content__detail p.data')->eq(3)->text();
// 提取导演
$strDaoyan = $QueryList->find('.stui-content__detail p.data')->eq(4)->text();
// 提取
$arrLeixinData = $QueryList->find('.stui-content__detail p.data')->eq(5)->text();
// 提取
$arrYuyanData = $QueryList->find('.stui-content__detail p.data')->eq(6)->text();
$strLeixin = '';
$strDiqu = '';
$strYears = '';
$strKuozhang = '';
// 使用正则表达式提取内容
$patternLeixin = '/类型：(.*?)扩展：(.*?)地区：(.*?)年份：(.*)/';
if (preg_match($patternLeixin, $arrLeixinData, $matches)) {
$strLeixin = trim($matches[1]);
$strKuozhang = trim($matches[2]);
$strDiqu = trim($matches[3]);
$strYears = trim($matches[4]);
}
$strLanguage = '';
// 使用正则表达式提取内容
$patternYuyan = '/语言：(.*?)集数：(.*?)时长：(.*?)点击：(.*)/';
if (preg_match($patternYuyan, $arrYuyanData, $matches)) {
$strLanguage = trim($matches[1]);
// $strKuozhang = trim($matches[2]);
// $strDiqu = trim($matches[3]);
// $strYears = trim($matches[4]);
}
if (strlen($strLeixin) > 0) {
VideoLeixinModel::addVideoLeixin(['vc_id' => $intClassId, 'vl_name' => $strLeixin]);
}
if (strlen($strDiqu) > 0) {
VideoDiquModel::add(['vc_id' => $intClassId, 'vd_name' => $strDiqu]);
}
if (strlen($strYears) > 0) {
VideoYearsModel::add(['vc_id' => $intClassId, 'vy_name' => $strYears]);
}
// 提取简介
$strDescription = $QueryList->find('.stui-content__desc')->text();
$arrVideoInfo = [
'v_name' => $strVideoName,
'v_cover' => json_encode($arrCover),
'vc_id' => $intClassId,
'v_pinyin' => trim(str_replace('别名：', '', $strPinyinId)),
'v_leixin' => $strLeixin,
'v_juqing' => '',
'v_diqu' => $strDiqu,
'v_years' => $strYears,
'v_language' => $strLanguage,
'v_daoyan' => trim(str_replace('主演：', '', $strZhuyan)),
'v_zhuyan' => trim(str_replace('导演：', '', $strDaoyan)),
'update_time' => $strLastDate,
'v_description' => $strDescription,
'v_pingfen' => $strPinfen,
'v_orurl' => $strUrl,
'v_jiaobiao' => $strJiaoBiao,
'v_md5' => md5($intClassId . $strVideoName . getUriByUrl($strUrl)),
];
//var_dump($arrVideoInfo);
// // 检查是否找到元素
if ($QueryList->find('.stui-content__playlist li')->count() > 0) {
// var_dump("视频入库成功--" . $strVideoName);
// $VideoInfoModel = VideoInfoModel::addVideoXiangQing($arrVideoInfo);
$resultVideoInfoModel = VideoInfoModel::addVideoXiangQing($arrVideoInfo);;
if ($resultVideoInfoModel['isNew']) {
var_dump("数据已新增：", $resultVideoInfoModel['model']->v_id."视频入库成功--" . $strVideoName);
} else {
echo "数据已存在：", $resultVideoInfoModel['model']->v_id;
}
$VideoInfoModel = $resultVideoInfoModel['model'];
$intVid = $VideoInfoModel->v_id;
$intChapterIndex = 0;
$QueryList->find('.stui-content__playlist li')->map(function ($ZhangJie) use (&$arrZhangjie, &$intChapterIndex, &$intVid) {
//echo "Processing index $intChapterIndex: " . $ZhangJie->text() . PHP_EOL;
$strM3u8 = $ZhangJie->find('a')->eq(1)->text();
$arrM3u8 = explode('$', $strM3u8);
$strM3u8Url = $arrM3u8[1];
$strM3u8Name = $arrM3u8[0];
$arrM3u8Url = [
'code' => 'HEIMUER_SOURCE_PLAY',
'uri' => $strM3u8Url,
];
$arrVideoPlayurl = [
'v_id' => $intVid,
'vp_sort' => $intChapterIndex,
'vp_name' => $strM3u8Name,
'vp_url' => json_encode($arrM3u8Url),
'vp_md5' => md5($intVid . getUriByUrl($strM3u8Url)),
];
$result = VideoPlayurlModel::add($arrVideoPlayurl);;
if ($result['isNew']) {
var_dump("数据已新增：", $result['model']->vp_id."播放线路入库成功--" . $strM3u8Url) ;
} else {
echo "数据已存在：", $result['model']->vp_id ;
}
$intChapterIndex++;
});
var_dump("--") ;
var_dump("一个视频任务完成------------------------------------------------------------------------------------") ;
} else {
echo "No elements found for .listmain a" . PHP_EOL;
}
} catch (\Throwable $T) {
throw $T;
}
}
// sleep(10);
}
public function getNovelFengMian($arrData)
{
//var_dump('getNovelFengMian');
//var_dump($arrData);
$Client = $this->Client;
$TaskCore = $this->TaskCore;
// 获取 Redis 实例
$redis = Cache::store('redis')->handler();
$intXiaoShuoSourceId = $arrData['xiaoshuo_id'];
$intXiaoShuoSourceSeoId = $arrData['xiaoshuo_seo_id'];
$redisKey = "task:novel_source_id:" . $intXiaoShuoSourceId;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($arrData['xiaoshuo_url']);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$strSiteUrl = $this->getAilisiDomain();
$strCover = $QueryList->find('#fmimg img')->eq(0)->attr("src");
if (strpos($strCover, 'http') === false) {
$strCover = $strSiteUrl . $strCover;
}
$strCoverUri = $strCover;
try {
$strCoverUri = "";
$strCoverUri = ImageProcessor::getInstance()->downloadAndEncryptImage($strCover, 'xs');
$arrCover = [
'code' => 'AI_LI_SI_COVER',
// 'uri' => getUriByUrl($strCover),
'uri' => $strCoverUri,
];
$arrXiaoShuoInfo = [
'xsxq_feng_mian' => json_encode($arrCover),
];
$XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
} catch (\Throwable $t) {
$strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
echo $strErr;
}
} catch (\Throwable $T) {
throw $T;
}
}
public function getNovelReader($arrData)
{
$strMuluHref = $arrData['vp_url'];
var_dump($strMuluHref);
$Client = $this->Client;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strMuluHref);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strMuluHref, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
unset($Response);
$QueryList = QueryList::html($strResult);
//提取 <script> 标签内容
$scriptContent = $QueryList->find('.vidoe-data-show')->text();
$strM3u8Url = "";
// 使用正则表达式提取 "url" 的值
if (preg_match('/"url":"(https:[^"]+\.m3u8)"/', $scriptContent, $matches)) {
$strM3u8Url = str_replace('\/', '/', $matches[1]); // 替换转义斜杠
echo "提取的 M3U8 地址: $strM3u8Url\n";
$arrM3u8Url = [
'code' => 'DADI_YINGSHI_M3U8',
'uri' => $strM3u8Url,
];
$arrVideoPlayurl = [
'v_id' => $arrData['v_id'],
'vp_sort' => $arrData['vp_sort'],
'vp_name' => $arrData['vp_name'],
'vp_url' => json_encode($arrM3u8Url),
'vp_md5' => md5($arrData['vp_url'] . $arrData['vp_url']),
];
VideoPlayurlModel::add($arrVideoPlayurl);
unset($QueryList);
} else {
echo "未找到 M3U8 地址\n";
}
} catch (\Throwable $T) {
throw $T;
}
}
public function pullDadiYingshi2()
{
$strSiteUrl = $this->getHeiMuErDomain();
$TaskCore = $this->TaskCore;
//电影 307 https://heimuer.tv/index.php/vod/type/id/1/page/307.html
for ($i = 307; $i >= 1; $i--) {
if (!config('task.task_is_first') && $i > 1) {
continue;
}
echo "当前值: $i\n";
$arrTask = [
'callback' => [self::class, 'getVideoPage'],
'data' => [
'video_page_url' => $strSiteUrl . '/index.php/vod/type/id/1/page/' . $i . '.html',
'video_class_id' => 1
]
];
$TaskCore->set($arrTask);
}
//5-87 短剧 https://heimuer.tv/index.php/vod/type/id/27/page/87.html
for ($i = 87; $i >= 1; $i--) {
if (!config('task.task_is_first') && $i > 1) {
continue;
}
echo "当前值: $i\n";
$arrTask = [
'callback' => [self::class, 'getVideoPage'],
'data' => [
'video_page_url' => $strSiteUrl . '/index.php/vod/type/id/27/page/' . $i . '.html',
'video_class_id' => 5
]
];
$TaskCore->set($arrTask);
}
//电视剧 2--225 https://heimuer.tv/index.php/vod/type/id/2/page/225.html
for ($i = 225; $i >= 1; $i--) {
if (!config('task.task_is_first') && $i > 1) {
continue;
}
echo "当前值: $i\n";
$arrTask = [
'callback' => [self::class, 'getVideoPage'],
'data' => [
'video_page_url' => $strSiteUrl . '/index.php/vod/type/id/2/page/' . $i . '.html',
'video_class_id' => 2
]
];
$TaskCore->set($arrTask);
}
//综艺 78 https://heimuer.tv/index.php/vod/type/id/4/page/78.html
for ($i = 78; $i >= 1; $i--) {
if (!config('task.task_is_first') && $i > 1) {
continue;
}
echo "当前值: $i\n";
$arrTask = [
'callback' => [self::class, 'getVideoPage'],
'data' => [
'video_page_url' => $strSiteUrl . '/index.php/vod/type/id/4/page/' . $i . '.html',
'video_class_id' => 3
]
];
$TaskCore->set($arrTask);
}
// 动漫--100 https://heimuer.tv/index.php/vod/type/id/3/page/1.html
for ($i = 100; $i >= 1; $i--) {
if (!config('task.task_is_first') && $i > 1) {
continue;
}
echo "当前值: $i\n";
$arrTask = [
'callback' => [self::class, 'getVideoPage'],
'data' => [
'video_page_url' => $strSiteUrl . '/index.php/vod/type/id/3/page/' . $i . '.html',
'video_class_id' => 4
]
];
$TaskCore->set($arrTask);
}
}
}

850
code/app/task/collection/novel/Zw630XiaoShuoCol.php
View File

@@ -1,850 +0,0 @@
<?php
declare(strict_types=1);
namespace app\task\collection\novel;
use app\model\XiaoShuoFenLeiModel;
use app\model\XiaoShuoXiangQingModel;
use app\model\XiaoShuoZhangJieModel;
use app\model\NovelInfoSeoModel;
use app\task\collection\PullDataColBase;
use GuzzleHttp\Cookie\CookieJar;
use processor\ImageProcessor;
use think\facade\Cache;
use QL\QueryList;
use app\common\Base64Helper as qsbs;
/**
* @mixin think\Model https://www.630zw.org/
*/
class Zw630XiaoShuoCol extends PullDataColBase
{
public function pullXiaoShuo()
{
// $decoded = '美女';
// $encoded = qsbs::encode($decoded);
// $decodedString = qsbs::decode('5oC75LmL77yM57uT5bGA5piv5aW955qE77yM5amJ5YS/5bqU6K+l5LiN5Lya5Zyo5LmO6L+Z5Lqb57uG6IqC44CCPC9wPg==');
// var_dump($encoded);
// var_dump($decodedString);
// 首次抓取，全部抓取
if (config('task.task_is_first')) {
$TaskCore = $this->TaskCore;
// 527797
for ($i = config('task.task_start_number'); $i <= config('task.task_end_number'); $i++) {
//echo "当前值: $i\n";
$arrTask = [
'callback' => [self::class, 'getNovelInfo'],
'data' => [
'xs_source_id' => $i, //528049
'xs_source_seo_id' => 1,
]
];
$TaskCore->set($arrTask);
// var_dump($arrTask);
}
}else{
$this->pullFenlei();
}
}
public function pullFenlei()
{
$TaskCore = $this->TaskCore;
for ($intFenleiId = 1; $intFenleiId <= 9; $intFenleiId++) {
for ($intPage = 1; $intPage <= 10; $intPage++) {
// https://www.630zw.org/list/1/10.html
$arrTask = [
'callback' => [self::class, 'getFenLei'],
'data' => [
'xs_fenlei_url' => 'https://www.630zw.org/list/'.$intFenleiId.'/'.$intPage.'.html',
]
];
$TaskCore->set($arrTask);
// var_dump($arrTask);
}
}
}
public function getFenLei($arrData)
{
$TaskCore = $this->TaskCore;
$Client = $this->Client;
$strSiteUrl = $this->get630zwDomain();
$strUrl = $arrData['xs_fenlei_url']; //1 ;
var_dump($strUrl);
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
$QueryList = QueryList::html($strResult);
$arrNovel = [];
// 检查是否找到元素
if ($QueryList->find('.layout2 .txt-list-row5')->eq(0)->find('li')->count() > 0) {
$QueryList->find('.layout2 .txt-list-row5')->eq(0)->find('li')->map(function ($Novel) use (&$arrNovel) {
$strNovelHref = $Novel->find('.s2 a')->attr('href') ;
$intNovelId = extractFilename($strNovelHref);
var_dump($strNovelHref);
$arrNovel[] = [
'xs_source_id' => $intNovelId, //528049
'xs_source_seo_id' => 1,
];
});
} else {
echo 'qie h5' . PHP_EOL;
if ($QueryList->find('.sort_list')->eq(0)->find('li')->count() > 0) {
$QueryList->find('.sort_list')->eq(0)->find('li')->map(function ($Novel) use (&$arrNovel) {
$strNovelHref = $Novel->find('.s2 a')->attr('href') ;
$intNovelId = extractFilename($strNovelHref);
var_dump($strNovelHref);
$arrNovel[] = [
'xs_source_id' => $intNovelId, //528049
'xs_source_seo_id' => 1,
];
});
}
}
foreach ($arrNovel as $intNum => $arrNovelOne) {
$arrTask = [
'callback' => [self::class, 'getNovelInfo'],
'data' => $arrNovelOne,
];
$TaskCore->set($arrTask);
}
} catch (\Throwable $T) {
$arrTask = [
'callback' => [self::class, 'getFenLei'],
'data' => [
'xs_fenlei_url' => $strUrl,
]
];
$TaskCore->set($arrTask);
throw $T;
}
}
public function getNovelInfo($arrData)
{
$TaskCore = $this->TaskCore;
$Client = $this->Client;
$strSiteUrl = $this->get630zwDomain();
$strUrl = 'https://630zw.org/shu/' . $arrData['xs_source_id'] . '.html'; //'https://630zw.org/shu/93716.html';
$intXiaoShuoSourceId = $arrData['xs_source_id']; //20053;
$intXiaoShuoSourceSeoId = $arrData['xs_source_seo_id']; //1 ;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
$QueryList = QueryList::html($strResult);
// var_dump($QueryList);
// 提取所有 <meta> 标签的属性
$result = $QueryList->find('meta[property^="og:"]')->map(function ($item) {
return [
'property' => $item->attr('property'),
'content' => $item->attr('content'),
];
});
// 打印结果
// print_r($result->all());
$strZuoZhe = '';
$strMingZi = '';
$strLastDate = '';
$strJieShao = '';
$strCover = '';
$strFenlei = '';
$strZiShu = 0;
$strZhuangTai = '';
foreach ($result as $item) {
switch ($item['property']) {
case 'og:title':
$strMingZi = $item['content'];
break;
case 'og:description':
$strJieShao = preg_split('/<br\s*\/?>/i', $item['content'])[0];
break;
case 'og:image':
$strCover = $item['content'];
break;
case 'og:novel:category':
$strFenlei = $item['content'];
break;
case 'og:novel:author':
$strZuoZhe = $item['content'];
break;
case 'og:novel:status':
$strZhuangTai = $item['content'];
break;
case 'og:novel:update_time':
$strLastDate = $item['content'];
break;
}
}
$strLastDate = trim((string)$strLastDate);
if (!isValidDateTime($strLastDate)) {
$strLastDate = date('Y-m-d H:i:s');
}
$arrTag = [];
if (strpos($strCover, 'http') === false) {
$strCover = $strSiteUrl . $strCover;
}
$strCoverUri = $strCover;
try {
$strCoverUri = "";
$strCoverUri = ImageProcessor::getInstance()->downloadImage($intXiaoShuoSourceId, $strCover, 'xs/630zw');
} catch (\Throwable $t) {
$strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
echo $strErr;
}
$arrCover = [
'code' => '630_ZW_XS_COVER',
// 'uri' => getUriByUrl($strCover),
'uri' => $strCoverUri,
];
$strFenleiIId = 1;
switch ($strFenlei) {
case '玄幻修真':
$strName = "玄幻";
$strFenleiIId = 1;
break;
case '重生穿越':
$strName = "修真";
$strFenleiIId = 2;
break;
case '都市小说':
$strName = "都市";
$strFenleiIId = 3;
break;
case '军史小说':
$strName = "历史";
$strFenleiIId = 4;
break;
case '网游小说':
$strName = "网游";
$strFenleiIId = 5;
break;
case '科幻小说':
$strName = "科幻";
$strFenleiIId = 6;
break;
case '灵异小说':
$strName = "女频";
$strFenleiIId = 7;
break;
case '言情小说':
$strName = "其它";
$strFenleiIId = 8;
break;
case '其他小说':
$strName = "其它";
$strFenleiIId = 9;
break;
default:
$strName = "其它";
$strFenleiIId = 9;
break;
}
$arrXiaoShuoInfo = [
'xsxq_ming_zi' => $strMingZi,
'xsxq_zuozhe' => $strZuoZhe, //$arrData['xiaoshuo_zuozhe'],
'xsfl_id' => $strFenleiIId,
'xsxq_zhuang_tai' => $strZhuangTai,
'xsxq_zi_shu' => $strZiShu,
'xsxq_jie_shao' => $strJieShao,
'xsxq_feng_mian' => json_encode($arrCover),
'xsxq_geng_xin_shi_jian' => $strLastDate,
'xsxq_tag' => json_encode($arrTag, JSON_UNESCAPED_UNICODE),
'xsxq_source_id' => $intXiaoShuoSourceId,
'xsxq_source_seo_id' => $intXiaoShuoSourceId . '1',
'xsxq_source_code' => 'ZW630_XIAOSHUO_' . $intXiaoShuoSourceId . '_' . $intXiaoShuoSourceSeoId,
];
// print_r($arrXiaoShuoInfo);
var_dump($arrXiaoShuoInfo['xsxq_source_code']);
$XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::where('xsxq_source_code', $arrXiaoShuoInfo['xsxq_source_code'])->find();
$arrZhangjie = [];
# seo 小说入库 - 只有新小说才会走这个步骤
if (!$XiaoShuoXiangQingModel) {
$XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
$intXiaoShuolId = $XiaoShuoXiangQingModel->xsxq_id;
// 检查是否找到元素-seo 小说入库
if ($QueryList->find('.first_txt a')->count() > 0 && $intXiaoShuolId) {
$QueryList->find('.first_txt a')->map(function ($ZhangJie) use ($intXiaoShuolId) {
$url = $ZhangJie->attr('href');
// 检查是否以 "/kan/" 开头
if (strpos($url, '/kan/') === 0) { // strpos 返回位置，0 表示是以 "/kan/" 开头
// 处理符合条件的链接
// echo "Processing index : " . $ZhangJie->text() . PHP_EOL;
$arrXiaoShuoSeoInfo = [
'nis_name' => $ZhangJie->text(),
'xsxq_id' => $intXiaoShuolId,
'nis_md5' => md5($intXiaoShuolId . $ZhangJie->text()),
];
NovelInfoSeoModel::addXiaoShuoXiangQingSeo($arrXiaoShuoSeoInfo);
}
});
} else {
echo "No elements found for .first_txt a" . PHP_EOL;
var_dump($strResult);
}
// # 新小说全部章节入库
// $intChapterIndex = 1; // 定义一个计数器变量
// // 检查是否找到元素
// if ($QueryList->find('.section-list')->eq(1)->find('li')->count() > 0) {
// $QueryList->find('.section-list')->eq(1)->find('li')->map(function ($ZhangJie) use (&$arrZhangjie, &$intChapterIndex) {
// echo "Processing index $intChapterIndex: " . $ZhangJie->text() . PHP_EOL;
// $arrZhangjie[] = [
// 'paixu' => $intChapterIndex,
// 'mingzi' => $ZhangJie->find('a')->text(),
// 'url' => $ZhangJie->find('a')->attr('href'),
// ];
// $intChapterIndex++;
// });
// } else {
// echo "No elements found for .listmain a" . PHP_EOL;
// }
//unset($QueryList);
}else{
$intXiaoShuolId = $XiaoShuoXiangQingModel->xsxq_id;
// 获取最大章节索引
$intMaxXiaoShuoZhangJiePaixu = XiaoShuoZhangJieModel::where('xsxq_id', $intXiaoShuolId)
->max('xszj_pai_xu'); // 获取最大值
// var_dump('获取小说章节最大值-'.$intMaxXiaoShuoZhangJiePaixu);
}
# 新小说全部章节入库
$intChapterIndex = 1; // 定义一个计数器变量
// 检查是否找到元素
if ($QueryList->find('.section-list')->eq(1)->find('li')->count() > 0) {
$QueryList->find('.section-list')->eq(1)->find('li')->map(function ($ZhangJie) use (&$arrZhangjie, &$intChapterIndex) {
// echo "Processing index $intChapterIndex: " . $ZhangJie->text() . PHP_EOL;
$arrZhangjie[] = [
'paixu' => $intChapterIndex,
'mingzi' => $ZhangJie->find('a')->text(),
'url' => $ZhangJie->find('a')->attr('href'),
];
$intChapterIndex++;
});
$this->eachNovelReader($arrZhangjie,$strSiteUrl,$intXiaoShuolId,$strFenleiIId);
} else {
echo "No elements found for .listmain a" . PHP_EOL;
}
# 章节处理
# 新小说全部章节入库
# 旧小说，只更新最新章节，获取数据库最大章节索引，对比源总章节
// // 如果没有结果，max 返回 null，可以设置默认值
$intMaxXiaoShuoZhangJiePaixu = $intMaxXiaoShuoZhangJiePaixu ?? 0;
# 目录页面处理
$strMuluHref = $QueryList->find('.btn-mulu')->eq(0)->attr('href');
$strMuluHref = $strSiteUrl . $strMuluHref ;
$ResponseMulu = $Client->request('GET', $strMuluHref, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResultMulu = (string)$ResponseMulu->getBody();
$QueryListMulu = QueryList::html($strResultMulu);
$intMuluTotal = $QueryListMulu->find('.page_num select')->eq(0)->find('option')->count();
$strMuluEndHref = $QueryListMulu->find('.page_num select')->eq(0)->find('option')->eq($intMuluTotal-1)->attr('value');
$strMuluEndText = $QueryListMulu->find('.page_num select')->eq(0)->find('option')->eq($intMuluTotal-1)->text();
// 使用正则表达式匹配
preg_match_all('/\.section-list\.ycxsid>li:nth-child\(/', $strResultMulu, $matches);
preg_match_all('/\.section-list\.ycxsid>li:nth-last-child\(/', $strResultMulu, $matchesLast);
$intProLiNum = count($matches[0]);
$intLastLiNum = count($matchesLast[0]);
$arrZhangjieMulu = [];
//if($intMaxXiaoShuoZhangJiePaixu < 200){
$intChapterPageIndex = 101; // 定义一个计数器变量
// 检查是否找到元素
if ($QueryListMulu->find('.chapter-list')->eq(0)->find('li')->count() > 0) {
$QueryListMulu->find('.chapter-list')->eq(0)->find('li')->map(function ($ZhangJie) use (&$arrZhangjieMulu, &$intChapterPageIndex) {
$arrZhangjieMulu[] = [
'paixu' => $intChapterPageIndex,
'mingzi' => $ZhangJie->find('a')->text(),
'url' => $ZhangJie->find('a')->attr('href'),
];
$intChapterPageIndex ++ ;
});
$arrZhangjieMulu = array_slice($arrZhangjieMulu, $intProLiNum);
// 删除后 11 条数据
$arrZhangjieMulu = array_slice($arrZhangjieMulu, 0, count($arrZhangjieMulu) - $intLastLiNum);
$this->eachNovelReader($arrZhangjieMulu,$strSiteUrl,$intXiaoShuolId,$strFenleiIId);
var_dump("chapter章节任务投递成功 $strMuluHref");
} else {
// var_dump($QueryListMulu->find('.section-list')->eq(0)->find('li')->count());
//echo "No elements found for .chapter-list $strMuluHref" . PHP_EOL;
// 检查是否找到元素
$intListIndex = 1;
if ($QueryListMulu->find('.section-list')->eq(0)->find('li')->count() > 0) {
$QueryListMulu->find('.section-list')->eq(0)->find('li')->map(function ($ZhangJie) use (&$arrZhangjieMulu, &$intChapterPageIndex,&$intListIndex) {
if($intListIndex >14){
$arrZhangjieMulu[] = [
'paixu' => $intChapterPageIndex,
'mingzi' => $ZhangJie->find('a')->text(),
'url' => $ZhangJie->find('a')->attr('href'),
];
$intChapterPageIndex ++ ;
};
$intListIndex++ ;
});
$arrZhangjieMulu = array_slice($arrZhangjieMulu, $intProLiNum);
// 删除后 11 条数据
$arrZhangjieMulu = array_slice($arrZhangjieMulu, 0, count($arrZhangjieMulu) - $intLastLiNum);
$this->eachNovelReader($arrZhangjieMulu,$strSiteUrl,$intXiaoShuolId,$strFenleiIId);
var_dump("section章节任务投递成功 $strMuluHref");
} else {
var_dump("No elements found for .section-list $strMuluHref") ;
}
}
//}
// 输出最终结果
// var_dump(count($arrZhangjie));
// var_dump($QueryListMulu->find('.chapter-list li')->count());
// var_dump($intMuluTotal);
// var_dump($strMuluEndHref);
// var_dump($strMuluEndText);
// var_dump($intMaxXiaoShuoZhangJiePaixu); // 1010
preg_match('/(\d+)-(\d+)章/', $strMuluEndText, $matches);
$intMaxPaixu = $matches[2]; // 最大章节
$intMaxMuluPage = floor($intMaxPaixu / 100); // 最大目录页面
// $intMaxMuluPage = 2;
// 所有目录分页处理
for ($i = 2; $i <= $intMaxMuluPage; $i++) {
//echo "当前值: $i\n";
$arrTask = [
'callback' => [self::class, 'getNovelMulu'],
'data' => [
'strMuluHref' => $strSiteUrl . '/shu/' . $XiaoShuoXiangQingModel->xsxq_source_id . '_' . $i . '.html' ,
'xsxq_id' => $XiaoShuoXiangQingModel->xsxq_id,
'xsfl_id' => $strFenleiIId,
'intStartIndex' => $i*100 + 1,
]
];
$TaskCore->set($arrTask);
// var_dump($arrTask);
}
unset($QueryList);
unset($QueryListMulu);
} catch (\Throwable $T) {
$arrTask = [
'callback' => [self::class, 'getNovelInfo'],
'data' => [
'xs_source_id' => $arrData['xs_source_id'],
'xs_source_seo_id' => $intXiaoShuoSourceSeoId,
]
];
$TaskCore->set($arrTask);
throw $T;
}
}
public function getNovelMulu($arrData)
{
$TaskCore = $this->TaskCore;
$Client = $this->Client;
$strSiteUrl = $this->get630zwDomain();
$strUrl = $arrData['strMuluHref'];
$intXiaoShuoId = $arrData['xsxq_id'];
$intXiaoShuoFenleiId = $arrData['xsfl_id'];
$intStartIndex = $arrData['intStartIndex'];
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
$QueryList = QueryList::html($strResult);
// 使用正则表达式匹配
preg_match_all('/\.section-list\.ycxsid>li:nth-child\(/', $strResult, $matches);
preg_match_all('/\.section-list\.ycxsid>li:nth-last-child\(/', $strResult, $matchesLast);
$intProLiNum = count($matches[0]);
$intLastLiNum = count($matchesLast[0]);
$arrZhangjie = [];
if ($QueryList->find('.chapter-list')->eq(0)->find('li')->count() > 0) {
$QueryList->find('.chapter-list')->eq(0)->find('li')->map(function ($ZhangJie) use (&$arrZhangjie, &$intStartIndex) {
$arrZhangjie[] = [
'paixu' => $intStartIndex,
'mingzi' => $ZhangJie->find('a')->text(),
'url' => $ZhangJie->find('a')->attr('href'),
];
$intStartIndex ++ ;
// var_dump($intStartIndex);
});
$arrZhangjie = array_slice($arrZhangjie, $intProLiNum);
// 删除后 11 条数据
$arrZhangjie = array_slice($arrZhangjie, 0, count($arrZhangjie) - $intLastLiNum);
$this->eachNovelReader($arrZhangjie,$strSiteUrl,$intXiaoShuoId,$intXiaoShuoFenleiId);
var_dump("chapter章节任务投递成功 $strUrl");
} else {
// 检查是否找到元素
if ($QueryList->find('.section-list')->eq(0)->find('li')->count() > 0) {
$QueryList->find('.section-list')->eq(0)->find('li')->map(function ($ZhangJie) use (&$arrZhangjie, &$intStartIndex) {
$arrZhangjie[] = [
'paixu' => $intStartIndex,
'mingzi' => $ZhangJie->find('a')->text(),
'url' => $ZhangJie->find('a')->attr('href'),
];
$intStartIndex ++ ;
// var_dump($intStartIndex);
});
$arrZhangjie = array_slice($arrZhangjie, $intProLiNum);
// 删除后 11 条数据
$arrZhangjie = array_slice($arrZhangjie, 0, count($arrZhangjie) - $intLastLiNum);
$this->eachNovelReader($arrZhangjie,$strSiteUrl,$intXiaoShuoId,$intXiaoShuoFenleiId);
var_dump("section章节任务投递成功 $strUrl");
} else {
var_dump("No elements found for .section-list $strUrl") ;
}
}
unset($QueryList);
} catch (\Throwable $T) {
$arrTask = [
'callback' => [self::class, 'getNovelMulu'],
'data' => [
'strMuluHref' => $strUrl ,
'xsxq_id' => $intXiaoShuoId,
'xsfl_id' => $intXiaoShuoFenleiId,
'intStartIndex' => $intStartIndex,
]
];
$TaskCore->set($arrTask);
throw $T;
}
}
public function eachNovelReader($arrZhangjie,$strSiteUrl,$intXiaoShuoId,$intXiaoShuoFenleiId)
{
$TaskCore = $this->TaskCore;
// var_dump($arrZhangjie);
foreach ($arrZhangjie as $intZhangJieNum => $arrZhangjieOne) {
// 如果数据库里的章节最大索引大于当前索引，说明已经入库，不需要入队列了
// if($intMaxXiaoShuoZhangJiePaixu > $intZhangJieNum){
// var_dump("小说章节已经入库，不再推送任务");
// continue;// 如果任务已存在，直接跳过 break
// }
$strZhangJieUrl = 'https://www.630zw.org' . $arrZhangjieOne['url'];
$strZhangJieSourceId = extractFilename($strZhangJieUrl);
# 'AILISI_XIAOSHUO_ZHANGJIE_小说源id_小说源章节id'
$strXszjSourceCode = '630ZW_XIAOSHUO_ZHANGJIE_'. $intXiaoShuoId . '_' . $strZhangJieSourceId . '_1' ;
$arrTask = [
'callback' => [self::class, 'getNovelReader'],
'data' => [
'xszj_source_id' => $strZhangJieSourceId,
'xszj_paixu' => $arrZhangjieOne['paixu'],
'xszj_name' => $arrZhangjieOne['mingzi'],
'xszj_url' => $strZhangJieUrl,
'xszj_source_code' => $strXszjSourceCode,
'xsxq_id' => $intXiaoShuoId,
'xsfl_id' => $intXiaoShuoFenleiId,
'intZhangJieSort' => 1,
]
];
$TaskCore->set($arrTask);
}
}
public function getNovelFengMian($arrData)
{
//var_dump('getNovelFengMian');
//var_dump($arrData);
$Client = $this->Client;
$TaskCore = $this->TaskCore;
// 获取 Redis 实例
$redis = Cache::store('redis')->handler();
$intXiaoShuoSourceId = $arrData['xiaoshuo_id'];
$intXiaoShuoSourceSeoId = $arrData['xiaoshuo_seo_id'];
$redisKey = "task:novel_source_id:" . $intXiaoShuoSourceId;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($arrData['xiaoshuo_url']);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$strSiteUrl = $this->getAilisiDomain();
$strCover = $QueryList->find('#fmimg img')->eq(0)->attr("src");
if (strpos($strCover, 'http') === false) {
$strCover = $strSiteUrl . $strCover;
}
$strCoverUri = $strCover;
try {
$strCoverUri = "";
$strCoverUri = ImageProcessor::getInstance()->downloadAndEncryptImage($strCover, 'xs');
$arrCover = [
'code' => 'AI_LI_SI_COVER',
// 'uri' => getUriByUrl($strCover),
'uri' => $strCoverUri,
];
$arrXiaoShuoInfo = [
'xsxq_feng_mian' => json_encode($arrCover),
];
$XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
} catch (\Throwable $t) {
$strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
echo $strErr;
}
} catch (\Throwable $T) {
throw $T;
}
}
//https://www.630zw.org/shu/438374/156339465.html
public function getNovelReader($arrData)
{
$TaskCore = $this->TaskCore;
$strSiteUrl = 'https://www.630zw.org';
var_dump('getNovelReader------------------------------------------------------------------------------------');
$Client = $this->Client;
$intZhangJieSort = $arrData['intZhangJieSort'];
$strXszjSourceCode = $arrData['xszj_source_code'];
$strZhangJieUrl = $arrData['xszj_url'];
$intZhangJiePaiXu = $arrData['xszj_paixu'];
$intFenleiId = $arrData['xsfl_id'];
$intXiaoShuoId = $arrData['xsxq_id'];
$strZhangJieSourceId = $arrData['xszj_source_id'];
$strZhangJieName = $arrData['xszj_name'];
$XiaoShuoZhangJieModel = XiaoShuoZhangJieModel::where('xszj_source_code', $strXszjSourceCode)->find();
if ($XiaoShuoZhangJieModel) {
var_dump($XiaoShuoZhangJieModel->xszj_source_code);
var_dump('章节已经存在 paixu: '.$intZhangJiePaiXu);
var_dump('章节已经存在strXszjSourceCode: '.$strXszjSourceCode);
var_dump('章节已经存在strZhangJieUrl: '.$strZhangJieUrl);
// return true;
}
var_dump($strZhangJieUrl);
var_dump('章节排序：'.$intZhangJiePaiXu);
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strZhangJieUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
if (XiaoShuoZhangJieModel::checkZhangJieExists($strXszjSourceCode)) {
return true;
}
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strZhangJieUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
unset($Response);
$ZhangJieQueryList = QueryList::html($strResult);
// 正则表达式，匹配 `_数字.html`
$pattern = '/_(\d+)\.html$/';
$patternHref = '/(\/shu\/\d+\/\d+_\d+\.html)/';
$strNeiRong = '';
$strNextHref = '';
// 使用 preg_match() 进行匹配
if (preg_match($patternHref, $strResult, $matches)) {
// 提取匹配到的部分
echo "提取到的 URL 是: " . $matches[1];
$strNextHref = $matches[1];
} else {
echo "没有匹配到 URL";
}
preg_match_all("/qsbs\.bb\('([^']+)'\)/", $strResult, $matches);
// 输出提取到的加密字符串
if (isset($matches[1])) {
$strNeiRong = implode(';;;', $matches[1]);
// var_dump($strNeiRong);
}
// 判断字符串是否匹配
if (preg_match($pattern, $strNextHref, $matches) ){
$intNextPage = 2;
// 确保匹配成功后再访问索引
if (isset($matches[1])) {
$intNextPage = (int)$matches[1];
$intNextPage +=1;
var_dump('提取到的数字是：' . $intNextPage);
}
var_dump('提取到的数字是-：' . $intNextPage);
$arrTask = [
'callback' => [self::class, 'getNovelReader'],
'data' => [
'xszj_source_id' => $strZhangJieSourceId,
'xszj_paixu' => $intZhangJiePaiXu,
'xszj_name' => $strZhangJieName,
'xszj_url' => $strSiteUrl . $strNextHref,
'xszj_source_code' => '630ZW_XIAOSHUO_ZHANGJIE_'. $intXiaoShuoId . '_' . $strZhangJieSourceId . '_' . $intNextPage,
'xsxq_id' => $intXiaoShuoId,
'xsfl_id' => $intFenleiId,
'intZhangJieSort' => $intNextPage,
]
];
$TaskCore->set($arrTask);
}
# '/novel/123yqw/分类id/小说源id/小说源章节id.txt'
$strNeiRongTxtPath = config("filesystem.novel") . '/novel/630zwxs/' . $intFenleiId . '/' . $intXiaoShuoId .'/' . $intZhangJiePaiXu .'/' . $intZhangJieSort . '.txt';
//$strNeiRongTxtPath = "/www/novel/123yqw/1/11/111.txt";
// 调用函数
$result = saveCompressAndDeleteTxt($strNeiRongTxtPath, $strNeiRong);
// 输出状态
if ($result['status']) {
var_dump("Success: " . $result['message']);
} else {
var_dump("Error: " . $result['message']);
}
$arrXiaoShuoZhangJie = [
'xsxq_id' => $intXiaoShuoId,
'xszj_pai_xu' => $intZhangJiePaiXu,
'xszj_ming_zi' => $strZhangJieName,
'xszj_nei_rong' => $strNeiRongTxtPath,
'xszj_source_id' => $strZhangJieSourceId,
'xszj_source_code' => $strXszjSourceCode,
'xszj_sort' => $intZhangJieSort,
];
// var_dump($arrXiaoShuoZhangJie);
if($strZhangJieName){
XiaoShuoZhangJieModel::addXiaoShuoZhangJie($arrXiaoShuoZhangJie);
}
unset($ZhangJieQueryList);
// // 延时 3 秒
//sleep(3);
} catch (\Throwable $T) {
$arrTask = [
'callback' => [self::class, 'getNovelReader'],
'data' => [
'xszj_source_id' => $strZhangJieSourceId,
'xszj_paixu' => $intZhangJiePaiXu,
'xszj_name' => $strZhangJieName,
'xszj_url' => $strZhangJieUrl,
'xszj_source_code' => $strXszjSourceCode,
'xsxq_id' => $intXiaoShuoId,
'xsfl_id' => $intFenleiId,
'intZhangJieSort' => $intZhangJieSort,
]
];
$TaskCore->set($arrTask);
throw $T;
}
}
}

426
code/app/task/collection/novelone/NovelJiuYuFanXianCol.php
View File

@@ -1,426 +0,0 @@
<?php
declare(strict_types=1);
namespace app\task\collection\novelone;
use app\model\XiaoShuoXiangQingModel;
use app\model\XiaoShuoZhangJieModel;
use app\task\collection\PullDataColBase;
use GuzzleHttp\Cookie\CookieJar;
use processor\ImageProcessor;
use QL\QueryList;
use app\common\Base64Helper;
/**
* @ 抓取 gdbzkz.info 站点相关小说
*/
class NovelJiuYuFanXianCol extends PullDataColBase
{
public function pullXiaoShuo()
{
$TaskCore = $this->TaskCore;
// http://www.gdbzkz.info/jiuyufanxian/
$arrNovel = [
['xs_source_id' => 'jiuyufanxian'],
];
foreach ($arrNovel as $Novel) {
$arrTask = [
'callback' => [self::class, 'getNovelInfo'],
'data' => [
'xs_source_id' => $Novel['xs_source_id'],
'xs_source_seo_id' => 1,
]
];
$TaskCore->set($arrTask);
var_dump($arrTask);
}
}
public function getNovelInfo($arrData)
{
$TaskCore = $this->TaskCore;
$Client = $this->Client;
$strSiteUrl = 'http://www.gdbzkz.info';
$intXiaoShuoSourceId = $arrData['xs_source_id'];
$intXiaoShuoSourceSeoId = $arrData['xs_source_seo_id'];
$strUrl = $strSiteUrl . '/' . $intXiaoShuoSourceId;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
$QueryList = QueryList::html($strResult);
// var_dump($QueryList);
// 提取所有 <meta> 标签的属性
$result = $QueryList->find('meta[property^="og:"]')->map(function ($item) {
return [
'property' => $item->attr('property'),
'content' => $item->attr('content'),
];
});
// 打印结果
// print_r($result->all());
$strZuoZhe = '';
$strMingZi = '';
$strLastDate = '';
$strJieShao = '';
$strCover = '';
$strFenlei = '';
$strZiShu = 0;
$strZhuangTai = '';
foreach ($result as $item) {
switch ($item['property']) {
case 'og:title':
$strMingZi = $item['content'];
break;
case 'og:description':
$strJieShao = $item['content'];
break;
case 'og:image':
$strCover = $item['content'];
break;
case 'og:novel:category':
$strFenlei = $item['content'];
break;
case 'og:novel:author':
$strZuoZhe = $item['content'];
break;
case 'og:novel:status':
$strZhuangTai = $item['content'];
break;
case 'og:novel:update_time':
$strLastDate = $item['content'];
break;
}
}
$strLastDate = trim((string)$strLastDate);
if (!isValidDateTime($strLastDate)) {
$strLastDate = date('Y-m-d H:i:s');
}
$arrTag = [];
if (strpos($strCover, 'http') === false) {
$strCover = $strSiteUrl . $strCover;
}
$strCoverUri = $strCover;
try {
$strCoverUri = "";
$strCoverUri = ImageProcessor::getInstance()->downloadImage($intXiaoShuoSourceId, $strCover, 'xs');
} catch (\Throwable $t) {
$strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
echo $strErr;
}
$arrCover = [
'code' => 'BI_QU_XS_COVER',
// 'uri' => getUriByUrl($strCover),
'uri' => $strCoverUri,
];
$strFenleiIId = 1;
switch ($strFenlei) {
case '玄幻小说':
$strName = "玄幻";
$strFenleiIId = 1;
break;
case '修真小说':
$strName = "修真";
$strFenleiIId = 2;
break;
case '都市小说':
$strName = "都市";
$strFenleiIId = 3;
break;
case '历史小说':
$strName = "历史";
$strFenleiIId = 4;
break;
case '网游小说':
$strName = "网游";
$strFenleiIId = 5;
break;
case '科幻小说':
$strName = "科幻";
$strFenleiIId = 6;
break;
case '女频小说':
$strName = "女频";
$strFenleiIId = 7;
break;
case '其它小说':
$strName = "其它";
$strFenleiIId = 8;
break;
default:
$strName = "其它";
$strFenleiIId = 8;
break;
}
$arrXiaoShuoInfo = [
'xsxq_ming_zi' => $strMingZi,
'xsxq_zuozhe' => $strZuoZhe, //$arrData['xiaoshuo_zuozhe'],
'xsfl_id' => $strFenleiIId,
'xsxq_zhuang_tai' => $strZhuangTai,
'xsxq_zi_shu' => $strZiShu,
'xsxq_jie_shao' => $strJieShao,
'xsxq_feng_mian' => json_encode($arrCover),
'xsxq_geng_xin_shi_jian' => $strLastDate,
'xsxq_tag' => json_encode($arrTag, JSON_UNESCAPED_UNICODE),
'xsxq_source_id' => $intXiaoShuoSourceId,
'xsxq_source_seo_id' => $intXiaoShuoSourceId . '1',
'xsxq_source_code' => 'BIQU_XIAOSHUO_' . $intXiaoShuoSourceId . '_' . $intXiaoShuoSourceSeoId,
];
// print_r($arrXiaoShuoInfo);
var_dump($arrXiaoShuoInfo['xsxq_source_code']);
$XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
$arrZhangjie = [];
$intChapterIndex = 0; // 定义一个计数器变量
// 检查是否找到元素
if ($QueryList->find('.listmain a')->count() > 0) {
$QueryList->find('.listmain a')->map(function ($ZhangJie) use (&$arrZhangjie, &$intChapterIndex) {
echo "Processing index $intChapterIndex: " . $ZhangJie->text() . PHP_EOL;
// 从索引6开始
if ($intChapterIndex >= 12) {
$arrZhangjie[] = [
'mingzi' => $ZhangJie->text(),
'url' => $ZhangJie->attr('href'),
];
}
$intChapterIndex++;
});
// 输出最终结果
// var_dump($arrZhangjie);
} else {
echo "No elements found for .listmain a" . PHP_EOL;
}
//unset($QueryList);
// 获取最大章节索引
$intMaxXiaoShuoZhangJiePaixu = XiaoShuoZhangJieModel::where('xsxq_id', $XiaoShuoXiangQingModel->xsxq_id)
->max('xszj_pai_xu'); // 获取最大值
var_dump('获取小说章节最大值-' . $intMaxXiaoShuoZhangJiePaixu);
// 如果没有结果，max 返回 null，可以设置默认值
$intMaxXiaoShuoZhangJiePaixu = $intMaxXiaoShuoZhangJiePaixu ?? 0;
foreach ($arrZhangjie as $intZhangJieNum => $arrZhangjieOne) {
var_dump($intZhangJieNum);
// 如果数据库里的章节最大索引大于当前索引，说明已经入库，不需要入队列了
// if ($intMaxXiaoShuoZhangJiePaixu > $intZhangJieNum) {
// var_dump("小说章节已经入库，不再推送任务");
// continue; // 如果任务已存在，直接跳过 break
// }
$strZhangJieUrl = $strSiteUrl . $arrZhangjieOne['url'];
$strZhangJieSourceId = extractFilename($strZhangJieUrl);
# 'AILISI_XIAOSHUO_ZHANGJIE_小说源id_小说源章节id'
$strXszjSourceCode = 'gdbzkz_XIAOSHUO_ZHANGJIE_' . $XiaoShuoXiangQingModel->xsxq_source_id . '_' . $strZhangJieSourceId . '_1';
$arrTask = [
'callback' => [self::class, 'getNovelReader'],
'data' => [
'xszj_source_id' => $strZhangJieSourceId,
'xszj_paixu' => $intZhangJieNum += 1,
'xszj_name' => $arrZhangjieOne['mingzi'],
'xszj_url' => $strZhangJieUrl,
'xszj_source_code' => $strXszjSourceCode,
'xsxq_id' => $XiaoShuoXiangQingModel->xsxq_id,
'xsfl_id' => $XiaoShuoXiangQingModel->xsfl_id,
'intZhangJieSort' => 1,
]
];
$TaskCore->set($arrTask);
}
var_dump("章节任务投递成功");
} catch (\Throwable $T) {
throw $T;
}
}
public function getNovelReader($arrData)
{
$TaskCore = $this->TaskCore;
$strSiteUrl = 'http://www.gdbzkz.info';
var_dump('getNovelReader------------------------------------------------------------------------------------');
$Client = $this->Client;
$intZhangJieSort = $arrData['intZhangJieSort'];
$strXszjSourceCode = $arrData['xszj_source_code'];
$strZhangJieUrl = $arrData['xszj_url'];
$intZhangJiePaiXu = $arrData['xszj_paixu'];
$intFenleiId = $arrData['xsfl_id'];
$intXiaoShuoId = $arrData['xsxq_id'];
$strZhangJieSourceId = $arrData['xszj_source_id'];
$strZhangJieName = $arrData['xszj_name'];
$XiaoShuoZhangJieModel = XiaoShuoZhangJieModel::where('xszj_source_code', $strXszjSourceCode)->find();
if ($XiaoShuoZhangJieModel) {
// var_dump($XiaoShuoZhangJieModel->xszj_source_code);
// var_dump('章节已经存在 paixu: ' . $intZhangJiePaiXu);
// var_dump('章节已经存在strXszjSourceCode: ' . $strXszjSourceCode);
// var_dump('章节已经存在strZhangJieUrl: ' . $strZhangJieUrl);
return true;
}
var_dump($strZhangJieUrl);
var_dump('章节排序：' . $intZhangJiePaiXu);
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strZhangJieUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
if (XiaoShuoZhangJieModel::checkZhangJieExists($strXszjSourceCode)) {
return true;
}
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strZhangJieUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
unset($Response);
$ZhangJieQueryList = QueryList::html($strResult);
$strNeiRong = '';
// 获取原始 HTML
$strNeiRong = $ZhangJieQueryList->find('#content')->html();
// 解码 HTML 实体
$strNeiRong = html_entity_decode($strNeiRong, ENT_QUOTES, 'UTF-8');
// 替换不间断空格 (\xc2\xa0) 和 &nbsp; 为普通空格
$strNeiRong = preg_replace('/\xc2\xa0/', ' ', $strNeiRong); // 替换 UTF-8 不间断空格
$strNeiRong = preg_replace('/&nbsp;/', ' ', $strNeiRong); // 替换 &nbsp;
// 清理多余空格和换行符
$strNeiRong = preg_replace('/\s+/', ' ', $strNeiRong);
// 1. 找到所有的 <br> 位置
$br_positions = [];
$offset = 0;
while (($pos = strpos($strNeiRong, "<br", $offset)) !== false) {
$br_positions[] = $pos;
$offset = $pos + 1;
}
// 2. 找到倒数第三个 <br> 的位置
if (count($br_positions) >= 3) {
$cut_position = $br_positions[count($br_positions) - 3]; // 倒数第三个 <br> 的索引
$strNeiRong = substr($strNeiRong, 0, $cut_position); // 截取前面部分，删除后面的内容
}
$arrNeiRong = explode('<br> <br>', $strNeiRong);
$arrEnCode = [];
foreach ($arrNeiRong as $intNeiRongNum => $strNeiRongOne) {
$strEnCode = Base64Helper::encode('<p>' . $strNeiRongOne . '</p>');
$arrEnCode[] = $strEnCode;
}
$strNeiRong = implode(';;;', $arrEnCode);
# '/novel/123yqw/分类id/小说源id/小说源章节id.txt'
$strNeiRongTxtPath = config("filesystem.novel") . '/novel/630zwxs/' . $intFenleiId . '/' . $intXiaoShuoId . '/' . $intZhangJiePaiXu . '/' . $intZhangJieSort . '.txt';
//$strNeiRongTxtPath = "/www/novel/123yqw/1/11/111.txt";
// 调用函数
$result = saveCompressAndDeleteTxt($strNeiRongTxtPath, $strNeiRong);
// 输出状态
if ($result['status']) {
var_dump("Success: " . $result['message']);
} else {
var_dump("Error: " . $result['message']);
}
$arrXiaoShuoZhangJie = [
'xsxq_id' => $intXiaoShuoId,
'xszj_pai_xu' => $intZhangJiePaiXu,
'xszj_ming_zi' => $strZhangJieName,
'xszj_nei_rong' => $strNeiRongTxtPath,
'xszj_source_id' => $strZhangJieSourceId,
'xszj_source_code' => $strXszjSourceCode,
'xszj_sort' => $intZhangJieSort,
];
var_dump($arrXiaoShuoZhangJie);
if ($strZhangJieName) {
XiaoShuoZhangJieModel::addXiaoShuoZhangJie($arrXiaoShuoZhangJie);
}
unset($ZhangJieQueryList);
// // 延时 3 秒
//sleep(3);
} catch (\Throwable $T) {
$arrTask = [
'callback' => [self::class, 'getNovelReader'],
'data' => [
'xszj_source_id' => $strZhangJieSourceId,
'xszj_paixu' => $intZhangJiePaiXu,
'xszj_name' => $strZhangJieName,
'xszj_url' => $strZhangJieUrl,
'xszj_source_code' => $strXszjSourceCode,
'xsxq_id' => $intXiaoShuoId,
'xsfl_id' => $intFenleiId,
'intZhangJieSort' => $intZhangJieSort,
]
];
$TaskCore->set($arrTask);
throw $T;
}
}
}

441
code/app/task/collection/novelone/NovelWenXueGuanCol.php
View File

@@ -1,441 +0,0 @@
<?php
declare(strict_types=1);
namespace app\task\collection\novelone;
use app\model\XiaoShuoXiangQingModel;
use app\model\XiaoShuoZhangJieModel;
use app\task\collection\PullDataColBase;
use GuzzleHttp\Cookie\CookieJar;
use processor\ImageProcessor;
use QL\QueryList;
use app\common\Base64Helper;
/**
* @ 抓取 bqvvxg8.cc 站点相关小说
*/
class NovelWenXueGuanCol extends PullDataColBase
{
public function pullXiaoShuo()
{
$TaskCore = $this->TaskCore;
// https://www.bqvvxg8.cc/wenzhang/86/86114/
$arrNovel = [
['xs_source_id' => 'wenxueguan86114', 'xs_url' => 'https://www.bqvvxg8.cc/wenzhang/86/86114/'],
['xs_source_id' => 'wenxueguan90205', 'xs_url' => 'https://www.bqvvxg8.cc/wenzhang/90/90205/'],
];
foreach ($arrNovel as $Novel) {
$arrTask = [
'callback' => [self::class, 'getNovelInfo'],
'data' => [
'xs_url' => $Novel['xs_url'],
'xs_source_id' => $Novel['xs_source_id'],
'xs_source_seo_id' => 1,
]
];
$TaskCore->set($arrTask);
var_dump($arrTask);
}
}
public function getNovelInfo($arrData)
{
$TaskCore = $this->TaskCore;
$Client = $this->Client;
$strSiteUrl = 'https://www.bqvvxg8.cc';
$intXiaoShuoSourceId = $arrData['xs_source_id'];
$intXiaoShuoSourceSeoId = $arrData['xs_source_seo_id'];
$strUrl = $arrData['xs_url'];
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
$QueryList = QueryList::html($strResult);
// var_dump($QueryList);
// 提取所有 <meta> 标签的属性
$result = $QueryList->find('meta[property^="og:"]')->map(function ($item) {
return [
'property' => $item->attr('property'),
'content' => $item->attr('content'),
];
});
// 打印结果
// print_r($result->all());
$strZuoZhe = '';
$strMingZi = '';
$strLastDate = '';
$strJieShao = '';
$strCover = '';
$strFenlei = '';
$strZiShu = 0;
$strZhuangTai = '';
foreach ($result as $item) {
switch ($item['property']) {
case 'og:title':
$strMingZi = $item['content'];
break;
case 'og:description':
$strJieShao = $item['content'];
break;
case 'og:image':
$strCover = $item['content'];
break;
case 'og:novel:category':
$strFenlei = $item['content'];
break;
case 'og:novel:author':
$strZuoZhe = $item['content'];
break;
case 'og:novel:status':
$strZhuangTai = $item['content'] . '中';
break;
case 'og:novel:update_time':
$strLastDate = $item['content'];
break;
}
}
$strLastDate = trim((string)$strLastDate);
if (!isValidDateTime($strLastDate)) {
$strLastDate = date('Y-m-d H:i:s');
}
$arrTag = [];
if (strpos($strCover, 'http') === false) {
$strCover = $strSiteUrl . $strCover;
}
$strCoverUri = $strCover;
try {
$strCoverUri = "";
$strCoverUri = ImageProcessor::getInstance()->downloadImage($intXiaoShuoSourceId, $strCover, 'xs');
} catch (\Throwable $t) {
$strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
echo $strErr;
}
$arrCover = [
'code' => 'BI_QU_XS_COVER',
// 'uri' => getUriByUrl($strCover),
'uri' => $strCoverUri,
];
$strFenleiIId = 1;
switch ($strFenlei) {
case '玄幻小说':
$strName = "玄幻";
$strFenleiIId = 1;
break;
case '修真小说':
$strName = "修真";
$strFenleiIId = 2;
break;
case '言情小说':
$strName = "都市";
$strFenleiIId = 3;
break;
case '历史小说':
$strName = "历史";
$strFenleiIId = 4;
break;
case '网游小说':
$strName = "网游";
$strFenleiIId = 5;
break;
case '科幻小说':
$strName = "科幻";
$strFenleiIId = 6;
break;
case '女频小说':
$strName = "女频";
$strFenleiIId = 7;
break;
case '其它小说':
$strName = "其它";
$strFenleiIId = 8;
break;
default:
$strName = "其它";
$strFenleiIId = 8;
break;
}
$arrXiaoShuoInfo = [
'xsxq_ming_zi' => $strMingZi,
'xsxq_zuozhe' => $strZuoZhe, //$arrData['xiaoshuo_zuozhe'],
'xsfl_id' => $strFenleiIId,
'xsxq_zhuang_tai' => $strZhuangTai,
'xsxq_zi_shu' => $strZiShu,
'xsxq_jie_shao' => $strJieShao,
'xsxq_feng_mian' => json_encode($arrCover),
'xsxq_geng_xin_shi_jian' => $strLastDate,
'xsxq_tag' => json_encode($arrTag, JSON_UNESCAPED_UNICODE),
'xsxq_source_id' => $intXiaoShuoSourceId,
'xsxq_source_seo_id' => $intXiaoShuoSourceId . '1',
'xsxq_source_code' => 'WENXUEGUANG_XIAOSHUO_' . $intXiaoShuoSourceId . '_' . $intXiaoShuoSourceSeoId,
];
// print_r($arrXiaoShuoInfo);
var_dump($arrXiaoShuoInfo['xsxq_source_code']);
$XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
$arrZhangjie = [];
$intChapterIndex = 0; // 定义一个计数器变量
// 检查是否找到元素
if ($QueryList->find('.listmain a')->count() > 0) {
$QueryList->find('.listmain a')->map(function ($ZhangJie) use (&$arrZhangjie, &$intChapterIndex) {
echo "Processing index $intChapterIndex: " . $ZhangJie->text() . PHP_EOL;
// 从索引6开始
if ($intChapterIndex >= 12) {
$arrZhangjie[] = [
'mingzi' => $ZhangJie->text(),
'url' => $ZhangJie->attr('href'),
];
}
$intChapterIndex++;
});
// 输出最终结果
var_dump($arrZhangjie);
} else {
echo "No elements found for .listmain a" . PHP_EOL;
}
//unset($QueryList);
// 获取最大章节索引
$intMaxXiaoShuoZhangJiePaixu = XiaoShuoZhangJieModel::where('xsxq_id', $XiaoShuoXiangQingModel->xsxq_id)
->max('xszj_pai_xu'); // 获取最大值
var_dump('获取小说章节最大值-' . $intMaxXiaoShuoZhangJiePaixu);
// 如果没有结果，max 返回 null，可以设置默认值
$intMaxXiaoShuoZhangJiePaixu = $intMaxXiaoShuoZhangJiePaixu ?? 0;
foreach ($arrZhangjie as $intZhangJieNum => $arrZhangjieOne) {
var_dump($intZhangJieNum);
// 如果数据库里的章节最大索引大于当前索引，说明已经入库，不需要入队列了
// if ($intMaxXiaoShuoZhangJiePaixu > $intZhangJieNum) {
// var_dump("小说章节已经入库，不再推送任务");
// continue; // 如果任务已存在，直接跳过 break
// }
$strZhangJieUrl = $strSiteUrl . $arrZhangjieOne['url'];
$strZhangJieSourceId = extractFilename($strZhangJieUrl);
# 'AILISI_XIAOSHUO_ZHANGJIE_小说源id_小说源章节id'
$strXszjSourceCode = 'gdbzkz_XIAOSHUO_ZHANGJIE_' . $XiaoShuoXiangQingModel->xsxq_source_id . '_' . $strZhangJieSourceId . '_1';
$arrTask = [
'callback' => [self::class, 'getNovelReader'],
'data' => [
'xszj_source_id' => $strZhangJieSourceId,
'xszj_paixu' => $intZhangJieNum += 1,
'xszj_name' => $arrZhangjieOne['mingzi'],
'xszj_url' => $strZhangJieUrl,
'xszj_source_code' => $strXszjSourceCode,
'xsxq_id' => $XiaoShuoXiangQingModel->xsxq_id,
'xsfl_id' => $XiaoShuoXiangQingModel->xsfl_id,
'intZhangJieSort' => 1,
]
];
$TaskCore->set($arrTask);
}
var_dump("章节任务投递成功");
} catch (\Throwable $T) {
throw $T;
}
}
public function getNovelReader($arrData)
{
$TaskCore = $this->TaskCore;
$strSiteUrl = 'https://www.bqvvxg8.cc';
var_dump('getNovelReader------------------------------------------------------------------------------------');
$Client = $this->Client;
$intZhangJieSort = $arrData['intZhangJieSort'];
$strXszjSourceCode = $arrData['xszj_source_code'];
$strZhangJieUrl = $arrData['xszj_url'];
$intZhangJiePaiXu = $arrData['xszj_paixu'];
$intFenleiId = $arrData['xsfl_id'];
$intXiaoShuoId = $arrData['xsxq_id'];
$strZhangJieSourceId = $arrData['xszj_source_id'];
$strZhangJieName = $arrData['xszj_name'];
$XiaoShuoZhangJieModel = XiaoShuoZhangJieModel::where('xszj_source_code', $strXszjSourceCode)->find();
if ($XiaoShuoZhangJieModel) {
// var_dump($XiaoShuoZhangJieModel->xszj_source_code);
// var_dump('章节已经存在 paixu: ' . $intZhangJiePaiXu);
// var_dump('章节已经存在strXszjSourceCode: ' . $strXszjSourceCode);
// var_dump('章节已经存在strZhangJieUrl: ' . $strZhangJieUrl);
return true;
}
var_dump($strZhangJieUrl);
var_dump('章节排序：' . $intZhangJiePaiXu);
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strZhangJieUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
if (XiaoShuoZhangJieModel::checkZhangJieExists($strXszjSourceCode)) {
return true;
}
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strZhangJieUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
'Accept-Charset' => 'GBK, UTF-8, *;q=0.8',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
// 先尝试自动检测编码
$strEncoding = mb_detect_encoding($strResult, ['GBK', 'GB2312', 'BIG5', 'UTF-8'], true);
// 如果不是 UTF-8，转换为 UTF-8
if ($strEncoding && $strEncoding !== 'UTF-8') {
$strResult = mb_convert_encoding($strResult, 'UTF-8', $strEncoding);
} else {
// 备用方案，强制转换
$strResult = @iconv('GBK', 'UTF-8//IGNORE', $strResult);
}
// **去掉可能影响解析的 meta 标签**
$strResult = preg_replace('/<meta[^>]+charset=[^>]+>/i', '', $strResult);
// **第一步：使用 QueryList**
$ZhangJieQueryList = QueryList::html($strResult);
$strNeiRong = '';
// 获取原始 HTML
$strNeiRong = $ZhangJieQueryList->find('#content')->html();
// 解码 HTML 实体
$strNeiRong = html_entity_decode($strNeiRong, ENT_QUOTES, 'UTF-8');
// 替换不间断空格 (\xc2\xa0) 和 &nbsp; 为普通空格
$strNeiRong = preg_replace('/\xc2\xa0/', ' ', $strNeiRong); // 替换 UTF-8 不间断空格
$strNeiRong = preg_replace('/&nbsp;/', ' ', $strNeiRong); // 替换 &nbsp;
// 清理多余空格和换行符
$strNeiRong = preg_replace('/\s+/', ' ', $strNeiRong);
// 1. 找到所有的 <br> 位置
$br_positions = [];
$offset = 0;
while (($pos = strpos($strNeiRong, "<br", $offset)) !== false) {
$br_positions[] = $pos;
$offset = $pos + 1;
}
// 2. 找到倒数第三个 <br> 的位置
if (count($br_positions) >= 3) {
$cut_position = $br_positions[count($br_positions) - 3]; // 倒数第三个 <br> 的索引
$strNeiRong = substr($strNeiRong, 0, $cut_position); // 截取前面部分，删除后面的内容
}
$arrNeiRong = explode('<script>app2();</script><br>', $strNeiRong);
$strNeiRong = $arrNeiRong[1];
$arrNeiRong = explode('<br><br>', $strNeiRong);
$arrEnCode = [];
foreach ($arrNeiRong as $intNeiRongNum => $strNeiRongOne) {
$strEnCode = Base64Helper::encode('<p>' . $strNeiRongOne . '</p>');
$arrEnCode[] = $strEnCode;
}
$strNeiRong = implode(';;;', $arrEnCode);
# '/novel/123yqw/分类id/小说源id/小说源章节id.txt'
$strNeiRongTxtPath = config("filesystem.novel") . '/novel/630zwxs/' . $intFenleiId . '/' . $intXiaoShuoId . '/' . $intZhangJiePaiXu . '/' . $intZhangJieSort . '.txt';
// 调用函数
$result = saveCompressAndDeleteTxt($strNeiRongTxtPath, $strNeiRong);
// 输出状态
if ($result['status']) {
var_dump("Success: " . $result['message']);
} else {
var_dump("Error: " . $result['message']);
}
$arrXiaoShuoZhangJie = [
'xsxq_id' => $intXiaoShuoId,
'xszj_pai_xu' => $intZhangJiePaiXu,
'xszj_ming_zi' => $strZhangJieName,
'xszj_nei_rong' => $strNeiRongTxtPath,
'xszj_source_id' => $strZhangJieSourceId,
'xszj_source_code' => $strXszjSourceCode,
'xszj_sort' => $intZhangJieSort,
];
if ($strZhangJieName) {
XiaoShuoZhangJieModel::addXiaoShuoZhangJie($arrXiaoShuoZhangJie);
}
unset($ZhangJieQueryList);
//sleep(3);
} catch (\Throwable $T) {
$arrTask = [
'callback' => [self::class, 'getNovelReader'],
'data' => [
'xszj_source_id' => $strZhangJieSourceId,
'xszj_paixu' => $intZhangJiePaiXu,
'xszj_name' => $strZhangJieName,
'xszj_url' => $strZhangJieUrl,
'xszj_source_code' => $strXszjSourceCode,
'xsxq_id' => $intXiaoShuoId,
'xsfl_id' => $intFenleiId,
'intZhangJieSort' => $intZhangJieSort,
]
];
$TaskCore->set($arrTask);
throw $T;
}
}
}

137
code/app/task/collection/seo/GenerateSiteMapCol.php
View File

@@ -1,137 +0,0 @@
<?php
declare(strict_types=1);
namespace app\task\collection\seo;
use app\task\collection\PullDataColBase;
use app\model\VideoInfoModel;
use app\model\SystemConfigModel;
use app\model\VideoClassModel;
class GenerateSiteMapCol extends PullDataColBase
{
public function generate()
{
$strSiteDomain = SystemConfigModel::getValByCode('YONG_JIU_DOMAIN');
// 每个 Sitemap 包含的最大 URL 数量
$chunkSize = 40000;
// 从数据库获取所有视频总数
$totalVideoCount = VideoInfoModel::count();
// 获取分类信息
$categories = VideoClassModel::getDataByCache();
// 生成视频详情的 Sitemap
$videoSitemapFiles = $this->generateVideoSitemaps($chunkSize, $totalVideoCount, $strSiteDomain);
// 生成分类的 Sitemap
$categorySitemapFiles = $this->generateCategorySitemaps($categories, $strSiteDomain);
// 合并所有 Sitemap 文件路径
$allSitemapFiles = array_merge($videoSitemapFiles, $categorySitemapFiles);
// 生成 Sitemap Index 文件
$this->buildSitemapIndex($allSitemapFiles, $strSiteDomain);
var_dump(['message' => '所有 Sitemap 生成成功', 'files' => $allSitemapFiles]);
}
private function generateVideoSitemaps(int $chunkSize, int $totalCount, string $strSiteDomain): array
{
$totalFiles = ceil($totalCount / $chunkSize);
$sitemapFiles = [];
for ($i = 0; $i < $totalFiles; $i++) {
$videos = VideoInfoModel::field('v_pinyin')
->limit($i * $chunkSize, $chunkSize)
->select()
->toArray();
$sitemapXml = $this->buildSitemap($videos, $strSiteDomain);
$filePath = public_path() . "sitemap_video_{$i}.xml";
file_put_contents($filePath, $sitemapXml);
$sitemapFiles[] = $filePath;
}
return $sitemapFiles;
}
private function generateCategorySitemaps(array $categories, string $strSiteDomain): array
{
$sitemapFiles = [];
foreach ($categories as $category) {
$vcId = $category['vc_id'];
$vcNicheng = $category['vc_nicheng'];
$vcName = $category['vc_name'];
$videoCount = VideoInfoModel::where('vc_id', $vcId)->count();
$totalPages = ceil($videoCount / 30);
$categoryUrls = [];
$categoryUrls[] = [
'url' => "/vodtype/{$vcNicheng}.html",
'title' => "{$vcName} - 首页",
'is_home' => true,
];
for ($page = 2; $page <= $totalPages; $page++) {
$categoryUrls[] = [
'url' => "/vodshow/{$vcNicheng}--------{$page}---.html",
'title' => "{$vcName} - 第{$page}页",
'is_home' => false,
];
}
$sitemapXml = $this->buildSitemap($categoryUrls, $strSiteDomain, true);
$filePath = public_path() . "sitemap_category_{$vcNicheng}.xml";
file_put_contents($filePath, $sitemapXml);
$sitemapFiles[] = $filePath;
}
return $sitemapFiles;
}
private function buildSitemap(array $items, string $strSiteDomain, bool $isCategory = false): string
{
$xmlHeader = '<?xml version="1.0" encoding="UTF-8"?>';
$urlsetStart = '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">';
$urlsetEnd = '</urlset>';
$urlEntries = '';
foreach ($items as $item) {
$urlEntries .= '<url>';
$urlEntries .= '<loc>https://' . $strSiteDomain . ($isCategory ? $item['url'] : '/voddetail/' . htmlspecialchars($item['v_pinyin'] . '.html', ENT_QUOTES)) . '</loc>';
$urlEntries .= '<changefreq>daily</changefreq>';
$urlEntries .= '<priority>0.8</priority>';
$urlEntries .= '</url>';
}
return $xmlHeader . $urlsetStart . $urlEntries . $urlsetEnd;
}
private function buildSitemapIndex(array $sitemapFiles, string $strSiteDomain): void
{
$xmlHeader = '<?xml version="1.0" encoding="UTF-8"?>';
$sitemapIndexStart = '<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">';
$sitemapIndexEnd = '</sitemapindex>';
$sitemapEntries = '';
foreach ($sitemapFiles as $file) {
$url = 'https://' . $strSiteDomain . '/' . basename($file);
$sitemapEntries .= '<sitemap>';
$sitemapEntries .= '<loc>' . htmlspecialchars($url, ENT_QUOTES) . '</loc>';
$sitemapEntries .= '<lastmod>' . date('Y-m-d') . '</lastmod>';
$sitemapEntries .= '</sitemap>';
}
$sitemapIndexXml = $xmlHeader . $sitemapIndexStart . $sitemapEntries . $sitemapIndexEnd;
file_put_contents(public_path() . 'sitemap_index.xml', $sitemapIndexXml);
var_dump(['message' => 'Sitemap Index 生成成功', 'file' => 'sitemap_index.xml']);
}
}

188
code/app/task/collection/seo/GenerateSiteMapNovel.php
View File

@@ -1,188 +0,0 @@
<?php
declare(strict_types=1);
namespace app\task\collection\seo;
use app\task\collection\PullDataColBase;
use app\model\XiaoShuoXiangQingModel;
use app\model\XiaoShuoZhangJieModel;
use app\model\SystemConfigModel;
use app\model\SiteModel;
use app\model\XiaoShuoFenLeiModel;
class GenerateSiteMapNovel extends PullDataColBase
{
public function generate()
{
$SiteModel = SiteModel::getValById(config('app.default_app_id'));
$strSiteDomain = $SiteModel['site_domain'];
$strFenleiPath = $SiteModel['site_fenlei_path'];
$strXqPath = $SiteModel['site_xiangqing_path'];
$strRankPath = $SiteModel['site_rank_path'];
$strTjPath = $SiteModel['site_tui_jian_path'];
$strSeoPath = $SiteModel['site_seo_path'];
$strZdyPath = $SiteModel['site_zdy_path'];
$strZjPath = $SiteModel['site_zhangjie_path'];
// 每个 Sitemap 包含的最大 URL 数量
$chunkSize = 40000;
// 从数据库获取所有总数
$totalNovelCount = XiaoShuoXiangQingModel::count();
// 从数据库获取所有总数
$totalNovelZjCount = XiaoShuoZhangJieModel::count();
// 获取分类信息
$categories = XiaoShuoFenLeiModel::getDataByCache();
// 生成详情的 Sitemap
$novelSitemapFiles = $this->generateNovelSitemaps($chunkSize, $totalNovelCount, $strSiteDomain);
// 生成章节的 Sitemap
$novelZjSitemapFiles = $this->generateNovelZjSitemaps($chunkSize, $totalNovelZjCount, $strSiteDomain);
// 生成分类的 Sitemap
$categorySitemapFiles = $this->generateCategorySitemaps($categories, $strSiteDomain);
// 合并所有 Sitemap 文件路径
$allSitemapFiles = array_merge($novelSitemapFiles, $categorySitemapFiles,$novelZjSitemapFiles);
// 生成 Sitemap Index 文件
$this->buildSitemapIndex($allSitemapFiles, $strSiteDomain);
var_dump(['message' => '所有 Sitemap 生成成功', 'files' => $allSitemapFiles]);
}
private function generateNovelZjSitemaps(int $chunkSize, int $totalCount, string $strSiteDomain): array
{
$totalFiles = ceil($totalCount / $chunkSize);
$sitemapFiles = [];
for ($i = 0; $i < $totalFiles; $i++) {
$Novels = XiaoShuoZhangJieModel::field(['xsxq_id','xszj_id']) //intNovelId-:intNovelSourceId-:intNovelSourceSeoId
->limit($i * $chunkSize, $chunkSize)
->select()
->toArray();
$sitemapXml = $this->buildZjSitemap($Novels, $strSiteDomain);
$filePath = public_path() . "sitemap_novel_zj_{$i}.xml";
file_put_contents($filePath, $sitemapXml);
$sitemapFiles[] = $filePath;
}
return $sitemapFiles;
}
private function generateNovelSitemaps(int $chunkSize, int $totalCount, string $strSiteDomain): array
{
$totalFiles = ceil($totalCount / $chunkSize);
$sitemapFiles = [];
for ($i = 0; $i < $totalFiles; $i++) {
$Novels = XiaoShuoXiangQingModel::field(['xsxq_id','xsxq_source_id','xsxq_source_seo_id']) //intNovelId-:intNovelSourceId-:intNovelSourceSeoId
->limit($i * $chunkSize, $chunkSize)
->select()
->toArray();
$sitemapXml = $this->buildSitemap($Novels, $strSiteDomain);
$filePath = public_path() . "sitemap_novel_{$i}.xml";
file_put_contents($filePath, $sitemapXml);
$sitemapFiles[] = $filePath;
}
return $sitemapFiles;
}
private function generateCategorySitemaps(array $categories, string $strSiteDomain): array
{
$sitemapFiles = [];
foreach ($categories as $category) {
$vcId = $category['xsfl_id'];
$vcName = $category['xsfl_name'];
$intNovelCount = XiaoShuoFenLeiModel::where('xsfl_id', $vcId)->count();
$totalPages = ceil($intNovelCount / 30);
$categoryUrls = [];
for ($type = 0; $type <= 3; $type++) {
for ($page = 1; $page <= $totalPages; $page++) {
$categoryUrls[] = [
// /fenlei/1-0-1.html
'url' => "/fenlei/{$vcId}-{$type}-{$page}.html",
'title' => "{$vcName} - 第{$page}页",
'is_home' => false,
];
}
}
$sitemapXml = $this->buildSitemap($categoryUrls, $strSiteDomain, true);
$filePath = public_path() . "sitemap_category_{$vcId}.xml";
file_put_contents($filePath, $sitemapXml);
$sitemapFiles[] = $filePath;
}
return $sitemapFiles;
}
private function buildZjSitemap(array $items, string $strSiteDomain, bool $isCategory = false): string
{
$xmlHeader = '<?xml version="1.0" encoding="UTF-8"?>';
$urlsetStart = '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">';
$urlsetEnd = '</urlset>';
// zj/18648-206.html
$urlEntries = '';
foreach ($items as $item) {
$urlEntries .= '<url>';
$urlEntries .= '<loc>https://' . $strSiteDomain . ($isCategory ? $item['url'] : '/zj/' . htmlspecialchars($item['xsxq_id'].'-'.$item['xszj_id']. '.html', ENT_QUOTES)) . '</loc>';
$urlEntries .= '<changefreq>daily</changefreq>';
$urlEntries .= '<priority>0.8</priority>';
$urlEntries .= '</url>';
}
return $xmlHeader . $urlsetStart . $urlEntries . $urlsetEnd;
}
private function buildSitemap(array $items, string $strSiteDomain, bool $isCategory = false): string
{
$xmlHeader = '<?xml version="1.0" encoding="UTF-8"?>';
$urlsetStart = '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">';
$urlsetEnd = '</urlset>';
// /xq/20052-16107-161071.html
$urlEntries = '';
foreach ($items as $item) {
$urlEntries .= '<url>';
$urlEntries .= '<loc>https://' . $strSiteDomain . ($isCategory ? $item['url'] : '/xq/' . htmlspecialchars($item['xsxq_id'].'-'.$item['xsxq_source_id'].'-'.$item['xsxq_source_seo_id']. '.html', ENT_QUOTES)) . '</loc>';
$urlEntries .= '<changefreq>daily</changefreq>';
$urlEntries .= '<priority>1.0</priority>';
$urlEntries .= '</url>';
}
return $xmlHeader . $urlsetStart . $urlEntries . $urlsetEnd;
}
private function buildSitemapIndex(array $sitemapFiles, string $strSiteDomain): void
{
$xmlHeader = '<?xml version="1.0" encoding="UTF-8"?>';
$sitemapIndexStart = '<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">';
$sitemapIndexEnd = '</sitemapindex>';
$sitemapEntries = '';
foreach ($sitemapFiles as $file) {
$url = 'https://' . $strSiteDomain . '/' . basename($file);
$sitemapEntries .= '<sitemap>';
$sitemapEntries .= '<loc>' . htmlspecialchars($url, ENT_QUOTES) . '</loc>';
$sitemapEntries .= '<lastmod>' . date('Y-m-d') . '</lastmod>';
$sitemapEntries .= '</sitemap>';
}
$sitemapIndexXml = $xmlHeader . $sitemapIndexStart . $sitemapEntries . $sitemapIndexEnd;
file_put_contents(public_path() . 'sitemap_index.xml', $sitemapIndexXml);
var_dump(['message' => 'Sitemap Index 生成成功', 'file' => 'sitemap_index.xml']);
}
}

242
code/app/task/collection/seo/GenerateSiteMapNovelSk.php
View File

@@ -1,242 +0,0 @@
<?php
declare(strict_types=1);
namespace app\task\collection\seo;
use app\task\collection\PullDataColBase;
use app\model\XiaoShuoXiangQingModel;
use app\model\NovelInfoSeoModel;
use app\model\XiaoShuoZhangJieModel;
use app\model\SiteModel;
use app\model\XiaoShuoFenLeiModel;
class GenerateSiteMapNovelSk extends PullDataColBase
{
public function generate()
{
$SiteModel = SiteModel::getValById(config('app.default_app_id'));
// $strSiteDomain = SiteModel::getValBySiteIdByValCode('site_domain');
$strSiteDomain = $SiteModel['site_domain'];
$strFenleiPath = $SiteModel['site_fenlei_path'];
$strXqPath = $SiteModel['site_xiangqing_path'];
$strRankPath = $SiteModel['site_rank_path'];
$strTjPath = $SiteModel['site_tui_jian_path'];
$strSeoPath = $SiteModel['site_seo_path'];
$strZdyPath = $SiteModel['site_zdy_path'];
$strZjPath = $SiteModel['site_zhangjie_path'];
// 每个 Sitemap 包含的最大 URL 数量
$chunkSize = 40000;
// 从数据库获取所有总数
$totalNovelCount = XiaoShuoXiangQingModel::count();
// 从数据库获取所有总数
$totalNovelSeoCount = NovelInfoSeoModel::count();
// 从数据库获取所有总数
$totalNovelZjCount = XiaoShuoZhangJieModel::count();
// 获取分类信息
$categories = XiaoShuoFenLeiModel::getDataByCache();
// 生成详情的 Sitemap 小说详情
$novelSitemapFiles = $this->generateNovelSitemaps($chunkSize, $totalNovelCount, $strSiteDomain, $strXqPath);
// 生成详情的 Sitemap 小说seo
$novelSeoSitemapFiles = $this->generateNovelSeoSitemaps($chunkSize, $totalNovelSeoCount, $strSiteDomain, $strSeoPath);
// 生成章节的 Sitemap 小说章节
$novelZjSitemapFiles = $this->generateNovelZjSitemaps($chunkSize, $totalNovelZjCount, $strSiteDomain, $strXqPath);
// 生成分类的 Sitemap 小说分类
$categorySitemapFiles = $this->generateCategorySitemaps($categories, $strSiteDomain, $strFenleiPath, $strXqPath);
// 合并所有 Sitemap 文件路径
$allSitemapFiles = array_merge($novelSitemapFiles, $novelSeoSitemapFiles, $categorySitemapFiles, $novelZjSitemapFiles);
// 生成 Sitemap Index 文件
$this->buildSitemapIndex($allSitemapFiles, $strSiteDomain);
var_dump(['message' => '所有 Sitemap 生成成功', 'files' => $allSitemapFiles]);
}
# 生成小说章节 xml文件
private function generateNovelZjSitemaps(int $chunkSize, int $totalCount, string $strSiteDomain, string $strXqPath): array
{
$totalFiles = ceil($totalCount / $chunkSize);
$sitemapFiles = [];
for ($i = 0; $i < $totalFiles; $i++) {
$Novels = XiaoShuoZhangJieModel::field(['xsxq_id', 'xszj_id']) //intNovelId-:intNovelSourceId-:intNovelSourceSeoId
->limit($i * $chunkSize, $chunkSize)
->select()
->toArray();
$sitemapXml = $this->buildZjSitemap($Novels, $strSiteDomain, $strXqPath);
$filePath = public_path() . "sitemap_novel_zj_{$i}.xml";
file_put_contents($filePath, $sitemapXml);
$sitemapFiles[] = $filePath;
}
return $sitemapFiles;
}
# 生成小说详情 xml文件
private function generateNovelSitemaps(int $chunkSize, int $totalCount, string $strSiteDomain, string $strXqPath): array
{
$totalFiles = ceil($totalCount / $chunkSize);
$sitemapFiles = [];
for ($i = 0; $i < $totalFiles; $i++) {
$Novels = XiaoShuoXiangQingModel::field(['xsxq_id', 'xsxq_source_id', 'xsxq_source_seo_id']) //intNovelId-:intNovelSourceId-:intNovelSourceSeoId
->limit($i * $chunkSize, $chunkSize)
->select()
->toArray();
$sitemapXml = $this->buildSitemap($Novels, $strSiteDomain, $strXqPath);
$filePath = public_path() . "sitemap_novel_{$i}.xml";
file_put_contents($filePath, $sitemapXml);
$sitemapFiles[] = $filePath;
}
return $sitemapFiles;
}
# 生成小说详情 xml文件
private function generateNovelSeoSitemaps(int $chunkSize, int $totalCount, string $strSiteDomain, string $strSeoPath): array
{
$totalFiles = ceil($totalCount / $chunkSize);
$sitemapFiles = [];
for ($i = 0; $i < $totalFiles; $i++) {
$Novels = NovelInfoSeoModel::field(['xsxq_id', 'nis_id']) //show/nis_id
->limit($i * $chunkSize, $chunkSize)
->select()
->toArray();
$sitemapXml = $this->buildXiaoShuoSeoSitemap($Novels, $strSiteDomain, $strSeoPath);
$filePath = public_path() . "sitemap_novel_seo_{$i}.xml";
file_put_contents($filePath, $sitemapXml);
$sitemapFiles[] = $filePath;
}
return $sitemapFiles;
}
// 生成分类链接
private function generateCategorySitemaps(array $categories, string $strSiteDomain, string $strFenleiPath, string $strXqPath): array
{
$sitemapFiles = [];
foreach ($categories as $category) {
$vcId = $category['xsfl_id'];
$vcName = $category['xsfl_name'];
$intNovelCount = XiaoShuoFenLeiModel::where('xsfl_id', $vcId)->count();
$totalPages = ceil($intNovelCount / 30);
$categoryUrls = [];
for ($status = 0; $status <= 2; $status++) {
for ($sort = 0; $sort <= 4; $sort++) {
for ($page = 1; $page <= $totalPages; $page++) {
$categoryUrls[] = [
// /fl/:intCId-:intSort-[:intPage]
'url' => "/{$strFenleiPath}/{$vcId}-{$sort}-{$page}.html",
'title' => "{$vcName} - 第{$page}页",
'is_home' => false,
];
}
}
}
$sitemapXml = $this->buildSitemap($categoryUrls, $strSiteDomain, $strXqPath, true);
$filePath = public_path() . "sitemap_category_{$vcId}.xml";
file_put_contents($filePath, $sitemapXml);
$sitemapFiles[] = $filePath;
}
return $sitemapFiles;
}
//生成章节链接
private function buildZjSitemap(array $items, string $strSiteDomain, string $strXqPath, bool $isCategory = false): string
{
$xmlHeader = '<?xml version="1.0" encoding="UTF-8"?>';
$urlsetStart = '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">';
$urlsetEnd = '</urlset>';
// /xq/:intNovelId/:intZhangJiePaiXu-[:strSort]
$urlEntries = '';
foreach ($items as $item) {
$urlEntries .= '<url>';
$urlEntries .= '<loc>https://' . $strSiteDomain . ($isCategory ? $item['url'] : '/' . $strXqPath . '/' . htmlspecialchars($item['xsxq_id'] . '-' . $item['xszj_pai_xu']. '-' . $item['xszj_sort'] . '.html', ENT_QUOTES)) . '</loc>';
$urlEntries .= '<changefreq>daily</changefreq>';
$urlEntries .= '<priority>0.8</priority>';
$urlEntries .= '</url>';
}
return $xmlHeader . $urlsetStart . $urlEntries . $urlsetEnd;
}
// 生成 详情链接
private function buildSitemap(array $items, string $strSiteDomain, string $strXqPath, bool $isCategory = false): string
{
$xmlHeader = '<?xml version="1.0" encoding="UTF-8"?>';
$urlsetStart = '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">';
$urlsetEnd = '</urlset>';
// /xq/:intNovelId
$urlEntries = '';
foreach ($items as $item) {
$urlEntries .= '<url>';
$urlEntries .= '<loc>https://' . $strSiteDomain . ($isCategory ? $item['url'] : '/' . $strXqPath . '/' . htmlspecialchars($item['xsxq_id'] . '.html', ENT_QUOTES)) . '</loc>';
$urlEntries .= '<changefreq>daily</changefreq>';
$urlEntries .= '<priority>1.0</priority>';
$urlEntries .= '</url>';
}
return $xmlHeader . $urlsetStart . $urlEntries . $urlsetEnd;
}
// 生成详情 seo 链接
private function buildXiaoShuoSeoSitemap(array $items, string $strSiteDomain, string $strSeoPath, bool $isCategory = false): string
{
$xmlHeader = '<?xml version="1.0" encoding="UTF-8"?>';
$urlsetStart = '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">';
$urlsetEnd = '</urlset>';
// /show/:intNovelId
$urlEntries = '';
foreach ($items as $item) {
$urlEntries .= '<url>';
$urlEntries .= '<loc>https://' . $strSiteDomain . ($isCategory ? $item['url'] : '/' . $strSeoPath . '/' . htmlspecialchars($item['nis_id'] . '.html', ENT_QUOTES)) . '</loc>';
$urlEntries .= '<changefreq>daily</changefreq>';
$urlEntries .= '<priority>1.0</priority>';
$urlEntries .= '</url>';
}
return $xmlHeader . $urlsetStart . $urlEntries . $urlsetEnd;
}
// 生成 xml 文件
private function buildSitemapIndex(array $sitemapFiles, string $strSiteDomain): void
{
$xmlHeader = '<?xml version="1.0" encoding="UTF-8"?>';
$sitemapIndexStart = '<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">';
$sitemapIndexEnd = '</sitemapindex>';
$sitemapEntries = '';
foreach ($sitemapFiles as $file) {
$url = 'https://' . $strSiteDomain . '/' . basename($file);
$sitemapEntries .= '<sitemap>';
$sitemapEntries .= '<loc>' . htmlspecialchars($url, ENT_QUOTES) . '</loc>';
$sitemapEntries .= '<lastmod>' . date('Y-m-d') . '</lastmod>';
$sitemapEntries .= '</sitemap>';
}
$sitemapIndexXml = $xmlHeader . $sitemapIndexStart . $sitemapEntries . $sitemapIndexEnd;
file_put_contents(public_path() . 'sitemap_index.xml', $sitemapIndexXml);
var_dump(['message' => 'Sitemap Index 生成成功', 'file' => 'sitemap_index.xml']);
}
}

242
code/app/task/collection/seo/GenerateSiteMapNovelZw.php
View File

@@ -1,242 +0,0 @@
<?php
declare(strict_types=1);
namespace app\task\collection\seo;
use app\task\collection\PullDataColBase;
use app\model\XiaoShuoXiangQingModel;
use app\model\NovelInfoSeoModel;
use app\model\XiaoShuoZhangJieModel;
use app\model\SiteModel;
use app\model\XiaoShuoFenLeiModel;
class GenerateSiteMapNovelZw extends PullDataColBase
{
public function generate()
{
$SiteModel = SiteModel::getValById(config('app.default_app_id'));
// $strSiteDomain = SiteModel::getValBySiteIdByValCode('site_domain');
$strSiteDomain = $SiteModel['site_domain'];
$strFenleiPath = $SiteModel['site_fenlei_path'];
$strXqPath = $SiteModel['site_xiangqing_path'];
$strRankPath = $SiteModel['site_rank_path'];
$strTjPath = $SiteModel['site_tui_jian_path'];
$strSeoPath = $SiteModel['site_seo_path'];
$strZdyPath = $SiteModel['site_zdy_path'];
$strZjPath = $SiteModel['site_zhangjie_path'];
// 每个 Sitemap 包含的最大 URL 数量
$chunkSize = 40000;
// 从数据库获取所有总数
$totalNovelCount = XiaoShuoXiangQingModel::count();
// 从数据库获取所有总数
$totalNovelSeoCount = NovelInfoSeoModel::count();
// 从数据库获取所有总数
$totalNovelZjCount = XiaoShuoZhangJieModel::count();
// 获取分类信息
$categories = XiaoShuoFenLeiModel::getDataByCache();
// 生成详情的 Sitemap 小说详情
$novelSitemapFiles = $this->generateNovelSitemaps($chunkSize, $totalNovelCount, $strSiteDomain, $strXqPath);
// 生成详情的 Sitemap 小说seo
$novelSeoSitemapFiles = $this->generateNovelSeoSitemaps($chunkSize, $totalNovelSeoCount, $strSiteDomain, $strSeoPath);
// 生成章节的 Sitemap 小说章节
$novelZjSitemapFiles = $this->generateNovelZjSitemaps($chunkSize, $totalNovelZjCount, $strSiteDomain, $strXqPath);
// 生成分类的 Sitemap 小说分类
$categorySitemapFiles = $this->generateCategorySitemaps($categories, $strSiteDomain, $strFenleiPath, $strXqPath);
// 合并所有 Sitemap 文件路径
$allSitemapFiles = array_merge($novelSitemapFiles, $novelSeoSitemapFiles, $categorySitemapFiles, $novelZjSitemapFiles);
// 生成 Sitemap Index 文件
$this->buildSitemapIndex($allSitemapFiles, $strSiteDomain);
var_dump(['message' => '所有 Sitemap 生成成功', 'files' => $allSitemapFiles]);
}
# 生成小说章节 xml文件
private function generateNovelZjSitemaps(int $chunkSize, int $totalCount, string $strSiteDomain, string $strXqPath): array
{
$totalFiles = ceil($totalCount / $chunkSize);
$sitemapFiles = [];
for ($i = 0; $i < $totalFiles; $i++) {
$Novels = XiaoShuoZhangJieModel::field(['xsxq_id', 'xszj_id']) //intNovelId-:intNovelSourceId-:intNovelSourceSeoId
->limit($i * $chunkSize, $chunkSize)
->select()
->toArray();
$sitemapXml = $this->buildZjSitemap($Novels, $strSiteDomain, $strXqPath);
$filePath = public_path() . "sitemap_novel_zj_{$i}.xml";
file_put_contents($filePath, $sitemapXml);
$sitemapFiles[] = $filePath;
}
return $sitemapFiles;
}
# 生成小说详情 xml文件
private function generateNovelSitemaps(int $chunkSize, int $totalCount, string $strSiteDomain, string $strXqPath): array
{
$totalFiles = ceil($totalCount / $chunkSize);
$sitemapFiles = [];
for ($i = 0; $i < $totalFiles; $i++) {
$Novels = XiaoShuoXiangQingModel::field(['xsxq_id', 'xsxq_source_id', 'xsxq_source_seo_id']) //intNovelId-:intNovelSourceId-:intNovelSourceSeoId
->limit($i * $chunkSize, $chunkSize)
->select()
->toArray();
$sitemapXml = $this->buildSitemap($Novels, $strSiteDomain, $strXqPath);
$filePath = public_path() . "sitemap_novel_{$i}.xml";
file_put_contents($filePath, $sitemapXml);
$sitemapFiles[] = $filePath;
}
return $sitemapFiles;
}
# 生成小说详情 xml文件
private function generateNovelSeoSitemaps(int $chunkSize, int $totalCount, string $strSiteDomain, string $strSeoPath): array
{
$totalFiles = ceil($totalCount / $chunkSize);
$sitemapFiles = [];
for ($i = 0; $i < $totalFiles; $i++) {
$Novels = NovelInfoSeoModel::field(['xsxq_id', 'nis_id']) //show/nis_id
->limit($i * $chunkSize, $chunkSize)
->select()
->toArray();
$sitemapXml = $this->buildXiaoShuoSeoSitemap($Novels, $strSiteDomain, $strSeoPath);
$filePath = public_path() . "sitemap_novel_seo_{$i}.xml";
file_put_contents($filePath, $sitemapXml);
$sitemapFiles[] = $filePath;
}
return $sitemapFiles;
}
// 生成分类链接
private function generateCategorySitemaps(array $categories, string $strSiteDomain, string $strFenleiPath, string $strXqPath): array
{
$sitemapFiles = [];
foreach ($categories as $category) {
$vcId = $category['xsfl_id'];
$vcName = $category['xsfl_name'];
$intNovelCount = XiaoShuoFenLeiModel::where('xsfl_id', $vcId)->count();
$totalPages = ceil($intNovelCount / 30);
$categoryUrls = [];
for ($status = 0; $status <= 2; $status++) {
for ($sort = 0; $sort <= 4; $sort++) {
for ($page = 1; $page <= $totalPages; $page++) {
$categoryUrls[] = [
// /fl/:intCId-:intSort-[:intPage]
'url' => "/{$strFenleiPath}/{$vcId}-{$sort}-{$page}.html",
'title' => "{$vcName} - 第{$page}页",
'is_home' => false,
];
}
}
}
$sitemapXml = $this->buildSitemap($categoryUrls, $strSiteDomain, $strXqPath, true);
$filePath = public_path() . "sitemap_category_{$vcId}.xml";
file_put_contents($filePath, $sitemapXml);
$sitemapFiles[] = $filePath;
}
return $sitemapFiles;
}
//生成章节链接
private function buildZjSitemap(array $items, string $strSiteDomain, string $strXqPath, bool $isCategory = false): string
{
$xmlHeader = '<?xml version="1.0" encoding="UTF-8"?>';
$urlsetStart = '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">';
$urlsetEnd = '</urlset>';
// /xq/:intNovelId/:intZhangJiePaiXu-[:strSort]
$urlEntries = '';
foreach ($items as $item) {
$urlEntries .= '<url>';
$urlEntries .= '<loc>https://' . $strSiteDomain . ($isCategory ? $item['url'] : '/' . $strXqPath . '/' . htmlspecialchars($item['xsxq_id'] . '-' . $item['xszj_pai_xu']. '-' . $item['xszj_sort'] . '.html', ENT_QUOTES)) . '</loc>';
$urlEntries .= '<changefreq>daily</changefreq>';
$urlEntries .= '<priority>0.8</priority>';
$urlEntries .= '</url>';
}
return $xmlHeader . $urlsetStart . $urlEntries . $urlsetEnd;
}
// 生成 详情链接
private function buildSitemap(array $items, string $strSiteDomain, string $strXqPath, bool $isCategory = false): string
{
$xmlHeader = '<?xml version="1.0" encoding="UTF-8"?>';
$urlsetStart = '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">';
$urlsetEnd = '</urlset>';
// /xq/:intNovelId
$urlEntries = '';
foreach ($items as $item) {
$urlEntries .= '<url>';
$urlEntries .= '<loc>https://' . $strSiteDomain . ($isCategory ? $item['url'] : '/' . $strXqPath . '/' . htmlspecialchars($item['xsxq_id'] . '.html', ENT_QUOTES)) . '</loc>';
$urlEntries .= '<changefreq>daily</changefreq>';
$urlEntries .= '<priority>1.0</priority>';
$urlEntries .= '</url>';
}
return $xmlHeader . $urlsetStart . $urlEntries . $urlsetEnd;
}
// 生成详情 seo 链接
private function buildXiaoShuoSeoSitemap(array $items, string $strSiteDomain, string $strSeoPath, bool $isCategory = false): string
{
$xmlHeader = '<?xml version="1.0" encoding="UTF-8"?>';
$urlsetStart = '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">';
$urlsetEnd = '</urlset>';
// /show/:intNovelId
$urlEntries = '';
foreach ($items as $item) {
$urlEntries .= '<url>';
$urlEntries .= '<loc>https://' . $strSiteDomain . ($isCategory ? $item['url'] : '/' . $strSeoPath . '/' . htmlspecialchars($item['nis_id'] . '.html', ENT_QUOTES)) . '</loc>';
$urlEntries .= '<changefreq>daily</changefreq>';
$urlEntries .= '<priority>1.0</priority>';
$urlEntries .= '</url>';
}
return $xmlHeader . $urlsetStart . $urlEntries . $urlsetEnd;
}
// 生成 xml 文件
private function buildSitemapIndex(array $sitemapFiles, string $strSiteDomain): void
{
$xmlHeader = '<?xml version="1.0" encoding="UTF-8"?>';
$sitemapIndexStart = '<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">';
$sitemapIndexEnd = '</sitemapindex>';
$sitemapEntries = '';
foreach ($sitemapFiles as $file) {
$url = 'https://' . $strSiteDomain . '/' . basename($file);
$sitemapEntries .= '<sitemap>';
$sitemapEntries .= '<loc>' . htmlspecialchars($url, ENT_QUOTES) . '</loc>';
$sitemapEntries .= '<lastmod>' . date('Y-m-d') . '</lastmod>';
$sitemapEntries .= '</sitemap>';
}
$sitemapIndexXml = $xmlHeader . $sitemapIndexStart . $sitemapEntries . $sitemapIndexEnd;
file_put_contents(public_path() . 'sitemap_index.xml', $sitemapIndexXml);
var_dump(['message' => 'Sitemap Index 生成成功', 'file' => 'sitemap_index.xml']);
}
}

538
code/app/task/collection/tianlai/TianLaiXiaoShuoCol.php
View File

@@ -1,538 +0,0 @@
<?php
declare(strict_types=1);
namespace app\task\collection\tianlai;
use app\model\XiaoShuoFenLeiModel;
use app\model\XiaoShuoXiangQingModel;
use app\model\XiaoShuoZhangJieModel;
use app\task\collection\PullDataColBase;
use GuzzleHttp\Cookie\CookieJar;
use processor\ImageProcessor;
use think\facade\Cache;
use QL\QueryList;
/**
* @mixin think\Model
*/
class TianLaiXiaoShuoCol extends PullDataColBase
{
public function pullXiaoShuo()
{
$TaskCore = $this->TaskCore;
$strUrl = $this->getTianLaiDomain();
var_dump($strUrl);
$Client = $this->Client;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
// $proxy = '127.0.0.1:8080'; // 代理地址
// $response = fetchContentWithCurl($strUrl, [], 10, $proxy);
$strResult = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$strResult->getBody();
var_dump($strResult);
// exit;
// $strResult = $response['data'];
$QueryList = QueryList::html($strResult);
$QueryList->find('.nav a')->map(function ($FenLeiQueryList) use ($strUrl, $TaskCore) {
$strName = trim($FenLeiQueryList->text());
switch ($strName) {
case '都市言情':
$strName = "都市言情";
break;
case '历史军事':
$strName = "历史军事";
break;
case '玄幻奇幻':
$strName = "玄幻奇幻";
break;
case '武侠修真':
$strName = "武侠修真";
break;
case '女生纯爱':
$strName = "女频言情";
break;
case '科幻网游':
$strName = "科幻网游";
break;
case '悬疑灵异':
$strName = "悬疑灵异";
break;
default:
$strName = "其它";
break;
}
$strFenLeiUrl = $strUrl . $FenLeiQueryList->attr('href');
// $intFenLeiId = extractFilename($strFenLeiUrl."index.html");
$intFenLeiId = extractNumberFromUrl($strFenLeiUrl);
if($intFenLeiId !== null){
$arrClass = [
'xsfl_source_id' => $intFenLeiId,
'xsfl_source_code' => 'TIANLAI_FEN_LEI_ID_' . $intFenLeiId,
'xsfl_name' => $strName,
];
var_dump($arrClass);
$XiaoShuoFenLeiModel = XiaoShuoFenLeiModel::addXiaoShuoFenLei($arrClass);
$arrTask = [
'callback' => [self::class, 'getNovelPage'],
'data' => [
'url' => $strFenLeiUrl,
'xsfl_source_id' => $intFenLeiId,
'xsfl_name' => $strName,
'xsfl_id' => $XiaoShuoFenLeiModel->xsfl_id,
'xsfl_page' => 1,
'xsfl_max_page' => 3000,
]
];
$TaskCore->set($arrTask);
}
});
} catch (\Throwable $T) {
//var_dump($T);
throw $T;
}
}
public function getNovelPage($arrData)
{
var_dump('getNovelPage');
var_dump($arrData);
$strBasePage = $arrData['url'];
$Client = $this->Client;
$TaskCore = $this->TaskCore;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strBasePage);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$strSiteUrl = $this->getTianLaiDomain();
$intPage = $arrData['xsfl_page'];
$intMaxPage = $arrData['xsfl_max_page'];
$strUrl = $strBasePage;
var_dump($strUrl);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$strResult = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => $userAgent
],
'cookies' => $CookieJar,
#'proxy' => $proxy,
'timeout' => 10,
]);
$strResult = (string)$strResult->getBody();
$QueryList = QueryList::html($strResult);
$intMaxPage = $QueryList->find('#newscontent h2 span')->eq(1)->text();
$intMaxPage = ceil($intMaxPage / 100);
var_dump($intMaxPage);
$intMaxPage = 2;
var_dump('$strUrl');
// 如果是首页 则根据最大页面吧所有分页 追加到任务
if($intPage == 1){
for ($intCurrentPage = 2; $intCurrentPage <= $intMaxPage; $intCurrentPage++) {
$strNewsFenleiUrl = replaceLastPageNumber($strBasePage, $intCurrentPage);
var_dump($strNewsFenleiUrl);
$arrTask = [
'callback' => [self::class, 'getNovelPage'],
'data' => [
'url' => $strNewsFenleiUrl,
'xsfl_source_id' => $arrData['xsfl_source_id'],
'xsfl_name' => $arrData['xsfl_name'],
'xsfl_id' => $arrData['xsfl_id'],
'xsfl_page' => $intCurrentPage,
'xsfl_max_page' => $intMaxPage,
]
];
$TaskCore->set($arrTask);
}
}
var_dump('提取小说');
$XiaoShuoListQueryList = $QueryList->find('#newscontent li');
if (!$XiaoShuoListQueryList || $XiaoShuoListQueryList->count() <= 0) {
var_dump('这个分类已经市最后一页');
return true;
}
var_dump($XiaoShuoListQueryList->count());
var_dump('提取小说2');
// var_dump($XiaoShuoListQueryList);
$XiaoShuoListQueryList->map(function ($XiaoShuoQueryList) use ($strSiteUrl, $arrData, $TaskCore, $strUrl) {
// 提取小说链接
// var_dump($XiaoShuoQueryList->html());exit;
var_dump('提取小说链接');
$A = $XiaoShuoQueryList->find('a')->eq(0);
$strXiaoShuoUrl = $A->attr('href');
$strXiaoShuoUrl = strpos($strXiaoShuoUrl, 'https:') === 0 ? $strXiaoShuoUrl : 'https:' . $strXiaoShuoUrl;
var_dump('$A');
var_dump($A);
var_dump($strXiaoShuoUrl);
exit;
// var_dump($A);
// 提取作者信息
$strZuoZhe = trim($XiaoShuoQueryList->find('.s4')->eq(0)->text());
// 提取小说 ID
$arrXiaoShuoId = extractNumbersFromUrl($strXiaoShuoUrl);
if (!isset($arrXiaoShuoId['first'], $arrXiaoShuoId['second'])) {
return; // 如果 ID 提取失败，直接跳过
}
// 准备任务数据
$arrXiaoShuo = array_merge($arrData, [
'xiaoshuo_url' => $strXiaoShuoUrl,
'xiaoshuo_id' => $arrXiaoShuoId['first'],
'xiaoshuo_seo_id' => $arrXiaoShuoId['second'],
'xiaoshuo_zuozhe' => $strZuoZhe,
'url' => $strUrl
]);
var_dump($arrXiaoShuo);
$arrTask = [
'callback' => [self::class, 'getNovelInfo'],
'data' => $arrXiaoShuo,
];
// 将任务加入队列
$TaskCore->set($arrTask);
var_dump("将任务加入队列getNovelInfo-{$strXiaoShuoUrl}");
});
var_dump('提取小说end');
} catch (\Throwable $T) {
throw $T;
}
}
public function getNovelInfo($arrData)
{
var_dump('getNovelInfo');
var_dump($arrData);
$Client = $this->Client;
$TaskCore = $this->TaskCore;
// 获取 Redis 实例
$redis = Cache::store('redis')->handler();
$intXiaoShuoSourceId = $arrData['xiaoshuo_id'];
$intXiaoShuoSourceSeoId = $arrData['xiaoshuo_seo_id'];
$redisKey = "task:novel_source_id:" . $intXiaoShuoSourceId;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($arrData['xiaoshuo_url']);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$strSiteUrl = $this->getTianLaiDomain();
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $arrData['xiaoshuo_url'], [
'headers' => [
'User-Agent' => $userAgent
],
'cookies' => $CookieJar,
#'proxy' => $proxy,
'timeout' => 10,
]);
$strResult = (string)$Response->getBody();
unset($Response);
$QueryList = QueryList::html($strResult);
$strMingZi = $QueryList->find('#info h1')->eq(0)->text();
$strZhuangTai = "状态：连载中";
$strZiShu = 0;
$strZhuangTai = getStringAfterWord($strZhuangTai, '状态：');
// 提取作者信息
$strZuoZhe = trim($QueryList->find('p a')->eq(0)->text());
// $strLastDate = $QueryList->find('meta[property="og:novel:update_time"]')->attr('content');
$strLastDate = $QueryList->find('#info p')->eq(2)->text();
$strLastDate = getStringAfterWord($strLastDate, '最后更新：');
$strLastDate = trim((string)$strLastDate);
if (!isValidDateTime($strLastDate)) {
$strLastDate = date('Y-m-d H:i:s');
}
$arrTag = [];
// $QueryList->find('.tg span')->map(function ($TagQuery) use (&$arrTag) {
// $arrTag[] = trim($TagQuery->text());
// });
$strJieShao = $QueryList->find('#intro')->eq(0)->text();
$strJieShao = removeScriptTags('/<script>.*?<\/script>/is',$strJieShao) ;
$strJieShao = removeScriptTags('/myJs\.bookJs\(\);/',$strJieShao) ;
$strCover = $QueryList->find('#fmimg img')->eq(0)->attr("src");
// if (strpos($strCover, 'http') === false) {
// $strCover = $strSiteUrl . $strCover;
// }
$strCoverUri = $strCover;
// try {
// $strCoverUri = "";
// $strCoverUri = ImageProcessor::getInstance()->downloadAndEncryptImage($strCover, 'xs');
// } catch (\Throwable $t) {
// $strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
// echo $strErr;
// }
$arrCover = [
'code' => 'AI_LI_SI_COVER',
// 'uri' => getUriByUrl($strCover),
'uri' => $strCoverUri,
];
$arrXiaoShuoInfo = [
'xsxq_ming_zi' => $strMingZi,
'xsxq_zuozhe' => $strZuoZhe , //$arrData['xiaoshuo_zuozhe'],
'xsfl_id' => $arrData['xsfl_id'],
'xsxq_zhuang_tai' => $strZhuangTai,
'xsxq_zi_shu' => $strZiShu,
'xsxq_jie_shao' => $strJieShao,
'xsxq_feng_mian' => json_encode($arrCover),
'xsxq_geng_xin_shi_jian' => $strLastDate,
'xsxq_tag' => json_encode($arrTag, JSON_UNESCAPED_UNICODE),
'xsxq_source_id' => $intXiaoShuoSourceId,
'xsxq_source_seo_id' => $intXiaoShuoSourceSeoId,
'xsxq_source_code' => 'AILISI_XIAOSHUO_' . $intXiaoShuoSourceId . '_' . $intXiaoShuoSourceSeoId,
];
// print_r($arrXiaoShuoInfo);
var_dump($arrXiaoShuoInfo['xsxq_source_code']);
$XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
/*
if ($redis->exists($redisKey)) {
var_dump('小说任务已存在，直接跳过');
return true; // 如果任务已存在，直接跳过
}
// 在 Redis 中标记任务为已存在
$redis->set($redisKey, 1);
$arrZhangjie = [];
$QueryList->find('#list a')->map(function ($ZhangJie) use (&$arrZhangjie) {
$arrZhangjie[] = [
'mingzi' => $ZhangJie->text(),
'url' => $ZhangJie->attr('href'),
];
});
unset($QueryList);
// 获取最大章节索引
$intMaxXiaoShuoZhangJiePaixu = XiaoShuoZhangJieModel::where('xsxq_id', $intXiaoShuoSourceId)
->max('xszj_pai_xu'); // 获取最大值
// 如果没有结果，max 返回 null，可以设置默认值
$intMaxXiaoShuoZhangJiePaixu = $intMaxXiaoShuoZhangJiePaixu ?? 0;
foreach ($arrZhangjie as $intZhangJieNum => $arrZhangjieOne) {
// 如果数据库里的章节最大索引大于当前索引，说明已经入库，不需要入队列了
if($intMaxXiaoShuoZhangJiePaixu > $intZhangJieNum){
var_dump("小说章节已经入库，不再推送任务");
continue;// 如果任务已存在，直接跳过 break
}
$strZhangJieUrl = "https:" . $arrZhangjieOne['url'];
$strZhangJieId = extractFilename($strZhangJieUrl);
# 'AILISI_XIAOSHUO_ZHANGJIE_小说源id_小说源章节id'
$strXszjSourceCode = 'AILISI_XIAOSHUO_ZHANGJIE_'. $XiaoShuoXiangQingModel->xsxq_source_id . '_' . $strZhangJieId;
$arrTask = [
'callback' => [self::class, 'getNovelReader'],
'data' => [
'xszj_source_id' => $strZhangJieId,
'xszj_paixu' => $intZhangJieNum,
'xszj_url' => $strZhangJieUrl,
'xszj_source_code' => $strXszjSourceCode,
'xsxq_source_id' => $XiaoShuoXiangQingModel->xsxq_source_id,
'xsfl_id' => $arrData['xsfl_id'],
]
];
$TaskCore->set($arrTask);
//var_dump("章节任务投递成功".$strXszjSourceCode);
}
// 任务成功后删除 Redis 标记
$redis->del($redisKey);
*/
} catch (\Throwable $T) {
throw $T;
}
}
public function getNovelFengMian($arrData)
{
//var_dump('getNovelFengMian');
//var_dump($arrData);
$Client = $this->Client;
$TaskCore = $this->TaskCore;
// 获取 Redis 实例
$redis = Cache::store('redis')->handler();
$intXiaoShuoSourceId = $arrData['xiaoshuo_id'];
$intXiaoShuoSourceSeoId = $arrData['xiaoshuo_seo_id'];
$redisKey = "task:novel_source_id:" . $intXiaoShuoSourceId;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($arrData['xiaoshuo_url']);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$strSiteUrl = $this->getTianLaiDomain();
$strCover = $QueryList->find('#fmimg img')->eq(0)->attr("src");
if (strpos($strCover, 'http') === false) {
$strCover = $strSiteUrl . $strCover;
}
$strCoverUri = $strCover;
try {
$strCoverUri = "";
$strCoverUri = ImageProcessor::getInstance()->downloadAndEncryptImage($strCover, 'xs');
$arrCover = [
'code' => 'AI_LI_SI_COVER',
// 'uri' => getUriByUrl($strCover),
'uri' => $strCoverUri,
];
$arrXiaoShuoInfo = [
'xsxq_feng_mian' => json_encode($arrCover),
];
$XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
} catch (\Throwable $t) {
$strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
echo $strErr;
}
} catch (\Throwable $T) {
throw $T;
}
}
public function getNovelReader($arrData)
{
//var_dump('getNovelReader');
//var_dump($arrData);
$Client = $this->Client;
$strXszjSourceCode = $arrData['xszj_source_code'];
$strZhangJieUrl = $arrData['xszj_url'];
$intZhangJiePaiXu = $arrData['xszj_paixu'];
$intFenleiId = $arrData['xsfl_id'];
$intXiaoShuoSourceID = $arrData['xsxq_source_id'];
$strZhangJieSourceId = $arrData['xszj_source_id'];
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strZhangJieUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
if (XiaoShuoZhangJieModel::checkZhangJieExists($strXszjSourceCode)) {
return true;
}
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strZhangJieUrl, [
'headers' => [
'User-Agent' => $userAgent
],
'cookies' => $CookieJar,
#'proxy' => $proxy,
'timeout' => 10,
]);
$strResult = (string)$Response->getBody();
unset($Response);
$ZhangJieQueryList = QueryList::html($strResult);
$strZhangJieName = $ZhangJieQueryList->find('h1')->text();
$strNeiRong = $ZhangJieQueryList->find('#content')->eq(0)->text();
# '/novel/123yqw/分类id/小说源id/小说源章节id.txt'
$strNeiRongTxtPath = config("filesystem.novel").'/novel/123yqw/'.$intFenleiId.'/'.$intXiaoShuoSourceID.'/'.$strZhangJieSourceId.'.txt';
//$strNeiRongTxtPath = "/www/novel/123yqw/1/11/111.txt";
// 调用函数
$result = saveCompressAndDeleteTxt($strNeiRongTxtPath, $strNeiRong);
// 输出状态
if ($result['status']) {
var_dump("Success: " . $result['message']) ;
} else {
var_dump("Error: " . $result['message']) ;
}
$arrXiaoShuoZhangJie = [
'xsxq_id' => $intXiaoShuoSourceID,
'xszj_pai_xu' => $intZhangJiePaiXu,
'xszj_ming_zi' => $strZhangJieName,
'xszj_nei_rong' => $strNeiRongTxtPath,
'xszj_source_id' => $strZhangJieSourceId,
'xszj_source_code' => $strXszjSourceCode,
];
XiaoShuoZhangJieModel::addXiaoShuoZhangJie($arrXiaoShuoZhangJie);
unset($ZhangJieQueryList);
} catch (\Throwable $T) {
throw $T;
}
}
}

90
code/app/task/collection/visitlogs/ContentStatsCol.php
View File

@@ -1,90 +0,0 @@
<?php
declare(strict_types=1);
namespace app\task\collection\visitlogs;
use app\model\XiaoShuoFenLeiModel;
use app\model\XiaoShuoXiangQingModel;
use app\model\XiaoShuoZhangJieModel;
use app\task\collection\PullDataColBase;
use GuzzleHttp\Cookie\CookieJar;
use processor\ImageProcessor;
use think\facade\Cache;
use QL\QueryList;
use app\model\VisitLogsModel;
use app\model\ContentStatsModel;
/**
* @mixin think\Model
*/
class ContentStatsCol extends PullDataColBase
{
public function run()
{
var_dump('ContentStatsCol');
try {
// 定义时间范围统计的周期类型和 SQL 表达式
$arrPeriods = [
'day' => "DATE(vil_timestamp)", // 按日统计
'week' => "YEARWEEK(vil_timestamp)", // 按周统计
'month' => "DATE_FORMAT(vil_timestamp, '%Y-%m')", // 按月统计
'quarter' => "CONCAT(YEAR(vil_timestamp), '-', QUARTER(vil_timestamp))", // 按季度统计
'year' => "YEAR(vil_timestamp)" // 按年统计
];
foreach ($arrPeriods as $periodName => $groupExpression) {
var_dump($groupExpression);
$results = VisitLogsModel::where('vil_is_spider', 0) // 排除爬虫访问
->field([
'vil_content_type',
'vil_content_id',
"$groupExpression" => 'stat_date', // 根据周期分组
'COUNT(*)' => 'pv', // PV
'COUNT(DISTINCT vil_ip)' => 'uv' // UV
])
->group('vil_content_type, vil_content_id, stat_date')
->select()->toArray();
// var_dump($results);
foreach ($results as $result) {
$ContentStatsModel = [
'cs_content_type' => $result['vil_content_type'],
'cs_content_id' => $result['vil_content_id'],
'cs_stat_date' => $result['stat_date'],
'cs_stat_period' => $periodName, // 周期类型（day, week, month, quarter, year）
'cs_pv' => $result['pv'],
'cs_uv' => $result['uv'],
'cs_md5' => md5($result['vil_content_type'].$result['vil_content_id'].$result['stat_date'].$periodName),
'updated_at' => date('Y-m-d H:i:s'),
];
var_dump($ContentStatsModel);
ContentStatsModel::add($ContentStatsModel);
// var_dump($r);
// 插入或更新到统计表
// ContentStatsModel::insertOrUpdate([
// 'cs_content_type' => $result['vil_content_type'],
// 'cs_content_id' => $result['vil_content_id'],
// 'cs_stat_date' => $result['stat_date'],
// 'cs_stat_period' => $periodName, // 周期类型（day, week, month, quarter, year）
// 'cs_pv' => $result['pv'],
// 'cs_uv' => $result['uv'],
// 'updated_at' => date('Y-m-d H:i:s'),
// ], [
// 'cs_content_type' => $result['vil_content_type'],
// 'cs_content_id' => $result['vil_content_id'],
// 'cs_stat_date' => $result['stat_date'],
// 'cs_stat_period' => $periodName,
// ]);
}
}
} catch (\Exception $e) {
// 记录异常日志
//\think\facade\Log::error('ContentStatsTask error: ' . $e->getMessage());
}
}
}

47
code/app/task/collection/visitlogs/VisitLogsCol.php
View File

@@ -1,47 +0,0 @@
<?php
declare(strict_types=1);
namespace app\task\collection\visitlogs;
use app\model\XiaoShuoFenLeiModel;
use app\model\XiaoShuoXiangQingModel;
use app\model\XiaoShuoZhangJieModel;
use app\task\collection\PullDataColBase;
use GuzzleHttp\Cookie\CookieJar;
use processor\ImageProcessor;
use think\facade\Cache;
use QL\QueryList;
use app\model\VisitLogsModel;
/**
* @mixin think\Model
*/
class VisitLogsCol extends PullDataColBase
{
public function add($arrData)
{
try {
// 'vil_ip', 'vil_url', 'vil_content_id', 'vil_content_type', 'vil_method'
if (empty($arrData['vil_ip']) || empty($arrData['vil_url']) || empty($arrData['vil_content_id']) || empty($arrData['vil_content_type']) || empty($arrData['vil_method'])) {
var_dump("信息不全!");
}else{
$strLocation = '';
// if(!empty($arrData['vil_ip'])){
// $strLocation = getLocationByIp($arrData['vil_ip']);
// }
$arrData['vil_location'] = $strLocation;
VisitLogsModel::add($arrData);
}
} catch (\Throwable $T) {
//var_dump($T);
throw $T;
}
}
}

375
code/app/task/collection/yzzy/YouZhiYingShiCol.php
View File

@@ -1,375 +0,0 @@
<?php
declare(strict_types=1);
namespace app\task\collection\yzzy;
use app\model\VideoYuyanModel;
use app\model\VideoPlayurlModel;
use app\model\VideoLeixinModel;
use app\model\VideoDiquModel;
use app\model\VideoYearsModel;
use app\model\VideoJuqingModel;
use app\model\XiaoShuoFenLeiModel;
use app\model\XiaoShuoXiangQingModel;
use app\model\VideoInfoModel;
use app\model\XiaoShuoZhangJieModel;
use app\task\collection\PullDataColBase;
use GuzzleHttp\Cookie\CookieJar;
use processor\ImageProcessor;
use think\facade\Cache;
use QL\QueryList;
/**
* @mixin think\Model 优质资源
*/
class YouZhiYingShiCol extends PullDataColBase
{
public function pullYouZhiYingshi()
{
// https://1080zyk1.com/?m=vod-type-id-1-pg-1.html
// https://1080zyk1.com/?m=vod-type-id-2-pg-2.html
// https://1080zyk1.com/?m=vod-type-id-3-pg-2.html
// https://1080zyk1.com/?m=vod-type-id-4-pg-2.html
// https://1080zyk1.com/?m=vod-type-id-83-pg-2.html
// https://1080zyk1.com/?m=vod-type-id-94-pg-2.html
$strSiteUrl = $this->YouZhiZiYuanDomain();
// 配置分类任务参数
$categories = [
['id' => 1, 'video_class_id' => 1, 'name' => '电影'],
['id' => 83, 'video_class_id' => 5, 'name' => '短剧'],
['id' => 2, 'video_class_id' => 2, 'name' => '电视剧'],
['id' => 3, 'video_class_id' => 3, 'name' => '综艺'],
['id' => 4, 'video_class_id' => 4, 'name' => '动漫'],
['id' => 94, 'video_class_id' => 6, 'name' => '体育'],
];
foreach ($categories as $category) {
$intClassId = $category['video_class_id'];
$intCategoryId = $category['id'];
// 先抓第一页
$strUrl = $strSiteUrl . '/?m=vod-type-id-' . $intCategoryId . '-pg-1.html';
var_dump($strUrl);
$maxPage = $this->getMaxPage($strUrl,$intClassId);
// 首次抓取，全部抓取
if (config('task.task_is_first')) {
$this->createTasks($strSiteUrl, $intCategoryId, $intClassId, $maxPage);
}
}
}
/**
* 获取分类的最大页码且抓取第一页
*/
protected function getMaxPage($strUrl, $intClassId)
{
$TaskCore = $this->TaskCore;
$Client = $this->Client;
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$Response = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'Accept-Language' => 'en-US,en;q=0.5',
'cookie' => '__sk_Vm8nTqM7LqSCWmR6__=be3bd4f4e0483619e1a434ec1b18a938',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 300,
]);
$strResult = (string)$Response->getBody();
$QueryList = QueryList::html($strResult);
// 检查是否找到元素
$intVideoListIndex = 0;
if ($QueryList->find('.xing_vb ul')->count() > 1) {
$QueryList->find('.xing_vb ul')->map(function ($Video) use ($intClassId, &$TaskCore,&$intVideoListIndex) {
if($intVideoListIndex >=1){
$strHref = $Video->find('.xing_vb4 a')->attr('href');
$strName = '';
if (!empty($strHref) && strlen($strHref) > 0) {
$arrTask = [
'callback' => [self::class, 'getVideoInfo'],
'data' => [
'class_id' => $intClassId,
'video_href' => $strHref,
'video_name' => $strName,
]
];
// var_dump($arrTask);
$TaskCore->set($arrTask);
echo "视频投递成功 : " . $Video->text() . PHP_EOL;
}
}
$intVideoListIndex ++ ;
});
} else {
echo "No elements found for .xing_vb ul" . PHP_EOL;
}
// 获取分页部分最后一个页码链接
$lastPageHref = $QueryList->find('.pages a')->eq($QueryList->find('.pages a')->count() - 1)->attr('href');
// 提取最大页码
preg_match('/-pg-(\d+)\.html/', $lastPageHref, $matches);
$maxPage = isset($matches[1]) ? (int)$matches[1] : 1; // 默认返回 1
return $maxPage;
}
/**
* 创建任务
*/
protected function createTasks($strSiteUrl, $intCategoryId, $videoClassId, $maxPage)
{
$TaskCore = $this->TaskCore;
for ($i = $maxPage; $i >= 1; $i--) {
echo "当前分类: $intCategoryId, 当前页: $i\n";
$arrTask = [
'callback' => [self::class, 'getVideoPage'],
'data' => [
'video_page_url' => $strSiteUrl . '/?m=vod-type-id-' . $intCategoryId . '-pg-' . $i . '.html',
'video_class_id' => $videoClassId,
]
];
$TaskCore->set($arrTask);
}
}
# 获取分页里面的视频列表
public function getVideoPage($arrData)
{
$TaskCore = $this->TaskCore;
$Client = $this->Client;
$strUrl = $arrData['video_page_url'];
$intClassId = $arrData['video_class_id'];
$strSiteUrl = $this->YouZhiZiYuanDomain();
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
var_dump($strUrl);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'Accept-Language' => 'en-US,en;q=0.5',
'cookie' => '__sk_Vm8nTqM7LqSCWmR6__=be3bd4f4e0483619e1a434ec1b18a938',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 300,
]);
$strResult = (string)$Response->getBody();
$QueryList = QueryList::html($strResult);
$intVideoListIndex = 0;
if ($QueryList->find('.xing_vb ul')->count() > 1) {
$QueryList->find('.xing_vb ul')->map(function ($Video) use ($intClassId, &$TaskCore,&$intVideoListIndex) {
if($intVideoListIndex >=1){
$strHref = $Video->find('.xing_vb4 a')->attr('href');
$strName = '';
if (!empty($strHref) && strlen($strHref) > 0) {
$arrTask = [
'callback' => [self::class, 'getVideoInfo'],
'data' => [
'class_id' => $intClassId,
'video_href' => $strHref,
'video_name' => $strName,
]
];
// var_dump($arrTask);
$TaskCore->set($arrTask);
echo "视频投递成功 : " . $strHref . PHP_EOL;
}
}
$intVideoListIndex ++ ;
});
} else {
echo "No elements found for .xing_vb ul" . PHP_EOL;
}
}
public function getVideoInfo($arrData)
{
$TaskCore = $this->TaskCore;
$Client = $this->Client;
$strSiteUrl = $this->YouZhiZiYuanDomain();
$strUrl = $strSiteUrl . $arrData['video_href'];
$intClassId = $arrData['class_id'];
$strVideoName = $arrData['video_name'] ?? '';
var_dump($strUrl);
if (!empty($strUrl) && strlen($strUrl) > 0) {
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
$QueryList = QueryList::html($strResult);
// 提取别名
$strPinyinId = 'yzzy'.extractIdFromUrl($strUrl);
$strVideoName = $QueryList->find('.vodh h2')->text();
$strJiaoBiao = $QueryList->find('.vodh span')->text();
$strPinfen = $QueryList->find('.vodh label')->text();
$strLeixin = $QueryList->find('.nvc dd a')->eq(1)->text();
$strCover = $QueryList->find('.vodImg img')->eq(0)->attr("src");
if (strpos($strCover, 'http') === false) {
$strCover = $strSiteUrl . $strCover;
}
$strCoverUri = $strCover;
$arrCover = [
'code' => 'YOUZHI_SOURCE_COVER',
'uri' => $strCoverUri,
];
// 提取导演
$strDaoyan = $QueryList->find('.vodInfo .vodinfobox ul li')->eq(1)->find('span')->text();
// 提取主演 多个逗号分割
$strZhuyan = $QueryList->find('.vodInfo .vodinfobox li')->eq(2)->find('span')->text();
// 提取类型
$arrLeixinData = $QueryList->find('.vodInfo .vodinfobox li')->eq(3)->find('span')->text();
// 切割后移除空元素
$arrJuqing = preg_split('/\s+/u', trim($arrLeixinData));
array_shift($arrJuqing);
// **在过滤后将“类型”数据入库**
foreach ($arrJuqing as $value) {
if(strlen($value) > 0){
VideoJuqingModel::add(['vc_id' => $intClassId, 'vj_name' => $value]);
}
}
// 提取地区
$strDiqu = $QueryList->find('.vodinfobox li span')->eq(4)->text();
// 提取语言
$strLanguage = $QueryList->find('.vodinfobox li span')->eq(5)->text();
// 提取年份
$strYears = $QueryList->find('.vodinfobox li span')->eq(6)->text();
// 提取更新时间
$strLastDate = $QueryList->find('.vodinfobox li span')->eq(8)->text();
if (strlen($strLeixin) > 0) {
VideoLeixinModel::addVideoLeixin(['vc_id' => $intClassId, 'vl_name' => $strLeixin]);
}
if (strlen($strDiqu) > 0) {
VideoDiquModel::add(['vc_id' => $intClassId, 'vd_name' => $strDiqu]);
}
if (strlen($strYears) > 0) {
VideoYearsModel::add(['vc_id' => $intClassId, 'vy_name' => $strYears]);
}
if (strlen($strLanguage) > 0) {
VideoYuyanModel::add(['vc_id' => $intClassId, 'vla_name' => $strLanguage]);
}
// 提取简介
$strDescription = $QueryList->find('.vodplayinfo')->eq(0)->text();
if(empty($strVideoName)){
var_dump('$strVideoName------');
var_dump($strVideoName);
return true;
}
$arrVideoInfo = [
'v_name' => $strVideoName,
'v_cover' => json_encode($arrCover),
'vc_id' => $intClassId,
'v_pinyin' => $strPinyinId,
'v_leixin' => $strLeixin,
'v_juqing' => implode(',', $arrJuqing),
'v_diqu' => $strDiqu,
'v_years' => $strYears,
'v_language' => $strLanguage,
'v_daoyan' => trim(str_replace('主演：', '', $strZhuyan)),
'v_zhuyan' => trim(str_replace('导演：', '', $strDaoyan)),
'update_time' => $strLastDate,
'v_description' => $strDescription,
'v_pingfen' => $strPinfen,
'v_orurl' => $strUrl,
'v_jiaobiao' => $strJiaoBiao,
'v_md5' => md5($intClassId .$strPinyinId. $strVideoName . getUriByUrl($strUrl)),
];
// var_dump($arrVideoInfo);
// // 检查是否找到元素
if ($QueryList->find('#play_2 ul li')->count() > 0) {
// var_dump("视频入库成功--" . $strVideoName);
$resultVideoInfoModel = VideoInfoModel::addVideoXiangQing($arrVideoInfo);;
if ($resultVideoInfoModel['isNew']) {
var_dump("视频详情数据已新增：", $resultVideoInfoModel['model']->v_id."视频入库成功--" . $strVideoName);
} else {
echo "视频详情数据已存在：", $resultVideoInfoModel['model']->v_id;
}
$VideoInfoModel = $resultVideoInfoModel['model'];
$intVid = $VideoInfoModel->v_id;
$intChapterIndex = 0;
$QueryList->find('#play_2 ul li')->map(function ($ZhangJie) use (&$arrZhangjie, &$intChapterIndex, &$intVid,&$intClassId) {
//echo "Processing index $intChapterIndex: " . $ZhangJie->text() . PHP_EOL;
$strM3u8 = $ZhangJie->find('a')->text();
$arrM3u8 = explode('$', $strM3u8);
$strM3u8Url = $arrM3u8[1];
$strM3u8Name = $arrM3u8[0];
$arrM3u8Url = [
'code' => 'YOUZHI_SOURCE_PLAY',
'uri' => $strM3u8Url,
];
$arrVideoPlayurl = [
'v_id' => $intVid,
'vp_sort' => $intChapterIndex,
'vp_name' => $strM3u8Name,
'vp_url' => json_encode($arrM3u8Url),
'vp_md5' => md5($intClassId.$intVid . getUriByUrl($strM3u8Url)),
];
$result = VideoPlayurlModel::add($arrVideoPlayurl);
if ($result['isNew']) {
var_dump("播放地址已新增：", $result['model']->vp_id."播放线路入库成功--" . $strM3u8Url) ;
} else {
echo "播放地址已存在：", $result['model']->vp_id ;
}
$intChapterIndex++;
});
var_dump("--") ;
var_dump("一个视频任务完成------------------------------------------------------------------------------------") ;
} else {
echo "No elements found for .listmain a" . PHP_EOL;
}
} catch (\Throwable $T) {
throw $T;
}
}
// sleep(10);
}
}

477
code/app/task/collection/zw630/Zw630Col.php Normal file
View File

@@ -0,0 +1,477 @@
<?php
declare(strict_types=1);
namespace app\task\collection\zw630;
use app\model\XiaoShuoXiangQingModel;
use app\task\collection\PullDataColBase;
use GuzzleHttp\Cookie\CookieJar;
use processor\ImageProcessor;
use think\facade\Cache;
use QL\QueryList;
/**
* @mixin think\Model
*/
class Zw630Col extends PullDataColBase
{
public function pull630ZwNovel()
{
// 全局扫描任务
$QueueManage = $this->QueueManage;
for ($i = 1; $i <= 531208; $i++) {
$arrTask = [
'callback' => [self::class, 'getNovelInfo'],
'data' => [
'n_id' => $i,
]
];
print_r($arrTask);
// $QueueManage->set($arrTask);
}
}
public function getNovelInfo($arrData)
{
//var_dump('getNovelFengMian');
//var_dump($arrData);
$Client = $this->Client;
$QueueManage = $this->QueueManage;
// 获取 Redis 实例
$redis = Cache::store('redis')->handler();
$intXiaoShuoSourceId = $arrData['xiaoshuo_id'];
$intXiaoShuoSourceSeoId = $arrData['xiaoshuo_seo_id'];
$redisKey = "task:novel_source_id:" . $intXiaoShuoSourceId;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($arrData['xiaoshuo_url']);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$strSiteUrl = $this->getAilisiDomain();
$strCover = $QueryList->find('#fmimg img')->eq(0)->attr("src");
if (strpos($strCover, 'http') === false) {
$strCover = $strSiteUrl . $strCover;
}
$strCoverUri = $strCover;
try {
$strCoverUri = "";
$strCoverUri = ImageProcessor::getInstance()->downloadAndEncryptImage($strCover, 'xs');
$arrCover = [
'code' => 'AI_LI_SI_COVER',
// 'uri' => getUriByUrl($strCover),
'uri' => $strCoverUri,
];
$arrXiaoShuoInfo = [
'xsxq_feng_mian' => json_encode($arrCover),
];
$XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
} catch (\Throwable $t) {
$strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
echo $strErr;
}
} catch (\Throwable $T) {
throw $T;
}
}
// public function getNovelInfo($arrData)
// {
// $QueueManage = $this->QueueManage;
// $Client = $this->Client;
// $strSiteUrl = $this->getBiquxsDomain();
// $strUrl = 'http://www.biquxs.com/book/' . $arrData['xs_source_id'] ; //'http://www.biquxs.com/book/20053';
// $intXiaoShuoSourceId = $arrData['xs_source_id']; //20053;
// $intXiaoShuoSourceSeoId = $arrData['xs_source_seo_id']; //1 ;
// try {
// $arrCookie = $this->getAilisiCookie();
// $strHost = getDomainFromUrl($strUrl);
// $CookieJar = CookieJar::fromArray($arrCookie, $strHost);
// $proxy = getRandomProxies();
// $userAgent = getRandomUserAgent();
// $Response = $Client->request('GET', $strUrl, [
// 'headers' => [
// 'User-Agent' => $userAgent,
// 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
// 'Accept-Language' => 'en-US,en;q=0.5',
// ],
// 'cookies' => $CookieJar,
// // 'proxy' => $proxy,
// 'timeout' => 30,
// ]);
// $strResult = (string)$Response->getBody();
// $QueryList = QueryList::html($strResult);
// // var_dump($QueryList);
// // 提取所有 <meta> 标签的属性
// $result = $QueryList->find('meta[property^="og:"]')->map(function ($item) {
// return [
// 'property' => $item->attr('property'),
// 'content' => $item->attr('content'),
// ];
// });
// // 打印结果
// // print_r($result->all());
// $strZuoZhe = '';
// $strMingZi = '';
// $strLastDate = '';
// $strJieShao = '';
// $strCover = '';
// $strFenlei = '';
// $strZiShu = 0;
// $strZhuangTai = $QueryList->find('#info p')->eq(1)->text();
// $strZhuangTai = getStringAfterWord($strZhuangTai, '状态：');
// foreach ($result as $item) {
// switch ($item['property']) {
// case 'og:title':
// $strMingZi = $item['content'];
// break;
// case 'og:description':
// $strJieShao = $item['content'];
// break;
// case 'og:image':
// $strCover = $item['content'];
// break;
// case 'og:book:category':
// $strFenlei = $item['content'];
// break;
// case 'og:book:author':
// $strZuoZhe = $item['content'];
// break;
// case 'og:book:update_time':
// $strLastDate = $item['content'];
// break;
// }
// }
// $strLastDate = trim((string)$strLastDate);
// if (!isValidDateTime($strLastDate)) {
// $strLastDate = date('Y-m-d H:i:s');
// }
// $arrTag = [];
// if (strpos($strCover, 'http') === false) {
// $strCover = $strSiteUrl . $strCover;
// }
// $strCoverUri = $strCover;
// try {
// $strCoverUri = "";
// $strCoverUri = ImageProcessor::getInstance()->downloadImage( $intXiaoShuoSourceId,$strCover, 'xs');
// } catch (\Throwable $t) {
// $strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
// echo $strErr;
// }
// $arrCover = [
// 'code' => 'BI_QU_XS_COVER',
// // 'uri' => getUriByUrl($strCover),
// 'uri' => $strCoverUri,
// ];
// $strFenleiIId = 1 ;
// switch ($strFenlei) {
// case '玄幻小说':
// $strName = "玄幻";
// $strFenleiIId = 1;
// break;
// case '修真小说':
// $strName = "修真";
// $strFenleiIId = 2;
// break;
// case '都市小说':
// $strName = "都市";
// $strFenleiIId = 3;
// break;
// case '历史小说':
// $strName = "历史";
// $strFenleiIId = 4;
// break;
// case '网游小说':
// $strName = "网游";
// $strFenleiIId = 5;
// break;
// case '科幻小说':
// $strName = "科幻";
// $strFenleiIId = 6;
// break;
// case '女频小说':
// $strName = "女频";
// $strFenleiIId = 7;
// break;
// case '其它小说':
// $strName = "其它";
// $strFenleiIId = 8;
// break;
// default:
// $strName = "其它";
// $strFenleiIId = 8;
// break;
// }
// $arrXiaoShuoInfo = [
// 'xsxq_ming_zi' => $strMingZi,
// 'xsxq_zuozhe' => $strZuoZhe , //$arrData['xiaoshuo_zuozhe'],
// 'xsfl_id' => $strFenleiIId,
// 'xsxq_zhuang_tai' => $strZhuangTai,
// 'xsxq_zi_shu' => $strZiShu,
// 'xsxq_jie_shao' => $strJieShao,
// 'xsxq_feng_mian' => json_encode($arrCover),
// 'xsxq_geng_xin_shi_jian' => $strLastDate,
// 'xsxq_tag' => json_encode($arrTag, JSON_UNESCAPED_UNICODE),
// 'xsxq_source_id' => $intXiaoShuoSourceId,
// 'xsxq_source_seo_id' => $intXiaoShuoSourceId.'1',
// 'xsxq_source_code' => 'BIQU_XIAOSHUO_' . $intXiaoShuoSourceId . '_' . $intXiaoShuoSourceSeoId,
// ];
// // print_r($arrXiaoShuoInfo);
// var_dump($arrXiaoShuoInfo['xsxq_source_code']);
// $XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
// $arrZhangjie = [];
// $intChapterIndex = 0; // 定义一个计数器变量
// // 检查是否找到元素
// if ($QueryList->find('.listmain a')->count() > 0) {
// $QueryList->find('.listmain a')->map(function ($ZhangJie) use (&$arrZhangjie, &$intChapterIndex) {
// echo "Processing index $intChapterIndex: " . $ZhangJie->text() . PHP_EOL;
// // 从索引6开始
// if ($intChapterIndex >= 5) {
// $arrZhangjie[] = [
// 'mingzi' => $ZhangJie->text(),
// 'url' => $ZhangJie->attr('href'),
// ];
// }
// $intChapterIndex++;
// });
// // 输出最终结果
// var_dump($arrZhangjie);
// } else {
// echo "No elements found for .listmain a" . PHP_EOL;
// }
// //unset($QueryList);
// // 获取最大章节索引
// $intMaxXiaoShuoZhangJiePaixu = XiaoShuoZhangJieModel::where('xsxq_id', $intXiaoShuoSourceId)
// ->max('xszj_pai_xu'); // 获取最大值
// var_dump('获取小说章节最大值-'.$intMaxXiaoShuoZhangJiePaixu);
// // 如果没有结果，max 返回 null，可以设置默认值
// $intMaxXiaoShuoZhangJiePaixu = $intMaxXiaoShuoZhangJiePaixu ?? 0;
// foreach ($arrZhangjie as $intZhangJieNum => $arrZhangjieOne) {
// var_dump($intZhangJieNum);
// // 如果数据库里的章节最大索引大于当前索引，说明已经入库，不需要入队列了
// if($intMaxXiaoShuoZhangJiePaixu > $intZhangJieNum){
// var_dump("小说章节已经入库，不再推送任务");
// continue;// 如果任务已存在，直接跳过 break
// }
// $strZhangJieUrl = $strSiteUrl . $arrZhangjieOne['url'];
// $strZhangJieId = extractFilename($strZhangJieUrl);
// # 'AILISI_XIAOSHUO_ZHANGJIE_小说源id_小说源章节id'
// $strXszjSourceCode = 'AILISI_XIAOSHUO_ZHANGJIE_'. $XiaoShuoXiangQingModel->xsxq_source_id . '_' . $strZhangJieId;
// $arrTask = [
// 'callback' => [self::class, 'getNovelReader'],
// 'data' => [
// 'xszj_source_id' => $strZhangJieId,
// 'xszj_paixu' => $intZhangJieNum,
// 'xszj_name' => $arrZhangjieOne['mingzi'],
// 'xszj_url' => $strZhangJieUrl,
// 'xszj_source_code' => $strXszjSourceCode,
// 'xsxq_source_id' => $XiaoShuoXiangQingModel->xsxq_source_id,
// 'xsfl_id' => $strFenleiIId,
// ]
// ];
// $QueueManage->set($arrTask);
// }
// var_dump("章节任务投递成功");
// } catch (\Throwable $T) {
// throw $T;
// }
// }
// public function getNovelFengMian($arrData)
// {
// //var_dump('getNovelFengMian');
// //var_dump($arrData);
// $Client = $this->Client;
// $QueueManage = $this->QueueManage;
// // 获取 Redis 实例
// $redis = Cache::store('redis')->handler();
// $intXiaoShuoSourceId = $arrData['xiaoshuo_id'];
// $intXiaoShuoSourceSeoId = $arrData['xiaoshuo_seo_id'];
// $redisKey = "task:novel_source_id:" . $intXiaoShuoSourceId;
// try {
// $arrCookie = $this->getAilisiCookie();
// $strHost = getDomainFromUrl($arrData['xiaoshuo_url']);
// $CookieJar = CookieJar::fromArray($arrCookie, $strHost);
// $strSiteUrl = $this->getAilisiDomain();
// $strCover = $QueryList->find('#fmimg img')->eq(0)->attr("src");
// if (strpos($strCover, 'http') === false) {
// $strCover = $strSiteUrl . $strCover;
// }
// $strCoverUri = $strCover;
// try {
// $strCoverUri = "";
// $strCoverUri = ImageProcessor::getInstance()->downloadAndEncryptImage($strCover, 'xs');
// $arrCover = [
// 'code' => 'AI_LI_SI_COVER',
// // 'uri' => getUriByUrl($strCover),
// 'uri' => $strCoverUri,
// ];
// $arrXiaoShuoInfo = [
// 'xsxq_feng_mian' => json_encode($arrCover),
// ];
// $XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
// } catch (\Throwable $t) {
// $strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
// echo $strErr;
// }
// } catch (\Throwable $T) {
// throw $T;
// }
// }
// public function getNovelReader($arrData)
// {
// var_dump('getNovelReader');
// var_dump($arrData['xszj_url']);
// $Client = $this->Client;
// $strXszjSourceCode = $arrData['xszj_source_code'];
// $strZhangJieUrl = $arrData['xszj_url'];
// $intZhangJiePaiXu = $arrData['xszj_paixu'];
// $intFenleiId = $arrData['xsfl_id'];
// $intXiaoShuoSourceID = $arrData['xsxq_source_id'];
// $strZhangJieSourceId = $arrData['xszj_source_id'];
// $strZhangJieName = $arrData['xszj_name'];
// try {
// $arrCookie = $this->getAilisiCookie();
// $strHost = getDomainFromUrl($strZhangJieUrl);
// $CookieJar = CookieJar::fromArray($arrCookie, $strHost);
// if (XiaoShuoZhangJieModel::checkZhangJieExists($strXszjSourceCode)) {
// return true;
// }
// $proxy = getRandomProxies();
// $userAgent = getRandomUserAgent();
// $Response = $Client->request('GET', $strZhangJieUrl, [
// 'headers' => [
// 'User-Agent' => $userAgent,
// 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
// 'Accept-Language' => 'en-US,en;q=0.5',
// ],
// 'cookies' => $CookieJar,
// // 'proxy' => $proxy,
// 'timeout' => 30,
// ]);
// $strResult = (string)$Response->getBody();
// unset($Response);
// $ZhangJieQueryList = QueryList::html($strResult);
// // 获取原始 HTML
// $strNeiRong = $ZhangJieQueryList->find('#content')->eq(0)->html();
// // 解码 HTML 实体
// $strNeiRong = html_entity_decode($strNeiRong, ENT_QUOTES, 'UTF-8');
// // 去掉 <p> 标签中的 class="content_detail" 属性
// $strNeiRong = preg_replace('/<p\s+class="content_detail"/i', '<p', $strNeiRong);
// // 清理多余的 HTML 标签
// $strNeiRong = strip_tags($strNeiRong, '<p>');
// // 替换不间断空格 (\xc2\xa0) 和 &nbsp; 为普通空格
// $strNeiRong = preg_replace('/\xc2\xa0/', ' ', $strNeiRong); // 替换 UTF-8 不间断空格
// $strNeiRong = preg_replace('/&nbsp;/', ' ', $strNeiRong); // 替换 &nbsp;
// // 清理多余空格和换行符
// $strNeiRong = preg_replace('/\s+/', ' ', $strNeiRong);
// $strNeiRong = preg_replace('/<p>\s+/', '<p>', $strNeiRong); // 清理 <p> 标签内部多余空格
// $strNeiRong = preg_replace('/\s+<\/p>/', '</p>', $strNeiRong); // 清理 <p> 标签结尾多余空格
// // 输出结果
// // var_dump($strNeiRong);
// # '/novel/123yqw/分类id/小说源id/小说源章节id.txt'
// $strNeiRongTxtPath = config("filesystem.novel").'/novel/biquxs/'.$intFenleiId.'/'.$intXiaoShuoSourceID.'/'.$strZhangJieSourceId.'.txt';
// //$strNeiRongTxtPath = "/www/novel/123yqw/1/11/111.txt";
// // 调用函数
// $result = saveCompressAndDeleteTxt($strNeiRongTxtPath, $strNeiRong);
// // 输出状态
// if ($result['status']) {
// var_dump("Success: " . $result['message']) ;
// } else {
// var_dump("Error: " . $result['message']) ;
// }
// $arrXiaoShuoZhangJie = [
// 'xsxq_id' => $intXiaoShuoSourceID,
// 'xszj_pai_xu' => $intZhangJiePaiXu,
// 'xszj_ming_zi' => $strZhangJieName,
// 'xszj_nei_rong' => $strNeiRongTxtPath,
// 'xszj_source_id' => $strZhangJieSourceId,
// 'xszj_source_code' => $strXszjSourceCode,
// ];
// XiaoShuoZhangJieModel::addXiaoShuoZhangJie($arrXiaoShuoZhangJie);
// unset($ZhangJieQueryList);
// // 延时 3 秒
// //sleep(3);
// } catch (\Throwable $T) {
// throw $T;
// }
// }
}