admin/SEONexus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

debug

Browse Source
This commit is contained in:
Default
2025-03-20 21:10:55 +08:00
parent 62d2e78347
commit b09e4fca11
41 changed files with 1369 additions and 6258 deletions
Download Patch File Download Diff File Expand all files Collapse all files

477
code/app/task/collection/zw630/Zw630Col.php Normal file
View File

@@ -0,0 +1,477 @@
<?php
declare(strict_types=1);
namespace app\task\collection\zw630;
use app\model\XiaoShuoXiangQingModel;
use app\task\collection\PullDataColBase;
use GuzzleHttp\Cookie\CookieJar;
use processor\ImageProcessor;
use think\facade\Cache;
use QL\QueryList;
/**
* @mixin think\Model
*/
class Zw630Col extends PullDataColBase
{
public function pull630ZwNovel()
{
// 全局扫描任务
$QueueManage = $this->QueueManage;
for ($i = 1; $i <= 531208; $i++) {
$arrTask = [
'callback' => [self::class, 'getNovelInfo'],
'data' => [
'n_id' => $i,
]
];
print_r($arrTask);
// $QueueManage->set($arrTask);
}
}
public function getNovelInfo($arrData)
{
//var_dump('getNovelFengMian');
//var_dump($arrData);
$Client = $this->Client;
$QueueManage = $this->QueueManage;
// 获取 Redis 实例
$redis = Cache::store('redis')->handler();
$intXiaoShuoSourceId = $arrData['xiaoshuo_id'];
$intXiaoShuoSourceSeoId = $arrData['xiaoshuo_seo_id'];
$redisKey = "task:novel_source_id:" . $intXiaoShuoSourceId;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($arrData['xiaoshuo_url']);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$strSiteUrl = $this->getAilisiDomain();
$strCover = $QueryList->find('#fmimg img')->eq(0)->attr("src");
if (strpos($strCover, 'http') === false) {
$strCover = $strSiteUrl . $strCover;
}
$strCoverUri = $strCover;
try {
$strCoverUri = "";
$strCoverUri = ImageProcessor::getInstance()->downloadAndEncryptImage($strCover, 'xs');
$arrCover = [
'code' => 'AI_LI_SI_COVER',
// 'uri' => getUriByUrl($strCover),
'uri' => $strCoverUri,
];
$arrXiaoShuoInfo = [
'xsxq_feng_mian' => json_encode($arrCover),
];
$XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
} catch (\Throwable $t) {
$strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
echo $strErr;
}
} catch (\Throwable $T) {
throw $T;
}
}
// public function getNovelInfo($arrData)
// {
// $QueueManage = $this->QueueManage;
// $Client = $this->Client;
// $strSiteUrl = $this->getBiquxsDomain();
// $strUrl = 'http://www.biquxs.com/book/' . $arrData['xs_source_id'] ; //'http://www.biquxs.com/book/20053';
// $intXiaoShuoSourceId = $arrData['xs_source_id']; //20053;
// $intXiaoShuoSourceSeoId = $arrData['xs_source_seo_id']; //1 ;
// try {
// $arrCookie = $this->getAilisiCookie();
// $strHost = getDomainFromUrl($strUrl);
// $CookieJar = CookieJar::fromArray($arrCookie, $strHost);
// $proxy = getRandomProxies();
// $userAgent = getRandomUserAgent();
// $Response = $Client->request('GET', $strUrl, [
// 'headers' => [
// 'User-Agent' => $userAgent,
// 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
// 'Accept-Language' => 'en-US,en;q=0.5',
// ],
// 'cookies' => $CookieJar,
// // 'proxy' => $proxy,
// 'timeout' => 30,
// ]);
// $strResult = (string)$Response->getBody();
// $QueryList = QueryList::html($strResult);
// // var_dump($QueryList);
// // 提取所有 <meta> 标签的属性
// $result = $QueryList->find('meta[property^="og:"]')->map(function ($item) {
// return [
// 'property' => $item->attr('property'),
// 'content' => $item->attr('content'),
// ];
// });
// // 打印结果
// // print_r($result->all());
// $strZuoZhe = '';
// $strMingZi = '';
// $strLastDate = '';
// $strJieShao = '';
// $strCover = '';
// $strFenlei = '';
// $strZiShu = 0;
// $strZhuangTai = $QueryList->find('#info p')->eq(1)->text();
// $strZhuangTai = getStringAfterWord($strZhuangTai, '状态：');
// foreach ($result as $item) {
// switch ($item['property']) {
// case 'og:title':
// $strMingZi = $item['content'];
// break;
// case 'og:description':
// $strJieShao = $item['content'];
// break;
// case 'og:image':
// $strCover = $item['content'];
// break;
// case 'og:book:category':
// $strFenlei = $item['content'];
// break;
// case 'og:book:author':
// $strZuoZhe = $item['content'];
// break;
// case 'og:book:update_time':
// $strLastDate = $item['content'];
// break;
// }
// }
// $strLastDate = trim((string)$strLastDate);
// if (!isValidDateTime($strLastDate)) {
// $strLastDate = date('Y-m-d H:i:s');
// }
// $arrTag = [];
// if (strpos($strCover, 'http') === false) {
// $strCover = $strSiteUrl . $strCover;
// }
// $strCoverUri = $strCover;
// try {
// $strCoverUri = "";
// $strCoverUri = ImageProcessor::getInstance()->downloadImage( $intXiaoShuoSourceId,$strCover, 'xs');
// } catch (\Throwable $t) {
// $strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
// echo $strErr;
// }
// $arrCover = [
// 'code' => 'BI_QU_XS_COVER',
// // 'uri' => getUriByUrl($strCover),
// 'uri' => $strCoverUri,
// ];
// $strFenleiIId = 1 ;
// switch ($strFenlei) {
// case '玄幻小说':
// $strName = "玄幻";
// $strFenleiIId = 1;
// break;
// case '修真小说':
// $strName = "修真";
// $strFenleiIId = 2;
// break;
// case '都市小说':
// $strName = "都市";
// $strFenleiIId = 3;
// break;
// case '历史小说':
// $strName = "历史";
// $strFenleiIId = 4;
// break;
// case '网游小说':
// $strName = "网游";
// $strFenleiIId = 5;
// break;
// case '科幻小说':
// $strName = "科幻";
// $strFenleiIId = 6;
// break;
// case '女频小说':
// $strName = "女频";
// $strFenleiIId = 7;
// break;
// case '其它小说':
// $strName = "其它";
// $strFenleiIId = 8;
// break;
// default:
// $strName = "其它";
// $strFenleiIId = 8;
// break;
// }
// $arrXiaoShuoInfo = [
// 'xsxq_ming_zi' => $strMingZi,
// 'xsxq_zuozhe' => $strZuoZhe , //$arrData['xiaoshuo_zuozhe'],
// 'xsfl_id' => $strFenleiIId,
// 'xsxq_zhuang_tai' => $strZhuangTai,
// 'xsxq_zi_shu' => $strZiShu,
// 'xsxq_jie_shao' => $strJieShao,
// 'xsxq_feng_mian' => json_encode($arrCover),
// 'xsxq_geng_xin_shi_jian' => $strLastDate,
// 'xsxq_tag' => json_encode($arrTag, JSON_UNESCAPED_UNICODE),
// 'xsxq_source_id' => $intXiaoShuoSourceId,
// 'xsxq_source_seo_id' => $intXiaoShuoSourceId.'1',
// 'xsxq_source_code' => 'BIQU_XIAOSHUO_' . $intXiaoShuoSourceId . '_' . $intXiaoShuoSourceSeoId,
// ];
// // print_r($arrXiaoShuoInfo);
// var_dump($arrXiaoShuoInfo['xsxq_source_code']);
// $XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
// $arrZhangjie = [];
// $intChapterIndex = 0; // 定义一个计数器变量
// // 检查是否找到元素
// if ($QueryList->find('.listmain a')->count() > 0) {
// $QueryList->find('.listmain a')->map(function ($ZhangJie) use (&$arrZhangjie, &$intChapterIndex) {
// echo "Processing index $intChapterIndex: " . $ZhangJie->text() . PHP_EOL;
// // 从索引6开始
// if ($intChapterIndex >= 5) {
// $arrZhangjie[] = [
// 'mingzi' => $ZhangJie->text(),
// 'url' => $ZhangJie->attr('href'),
// ];
// }
// $intChapterIndex++;
// });
// // 输出最终结果
// var_dump($arrZhangjie);
// } else {
// echo "No elements found for .listmain a" . PHP_EOL;
// }
// //unset($QueryList);
// // 获取最大章节索引
// $intMaxXiaoShuoZhangJiePaixu = XiaoShuoZhangJieModel::where('xsxq_id', $intXiaoShuoSourceId)
// ->max('xszj_pai_xu'); // 获取最大值
// var_dump('获取小说章节最大值-'.$intMaxXiaoShuoZhangJiePaixu);
// // 如果没有结果，max 返回 null，可以设置默认值
// $intMaxXiaoShuoZhangJiePaixu = $intMaxXiaoShuoZhangJiePaixu ?? 0;
// foreach ($arrZhangjie as $intZhangJieNum => $arrZhangjieOne) {
// var_dump($intZhangJieNum);
// // 如果数据库里的章节最大索引大于当前索引，说明已经入库，不需要入队列了
// if($intMaxXiaoShuoZhangJiePaixu > $intZhangJieNum){
// var_dump("小说章节已经入库，不再推送任务");
// continue;// 如果任务已存在，直接跳过 break
// }
// $strZhangJieUrl = $strSiteUrl . $arrZhangjieOne['url'];
// $strZhangJieId = extractFilename($strZhangJieUrl);
// # 'AILISI_XIAOSHUO_ZHANGJIE_小说源id_小说源章节id'
// $strXszjSourceCode = 'AILISI_XIAOSHUO_ZHANGJIE_'. $XiaoShuoXiangQingModel->xsxq_source_id . '_' . $strZhangJieId;
// $arrTask = [
// 'callback' => [self::class, 'getNovelReader'],
// 'data' => [
// 'xszj_source_id' => $strZhangJieId,
// 'xszj_paixu' => $intZhangJieNum,
// 'xszj_name' => $arrZhangjieOne['mingzi'],
// 'xszj_url' => $strZhangJieUrl,
// 'xszj_source_code' => $strXszjSourceCode,
// 'xsxq_source_id' => $XiaoShuoXiangQingModel->xsxq_source_id,
// 'xsfl_id' => $strFenleiIId,
// ]
// ];
// $QueueManage->set($arrTask);
// }
// var_dump("章节任务投递成功");
// } catch (\Throwable $T) {
// throw $T;
// }
// }
// public function getNovelFengMian($arrData)
// {
// //var_dump('getNovelFengMian');
// //var_dump($arrData);
// $Client = $this->Client;
// $QueueManage = $this->QueueManage;
// // 获取 Redis 实例
// $redis = Cache::store('redis')->handler();
// $intXiaoShuoSourceId = $arrData['xiaoshuo_id'];
// $intXiaoShuoSourceSeoId = $arrData['xiaoshuo_seo_id'];
// $redisKey = "task:novel_source_id:" . $intXiaoShuoSourceId;
// try {
// $arrCookie = $this->getAilisiCookie();
// $strHost = getDomainFromUrl($arrData['xiaoshuo_url']);
// $CookieJar = CookieJar::fromArray($arrCookie, $strHost);
// $strSiteUrl = $this->getAilisiDomain();
// $strCover = $QueryList->find('#fmimg img')->eq(0)->attr("src");
// if (strpos($strCover, 'http') === false) {
// $strCover = $strSiteUrl . $strCover;
// }
// $strCoverUri = $strCover;
// try {
// $strCoverUri = "";
// $strCoverUri = ImageProcessor::getInstance()->downloadAndEncryptImage($strCover, 'xs');
// $arrCover = [
// 'code' => 'AI_LI_SI_COVER',
// // 'uri' => getUriByUrl($strCover),
// 'uri' => $strCoverUri,
// ];
// $arrXiaoShuoInfo = [
// 'xsxq_feng_mian' => json_encode($arrCover),
// ];
// $XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
// } catch (\Throwable $t) {
// $strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
// echo $strErr;
// }
// } catch (\Throwable $T) {
// throw $T;
// }
// }
// public function getNovelReader($arrData)
// {
// var_dump('getNovelReader');
// var_dump($arrData['xszj_url']);
// $Client = $this->Client;
// $strXszjSourceCode = $arrData['xszj_source_code'];
// $strZhangJieUrl = $arrData['xszj_url'];
// $intZhangJiePaiXu = $arrData['xszj_paixu'];
// $intFenleiId = $arrData['xsfl_id'];
// $intXiaoShuoSourceID = $arrData['xsxq_source_id'];
// $strZhangJieSourceId = $arrData['xszj_source_id'];
// $strZhangJieName = $arrData['xszj_name'];
// try {
// $arrCookie = $this->getAilisiCookie();
// $strHost = getDomainFromUrl($strZhangJieUrl);
// $CookieJar = CookieJar::fromArray($arrCookie, $strHost);
// if (XiaoShuoZhangJieModel::checkZhangJieExists($strXszjSourceCode)) {
// return true;
// }
// $proxy = getRandomProxies();
// $userAgent = getRandomUserAgent();
// $Response = $Client->request('GET', $strZhangJieUrl, [
// 'headers' => [
// 'User-Agent' => $userAgent,
// 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
// 'Accept-Language' => 'en-US,en;q=0.5',
// ],
// 'cookies' => $CookieJar,
// // 'proxy' => $proxy,
// 'timeout' => 30,
// ]);
// $strResult = (string)$Response->getBody();
// unset($Response);
// $ZhangJieQueryList = QueryList::html($strResult);
// // 获取原始 HTML
// $strNeiRong = $ZhangJieQueryList->find('#content')->eq(0)->html();
// // 解码 HTML 实体
// $strNeiRong = html_entity_decode($strNeiRong, ENT_QUOTES, 'UTF-8');
// // 去掉 <p> 标签中的 class="content_detail" 属性
// $strNeiRong = preg_replace('/<p\s+class="content_detail"/i', '<p', $strNeiRong);
// // 清理多余的 HTML 标签
// $strNeiRong = strip_tags($strNeiRong, '<p>');
// // 替换不间断空格 (\xc2\xa0) 和 &nbsp; 为普通空格
// $strNeiRong = preg_replace('/\xc2\xa0/', ' ', $strNeiRong); // 替换 UTF-8 不间断空格
// $strNeiRong = preg_replace('/&nbsp;/', ' ', $strNeiRong); // 替换 &nbsp;
// // 清理多余空格和换行符
// $strNeiRong = preg_replace('/\s+/', ' ', $strNeiRong);
// $strNeiRong = preg_replace('/<p>\s+/', '<p>', $strNeiRong); // 清理 <p> 标签内部多余空格
// $strNeiRong = preg_replace('/\s+<\/p>/', '</p>', $strNeiRong); // 清理 <p> 标签结尾多余空格
// // 输出结果
// // var_dump($strNeiRong);
// # '/novel/123yqw/分类id/小说源id/小说源章节id.txt'
// $strNeiRongTxtPath = config("filesystem.novel").'/novel/biquxs/'.$intFenleiId.'/'.$intXiaoShuoSourceID.'/'.$strZhangJieSourceId.'.txt';
// //$strNeiRongTxtPath = "/www/novel/123yqw/1/11/111.txt";
// // 调用函数
// $result = saveCompressAndDeleteTxt($strNeiRongTxtPath, $strNeiRong);
// // 输出状态
// if ($result['status']) {
// var_dump("Success: " . $result['message']) ;
// } else {
// var_dump("Error: " . $result['message']) ;
// }
// $arrXiaoShuoZhangJie = [
// 'xsxq_id' => $intXiaoShuoSourceID,
// 'xszj_pai_xu' => $intZhangJiePaiXu,
// 'xszj_ming_zi' => $strZhangJieName,
// 'xszj_nei_rong' => $strNeiRongTxtPath,
// 'xszj_source_id' => $strZhangJieSourceId,
// 'xszj_source_code' => $strXszjSourceCode,
// ];
// XiaoShuoZhangJieModel::addXiaoShuoZhangJie($arrXiaoShuoZhangJie);
// unset($ZhangJieQueryList);
// // 延时 3 秒
// //sleep(3);
// } catch (\Throwable $T) {
// throw $T;
// }
// }
}