admin/SEONexus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

debug

Browse Source
This commit is contained in:
Default
2025-03-15 20:12:21 +08:00
parent 105ffe11c7
commit eb9035a7ae
134 changed files with 15450 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files

538
code/app/task/collection/tianlai/TianLaiXiaoShuoCol.php Normal file
View File

@@ -0,0 +1,538 @@
<?php
declare(strict_types=1);
namespace app\task\collection\tianlai;
use app\admin\model\XiaoShuoFenLeiModel;
use app\admin\model\XiaoShuoXiangQingModel;
use app\admin\model\XiaoShuoZhangJieModel;
use app\task\collection\PullDataColBase;
use GuzzleHttp\Cookie\CookieJar;
use processor\ImageProcessor;
use think\facade\Cache;
use QL\QueryList;
/**
* @mixin think\Model
*/
class TianLaiXiaoShuoCol extends PullDataColBase
{
public function pullXiaoShuo()
{
$TaskCore = $this->TaskCore;
$strUrl = $this->getTianLaiDomain();
var_dump($strUrl);
$Client = $this->Client;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
// $proxy = '127.0.0.1:8080'; // 代理地址
// $response = fetchContentWithCurl($strUrl, [], 10, $proxy);
$strResult = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$strResult->getBody();
var_dump($strResult);
// exit;
// $strResult = $response['data'];
$QueryList = QueryList::html($strResult);
$QueryList->find('.nav a')->map(function ($FenLeiQueryList) use ($strUrl, $TaskCore) {
$strName = trim($FenLeiQueryList->text());
switch ($strName) {
case '都市言情':
$strName = "都市言情";
break;
case '历史军事':
$strName = "历史军事";
break;
case '玄幻奇幻':
$strName = "玄幻奇幻";
break;
case '武侠修真':
$strName = "武侠修真";
break;
case '女生纯爱':
$strName = "女频言情";
break;
case '科幻网游':
$strName = "科幻网游";
break;
case '悬疑灵异':
$strName = "悬疑灵异";
break;
default:
$strName = "其它";
break;
}
$strFenLeiUrl = $strUrl . $FenLeiQueryList->attr('href');
// $intFenLeiId = extractFilename($strFenLeiUrl."index.html");
$intFenLeiId = extractNumberFromUrl($strFenLeiUrl);
if($intFenLeiId !== null){
$arrClass = [
'xsfl_source_id' => $intFenLeiId,
'xsfl_source_code' => 'TIANLAI_FEN_LEI_ID_' . $intFenLeiId,
'xsfl_name' => $strName,
];
var_dump($arrClass);
$XiaoShuoFenLeiModel = XiaoShuoFenLeiModel::addXiaoShuoFenLei($arrClass);
$arrTask = [
'callback' => [self::class, 'getNovelPage'],
'data' => [
'url' => $strFenLeiUrl,
'xsfl_source_id' => $intFenLeiId,
'xsfl_name' => $strName,
'xsfl_id' => $XiaoShuoFenLeiModel->xsfl_id,
'xsfl_page' => 1,
'xsfl_max_page' => 3000,
]
];
$TaskCore->set($arrTask);
}
});
} catch (\Throwable $T) {
//var_dump($T);
throw $T;
}
}
public function getNovelPage($arrData)
{
var_dump('getNovelPage');
var_dump($arrData);
$strBasePage = $arrData['url'];
$Client = $this->Client;
$TaskCore = $this->TaskCore;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strBasePage);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$strSiteUrl = $this->getTianLaiDomain();
$intPage = $arrData['xsfl_page'];
$intMaxPage = $arrData['xsfl_max_page'];
$strUrl = $strBasePage;
var_dump($strUrl);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$strResult = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => $userAgent
],
'cookies' => $CookieJar,
#'proxy' => $proxy,
'timeout' => 10,
]);
$strResult = (string)$strResult->getBody();
$QueryList = QueryList::html($strResult);
$intMaxPage = $QueryList->find('#newscontent h2 span')->eq(1)->text();
$intMaxPage = ceil($intMaxPage / 100);
var_dump($intMaxPage);
$intMaxPage = 2;
var_dump('$strUrl');
// 如果是首页 则根据最大页面吧所有分页 追加到任务
if($intPage == 1){
for ($intCurrentPage = 2; $intCurrentPage <= $intMaxPage; $intCurrentPage++) {
$strNewsFenleiUrl = replaceLastPageNumber($strBasePage, $intCurrentPage);
var_dump($strNewsFenleiUrl);
$arrTask = [
'callback' => [self::class, 'getNovelPage'],
'data' => [
'url' => $strNewsFenleiUrl,
'xsfl_source_id' => $arrData['xsfl_source_id'],
'xsfl_name' => $arrData['xsfl_name'],
'xsfl_id' => $arrData['xsfl_id'],
'xsfl_page' => $intCurrentPage,
'xsfl_max_page' => $intMaxPage,
]
];
$TaskCore->set($arrTask);
}
}
var_dump('提取小说');
$XiaoShuoListQueryList = $QueryList->find('#newscontent li');
if (!$XiaoShuoListQueryList || $XiaoShuoListQueryList->count() <= 0) {
var_dump('这个分类已经市最后一页');
return true;
}
var_dump($XiaoShuoListQueryList->count());
var_dump('提取小说2');
// var_dump($XiaoShuoListQueryList);
$XiaoShuoListQueryList->map(function ($XiaoShuoQueryList) use ($strSiteUrl, $arrData, $TaskCore, $strUrl) {
// 提取小说链接
// var_dump($XiaoShuoQueryList->html());exit;
var_dump('提取小说链接');
$A = $XiaoShuoQueryList->find('a')->eq(0);
$strXiaoShuoUrl = $A->attr('href');
$strXiaoShuoUrl = strpos($strXiaoShuoUrl, 'https:') === 0 ? $strXiaoShuoUrl : 'https:' . $strXiaoShuoUrl;
var_dump('$A');
var_dump($A);
var_dump($strXiaoShuoUrl);
exit;
// var_dump($A);
// 提取作者信息
$strZuoZhe = trim($XiaoShuoQueryList->find('.s4')->eq(0)->text());
// 提取小说 ID
$arrXiaoShuoId = extractNumbersFromUrl($strXiaoShuoUrl);
if (!isset($arrXiaoShuoId['first'], $arrXiaoShuoId['second'])) {
return; // 如果 ID 提取失败，直接跳过
}
// 准备任务数据
$arrXiaoShuo = array_merge($arrData, [
'xiaoshuo_url' => $strXiaoShuoUrl,
'xiaoshuo_id' => $arrXiaoShuoId['first'],
'xiaoshuo_seo_id' => $arrXiaoShuoId['second'],
'xiaoshuo_zuozhe' => $strZuoZhe,
'url' => $strUrl
]);
var_dump($arrXiaoShuo);
$arrTask = [
'callback' => [self::class, 'getNovelInfo'],
'data' => $arrXiaoShuo,
];
// 将任务加入队列
$TaskCore->set($arrTask);
var_dump("将任务加入队列getNovelInfo-{$strXiaoShuoUrl}");
});
var_dump('提取小说end');
} catch (\Throwable $T) {
throw $T;
}
}
public function getNovelInfo($arrData)
{
var_dump('getNovelInfo');
var_dump($arrData);
$Client = $this->Client;
$TaskCore = $this->TaskCore;
// 获取 Redis 实例
$redis = Cache::store('redis')->handler();
$intXiaoShuoSourceId = $arrData['xiaoshuo_id'];
$intXiaoShuoSourceSeoId = $arrData['xiaoshuo_seo_id'];
$redisKey = "task:novel_source_id:" . $intXiaoShuoSourceId;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($arrData['xiaoshuo_url']);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$strSiteUrl = $this->getTianLaiDomain();
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $arrData['xiaoshuo_url'], [
'headers' => [
'User-Agent' => $userAgent
],
'cookies' => $CookieJar,
#'proxy' => $proxy,
'timeout' => 10,
]);
$strResult = (string)$Response->getBody();
unset($Response);
$QueryList = QueryList::html($strResult);
$strMingZi = $QueryList->find('#info h1')->eq(0)->text();
$strZhuangTai = "状态：连载中";
$strZiShu = 0;
$strZhuangTai = getStringAfterWord($strZhuangTai, '状态：');
// 提取作者信息
$strZuoZhe = trim($QueryList->find('p a')->eq(0)->text());
// $strLastDate = $QueryList->find('meta[property="og:novel:update_time"]')->attr('content');
$strLastDate = $QueryList->find('#info p')->eq(2)->text();
$strLastDate = getStringAfterWord($strLastDate, '最后更新：');
$strLastDate = trim((string)$strLastDate);
if (!isValidDateTime($strLastDate)) {
$strLastDate = date('Y-m-d H:i:s');
}
$arrTag = [];
// $QueryList->find('.tg span')->map(function ($TagQuery) use (&$arrTag) {
// $arrTag[] = trim($TagQuery->text());
// });
$strJieShao = $QueryList->find('#intro')->eq(0)->text();
$strJieShao = removeScriptTags('/<script>.*?<\/script>/is',$strJieShao) ;
$strJieShao = removeScriptTags('/myJs\.bookJs\(\);/',$strJieShao) ;
$strCover = $QueryList->find('#fmimg img')->eq(0)->attr("src");
// if (strpos($strCover, 'http') === false) {
// $strCover = $strSiteUrl . $strCover;
// }
$strCoverUri = $strCover;
// try {
// $strCoverUri = "";
// $strCoverUri = ImageProcessor::getInstance()->downloadAndEncryptImage($strCover, 'xs');
// } catch (\Throwable $t) {
// $strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
// echo $strErr;
// }
$arrCover = [
'code' => 'AI_LI_SI_COVER',
// 'uri' => getUriByUrl($strCover),
'uri' => $strCoverUri,
];
$arrXiaoShuoInfo = [
'xsxq_ming_zi' => $strMingZi,
'xsxq_zuozhe' => $strZuoZhe , //$arrData['xiaoshuo_zuozhe'],
'xsfl_id' => $arrData['xsfl_id'],
'xsxq_zhuang_tai' => $strZhuangTai,
'xsxq_zi_shu' => $strZiShu,
'xsxq_jie_shao' => $strJieShao,
'xsxq_feng_mian' => json_encode($arrCover),
'xsxq_geng_xin_shi_jian' => $strLastDate,
'xsxq_tag' => json_encode($arrTag, JSON_UNESCAPED_UNICODE),
'xsxq_source_id' => $intXiaoShuoSourceId,
'xsxq_source_seo_id' => $intXiaoShuoSourceSeoId,
'xsxq_source_code' => 'AILISI_XIAOSHUO_' . $intXiaoShuoSourceId . '_' . $intXiaoShuoSourceSeoId,
];
// print_r($arrXiaoShuoInfo);
var_dump($arrXiaoShuoInfo['xsxq_source_code']);
$XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
/*
if ($redis->exists($redisKey)) {
var_dump('小说任务已存在，直接跳过');
return true; // 如果任务已存在，直接跳过
}
// 在 Redis 中标记任务为已存在
$redis->set($redisKey, 1);
$arrZhangjie = [];
$QueryList->find('#list a')->map(function ($ZhangJie) use (&$arrZhangjie) {
$arrZhangjie[] = [
'mingzi' => $ZhangJie->text(),
'url' => $ZhangJie->attr('href'),
];
});
unset($QueryList);
// 获取最大章节索引
$intMaxXiaoShuoZhangJiePaixu = XiaoShuoZhangJieModel::where('xsxq_id', $intXiaoShuoSourceId)
->max('xszj_pai_xu'); // 获取最大值
// 如果没有结果，max 返回 null，可以设置默认值
$intMaxXiaoShuoZhangJiePaixu = $intMaxXiaoShuoZhangJiePaixu ?? 0;
foreach ($arrZhangjie as $intZhangJieNum => $arrZhangjieOne) {
// 如果数据库里的章节最大索引大于当前索引，说明已经入库，不需要入队列了
if($intMaxXiaoShuoZhangJiePaixu > $intZhangJieNum){
var_dump("小说章节已经入库，不再推送任务");
continue;// 如果任务已存在，直接跳过 break
}
$strZhangJieUrl = "https:" . $arrZhangjieOne['url'];
$strZhangJieId = extractFilename($strZhangJieUrl);
# 'AILISI_XIAOSHUO_ZHANGJIE_小说源id_小说源章节id'
$strXszjSourceCode = 'AILISI_XIAOSHUO_ZHANGJIE_'. $XiaoShuoXiangQingModel->xsxq_source_id . '_' . $strZhangJieId;
$arrTask = [
'callback' => [self::class, 'getNovelReader'],
'data' => [
'xszj_source_id' => $strZhangJieId,
'xszj_paixu' => $intZhangJieNum,
'xszj_url' => $strZhangJieUrl,
'xszj_source_code' => $strXszjSourceCode,
'xsxq_source_id' => $XiaoShuoXiangQingModel->xsxq_source_id,
'xsfl_id' => $arrData['xsfl_id'],
]
];
$TaskCore->set($arrTask);
//var_dump("章节任务投递成功".$strXszjSourceCode);
}
// 任务成功后删除 Redis 标记
$redis->del($redisKey);
*/
} catch (\Throwable $T) {
throw $T;
}
}
public function getNovelFengMian($arrData)
{
//var_dump('getNovelFengMian');
//var_dump($arrData);
$Client = $this->Client;
$TaskCore = $this->TaskCore;
// 获取 Redis 实例
$redis = Cache::store('redis')->handler();
$intXiaoShuoSourceId = $arrData['xiaoshuo_id'];
$intXiaoShuoSourceSeoId = $arrData['xiaoshuo_seo_id'];
$redisKey = "task:novel_source_id:" . $intXiaoShuoSourceId;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($arrData['xiaoshuo_url']);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$strSiteUrl = $this->getTianLaiDomain();
$strCover = $QueryList->find('#fmimg img')->eq(0)->attr("src");
if (strpos($strCover, 'http') === false) {
$strCover = $strSiteUrl . $strCover;
}
$strCoverUri = $strCover;
try {
$strCoverUri = "";
$strCoverUri = ImageProcessor::getInstance()->downloadAndEncryptImage($strCover, 'xs');
$arrCover = [
'code' => 'AI_LI_SI_COVER',
// 'uri' => getUriByUrl($strCover),
'uri' => $strCoverUri,
];
$arrXiaoShuoInfo = [
'xsxq_feng_mian' => json_encode($arrCover),
];
$XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
} catch (\Throwable $t) {
$strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
echo $strErr;
}
} catch (\Throwable $T) {
throw $T;
}
}
public function getNovelReader($arrData)
{
//var_dump('getNovelReader');
//var_dump($arrData);
$Client = $this->Client;
$strXszjSourceCode = $arrData['xszj_source_code'];
$strZhangJieUrl = $arrData['xszj_url'];
$intZhangJiePaiXu = $arrData['xszj_paixu'];
$intFenleiId = $arrData['xsfl_id'];
$intXiaoShuoSourceID = $arrData['xsxq_source_id'];
$strZhangJieSourceId = $arrData['xszj_source_id'];
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strZhangJieUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
if (XiaoShuoZhangJieModel::checkZhangJieExists($strXszjSourceCode)) {
return true;
}
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strZhangJieUrl, [
'headers' => [
'User-Agent' => $userAgent
],
'cookies' => $CookieJar,
#'proxy' => $proxy,
'timeout' => 10,
]);
$strResult = (string)$Response->getBody();
unset($Response);
$ZhangJieQueryList = QueryList::html($strResult);
$strZhangJieName = $ZhangJieQueryList->find('h1')->text();
$strNeiRong = $ZhangJieQueryList->find('#content')->eq(0)->text();
# '/novel/123yqw/分类id/小说源id/小说源章节id.txt'
$strNeiRongTxtPath = config("filesystem.novel").'/novel/123yqw/'.$intFenleiId.'/'.$intXiaoShuoSourceID.'/'.$strZhangJieSourceId.'.txt';
//$strNeiRongTxtPath = "/www/novel/123yqw/1/11/111.txt";
// 调用函数
$result = saveCompressAndDeleteTxt($strNeiRongTxtPath, $strNeiRong);
// 输出状态
if ($result['status']) {
var_dump("Success: " . $result['message']) ;
} else {
var_dump("Error: " . $result['message']) ;
}
$arrXiaoShuoZhangJie = [
'xsxq_id' => $intXiaoShuoSourceID,
'xszj_pai_xu' => $intZhangJiePaiXu,
'xszj_ming_zi' => $strZhangJieName,
'xszj_nei_rong' => $strNeiRongTxtPath,
'xszj_source_id' => $strZhangJieSourceId,
'xszj_source_code' => $strXszjSourceCode,
];
XiaoShuoZhangJieModel::addXiaoShuoZhangJie($arrXiaoShuoZhangJie);
unset($ZhangJieQueryList);
} catch (\Throwable $T) {
throw $T;
}
}
}