debug
This commit is contained in:
Default
427
code/app/task/collection/biquxs/BiquxsXiaoShuoCol.php
Normal file
427
code/app/task/collection/biquxs/BiquxsXiaoShuoCol.php
Normal file
@@ -0,0 +1,427 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace app\task\collection\biquxs;
|
||||
|
||||
|
||||
use app\admin\model\XiaoShuoFenLeiModel;
|
||||
use app\admin\model\XiaoShuoXiangQingModel;
|
||||
use app\admin\model\XiaoShuoZhangJieModel;
|
||||
use app\task\collection\PullDataColBase;
|
||||
use GuzzleHttp\Cookie\CookieJar;
|
||||
use processor\ImageProcessor;
|
||||
use think\facade\Cache;
|
||||
use QL\QueryList;
|
||||
|
||||
|
||||
/**
|
||||
* @mixin think\Model
|
||||
*/
|
||||
class BiquxsXiaoShuoCol extends PullDataColBase
|
||||
{
|
||||
public function pullXiaoShuo()
|
||||
{
|
||||
|
||||
$TaskCore = $this->TaskCore;
|
||||
// 20054
|
||||
for ($i = 1; $i <= 20054; $i++) {
|
||||
echo "当前值: $i\n";
|
||||
$arrTask = [
|
||||
'callback' => [self::class, 'getNovelInfo'],
|
||||
'data' => [
|
||||
'xs_source_id' => $i,
|
||||
'xs_source_seo_id' => 1,
|
||||
]
|
||||
];
|
||||
$TaskCore->set($arrTask);
|
||||
var_dump($arrTask);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public function getNovelInfo($arrData)
|
||||
{
|
||||
$TaskCore = $this->TaskCore;
|
||||
$Client = $this->Client;
|
||||
$strSiteUrl = $this->getBiquxsDomain();
|
||||
|
||||
$strUrl = 'http://www.biquxs.com/book/' . $arrData['xs_source_id'] ; //'http://www.biquxs.com/book/20053';
|
||||
$intXiaoShuoSourceId = $arrData['xs_source_id']; //20053;
|
||||
$intXiaoShuoSourceSeoId = $arrData['xs_source_seo_id']; //1 ;
|
||||
try {
|
||||
$arrCookie = $this->getAilisiCookie();
|
||||
$strHost = getDomainFromUrl($strUrl);
|
||||
|
||||
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
|
||||
|
||||
$proxy = getRandomProxies();
|
||||
$userAgent = getRandomUserAgent();
|
||||
$Response = $Client->request('GET', $strUrl, [
|
||||
'headers' => [
|
||||
'User-Agent' => $userAgent,
|
||||
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||
'Accept-Language' => 'en-US,en;q=0.5',
|
||||
],
|
||||
'cookies' => $CookieJar,
|
||||
// 'proxy' => $proxy,
|
||||
'timeout' => 30,
|
||||
]);
|
||||
|
||||
$strResult = (string)$Response->getBody();
|
||||
|
||||
$QueryList = QueryList::html($strResult);
|
||||
// var_dump($QueryList);
|
||||
// 提取所有 <meta> 标签的属性
|
||||
$result = $QueryList->find('meta[property^="og:"]')->map(function ($item) {
|
||||
return [
|
||||
'property' => $item->attr('property'),
|
||||
'content' => $item->attr('content'),
|
||||
];
|
||||
});
|
||||
|
||||
// 打印结果
|
||||
// print_r($result->all());
|
||||
$strZuoZhe = '';
|
||||
$strMingZi = '';
|
||||
$strLastDate = '';
|
||||
$strJieShao = '';
|
||||
$strCover = '';
|
||||
$strFenlei = '';
|
||||
$strZiShu = 0;
|
||||
$strZhuangTai = $QueryList->find('#info p')->eq(1)->text();
|
||||
$strZhuangTai = getStringAfterWord($strZhuangTai, '状态：');
|
||||
|
||||
foreach ($result as $item) {
|
||||
switch ($item['property']) {
|
||||
case 'og:title':
|
||||
$strMingZi = $item['content'];
|
||||
break;
|
||||
case 'og:description':
|
||||
$strJieShao = $item['content'];
|
||||
break;
|
||||
case 'og:image':
|
||||
$strCover = $item['content'];
|
||||
break;
|
||||
case 'og:book:category':
|
||||
$strFenlei = $item['content'];
|
||||
break;
|
||||
case 'og:book:author':
|
||||
$strZuoZhe = $item['content'];
|
||||
break;
|
||||
case 'og:book:update_time':
|
||||
$strLastDate = $item['content'];
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
$strLastDate = trim((string)$strLastDate);
|
||||
if (!isValidDateTime($strLastDate)) {
|
||||
$strLastDate = date('Y-m-d H:i:s');
|
||||
}
|
||||
|
||||
$arrTag = [];
|
||||
|
||||
if (strpos($strCover, 'http') === false) {
|
||||
$strCover = $strSiteUrl . $strCover;
|
||||
}
|
||||
$strCoverUri = $strCover;
|
||||
|
||||
try {
|
||||
$strCoverUri = "";
|
||||
$strCoverUri = ImageProcessor::getInstance()->downloadImage( $intXiaoShuoSourceId,$strCover, 'xs');
|
||||
} catch (\Throwable $t) {
|
||||
$strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
|
||||
echo $strErr;
|
||||
}
|
||||
|
||||
$arrCover = [
|
||||
'code' => 'BI_QU_XS_COVER',
|
||||
// 'uri' => getUriByUrl($strCover),
|
||||
'uri' => $strCoverUri,
|
||||
|
||||
];
|
||||
|
||||
$strFenleiIId = 1 ;
|
||||
switch ($strFenlei) {
|
||||
case '玄幻小说':
|
||||
$strName = "玄幻";
|
||||
$strFenleiIId = 1;
|
||||
break;
|
||||
case '修真小说':
|
||||
$strName = "修真";
|
||||
$strFenleiIId = 2;
|
||||
break;
|
||||
case '都市小说':
|
||||
$strName = "都市";
|
||||
$strFenleiIId = 3;
|
||||
break;
|
||||
case '历史小说':
|
||||
$strName = "历史";
|
||||
$strFenleiIId = 4;
|
||||
break;
|
||||
case '网游小说':
|
||||
$strName = "网游";
|
||||
$strFenleiIId = 5;
|
||||
break;
|
||||
case '科幻小说':
|
||||
$strName = "科幻";
|
||||
$strFenleiIId = 6;
|
||||
break;
|
||||
case '女频小说':
|
||||
$strName = "女频";
|
||||
$strFenleiIId = 7;
|
||||
break;
|
||||
case '其它小说':
|
||||
$strName = "其它";
|
||||
$strFenleiIId = 8;
|
||||
break;
|
||||
default:
|
||||
$strName = "其它";
|
||||
$strFenleiIId = 8;
|
||||
break;
|
||||
}
|
||||
|
||||
$arrXiaoShuoInfo = [
|
||||
'xsxq_ming_zi' => $strMingZi,
|
||||
'xsxq_zuozhe' => $strZuoZhe , //$arrData['xiaoshuo_zuozhe'],
|
||||
'xsfl_id' => $strFenleiIId,
|
||||
'xsxq_zhuang_tai' => $strZhuangTai,
|
||||
'xsxq_zi_shu' => $strZiShu,
|
||||
'xsxq_jie_shao' => $strJieShao,
|
||||
'xsxq_feng_mian' => json_encode($arrCover),
|
||||
'xsxq_geng_xin_shi_jian' => $strLastDate,
|
||||
'xsxq_tag' => json_encode($arrTag, JSON_UNESCAPED_UNICODE),
|
||||
'xsxq_source_id' => $intXiaoShuoSourceId,
|
||||
'xsxq_source_seo_id' => $intXiaoShuoSourceId.'1',
|
||||
'xsxq_source_code' => 'BIQU_XIAOSHUO_' . $intXiaoShuoSourceId . '_' . $intXiaoShuoSourceSeoId,
|
||||
];
|
||||
// print_r($arrXiaoShuoInfo);
|
||||
var_dump($arrXiaoShuoInfo['xsxq_source_code']);
|
||||
|
||||
$XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
|
||||
|
||||
$arrZhangjie = [];
|
||||
$intChapterIndex = 0; // 定义一个计数器变量
|
||||
|
||||
// 检查是否找到元素
|
||||
if ($QueryList->find('.listmain a')->count() > 0) {
|
||||
$QueryList->find('.listmain a')->map(function ($ZhangJie) use (&$arrZhangjie, &$intChapterIndex) {
|
||||
echo "Processing index $intChapterIndex: " . $ZhangJie->text() . PHP_EOL;
|
||||
|
||||
// 从索引6开始
|
||||
if ($intChapterIndex >= 5) {
|
||||
$arrZhangjie[] = [
|
||||
'mingzi' => $ZhangJie->text(),
|
||||
'url' => $ZhangJie->attr('href'),
|
||||
];
|
||||
}
|
||||
$intChapterIndex++;
|
||||
});
|
||||
|
||||
// 输出最终结果
|
||||
var_dump($arrZhangjie);
|
||||
} else {
|
||||
echo "No elements found for .listmain a" . PHP_EOL;
|
||||
}
|
||||
|
||||
//unset($QueryList);
|
||||
|
||||
// 获取最大章节索引
|
||||
$intMaxXiaoShuoZhangJiePaixu = XiaoShuoZhangJieModel::where('xsxq_id', $intXiaoShuoSourceId)
|
||||
->max('xszj_pai_xu'); // 获取最大值
|
||||
var_dump('获取小说章节最大值-'.$intMaxXiaoShuoZhangJiePaixu);
|
||||
|
||||
// 如果没有结果，max 返回 null，可以设置默认值
|
||||
$intMaxXiaoShuoZhangJiePaixu = $intMaxXiaoShuoZhangJiePaixu ?? 0;
|
||||
|
||||
foreach ($arrZhangjie as $intZhangJieNum => $arrZhangjieOne) {
|
||||
var_dump($intZhangJieNum);
|
||||
|
||||
// 如果数据库里的章节最大索引大于当前索引，说明已经入库，不需要入队列了
|
||||
if($intMaxXiaoShuoZhangJiePaixu > $intZhangJieNum){
|
||||
var_dump("小说章节已经入库，不再推送任务");
|
||||
continue;// 如果任务已存在，直接跳过 break
|
||||
}
|
||||
|
||||
$strZhangJieUrl = $strSiteUrl . $arrZhangjieOne['url'];
|
||||
|
||||
$strZhangJieId = extractFilename($strZhangJieUrl);
|
||||
|
||||
# 'AILISI_XIAOSHUO_ZHANGJIE_小说源id_小说源章节id'
|
||||
$strXszjSourceCode = 'AILISI_XIAOSHUO_ZHANGJIE_'. $XiaoShuoXiangQingModel->xsxq_source_id . '_' . $strZhangJieId;
|
||||
|
||||
$arrTask = [
|
||||
'callback' => [self::class, 'getNovelReader'],
|
||||
'data' => [
|
||||
'xszj_source_id' => $strZhangJieId,
|
||||
'xszj_paixu' => $intZhangJieNum,
|
||||
'xszj_name' => $arrZhangjieOne['mingzi'],
|
||||
'xszj_url' => $strZhangJieUrl,
|
||||
'xszj_source_code' => $strXszjSourceCode,
|
||||
'xsxq_source_id' => $XiaoShuoXiangQingModel->xsxq_source_id,
|
||||
'xsfl_id' => $strFenleiIId,
|
||||
]
|
||||
];
|
||||
$TaskCore->set($arrTask);
|
||||
|
||||
}
|
||||
var_dump("章节任务投递成功");
|
||||
|
||||
} catch (\Throwable $T) {
|
||||
throw $T;
|
||||
}
|
||||
}
|
||||
|
||||
public function getNovelFengMian($arrData)
|
||||
{
|
||||
//var_dump('getNovelFengMian');
|
||||
//var_dump($arrData);
|
||||
$Client = $this->Client;
|
||||
$TaskCore = $this->TaskCore;
|
||||
|
||||
// 获取 Redis 实例
|
||||
$redis = Cache::store('redis')->handler();
|
||||
$intXiaoShuoSourceId = $arrData['xiaoshuo_id'];
|
||||
$intXiaoShuoSourceSeoId = $arrData['xiaoshuo_seo_id'];
|
||||
$redisKey = "task:novel_source_id:" . $intXiaoShuoSourceId;
|
||||
|
||||
try {
|
||||
$arrCookie = $this->getAilisiCookie();
|
||||
$strHost = getDomainFromUrl($arrData['xiaoshuo_url']);
|
||||
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
|
||||
|
||||
$strSiteUrl = $this->getAilisiDomain();
|
||||
|
||||
|
||||
$strCover = $QueryList->find('#fmimg img')->eq(0)->attr("src");
|
||||
|
||||
if (strpos($strCover, 'http') === false) {
|
||||
$strCover = $strSiteUrl . $strCover;
|
||||
}
|
||||
$strCoverUri = $strCover;
|
||||
|
||||
try {
|
||||
$strCoverUri = "";
|
||||
$strCoverUri = ImageProcessor::getInstance()->downloadAndEncryptImage($strCover, 'xs');
|
||||
$arrCover = [
|
||||
'code' => 'AI_LI_SI_COVER',
|
||||
// 'uri' => getUriByUrl($strCover),
|
||||
'uri' => $strCoverUri,
|
||||
|
||||
];
|
||||
|
||||
$arrXiaoShuoInfo = [
|
||||
'xsxq_feng_mian' => json_encode($arrCover),
|
||||
|
||||
];
|
||||
$XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
|
||||
} catch (\Throwable $t) {
|
||||
$strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
|
||||
echo $strErr;
|
||||
}
|
||||
|
||||
|
||||
|
||||
} catch (\Throwable $T) {
|
||||
throw $T;
|
||||
}
|
||||
}
|
||||
|
||||
public function getNovelReader($arrData)
|
||||
{
|
||||
var_dump('getNovelReader');
|
||||
var_dump($arrData['xszj_url']);
|
||||
$Client = $this->Client;
|
||||
$strXszjSourceCode = $arrData['xszj_source_code'];
|
||||
$strZhangJieUrl = $arrData['xszj_url'];
|
||||
$intZhangJiePaiXu = $arrData['xszj_paixu'];
|
||||
$intFenleiId = $arrData['xsfl_id'];
|
||||
$intXiaoShuoSourceID = $arrData['xsxq_source_id'];
|
||||
$strZhangJieSourceId = $arrData['xszj_source_id'];
|
||||
$strZhangJieName = $arrData['xszj_name'];
|
||||
try {
|
||||
$arrCookie = $this->getAilisiCookie();
|
||||
$strHost = getDomainFromUrl($strZhangJieUrl);
|
||||
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
|
||||
|
||||
if (XiaoShuoZhangJieModel::checkZhangJieExists($strXszjSourceCode)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
$proxy = getRandomProxies();
|
||||
$userAgent = getRandomUserAgent();
|
||||
$Response = $Client->request('GET', $strZhangJieUrl, [
|
||||
'headers' => [
|
||||
'User-Agent' => $userAgent,
|
||||
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||
'Accept-Language' => 'en-US,en;q=0.5',
|
||||
],
|
||||
'cookies' => $CookieJar,
|
||||
// 'proxy' => $proxy,
|
||||
'timeout' => 30,
|
||||
]);
|
||||
|
||||
$strResult = (string)$Response->getBody();
|
||||
unset($Response);
|
||||
$ZhangJieQueryList = QueryList::html($strResult);
|
||||
|
||||
|
||||
// 获取原始 HTML
|
||||
$strNeiRong = $ZhangJieQueryList->find('#content')->eq(0)->html();
|
||||
|
||||
// 解码 HTML 实体
|
||||
$strNeiRong = html_entity_decode($strNeiRong, ENT_QUOTES, 'UTF-8');
|
||||
|
||||
// 去掉 <p> 标签中的 class="content_detail" 属性
|
||||
$strNeiRong = preg_replace('/<p\s+class="content_detail"/i', '<p', $strNeiRong);
|
||||
|
||||
// 清理多余的 HTML 标签
|
||||
$strNeiRong = strip_tags($strNeiRong, '<p>');
|
||||
|
||||
// 替换不间断空格 (\xc2\xa0) 和 为普通空格
|
||||
$strNeiRong = preg_replace('/\xc2\xa0/', ' ', $strNeiRong); // 替换 UTF-8 不间断空格
|
||||
$strNeiRong = preg_replace('/ /', ' ', $strNeiRong); // 替换
|
||||
|
||||
// 清理多余空格和换行符
|
||||
$strNeiRong = preg_replace('/\s+/', ' ', $strNeiRong);
|
||||
$strNeiRong = preg_replace('/<p>\s+/', '<p>', $strNeiRong); // 清理 <p> 标签内部多余空格
|
||||
$strNeiRong = preg_replace('/\s+<\/p>/', '</p>', $strNeiRong); // 清理 <p> 标签结尾多余空格
|
||||
|
||||
// 输出结果
|
||||
// var_dump($strNeiRong);
|
||||
|
||||
|
||||
# '/novel/123yqw/分类id/小说源id/小说源章节id.txt'
|
||||
$strNeiRongTxtPath = config("filesystem.novel").'/novel/biquxs/'.$intFenleiId.'/'.$intXiaoShuoSourceID.'/'.$strZhangJieSourceId.'.txt';
|
||||
//$strNeiRongTxtPath = "/www/novel/123yqw/1/11/111.txt";
|
||||
// 调用函数
|
||||
$result = saveCompressAndDeleteTxt($strNeiRongTxtPath, $strNeiRong);
|
||||
|
||||
// 输出状态
|
||||
if ($result['status']) {
|
||||
var_dump("Success: " . $result['message']) ;
|
||||
} else {
|
||||
var_dump("Error: " . $result['message']) ;
|
||||
}
|
||||
|
||||
$arrXiaoShuoZhangJie = [
|
||||
'xsxq_id' => $intXiaoShuoSourceID,
|
||||
'xszj_pai_xu' => $intZhangJiePaiXu,
|
||||
'xszj_ming_zi' => $strZhangJieName,
|
||||
'xszj_nei_rong' => $strNeiRongTxtPath,
|
||||
'xszj_source_id' => $strZhangJieSourceId,
|
||||
'xszj_source_code' => $strXszjSourceCode,
|
||||
];
|
||||
|
||||
XiaoShuoZhangJieModel::addXiaoShuoZhangJie($arrXiaoShuoZhangJie);
|
||||
unset($ZhangJieQueryList);
|
||||
// 延时 3 秒
|
||||
//sleep(3);
|
||||
|
||||
} catch (\Throwable $T) {
|
||||
throw $T;
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user