debug
This commit is contained in:
Default
426
code/app/task/collection/novelone/NovelJiuYuFanXianCol.php
Normal file
426
code/app/task/collection/novelone/NovelJiuYuFanXianCol.php
Normal file
@@ -0,0 +1,426 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace app\task\collection\novelone;
|
||||
|
||||
|
||||
use app\admin\model\XiaoShuoXiangQingModel;
|
||||
use app\admin\model\XiaoShuoZhangJieModel;
|
||||
use app\task\collection\PullDataColBase;
|
||||
use GuzzleHttp\Cookie\CookieJar;
|
||||
use processor\ImageProcessor;
|
||||
use QL\QueryList;
|
||||
use app\common\Base64Helper;
|
||||
|
||||
|
||||
/**
|
||||
* @ 抓取 gdbzkz.info 站点相关小说
|
||||
*/
|
||||
class NovelJiuYuFanXianCol extends PullDataColBase
|
||||
{
|
||||
public function pullXiaoShuo()
|
||||
{
|
||||
$TaskCore = $this->TaskCore;
|
||||
// http://www.gdbzkz.info/jiuyufanxian/
|
||||
|
||||
$arrNovel = [
|
||||
['xs_source_id' => 'jiuyufanxian'],
|
||||
];
|
||||
foreach ($arrNovel as $Novel) {
|
||||
$arrTask = [
|
||||
'callback' => [self::class, 'getNovelInfo'],
|
||||
'data' => [
|
||||
'xs_source_id' => $Novel['xs_source_id'],
|
||||
'xs_source_seo_id' => 1,
|
||||
]
|
||||
];
|
||||
$TaskCore->set($arrTask);
|
||||
var_dump($arrTask);
|
||||
}
|
||||
}
|
||||
|
||||
public function getNovelInfo($arrData)
|
||||
{
|
||||
$TaskCore = $this->TaskCore;
|
||||
$Client = $this->Client;
|
||||
$strSiteUrl = 'http://www.gdbzkz.info';
|
||||
|
||||
$intXiaoShuoSourceId = $arrData['xs_source_id'];
|
||||
$intXiaoShuoSourceSeoId = $arrData['xs_source_seo_id'];
|
||||
|
||||
$strUrl = $strSiteUrl . '/' . $intXiaoShuoSourceId;
|
||||
|
||||
try {
|
||||
$arrCookie = $this->getAilisiCookie();
|
||||
$strHost = getDomainFromUrl($strUrl);
|
||||
|
||||
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
|
||||
|
||||
$proxy = getRandomProxies();
|
||||
$userAgent = getRandomUserAgent();
|
||||
$Response = $Client->request('GET', $strUrl, [
|
||||
'headers' => [
|
||||
'User-Agent' => $userAgent,
|
||||
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||
'Accept-Language' => 'en-US,en;q=0.5',
|
||||
],
|
||||
'cookies' => $CookieJar,
|
||||
// 'proxy' => $proxy,
|
||||
'timeout' => 30,
|
||||
]);
|
||||
|
||||
$strResult = (string)$Response->getBody();
|
||||
|
||||
$QueryList = QueryList::html($strResult);
|
||||
// var_dump($QueryList);
|
||||
// 提取所有 <meta> 标签的属性
|
||||
$result = $QueryList->find('meta[property^="og:"]')->map(function ($item) {
|
||||
return [
|
||||
'property' => $item->attr('property'),
|
||||
'content' => $item->attr('content'),
|
||||
];
|
||||
});
|
||||
|
||||
// 打印结果
|
||||
// print_r($result->all());
|
||||
$strZuoZhe = '';
|
||||
$strMingZi = '';
|
||||
$strLastDate = '';
|
||||
$strJieShao = '';
|
||||
$strCover = '';
|
||||
$strFenlei = '';
|
||||
$strZiShu = 0;
|
||||
$strZhuangTai = '';
|
||||
|
||||
foreach ($result as $item) {
|
||||
switch ($item['property']) {
|
||||
case 'og:title':
|
||||
$strMingZi = $item['content'];
|
||||
break;
|
||||
case 'og:description':
|
||||
$strJieShao = $item['content'];
|
||||
break;
|
||||
case 'og:image':
|
||||
$strCover = $item['content'];
|
||||
break;
|
||||
case 'og:novel:category':
|
||||
$strFenlei = $item['content'];
|
||||
break;
|
||||
case 'og:novel:author':
|
||||
$strZuoZhe = $item['content'];
|
||||
break;
|
||||
case 'og:novel:status':
|
||||
$strZhuangTai = $item['content'];
|
||||
break;
|
||||
case 'og:novel:update_time':
|
||||
$strLastDate = $item['content'];
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
$strLastDate = trim((string)$strLastDate);
|
||||
if (!isValidDateTime($strLastDate)) {
|
||||
$strLastDate = date('Y-m-d H:i:s');
|
||||
}
|
||||
|
||||
$arrTag = [];
|
||||
|
||||
if (strpos($strCover, 'http') === false) {
|
||||
$strCover = $strSiteUrl . $strCover;
|
||||
}
|
||||
$strCoverUri = $strCover;
|
||||
|
||||
try {
|
||||
$strCoverUri = "";
|
||||
$strCoverUri = ImageProcessor::getInstance()->downloadImage($intXiaoShuoSourceId, $strCover, 'xs');
|
||||
} catch (\Throwable $t) {
|
||||
$strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
|
||||
echo $strErr;
|
||||
}
|
||||
|
||||
$arrCover = [
|
||||
'code' => 'BI_QU_XS_COVER',
|
||||
// 'uri' => getUriByUrl($strCover),
|
||||
'uri' => $strCoverUri,
|
||||
|
||||
];
|
||||
|
||||
$strFenleiIId = 1;
|
||||
switch ($strFenlei) {
|
||||
case '玄幻小说':
|
||||
$strName = "玄幻";
|
||||
$strFenleiIId = 1;
|
||||
break;
|
||||
case '修真小说':
|
||||
$strName = "修真";
|
||||
$strFenleiIId = 2;
|
||||
break;
|
||||
case '都市小说':
|
||||
$strName = "都市";
|
||||
$strFenleiIId = 3;
|
||||
break;
|
||||
case '历史小说':
|
||||
$strName = "历史";
|
||||
$strFenleiIId = 4;
|
||||
break;
|
||||
case '网游小说':
|
||||
$strName = "网游";
|
||||
$strFenleiIId = 5;
|
||||
break;
|
||||
case '科幻小说':
|
||||
$strName = "科幻";
|
||||
$strFenleiIId = 6;
|
||||
break;
|
||||
case '女频小说':
|
||||
$strName = "女频";
|
||||
$strFenleiIId = 7;
|
||||
break;
|
||||
case '其它小说':
|
||||
$strName = "其它";
|
||||
$strFenleiIId = 8;
|
||||
break;
|
||||
default:
|
||||
$strName = "其它";
|
||||
$strFenleiIId = 8;
|
||||
break;
|
||||
}
|
||||
|
||||
$arrXiaoShuoInfo = [
|
||||
'xsxq_ming_zi' => $strMingZi,
|
||||
'xsxq_zuozhe' => $strZuoZhe, //$arrData['xiaoshuo_zuozhe'],
|
||||
'xsfl_id' => $strFenleiIId,
|
||||
'xsxq_zhuang_tai' => $strZhuangTai,
|
||||
'xsxq_zi_shu' => $strZiShu,
|
||||
'xsxq_jie_shao' => $strJieShao,
|
||||
'xsxq_feng_mian' => json_encode($arrCover),
|
||||
'xsxq_geng_xin_shi_jian' => $strLastDate,
|
||||
'xsxq_tag' => json_encode($arrTag, JSON_UNESCAPED_UNICODE),
|
||||
'xsxq_source_id' => $intXiaoShuoSourceId,
|
||||
'xsxq_source_seo_id' => $intXiaoShuoSourceId . '1',
|
||||
'xsxq_source_code' => 'BIQU_XIAOSHUO_' . $intXiaoShuoSourceId . '_' . $intXiaoShuoSourceSeoId,
|
||||
];
|
||||
// print_r($arrXiaoShuoInfo);
|
||||
var_dump($arrXiaoShuoInfo['xsxq_source_code']);
|
||||
|
||||
$XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
|
||||
|
||||
$arrZhangjie = [];
|
||||
$intChapterIndex = 0; // 定义一个计数器变量
|
||||
|
||||
// 检查是否找到元素
|
||||
if ($QueryList->find('.listmain a')->count() > 0) {
|
||||
$QueryList->find('.listmain a')->map(function ($ZhangJie) use (&$arrZhangjie, &$intChapterIndex) {
|
||||
echo "Processing index $intChapterIndex: " . $ZhangJie->text() . PHP_EOL;
|
||||
|
||||
// 从索引6开始
|
||||
if ($intChapterIndex >= 12) {
|
||||
$arrZhangjie[] = [
|
||||
'mingzi' => $ZhangJie->text(),
|
||||
'url' => $ZhangJie->attr('href'),
|
||||
];
|
||||
}
|
||||
$intChapterIndex++;
|
||||
});
|
||||
|
||||
// 输出最终结果
|
||||
// var_dump($arrZhangjie);
|
||||
} else {
|
||||
echo "No elements found for .listmain a" . PHP_EOL;
|
||||
}
|
||||
|
||||
//unset($QueryList);
|
||||
|
||||
// 获取最大章节索引
|
||||
$intMaxXiaoShuoZhangJiePaixu = XiaoShuoZhangJieModel::where('xsxq_id', $XiaoShuoXiangQingModel->xsxq_id)
|
||||
->max('xszj_pai_xu'); // 获取最大值
|
||||
var_dump('获取小说章节最大值-' . $intMaxXiaoShuoZhangJiePaixu);
|
||||
|
||||
// 如果没有结果，max 返回 null，可以设置默认值
|
||||
$intMaxXiaoShuoZhangJiePaixu = $intMaxXiaoShuoZhangJiePaixu ?? 0;
|
||||
|
||||
foreach ($arrZhangjie as $intZhangJieNum => $arrZhangjieOne) {
|
||||
var_dump($intZhangJieNum);
|
||||
|
||||
// 如果数据库里的章节最大索引大于当前索引，说明已经入库，不需要入队列了
|
||||
// if ($intMaxXiaoShuoZhangJiePaixu > $intZhangJieNum) {
|
||||
// var_dump("小说章节已经入库，不再推送任务");
|
||||
// continue; // 如果任务已存在，直接跳过 break
|
||||
// }
|
||||
|
||||
$strZhangJieUrl = $strSiteUrl . $arrZhangjieOne['url'];
|
||||
|
||||
$strZhangJieSourceId = extractFilename($strZhangJieUrl);
|
||||
|
||||
# 'AILISI_XIAOSHUO_ZHANGJIE_小说源id_小说源章节id'
|
||||
$strXszjSourceCode = 'gdbzkz_XIAOSHUO_ZHANGJIE_' . $XiaoShuoXiangQingModel->xsxq_source_id . '_' . $strZhangJieSourceId . '_1';
|
||||
|
||||
$arrTask = [
|
||||
'callback' => [self::class, 'getNovelReader'],
|
||||
'data' => [
|
||||
'xszj_source_id' => $strZhangJieSourceId,
|
||||
'xszj_paixu' => $intZhangJieNum += 1,
|
||||
'xszj_name' => $arrZhangjieOne['mingzi'],
|
||||
'xszj_url' => $strZhangJieUrl,
|
||||
'xszj_source_code' => $strXszjSourceCode,
|
||||
'xsxq_id' => $XiaoShuoXiangQingModel->xsxq_id,
|
||||
'xsfl_id' => $XiaoShuoXiangQingModel->xsfl_id,
|
||||
'intZhangJieSort' => 1,
|
||||
]
|
||||
];
|
||||
$TaskCore->set($arrTask);
|
||||
}
|
||||
var_dump("章节任务投递成功");
|
||||
} catch (\Throwable $T) {
|
||||
throw $T;
|
||||
}
|
||||
}
|
||||
|
||||
public function getNovelReader($arrData)
|
||||
{
|
||||
$TaskCore = $this->TaskCore;
|
||||
$strSiteUrl = 'http://www.gdbzkz.info';
|
||||
var_dump('getNovelReader------------------------------------------------------------------------------------');
|
||||
|
||||
$Client = $this->Client;
|
||||
$intZhangJieSort = $arrData['intZhangJieSort'];
|
||||
$strXszjSourceCode = $arrData['xszj_source_code'];
|
||||
$strZhangJieUrl = $arrData['xszj_url'];
|
||||
$intZhangJiePaiXu = $arrData['xszj_paixu'];
|
||||
$intFenleiId = $arrData['xsfl_id'];
|
||||
$intXiaoShuoId = $arrData['xsxq_id'];
|
||||
$strZhangJieSourceId = $arrData['xszj_source_id'];
|
||||
$strZhangJieName = $arrData['xszj_name'];
|
||||
|
||||
$XiaoShuoZhangJieModel = XiaoShuoZhangJieModel::where('xszj_source_code', $strXszjSourceCode)->find();
|
||||
if ($XiaoShuoZhangJieModel) {
|
||||
// var_dump($XiaoShuoZhangJieModel->xszj_source_code);
|
||||
// var_dump('章节已经存在 paixu: ' . $intZhangJiePaiXu);
|
||||
// var_dump('章节已经存在strXszjSourceCode: ' . $strXszjSourceCode);
|
||||
// var_dump('章节已经存在strZhangJieUrl: ' . $strZhangJieUrl);
|
||||
return true;
|
||||
}
|
||||
var_dump($strZhangJieUrl);
|
||||
var_dump('章节排序：' . $intZhangJiePaiXu);
|
||||
|
||||
try {
|
||||
$arrCookie = $this->getAilisiCookie();
|
||||
$strHost = getDomainFromUrl($strZhangJieUrl);
|
||||
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
|
||||
|
||||
if (XiaoShuoZhangJieModel::checkZhangJieExists($strXszjSourceCode)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
$proxy = getRandomProxies();
|
||||
$userAgent = getRandomUserAgent();
|
||||
$Response = $Client->request('GET', $strZhangJieUrl, [
|
||||
'headers' => [
|
||||
'User-Agent' => $userAgent,
|
||||
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||
'Accept-Language' => 'en-US,en;q=0.5',
|
||||
],
|
||||
'cookies' => $CookieJar,
|
||||
// 'proxy' => $proxy,
|
||||
'timeout' => 30,
|
||||
]);
|
||||
|
||||
$strResult = (string)$Response->getBody();
|
||||
unset($Response);
|
||||
$ZhangJieQueryList = QueryList::html($strResult);
|
||||
|
||||
$strNeiRong = '';
|
||||
|
||||
// 获取原始 HTML
|
||||
$strNeiRong = $ZhangJieQueryList->find('#content')->html();
|
||||
|
||||
// 解码 HTML 实体
|
||||
$strNeiRong = html_entity_decode($strNeiRong, ENT_QUOTES, 'UTF-8');
|
||||
|
||||
// 替换不间断空格 (\xc2\xa0) 和 为普通空格
|
||||
$strNeiRong = preg_replace('/\xc2\xa0/', ' ', $strNeiRong); // 替换 UTF-8 不间断空格
|
||||
$strNeiRong = preg_replace('/ /', ' ', $strNeiRong); // 替换
|
||||
|
||||
// 清理多余空格和换行符
|
||||
$strNeiRong = preg_replace('/\s+/', ' ', $strNeiRong);
|
||||
|
||||
// 1. 找到所有的 <br> 位置
|
||||
$br_positions = [];
|
||||
$offset = 0;
|
||||
while (($pos = strpos($strNeiRong, "<br", $offset)) !== false) {
|
||||
$br_positions[] = $pos;
|
||||
$offset = $pos + 1;
|
||||
}
|
||||
|
||||
// 2. 找到倒数第三个 <br> 的位置
|
||||
if (count($br_positions) >= 3) {
|
||||
$cut_position = $br_positions[count($br_positions) - 3]; // 倒数第三个 <br> 的索引
|
||||
$strNeiRong = substr($strNeiRong, 0, $cut_position); // 截取前面部分，删除后面的内容
|
||||
}
|
||||
|
||||
|
||||
$arrNeiRong = explode('<br> <br>', $strNeiRong);
|
||||
|
||||
$arrEnCode = [];
|
||||
foreach ($arrNeiRong as $intNeiRongNum => $strNeiRongOne) {
|
||||
$strEnCode = Base64Helper::encode('<p>' . $strNeiRongOne . '</p>');
|
||||
$arrEnCode[] = $strEnCode;
|
||||
}
|
||||
|
||||
$strNeiRong = implode(';;;', $arrEnCode);
|
||||
|
||||
|
||||
# '/novel/123yqw/分类id/小说源id/小说源章节id.txt'
|
||||
$strNeiRongTxtPath = config("filesystem.novel") . '/novel/630zwxs/' . $intFenleiId . '/' . $intXiaoShuoId . '/' . $intZhangJiePaiXu . '/' . $intZhangJieSort . '.txt';
|
||||
//$strNeiRongTxtPath = "/www/novel/123yqw/1/11/111.txt";
|
||||
// 调用函数
|
||||
$result = saveCompressAndDeleteTxt($strNeiRongTxtPath, $strNeiRong);
|
||||
|
||||
// 输出状态
|
||||
if ($result['status']) {
|
||||
var_dump("Success: " . $result['message']);
|
||||
} else {
|
||||
var_dump("Error: " . $result['message']);
|
||||
}
|
||||
|
||||
$arrXiaoShuoZhangJie = [
|
||||
'xsxq_id' => $intXiaoShuoId,
|
||||
'xszj_pai_xu' => $intZhangJiePaiXu,
|
||||
'xszj_ming_zi' => $strZhangJieName,
|
||||
'xszj_nei_rong' => $strNeiRongTxtPath,
|
||||
'xszj_source_id' => $strZhangJieSourceId,
|
||||
'xszj_source_code' => $strXszjSourceCode,
|
||||
'xszj_sort' => $intZhangJieSort,
|
||||
|
||||
];
|
||||
var_dump($arrXiaoShuoZhangJie);
|
||||
if ($strZhangJieName) {
|
||||
XiaoShuoZhangJieModel::addXiaoShuoZhangJie($arrXiaoShuoZhangJie);
|
||||
}
|
||||
|
||||
unset($ZhangJieQueryList);
|
||||
// // 延时 3 秒
|
||||
//sleep(3);
|
||||
|
||||
|
||||
} catch (\Throwable $T) {
|
||||
$arrTask = [
|
||||
'callback' => [self::class, 'getNovelReader'],
|
||||
'data' => [
|
||||
'xszj_source_id' => $strZhangJieSourceId,
|
||||
'xszj_paixu' => $intZhangJiePaiXu,
|
||||
'xszj_name' => $strZhangJieName,
|
||||
'xszj_url' => $strZhangJieUrl,
|
||||
'xszj_source_code' => $strXszjSourceCode,
|
||||
'xsxq_id' => $intXiaoShuoId,
|
||||
'xsfl_id' => $intFenleiId,
|
||||
'intZhangJieSort' => $intZhangJieSort,
|
||||
]
|
||||
];
|
||||
|
||||
$TaskCore->set($arrTask);
|
||||
throw $T;
|
||||
}
|
||||
}
|
||||
}
|
||||
441
code/app/task/collection/novelone/NovelWenXueGuanCol.php
Normal file
441
code/app/task/collection/novelone/NovelWenXueGuanCol.php
Normal file
@@ -0,0 +1,441 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace app\task\collection\novelone;
|
||||
|
||||
|
||||
use app\admin\model\XiaoShuoXiangQingModel;
|
||||
use app\admin\model\XiaoShuoZhangJieModel;
|
||||
use app\task\collection\PullDataColBase;
|
||||
use GuzzleHttp\Cookie\CookieJar;
|
||||
use processor\ImageProcessor;
|
||||
use QL\QueryList;
|
||||
use app\common\Base64Helper;
|
||||
|
||||
|
||||
/**
|
||||
* @ 抓取 bqvvxg8.cc 站点相关小说
|
||||
*/
|
||||
class NovelWenXueGuanCol extends PullDataColBase
|
||||
{
|
||||
public function pullXiaoShuo()
|
||||
{
|
||||
$TaskCore = $this->TaskCore;
|
||||
// https://www.bqvvxg8.cc/wenzhang/86/86114/
|
||||
|
||||
$arrNovel = [
|
||||
['xs_source_id' => 'wenxueguan86114', 'xs_url' => 'https://www.bqvvxg8.cc/wenzhang/86/86114/'],
|
||||
['xs_source_id' => 'wenxueguan90205', 'xs_url' => 'https://www.bqvvxg8.cc/wenzhang/90/90205/'],
|
||||
];
|
||||
foreach ($arrNovel as $Novel) {
|
||||
$arrTask = [
|
||||
'callback' => [self::class, 'getNovelInfo'],
|
||||
'data' => [
|
||||
'xs_url' => $Novel['xs_url'],
|
||||
'xs_source_id' => $Novel['xs_source_id'],
|
||||
'xs_source_seo_id' => 1,
|
||||
]
|
||||
];
|
||||
$TaskCore->set($arrTask);
|
||||
var_dump($arrTask);
|
||||
}
|
||||
}
|
||||
|
||||
public function getNovelInfo($arrData)
|
||||
{
|
||||
$TaskCore = $this->TaskCore;
|
||||
$Client = $this->Client;
|
||||
$strSiteUrl = 'https://www.bqvvxg8.cc';
|
||||
$intXiaoShuoSourceId = $arrData['xs_source_id'];
|
||||
$intXiaoShuoSourceSeoId = $arrData['xs_source_seo_id'];
|
||||
|
||||
$strUrl = $arrData['xs_url'];
|
||||
|
||||
try {
|
||||
$arrCookie = $this->getAilisiCookie();
|
||||
$strHost = getDomainFromUrl($strUrl);
|
||||
|
||||
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
|
||||
|
||||
$proxy = getRandomProxies();
|
||||
$userAgent = getRandomUserAgent();
|
||||
$Response = $Client->request('GET', $strUrl, [
|
||||
'headers' => [
|
||||
'User-Agent' => $userAgent,
|
||||
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||
'Accept-Language' => 'en-US,en;q=0.5',
|
||||
],
|
||||
'cookies' => $CookieJar,
|
||||
// 'proxy' => $proxy,
|
||||
'timeout' => 30,
|
||||
]);
|
||||
|
||||
$strResult = (string)$Response->getBody();
|
||||
|
||||
$QueryList = QueryList::html($strResult);
|
||||
// var_dump($QueryList);
|
||||
// 提取所有 <meta> 标签的属性
|
||||
$result = $QueryList->find('meta[property^="og:"]')->map(function ($item) {
|
||||
return [
|
||||
'property' => $item->attr('property'),
|
||||
'content' => $item->attr('content'),
|
||||
];
|
||||
});
|
||||
|
||||
// 打印结果
|
||||
// print_r($result->all());
|
||||
$strZuoZhe = '';
|
||||
$strMingZi = '';
|
||||
$strLastDate = '';
|
||||
$strJieShao = '';
|
||||
$strCover = '';
|
||||
$strFenlei = '';
|
||||
$strZiShu = 0;
|
||||
$strZhuangTai = '';
|
||||
|
||||
foreach ($result as $item) {
|
||||
switch ($item['property']) {
|
||||
case 'og:title':
|
||||
$strMingZi = $item['content'];
|
||||
break;
|
||||
case 'og:description':
|
||||
$strJieShao = $item['content'];
|
||||
break;
|
||||
case 'og:image':
|
||||
$strCover = $item['content'];
|
||||
break;
|
||||
case 'og:novel:category':
|
||||
$strFenlei = $item['content'];
|
||||
break;
|
||||
case 'og:novel:author':
|
||||
$strZuoZhe = $item['content'];
|
||||
break;
|
||||
case 'og:novel:status':
|
||||
$strZhuangTai = $item['content'] . '中';
|
||||
break;
|
||||
case 'og:novel:update_time':
|
||||
$strLastDate = $item['content'];
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
$strLastDate = trim((string)$strLastDate);
|
||||
if (!isValidDateTime($strLastDate)) {
|
||||
$strLastDate = date('Y-m-d H:i:s');
|
||||
}
|
||||
|
||||
$arrTag = [];
|
||||
|
||||
if (strpos($strCover, 'http') === false) {
|
||||
$strCover = $strSiteUrl . $strCover;
|
||||
}
|
||||
$strCoverUri = $strCover;
|
||||
|
||||
try {
|
||||
$strCoverUri = "";
|
||||
$strCoverUri = ImageProcessor::getInstance()->downloadImage($intXiaoShuoSourceId, $strCover, 'xs');
|
||||
} catch (\Throwable $t) {
|
||||
$strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
|
||||
echo $strErr;
|
||||
}
|
||||
|
||||
$arrCover = [
|
||||
'code' => 'BI_QU_XS_COVER',
|
||||
// 'uri' => getUriByUrl($strCover),
|
||||
'uri' => $strCoverUri,
|
||||
|
||||
];
|
||||
|
||||
$strFenleiIId = 1;
|
||||
switch ($strFenlei) {
|
||||
case '玄幻小说':
|
||||
$strName = "玄幻";
|
||||
$strFenleiIId = 1;
|
||||
break;
|
||||
case '修真小说':
|
||||
$strName = "修真";
|
||||
$strFenleiIId = 2;
|
||||
break;
|
||||
case '言情小说':
|
||||
$strName = "都市";
|
||||
$strFenleiIId = 3;
|
||||
break;
|
||||
case '历史小说':
|
||||
$strName = "历史";
|
||||
$strFenleiIId = 4;
|
||||
break;
|
||||
case '网游小说':
|
||||
$strName = "网游";
|
||||
$strFenleiIId = 5;
|
||||
break;
|
||||
case '科幻小说':
|
||||
$strName = "科幻";
|
||||
$strFenleiIId = 6;
|
||||
break;
|
||||
case '女频小说':
|
||||
$strName = "女频";
|
||||
$strFenleiIId = 7;
|
||||
break;
|
||||
case '其它小说':
|
||||
$strName = "其它";
|
||||
$strFenleiIId = 8;
|
||||
break;
|
||||
default:
|
||||
$strName = "其它";
|
||||
$strFenleiIId = 8;
|
||||
break;
|
||||
}
|
||||
|
||||
$arrXiaoShuoInfo = [
|
||||
'xsxq_ming_zi' => $strMingZi,
|
||||
'xsxq_zuozhe' => $strZuoZhe, //$arrData['xiaoshuo_zuozhe'],
|
||||
'xsfl_id' => $strFenleiIId,
|
||||
'xsxq_zhuang_tai' => $strZhuangTai,
|
||||
'xsxq_zi_shu' => $strZiShu,
|
||||
'xsxq_jie_shao' => $strJieShao,
|
||||
'xsxq_feng_mian' => json_encode($arrCover),
|
||||
'xsxq_geng_xin_shi_jian' => $strLastDate,
|
||||
'xsxq_tag' => json_encode($arrTag, JSON_UNESCAPED_UNICODE),
|
||||
'xsxq_source_id' => $intXiaoShuoSourceId,
|
||||
'xsxq_source_seo_id' => $intXiaoShuoSourceId . '1',
|
||||
'xsxq_source_code' => 'WENXUEGUANG_XIAOSHUO_' . $intXiaoShuoSourceId . '_' . $intXiaoShuoSourceSeoId,
|
||||
];
|
||||
// print_r($arrXiaoShuoInfo);
|
||||
var_dump($arrXiaoShuoInfo['xsxq_source_code']);
|
||||
|
||||
$XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
|
||||
|
||||
$arrZhangjie = [];
|
||||
$intChapterIndex = 0; // 定义一个计数器变量
|
||||
|
||||
// 检查是否找到元素
|
||||
if ($QueryList->find('.listmain a')->count() > 0) {
|
||||
$QueryList->find('.listmain a')->map(function ($ZhangJie) use (&$arrZhangjie, &$intChapterIndex) {
|
||||
echo "Processing index $intChapterIndex: " . $ZhangJie->text() . PHP_EOL;
|
||||
|
||||
// 从索引6开始
|
||||
if ($intChapterIndex >= 12) {
|
||||
$arrZhangjie[] = [
|
||||
'mingzi' => $ZhangJie->text(),
|
||||
'url' => $ZhangJie->attr('href'),
|
||||
];
|
||||
}
|
||||
$intChapterIndex++;
|
||||
});
|
||||
|
||||
// 输出最终结果
|
||||
var_dump($arrZhangjie);
|
||||
} else {
|
||||
echo "No elements found for .listmain a" . PHP_EOL;
|
||||
}
|
||||
|
||||
//unset($QueryList);
|
||||
|
||||
// 获取最大章节索引
|
||||
$intMaxXiaoShuoZhangJiePaixu = XiaoShuoZhangJieModel::where('xsxq_id', $XiaoShuoXiangQingModel->xsxq_id)
|
||||
->max('xszj_pai_xu'); // 获取最大值
|
||||
var_dump('获取小说章节最大值-' . $intMaxXiaoShuoZhangJiePaixu);
|
||||
|
||||
// 如果没有结果，max 返回 null，可以设置默认值
|
||||
$intMaxXiaoShuoZhangJiePaixu = $intMaxXiaoShuoZhangJiePaixu ?? 0;
|
||||
|
||||
foreach ($arrZhangjie as $intZhangJieNum => $arrZhangjieOne) {
|
||||
var_dump($intZhangJieNum);
|
||||
|
||||
// 如果数据库里的章节最大索引大于当前索引，说明已经入库，不需要入队列了
|
||||
// if ($intMaxXiaoShuoZhangJiePaixu > $intZhangJieNum) {
|
||||
// var_dump("小说章节已经入库，不再推送任务");
|
||||
// continue; // 如果任务已存在，直接跳过 break
|
||||
// }
|
||||
|
||||
$strZhangJieUrl = $strSiteUrl . $arrZhangjieOne['url'];
|
||||
|
||||
$strZhangJieSourceId = extractFilename($strZhangJieUrl);
|
||||
|
||||
# 'AILISI_XIAOSHUO_ZHANGJIE_小说源id_小说源章节id'
|
||||
$strXszjSourceCode = 'gdbzkz_XIAOSHUO_ZHANGJIE_' . $XiaoShuoXiangQingModel->xsxq_source_id . '_' . $strZhangJieSourceId . '_1';
|
||||
|
||||
$arrTask = [
|
||||
'callback' => [self::class, 'getNovelReader'],
|
||||
'data' => [
|
||||
'xszj_source_id' => $strZhangJieSourceId,
|
||||
'xszj_paixu' => $intZhangJieNum += 1,
|
||||
'xszj_name' => $arrZhangjieOne['mingzi'],
|
||||
'xszj_url' => $strZhangJieUrl,
|
||||
'xszj_source_code' => $strXszjSourceCode,
|
||||
'xsxq_id' => $XiaoShuoXiangQingModel->xsxq_id,
|
||||
'xsfl_id' => $XiaoShuoXiangQingModel->xsfl_id,
|
||||
'intZhangJieSort' => 1,
|
||||
]
|
||||
];
|
||||
$TaskCore->set($arrTask);
|
||||
}
|
||||
var_dump("章节任务投递成功");
|
||||
} catch (\Throwable $T) {
|
||||
throw $T;
|
||||
}
|
||||
}
|
||||
|
||||
public function getNovelReader($arrData)
|
||||
{
|
||||
$TaskCore = $this->TaskCore;
|
||||
$strSiteUrl = 'https://www.bqvvxg8.cc';
|
||||
var_dump('getNovelReader------------------------------------------------------------------------------------');
|
||||
|
||||
$Client = $this->Client;
|
||||
$intZhangJieSort = $arrData['intZhangJieSort'];
|
||||
$strXszjSourceCode = $arrData['xszj_source_code'];
|
||||
$strZhangJieUrl = $arrData['xszj_url'];
|
||||
$intZhangJiePaiXu = $arrData['xszj_paixu'];
|
||||
$intFenleiId = $arrData['xsfl_id'];
|
||||
$intXiaoShuoId = $arrData['xsxq_id'];
|
||||
$strZhangJieSourceId = $arrData['xszj_source_id'];
|
||||
$strZhangJieName = $arrData['xszj_name'];
|
||||
|
||||
$XiaoShuoZhangJieModel = XiaoShuoZhangJieModel::where('xszj_source_code', $strXszjSourceCode)->find();
|
||||
if ($XiaoShuoZhangJieModel) {
|
||||
// var_dump($XiaoShuoZhangJieModel->xszj_source_code);
|
||||
// var_dump('章节已经存在 paixu: ' . $intZhangJiePaiXu);
|
||||
// var_dump('章节已经存在strXszjSourceCode: ' . $strXszjSourceCode);
|
||||
// var_dump('章节已经存在strZhangJieUrl: ' . $strZhangJieUrl);
|
||||
return true;
|
||||
}
|
||||
var_dump($strZhangJieUrl);
|
||||
var_dump('章节排序：' . $intZhangJiePaiXu);
|
||||
|
||||
try {
|
||||
$arrCookie = $this->getAilisiCookie();
|
||||
$strHost = getDomainFromUrl($strZhangJieUrl);
|
||||
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
|
||||
|
||||
if (XiaoShuoZhangJieModel::checkZhangJieExists($strXszjSourceCode)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
$proxy = getRandomProxies();
|
||||
$userAgent = getRandomUserAgent();
|
||||
$Response = $Client->request('GET', $strZhangJieUrl, [
|
||||
'headers' => [
|
||||
'User-Agent' => $userAgent,
|
||||
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||
'Accept-Language' => 'en-US,en;q=0.5',
|
||||
'Accept-Charset' => 'GBK, UTF-8, *;q=0.8',
|
||||
],
|
||||
'cookies' => $CookieJar,
|
||||
// 'proxy' => $proxy,
|
||||
'timeout' => 30,
|
||||
]);
|
||||
|
||||
$strResult = (string)$Response->getBody();
|
||||
|
||||
// 先尝试自动检测编码
|
||||
$strEncoding = mb_detect_encoding($strResult, ['GBK', 'GB2312', 'BIG5', 'UTF-8'], true);
|
||||
|
||||
// 如果不是 UTF-8，转换为 UTF-8
|
||||
if ($strEncoding && $strEncoding !== 'UTF-8') {
|
||||
$strResult = mb_convert_encoding($strResult, 'UTF-8', $strEncoding);
|
||||
} else {
|
||||
// 备用方案，强制转换
|
||||
$strResult = @iconv('GBK', 'UTF-8//IGNORE', $strResult);
|
||||
}
|
||||
|
||||
// **去掉可能影响解析的 meta 标签**
|
||||
$strResult = preg_replace('/<meta[^>]+charset=[^>]+>/i', '', $strResult);
|
||||
|
||||
// **第一步：使用 QueryList**
|
||||
$ZhangJieQueryList = QueryList::html($strResult);
|
||||
|
||||
$strNeiRong = '';
|
||||
|
||||
// 获取原始 HTML
|
||||
$strNeiRong = $ZhangJieQueryList->find('#content')->html();
|
||||
// 解码 HTML 实体
|
||||
$strNeiRong = html_entity_decode($strNeiRong, ENT_QUOTES, 'UTF-8');
|
||||
|
||||
// 替换不间断空格 (\xc2\xa0) 和 为普通空格
|
||||
$strNeiRong = preg_replace('/\xc2\xa0/', ' ', $strNeiRong); // 替换 UTF-8 不间断空格
|
||||
$strNeiRong = preg_replace('/ /', ' ', $strNeiRong); // 替换
|
||||
|
||||
// 清理多余空格和换行符
|
||||
$strNeiRong = preg_replace('/\s+/', ' ', $strNeiRong);
|
||||
|
||||
// 1. 找到所有的 <br> 位置
|
||||
$br_positions = [];
|
||||
$offset = 0;
|
||||
while (($pos = strpos($strNeiRong, "<br", $offset)) !== false) {
|
||||
$br_positions[] = $pos;
|
||||
$offset = $pos + 1;
|
||||
}
|
||||
|
||||
// 2. 找到倒数第三个 <br> 的位置
|
||||
if (count($br_positions) >= 3) {
|
||||
$cut_position = $br_positions[count($br_positions) - 3]; // 倒数第三个 <br> 的索引
|
||||
$strNeiRong = substr($strNeiRong, 0, $cut_position); // 截取前面部分，删除后面的内容
|
||||
}
|
||||
|
||||
$arrNeiRong = explode('<script>app2();</script><br>', $strNeiRong);
|
||||
|
||||
$strNeiRong = $arrNeiRong[1];
|
||||
|
||||
$arrNeiRong = explode('<br><br>', $strNeiRong);
|
||||
|
||||
$arrEnCode = [];
|
||||
foreach ($arrNeiRong as $intNeiRongNum => $strNeiRongOne) {
|
||||
$strEnCode = Base64Helper::encode('<p>' . $strNeiRongOne . '</p>');
|
||||
$arrEnCode[] = $strEnCode;
|
||||
}
|
||||
|
||||
$strNeiRong = implode(';;;', $arrEnCode);
|
||||
|
||||
# '/novel/123yqw/分类id/小说源id/小说源章节id.txt'
|
||||
$strNeiRongTxtPath = config("filesystem.novel") . '/novel/630zwxs/' . $intFenleiId . '/' . $intXiaoShuoId . '/' . $intZhangJiePaiXu . '/' . $intZhangJieSort . '.txt';
|
||||
// 调用函数
|
||||
$result = saveCompressAndDeleteTxt($strNeiRongTxtPath, $strNeiRong);
|
||||
|
||||
// 输出状态
|
||||
if ($result['status']) {
|
||||
var_dump("Success: " . $result['message']);
|
||||
} else {
|
||||
var_dump("Error: " . $result['message']);
|
||||
}
|
||||
|
||||
$arrXiaoShuoZhangJie = [
|
||||
'xsxq_id' => $intXiaoShuoId,
|
||||
'xszj_pai_xu' => $intZhangJiePaiXu,
|
||||
'xszj_ming_zi' => $strZhangJieName,
|
||||
'xszj_nei_rong' => $strNeiRongTxtPath,
|
||||
'xszj_source_id' => $strZhangJieSourceId,
|
||||
'xszj_source_code' => $strXszjSourceCode,
|
||||
'xszj_sort' => $intZhangJieSort,
|
||||
|
||||
];
|
||||
if ($strZhangJieName) {
|
||||
XiaoShuoZhangJieModel::addXiaoShuoZhangJie($arrXiaoShuoZhangJie);
|
||||
}
|
||||
|
||||
unset($ZhangJieQueryList);
|
||||
//sleep(3);
|
||||
|
||||
|
||||
} catch (\Throwable $T) {
|
||||
$arrTask = [
|
||||
'callback' => [self::class, 'getNovelReader'],
|
||||
'data' => [
|
||||
'xszj_source_id' => $strZhangJieSourceId,
|
||||
'xszj_paixu' => $intZhangJiePaiXu,
|
||||
'xszj_name' => $strZhangJieName,
|
||||
'xszj_url' => $strZhangJieUrl,
|
||||
'xszj_source_code' => $strXszjSourceCode,
|
||||
'xsxq_id' => $intXiaoShuoId,
|
||||
'xsfl_id' => $intFenleiId,
|
||||
'intZhangJieSort' => $intZhangJieSort,
|
||||
]
|
||||
];
|
||||
|
||||
$TaskCore->set($arrTask);
|
||||
throw $T;
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user