admin/SEONexus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

debug

Browse Source
This commit is contained in:
Default
2025-03-15 20:12:21 +08:00
parent 105ffe11c7
commit eb9035a7ae
134 changed files with 15450 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files

124
code/app/task/collection/PullDataColBase.php Normal file
View File

@@ -0,0 +1,124 @@
<?php
declare(strict_types=1);
namespace app\task\collection;
use app\admin\model\CaiJiPeiZhiModel;
use app\admin\model\SystemConfigModel;
use app\task\core\TaskCore;
use GuzzleHttp\Client;
/**
* @mixin think\Model
*/
class PullDataColBase
{
/**
* Undocumented variable
*
* @var \GuzzleHttp\Client
*/
protected $Client = NULL;
/**
* Undocumented variable
*
* @var \app\task\core\TaskCore
*/
protected $TaskCore = NULL;
protected $arrCaiJiPeiZhi = [];
public function __construct()
{
$this->Client = new Client();
$this->TaskCore = new TaskCore();
}
public function getAilisiDomain()
{
if (!key_exists('AILISI_DOMAIN', $this->arrCaiJiPeiZhi)) {
$this->arrCaiJiPeiZhi['AILISI_DOMAIN'] = CaiJiPeiZhiModel::getValByCode('AILISI_DOMAIN');
}
return $this->arrCaiJiPeiZhi['AILISI_DOMAIN'];
}
public function getAilisiCookie()
{
if (!key_exists('AILISI_COOKIE', $this->arrCaiJiPeiZhi)) {
$strCookie = CaiJiPeiZhiModel::getValByCode('AILISI_COOKIE');
$this->arrCaiJiPeiZhi['AILISI_COOKIE'] = parseCookies($strCookie);
}
return $this->arrCaiJiPeiZhi['AILISI_COOKIE'];
}
public function getTianLaiDomain()
{
if (!key_exists('TIANLAI_DOMAIN', $this->arrCaiJiPeiZhi)) {
$this->arrCaiJiPeiZhi['TIANLAI_DOMAIN'] = CaiJiPeiZhiModel::getValByCode('TIANLAI_DOMAIN');
}
return $this->arrCaiJiPeiZhi['TIANLAI_DOMAIN'];
}
public function getBiquxsDomain()
{
if (!key_exists('BIQUXS_DOMAIN', $this->arrCaiJiPeiZhi)) {
$this->arrCaiJiPeiZhi['BIQUXS_DOMAIN'] = CaiJiPeiZhiModel::getValByCode('BIQUXS_DOMAIN');
}
return $this->arrCaiJiPeiZhi['BIQUXS_DOMAIN'];
}
public function get5cccDomain()
{
if (!key_exists('5CCC_DOMAIN', $this->arrCaiJiPeiZhi)) {
$this->arrCaiJiPeiZhi['5CCC_DOMAIN'] = CaiJiPeiZhiModel::getValByCode('5CCC_DOMAIN');
}
return $this->arrCaiJiPeiZhi['5CCC_DOMAIN'];
}
public function get630zwDomain()
{
if (!key_exists('630zw_DOMAIN', $this->arrCaiJiPeiZhi)) {
$this->arrCaiJiPeiZhi['630zw_DOMAIN'] = CaiJiPeiZhiModel::getValByCode('630zw_DOMAIN');
}
return $this->arrCaiJiPeiZhi['630zw_DOMAIN'];
}
public function getDadiDomain()
{
if (!key_exists('DADI_DOMAIN', $this->arrCaiJiPeiZhi)) {
$this->arrCaiJiPeiZhi['DADI_DOMAIN'] = CaiJiPeiZhiModel::getValByCode('DADI_DOMAIN');
}
return $this->arrCaiJiPeiZhi['DADI_DOMAIN'];
}
public function getHeiMuErDomain()
{
if (!key_exists('HEIMUER_DOMAIN', $this->arrCaiJiPeiZhi)) {
$this->arrCaiJiPeiZhi['HEIMUER_DOMAIN'] = CaiJiPeiZhiModel::getValByCode('HEIMUER_DOMAIN');
}
return $this->arrCaiJiPeiZhi['HEIMUER_DOMAIN'];
}
public function YouZhiZiYuanDomain()
{
if (!key_exists('YZZY_DOMAIN', $this->arrCaiJiPeiZhi)) {
$this->arrCaiJiPeiZhi['YZZY_DOMAIN'] = CaiJiPeiZhiModel::getValByCode('YZZY_DOMAIN');
}
return $this->arrCaiJiPeiZhi['YZZY_DOMAIN'];
}
}

427
code/app/task/collection/biquxs/BiquxsXiaoShuoCol.php Normal file
View File

@@ -0,0 +1,427 @@
<?php
declare(strict_types=1);
namespace app\task\collection\biquxs;
use app\admin\model\XiaoShuoFenLeiModel;
use app\admin\model\XiaoShuoXiangQingModel;
use app\admin\model\XiaoShuoZhangJieModel;
use app\task\collection\PullDataColBase;
use GuzzleHttp\Cookie\CookieJar;
use processor\ImageProcessor;
use think\facade\Cache;
use QL\QueryList;
/**
* @mixin think\Model
*/
class BiquxsXiaoShuoCol extends PullDataColBase
{
public function pullXiaoShuo()
{
$TaskCore = $this->TaskCore;
// 20054
for ($i = 1; $i <= 20054; $i++) {
echo "当前值: $i\n";
$arrTask = [
'callback' => [self::class, 'getNovelInfo'],
'data' => [
'xs_source_id' => $i,
'xs_source_seo_id' => 1,
]
];
$TaskCore->set($arrTask);
var_dump($arrTask);
}
}
public function getNovelInfo($arrData)
{
$TaskCore = $this->TaskCore;
$Client = $this->Client;
$strSiteUrl = $this->getBiquxsDomain();
$strUrl = 'http://www.biquxs.com/book/' . $arrData['xs_source_id'] ; //'http://www.biquxs.com/book/20053';
$intXiaoShuoSourceId = $arrData['xs_source_id']; //20053;
$intXiaoShuoSourceSeoId = $arrData['xs_source_seo_id']; //1 ;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
$QueryList = QueryList::html($strResult);
// var_dump($QueryList);
// 提取所有 <meta> 标签的属性
$result = $QueryList->find('meta[property^="og:"]')->map(function ($item) {
return [
'property' => $item->attr('property'),
'content' => $item->attr('content'),
];
});
// 打印结果
// print_r($result->all());
$strZuoZhe = '';
$strMingZi = '';
$strLastDate = '';
$strJieShao = '';
$strCover = '';
$strFenlei = '';
$strZiShu = 0;
$strZhuangTai = $QueryList->find('#info p')->eq(1)->text();
$strZhuangTai = getStringAfterWord($strZhuangTai, '状态：');
foreach ($result as $item) {
switch ($item['property']) {
case 'og:title':
$strMingZi = $item['content'];
break;
case 'og:description':
$strJieShao = $item['content'];
break;
case 'og:image':
$strCover = $item['content'];
break;
case 'og:book:category':
$strFenlei = $item['content'];
break;
case 'og:book:author':
$strZuoZhe = $item['content'];
break;
case 'og:book:update_time':
$strLastDate = $item['content'];
break;
}
}
$strLastDate = trim((string)$strLastDate);
if (!isValidDateTime($strLastDate)) {
$strLastDate = date('Y-m-d H:i:s');
}
$arrTag = [];
if (strpos($strCover, 'http') === false) {
$strCover = $strSiteUrl . $strCover;
}
$strCoverUri = $strCover;
try {
$strCoverUri = "";
$strCoverUri = ImageProcessor::getInstance()->downloadImage( $intXiaoShuoSourceId,$strCover, 'xs');
} catch (\Throwable $t) {
$strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
echo $strErr;
}
$arrCover = [
'code' => 'BI_QU_XS_COVER',
// 'uri' => getUriByUrl($strCover),
'uri' => $strCoverUri,
];
$strFenleiIId = 1 ;
switch ($strFenlei) {
case '玄幻小说':
$strName = "玄幻";
$strFenleiIId = 1;
break;
case '修真小说':
$strName = "修真";
$strFenleiIId = 2;
break;
case '都市小说':
$strName = "都市";
$strFenleiIId = 3;
break;
case '历史小说':
$strName = "历史";
$strFenleiIId = 4;
break;
case '网游小说':
$strName = "网游";
$strFenleiIId = 5;
break;
case '科幻小说':
$strName = "科幻";
$strFenleiIId = 6;
break;
case '女频小说':
$strName = "女频";
$strFenleiIId = 7;
break;
case '其它小说':
$strName = "其它";
$strFenleiIId = 8;
break;
default:
$strName = "其它";
$strFenleiIId = 8;
break;
}
$arrXiaoShuoInfo = [
'xsxq_ming_zi' => $strMingZi,
'xsxq_zuozhe' => $strZuoZhe , //$arrData['xiaoshuo_zuozhe'],
'xsfl_id' => $strFenleiIId,
'xsxq_zhuang_tai' => $strZhuangTai,
'xsxq_zi_shu' => $strZiShu,
'xsxq_jie_shao' => $strJieShao,
'xsxq_feng_mian' => json_encode($arrCover),
'xsxq_geng_xin_shi_jian' => $strLastDate,
'xsxq_tag' => json_encode($arrTag, JSON_UNESCAPED_UNICODE),
'xsxq_source_id' => $intXiaoShuoSourceId,
'xsxq_source_seo_id' => $intXiaoShuoSourceId.'1',
'xsxq_source_code' => 'BIQU_XIAOSHUO_' . $intXiaoShuoSourceId . '_' . $intXiaoShuoSourceSeoId,
];
// print_r($arrXiaoShuoInfo);
var_dump($arrXiaoShuoInfo['xsxq_source_code']);
$XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
$arrZhangjie = [];
$intChapterIndex = 0; // 定义一个计数器变量
// 检查是否找到元素
if ($QueryList->find('.listmain a')->count() > 0) {
$QueryList->find('.listmain a')->map(function ($ZhangJie) use (&$arrZhangjie, &$intChapterIndex) {
echo "Processing index $intChapterIndex: " . $ZhangJie->text() . PHP_EOL;
// 从索引6开始
if ($intChapterIndex >= 5) {
$arrZhangjie[] = [
'mingzi' => $ZhangJie->text(),
'url' => $ZhangJie->attr('href'),
];
}
$intChapterIndex++;
});
// 输出最终结果
var_dump($arrZhangjie);
} else {
echo "No elements found for .listmain a" . PHP_EOL;
}
//unset($QueryList);
// 获取最大章节索引
$intMaxXiaoShuoZhangJiePaixu = XiaoShuoZhangJieModel::where('xsxq_id', $intXiaoShuoSourceId)
->max('xszj_pai_xu'); // 获取最大值
var_dump('获取小说章节最大值-'.$intMaxXiaoShuoZhangJiePaixu);
// 如果没有结果，max 返回 null，可以设置默认值
$intMaxXiaoShuoZhangJiePaixu = $intMaxXiaoShuoZhangJiePaixu ?? 0;
foreach ($arrZhangjie as $intZhangJieNum => $arrZhangjieOne) {
var_dump($intZhangJieNum);
// 如果数据库里的章节最大索引大于当前索引，说明已经入库，不需要入队列了
if($intMaxXiaoShuoZhangJiePaixu > $intZhangJieNum){
var_dump("小说章节已经入库，不再推送任务");
continue;// 如果任务已存在，直接跳过 break
}
$strZhangJieUrl = $strSiteUrl . $arrZhangjieOne['url'];
$strZhangJieId = extractFilename($strZhangJieUrl);
# 'AILISI_XIAOSHUO_ZHANGJIE_小说源id_小说源章节id'
$strXszjSourceCode = 'AILISI_XIAOSHUO_ZHANGJIE_'. $XiaoShuoXiangQingModel->xsxq_source_id . '_' . $strZhangJieId;
$arrTask = [
'callback' => [self::class, 'getNovelReader'],
'data' => [
'xszj_source_id' => $strZhangJieId,
'xszj_paixu' => $intZhangJieNum,
'xszj_name' => $arrZhangjieOne['mingzi'],
'xszj_url' => $strZhangJieUrl,
'xszj_source_code' => $strXszjSourceCode,
'xsxq_source_id' => $XiaoShuoXiangQingModel->xsxq_source_id,
'xsfl_id' => $strFenleiIId,
]
];
$TaskCore->set($arrTask);
}
var_dump("章节任务投递成功");
} catch (\Throwable $T) {
throw $T;
}
}
public function getNovelFengMian($arrData)
{
//var_dump('getNovelFengMian');
//var_dump($arrData);
$Client = $this->Client;
$TaskCore = $this->TaskCore;
// 获取 Redis 实例
$redis = Cache::store('redis')->handler();
$intXiaoShuoSourceId = $arrData['xiaoshuo_id'];
$intXiaoShuoSourceSeoId = $arrData['xiaoshuo_seo_id'];
$redisKey = "task:novel_source_id:" . $intXiaoShuoSourceId;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($arrData['xiaoshuo_url']);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$strSiteUrl = $this->getAilisiDomain();
$strCover = $QueryList->find('#fmimg img')->eq(0)->attr("src");
if (strpos($strCover, 'http') === false) {
$strCover = $strSiteUrl . $strCover;
}
$strCoverUri = $strCover;
try {
$strCoverUri = "";
$strCoverUri = ImageProcessor::getInstance()->downloadAndEncryptImage($strCover, 'xs');
$arrCover = [
'code' => 'AI_LI_SI_COVER',
// 'uri' => getUriByUrl($strCover),
'uri' => $strCoverUri,
];
$arrXiaoShuoInfo = [
'xsxq_feng_mian' => json_encode($arrCover),
];
$XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
} catch (\Throwable $t) {
$strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
echo $strErr;
}
} catch (\Throwable $T) {
throw $T;
}
}
public function getNovelReader($arrData)
{
var_dump('getNovelReader');
var_dump($arrData['xszj_url']);
$Client = $this->Client;
$strXszjSourceCode = $arrData['xszj_source_code'];
$strZhangJieUrl = $arrData['xszj_url'];
$intZhangJiePaiXu = $arrData['xszj_paixu'];
$intFenleiId = $arrData['xsfl_id'];
$intXiaoShuoSourceID = $arrData['xsxq_source_id'];
$strZhangJieSourceId = $arrData['xszj_source_id'];
$strZhangJieName = $arrData['xszj_name'];
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strZhangJieUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
if (XiaoShuoZhangJieModel::checkZhangJieExists($strXszjSourceCode)) {
return true;
}
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strZhangJieUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
unset($Response);
$ZhangJieQueryList = QueryList::html($strResult);
// 获取原始 HTML
$strNeiRong = $ZhangJieQueryList->find('#content')->eq(0)->html();
// 解码 HTML 实体
$strNeiRong = html_entity_decode($strNeiRong, ENT_QUOTES, 'UTF-8');
// 去掉 <p> 标签中的 class="content_detail" 属性
$strNeiRong = preg_replace('/<p\s+class="content_detail"/i', '<p', $strNeiRong);
// 清理多余的 HTML 标签
$strNeiRong = strip_tags($strNeiRong, '<p>');
// 替换不间断空格 (\xc2\xa0) 和 &nbsp; 为普通空格
$strNeiRong = preg_replace('/\xc2\xa0/', ' ', $strNeiRong); // 替换 UTF-8 不间断空格
$strNeiRong = preg_replace('/&nbsp;/', ' ', $strNeiRong); // 替换 &nbsp;
// 清理多余空格和换行符
$strNeiRong = preg_replace('/\s+/', ' ', $strNeiRong);
$strNeiRong = preg_replace('/<p>\s+/', '<p>', $strNeiRong); // 清理 <p> 标签内部多余空格
$strNeiRong = preg_replace('/\s+<\/p>/', '</p>', $strNeiRong); // 清理 <p> 标签结尾多余空格
// 输出结果
// var_dump($strNeiRong);
# '/novel/123yqw/分类id/小说源id/小说源章节id.txt'
$strNeiRongTxtPath = config("filesystem.novel").'/novel/biquxs/'.$intFenleiId.'/'.$intXiaoShuoSourceID.'/'.$strZhangJieSourceId.'.txt';
//$strNeiRongTxtPath = "/www/novel/123yqw/1/11/111.txt";
// 调用函数
$result = saveCompressAndDeleteTxt($strNeiRongTxtPath, $strNeiRong);
// 输出状态
if ($result['status']) {
var_dump("Success: " . $result['message']) ;
} else {
var_dump("Error: " . $result['message']) ;
}
$arrXiaoShuoZhangJie = [
'xsxq_id' => $intXiaoShuoSourceID,
'xszj_pai_xu' => $intZhangJiePaiXu,
'xszj_ming_zi' => $strZhangJieName,
'xszj_nei_rong' => $strNeiRongTxtPath,
'xszj_source_id' => $strZhangJieSourceId,
'xszj_source_code' => $strXszjSourceCode,
];
XiaoShuoZhangJieModel::addXiaoShuoZhangJie($arrXiaoShuoZhangJie);
unset($ZhangJieQueryList);
// 延时 3 秒
//sleep(3);
} catch (\Throwable $T) {
throw $T;
}
}
}

545
code/app/task/collection/ccc5/XiaoShuo5cccCol.php Normal file
View File

@@ -0,0 +1,545 @@
<?php
declare(strict_types=1);
namespace app\task\collection\ccc5;
use app\admin\model\XiaoShuoFenLeiModel;
use app\admin\model\XiaoShuoXiangQingModel;
use app\admin\model\XiaoShuoZhangJieModel;
use app\task\collection\PullDataColBase;
use GuzzleHttp\Cookie\CookieJar;
use processor\ImageProcessor;
use think\facade\Cache;
use QL\QueryList;
/**
* @mixin think\Model
*/
class XiaoShuo5cccCol extends PullDataColBase
{
public function pullXiaoShuo()
{
$TaskCore = $this->TaskCore;
// 20054
for ($i = 3; $i <= 3; $i++) {
echo "当前值: $i\n";
$arrTask = [
'callback' => [self::class, 'getNovelInfo'],
'data' => [
'xs_source_id' => $i,
'xs_source_seo_id' => 1,
]
];
$TaskCore->set($arrTask);
var_dump($arrTask);
}
}
public function getNovelInfo($arrData)
{
$TaskCore = $this->TaskCore;
$Client = $this->Client;
$strSiteUrl = $this->get5cccDomain();
$strUrl = 'https://www.5ccc.org/chapter/' . $arrData['xs_source_id'].'.html' ; //'http://www.biquxs.com/book/20053';
$intXiaoShuoSourceId = $arrData['xs_source_id']; //20053;
$intXiaoShuoSourceSeoId = $arrData['xs_source_seo_id']; //1 ;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
$QueryList = QueryList::html($strResult);
// var_dump($QueryList);
// 提取所有 <meta> 标签的属性
$result = $QueryList->find('meta[property^="og:"]')->map(function ($item) {
return [
'property' => $item->attr('property'),
'content' => $item->attr('content'),
];
});
// 打印结果
print_r($result->all());
$strZuoZhe = '';
$strMingZi = '';
$strLastDate = '';
$strJieShao = '';
$strCover = '';
$strFenlei = '';
$strZiShu = 0;
$strZhuangTai = "";
foreach ($result as $item) {
switch ($item['property']) {
case 'og:title':
$strMingZi = $item['content'];
break;
case 'og:description':
$strJieShao = $item['content'];
break;
case 'og:image':
$strCover = $item['content'];
break;
case 'og:novel:category':
$strFenlei = $item['content'];
break;
case 'og:novel:author':
$strZuoZhe = $item['content'];
break;
case 'og:novel:update_time':
$strLastDate = $item['content'];
break;
case 'og:novel:status':
$strZhuangTai = $item['content'];
break;
}
}
$strLastDate = trim((string)$strLastDate);
if (!isValidDateTime($strLastDate)) {
$strLastDate = date('Y-m-d H:i:s');
}
$arrTag = [];
$strCover = $QueryList->find('.imgbox img')->eq(0)->attr("src");
if (strpos($strCover, 'http') === false) {
$strCover = $strSiteUrl . $strCover;
}
$strCoverUri = $strCover;
try {
$strCoverUri = "";
$strCoverUri = ImageProcessor::getInstance()->downloadImage( $intXiaoShuoSourceId,$strCover, 'xs');
} catch (\Throwable $t) {
$strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
echo $strErr;
}
$arrCover = [
'code' => '5CCC_XS_COVER',
// 'uri' => getUriByUrl($strCover),
'uri' => $strCoverUri,
];
$strFenleiIId = 1 ;
switch ($strFenlei) {
case '玄幻修真':
$strName = "玄幻修真";
$strFenleiIId = 1;
break;
case '重生穿越':
$strName = "重生穿越";
$strFenleiIId = 2;
break;
case '都市小说':
$strName = "都市小说";
$strFenleiIId = 3;
break;
case '军史小说':
$strName = "军史小说";
$strFenleiIId = 4;
break;
case '网游小说':
$strName = "网游小说";
$strFenleiIId = 5;
break;
case '科幻小说':
$strName = "科幻小说";
$strFenleiIId = 6;
break;
case '灵异小说':
$strName = "灵异小说";
$strFenleiIId = 7;
break;
case '言情小说':
$strName = "言情小说";
$strFenleiIId = 8;
break;
case '其他小说':
$strName = "其他小说";
$strFenleiIId = 9;
break;
}
$arrXiaoShuoInfo = [
'xsxq_ming_zi' => $strMingZi,
'xsxq_zuozhe' => $strZuoZhe , //$arrData['xiaoshuo_zuozhe'],
'xsfl_id' => $strFenleiIId,
'xsxq_zhuang_tai' => $strZhuangTai,
'xsxq_zi_shu' => $strZiShu,
'xsxq_jie_shao' => $strJieShao,
'xsxq_feng_mian' => json_encode($arrCover),
'xsxq_geng_xin_shi_jian' => $strLastDate,
'xsxq_tag' => json_encode($arrTag, JSON_UNESCAPED_UNICODE),
'xsxq_source_id' => $intXiaoShuoSourceId,
'xsxq_source_seo_id' => $intXiaoShuoSourceId.'1',
'xsxq_source_code' => 'BIQU_XIAOSHUO_' . $intXiaoShuoSourceId . '_' . $intXiaoShuoSourceSeoId,
];
// print_r($arrXiaoShuoInfo);
var_dump($arrXiaoShuoInfo['xsxq_source_code']);
$XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
$strMuluHref = $QueryList->find('.btn-mulu')->eq(0)->attr("href");
$arrTask = [
'callback' => [self::class, 'getNovelReaderAll'],
'data' => [
'xszj_href' => $strMuluHref,
]
];
$TaskCore->set($arrTask);
/*
$arrZhangjie = [];
$intChapterIndex = 0; // 定义一个计数器变量
// 检查是否找到元素
if ($QueryList->find('.section-list a')->count() > 0) {
$QueryList->find('.section-list a')->map(function ($ZhangJie) use (&$arrZhangjie, &$intChapterIndex) {
echo "Processing index $intChapterIndex: " . $ZhangJie->text() . PHP_EOL;
// 从索引6开始
//if ($intChapterIndex >= 5) {
$arrZhangjie[] = [
'mingzi' => $ZhangJie->text(),
'url' => $ZhangJie->attr('href'),
];
//}
$intChapterIndex++;
});
// 输出最终结果
var_dump($arrZhangjie);
} else {
echo "No elements found for .section-list a" . PHP_EOL;
}
//unset($QueryList);
var_dump($arrZhangjie);
// 获取最大章节索引
$intMaxXiaoShuoZhangJiePaixu = XiaoShuoZhangJieModel::where('xsxq_id', $intXiaoShuoSourceId)
->max('xszj_pai_xu'); // 获取最大值
var_dump($intMaxXiaoShuoZhangJiePaixu);
// 如果没有结果，max 返回 null，可以设置默认值
$intMaxXiaoShuoZhangJiePaixu = $intMaxXiaoShuoZhangJiePaixu ?? 0;
foreach ($arrZhangjie as $intZhangJieNum => $arrZhangjieOne) {
var_dump($intZhangJieNum);
// 如果数据库里的章节最大索引大于当前索引，说明已经入库，不需要入队列了
if($intMaxXiaoShuoZhangJiePaixu > $intZhangJieNum){
var_dump("小说章节已经入库，不再推送任务");
continue;// 如果任务已存在，直接跳过 break
}
$strZhangJieUrl = $strSiteUrl . $arrZhangjieOne['url'];
$strZhangJieId = extractFilename($strZhangJieUrl);
# 'AILISI_XIAOSHUO_ZHANGJIE_小说源id_小说源章节id'
$strXszjSourceCode = '5CCC_XIAOSHUO_ZHANGJIE_'. $XiaoShuoXiangQingModel->xsxq_source_id . '_' . $strZhangJieId;
$arrTask = [
'callback' => [self::class, 'getNovelReader'],
'data' => [
'xszj_source_id' => $strZhangJieId,
'xszj_paixu' => $intZhangJieNum,
'xszj_name' => $arrZhangjieOne['mingzi'],
'xszj_url' => $strZhangJieUrl,
'xszj_source_code' => $strXszjSourceCode,
'xsxq_source_id' => $XiaoShuoXiangQingModel->xsxq_source_id,
'xsfl_id' => $strFenleiIId,
]
];
$TaskCore->set($arrTask);
var_dump("章节任务投递成功".$strXszjSourceCode);
}
*/
} catch (\Throwable $T) {
throw $T;
}
}
public function getNovelFengMian($arrData)
{
//var_dump('getNovelFengMian');
//var_dump($arrData);
$Client = $this->Client;
$TaskCore = $this->TaskCore;
// 获取 Redis 实例
$redis = Cache::store('redis')->handler();
$intXiaoShuoSourceId = $arrData['xiaoshuo_id'];
$intXiaoShuoSourceSeoId = $arrData['xiaoshuo_seo_id'];
$redisKey = "task:novel_source_id:" . $intXiaoShuoSourceId;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($arrData['xiaoshuo_url']);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$strSiteUrl = $this->getAilisiDomain();
$strCover = $QueryList->find('#fmimg img')->eq(0)->attr("src");
if (strpos($strCover, 'http') === false) {
$strCover = $strSiteUrl . $strCover;
}
$strCoverUri = $strCover;
try {
$strCoverUri = "";
$strCoverUri = ImageProcessor::getInstance()->downloadAndEncryptImage($strCover, 'xs');
$arrCover = [
'code' => 'AI_LI_SI_COVER',
// 'uri' => getUriByUrl($strCover),
'uri' => $strCoverUri,
];
$arrXiaoShuoInfo = [
'xsxq_feng_mian' => json_encode($arrCover),
];
$XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
} catch (\Throwable $t) {
$strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
echo $strErr;
}
} catch (\Throwable $T) {
throw $T;
}
}
public function getNovelReaderAll($arrData)
{
var_dump('getNovelReaderAll');
$strMuluHref = $arrData['xszj_href'];
var_dump($strMuluHref);
$Client = $this->Client;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strMuluHref);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strMuluHref, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
unset($Response);
$QueryList = QueryList::html($strResult);
$intChapterIndex = 0; // 定义一个计数器变量
var_dump($QueryList->find('.section-list a')->count());
// 检查是否找到元素
if ($QueryList->find('.section-list a')->count() > 0) {
$QueryList->find('.section-list a')->map(function ($ZhangJie) use (&$arrZhangjie, &$intChapterIndex) {
echo "Processing index $intChapterIndex: " . $ZhangJie->text() . PHP_EOL;
// 从索引6开始
//if ($intChapterIndex >= 5) {
$arrZhangjie[] = [
'mingzi' => $ZhangJie->text(),
'url' => $ZhangJie->attr('href'),
];
//}
$intChapterIndex++;
});
// 输出最终结果
var_dump($arrZhangjie);
} else {
echo "No elements found for .section-list a" . PHP_EOL;
}
//unset($QueryList);
var_dump($arrZhangjie);
/*
// 获取最大章节索引
$intMaxXiaoShuoZhangJiePaixu = XiaoShuoZhangJieModel::where('xsxq_id', $intXiaoShuoSourceId)
->max('xszj_pai_xu'); // 获取最大值
var_dump($intMaxXiaoShuoZhangJiePaixu);
// 如果没有结果，max 返回 null，可以设置默认值
$intMaxXiaoShuoZhangJiePaixu = $intMaxXiaoShuoZhangJiePaixu ?? 0;
foreach ($arrZhangjie as $intZhangJieNum => $arrZhangjieOne) {
var_dump($intZhangJieNum);
// 如果数据库里的章节最大索引大于当前索引，说明已经入库，不需要入队列了
if($intMaxXiaoShuoZhangJiePaixu > $intZhangJieNum){
var_dump("小说章节已经入库，不再推送任务");
continue;// 如果任务已存在，直接跳过 break
}
$strZhangJieUrl = $strSiteUrl . $arrZhangjieOne['url'];
$strZhangJieId = extractFilename($strZhangJieUrl);
# 'AILISI_XIAOSHUO_ZHANGJIE_小说源id_小说源章节id'
$strXszjSourceCode = '5CCC_XIAOSHUO_ZHANGJIE_'. $XiaoShuoXiangQingModel->xsxq_source_id . '_' . $strZhangJieId;
$arrTask = [
'callback' => [self::class, 'getNovelReader'],
'data' => [
'xszj_source_id' => $strZhangJieId,
'xszj_paixu' => $intZhangJieNum,
'xszj_name' => $arrZhangjieOne['mingzi'],
'xszj_url' => $strZhangJieUrl,
'xszj_source_code' => $strXszjSourceCode,
'xsxq_source_id' => $XiaoShuoXiangQingModel->xsxq_source_id,
'xsfl_id' => $strFenleiIId,
]
];
$TaskCore->set($arrTask);
var_dump("章节任务投递成功".$strXszjSourceCode);
}
*/
} catch (\Throwable $T) {
throw $T;
}
}
public function getNovelReader($arrData)
{
var_dump('getNovelReader');
var_dump($arrData['xszj_url']);
$Client = $this->Client;
$strXszjSourceCode = $arrData['xszj_source_code'];
$strZhangJieUrl = $arrData['xszj_url'];
$intZhangJiePaiXu = $arrData['xszj_paixu'];
$intFenleiId = $arrData['xsfl_id'];
$intXiaoShuoSourceID = $arrData['xsxq_source_id'];
$strZhangJieSourceId = $arrData['xszj_source_id'];
$strZhangJieName = $arrData['xszj_name'];
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strZhangJieUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
if (XiaoShuoZhangJieModel::checkZhangJieExists($strXszjSourceCode)) {
return true;
}
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strZhangJieUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
unset($Response);
$ZhangJieQueryList = QueryList::html($strResult);
// 获取原始 HTML
$strNeiRong = $ZhangJieQueryList->find('#content')->eq(0)->html();
// 解码 HTML 实体
$strNeiRong = html_entity_decode($strNeiRong, ENT_QUOTES, 'UTF-8');
// 去掉 <p> 标签中的 class="content_detail" 属性
$strNeiRong = preg_replace('/<p\s+class="content_detail"/i', '<p', $strNeiRong);
// 清理多余的 HTML 标签
$strNeiRong = strip_tags($strNeiRong, '<p>');
// 替换不间断空格 (\xc2\xa0) 和 &nbsp; 为普通空格
$strNeiRong = preg_replace('/\xc2\xa0/', ' ', $strNeiRong); // 替换 UTF-8 不间断空格
$strNeiRong = preg_replace('/&nbsp;/', ' ', $strNeiRong); // 替换 &nbsp;
// 清理多余空格和换行符
$strNeiRong = preg_replace('/\s+/', ' ', $strNeiRong);
$strNeiRong = preg_replace('/<p>\s+/', '<p>', $strNeiRong); // 清理 <p> 标签内部多余空格
$strNeiRong = preg_replace('/\s+<\/p>/', '</p>', $strNeiRong); // 清理 <p> 标签结尾多余空格
// 输出结果
// var_dump($strNeiRong);
# '/novel/123yqw/分类id/小说源id/小说源章节id.txt'
$strNeiRongTxtPath = config("filesystem.novel").'/novel/biquxs/'.$intFenleiId.'/'.$intXiaoShuoSourceID.'/'.$strZhangJieSourceId.'.txt';
//$strNeiRongTxtPath = "/www/novel/123yqw/1/11/111.txt";
// 调用函数
$result = saveCompressAndDeleteTxt($strNeiRongTxtPath, $strNeiRong);
// 输出状态
if ($result['status']) {
var_dump("Success: " . $result['message']) ;
} else {
var_dump("Error: " . $result['message']) ;
}
$arrXiaoShuoZhangJie = [
'xsxq_id' => $intXiaoShuoSourceID,
'xszj_pai_xu' => $intZhangJiePaiXu,
'xszj_ming_zi' => $strZhangJieName,
'xszj_nei_rong' => $strNeiRongTxtPath,
'xszj_source_id' => $strZhangJieSourceId,
'xszj_source_code' => $strXszjSourceCode,
];
XiaoShuoZhangJieModel::addXiaoShuoZhangJie($arrXiaoShuoZhangJie);
unset($ZhangJieQueryList);
// 延时 3 秒
//sleep(3);
} catch (\Throwable $T) {
throw $T;
}
}
}

498
code/app/task/collection/dadi/DaDiDianyingCol.php Normal file
View File

@@ -0,0 +1,498 @@
<?php
declare(strict_types=1);
namespace app\task\collection\dadi;
use app\admin\model\VideoPlayurlModel;
use app\admin\model\VideoLeixinModel;
use app\admin\model\VideoDiquModel;
use app\admin\model\VideoYearsModel;
use app\admin\model\VideoJuqingModel;
use app\admin\model\XiaoShuoFenLeiModel;
use app\admin\model\XiaoShuoXiangQingModel;
use app\admin\model\VideoInfoModel;
use app\admin\model\XiaoShuoZhangJieModel;
use app\task\collection\PullDataColBase;
use GuzzleHttp\Cookie\CookieJar;
use processor\ImageProcessor;
use think\facade\Cache;
use QL\QueryList;
/**
* @mixin think\Model
*/
class DaDiDianyingCol extends PullDataColBase
{
public function pullDadiYingshi()
{
$TaskCore = $this->TaskCore;
// 1-75
for ($i = 1; $i <= 75; $i++) {
echo "当前值: $i\n";
$arrTask = [
'callback' => [self::class, 'getVideoPage'],
'data' => [
'video_page_url' => 'https://www.czdadi.net/vodshow/dy--------'.$i.'---.html',
'video_class_id' => 1
]
];
$TaskCore->set($arrTask);
}
//5-25 https://www.czdadi.net/vodshow/duanju-----------.html
for ($i = 1; $i <= 25; $i++) {
echo "当前值: $i\n";
$arrTask = [
'callback' => [self::class, 'getVideoPage'],
'data' => [
'video_page_url' => 'https://www.czdadi.net/vodshow/duanju--------'.$i.'---.html',
'video_class_id' => 5
]
];
$TaskCore->set($arrTask);
}
// 2--18 https://www.czdadi.net/vodshow/dsj--------18---.html
for ($i = 1; $i <= 18; $i++) {
echo "当前值: $i\n";
$arrTask = [
'callback' => [self::class, 'getVideoPage'],
'data' => [
'video_page_url' => 'https://www.czdadi.net/vodshow/dsj--------'.$i.'---.html',
'video_class_id' => 2
]
];
$TaskCore->set($arrTask);
}
// 3--11 https://www.czdadi.net/vodshow/zy--------11---.html
for ($i = 1; $i <= 11; $i++) {
echo "当前值: $i\n";
$arrTask = [
'callback' => [self::class, 'getVideoPage'],
'data' => [
'video_page_url' => 'https://www.czdadi.net/vodshow/zy--------'.$i.'---.html',
'video_class_id' => 3
]
];
$TaskCore->set($arrTask);
}
// 4--29 https://www.czdadi.net/vodshow/dm-----------.html
for ($i = 1; $i <= 29; $i++) {
echo "当前值: $i\n";
$arrTask = [
'callback' => [self::class, 'getVideoPage'],
'data' => [
'video_page_url' => 'https://www.czdadi.net/vodshow/dm--------'.$i.'---.html',
'video_class_id' => 4
]
];
$TaskCore->set($arrTask);
}
}
# 获取分页里面的视频列表
public function getVideoPage($arrData)
{
$TaskCore = $this->TaskCore;
$Client = $this->Client;
$strUrl= $arrData['video_page_url'];
$intClassId = $arrData['video_class_id'];
$strSiteUrl = $this->get5cccDomain();
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
$QueryList = QueryList::html($strResult);
//$arrVideo = [];
// 检查是否找到元素
if ($QueryList->find('.stui-vodlist li .stui-vodlist__detail a')->count() > 0) {
$QueryList->find('.stui-vodlist li .stui-vodlist__detail a ')->map(function ($Video) use (&$arrVideo,$intClassId,&$TaskCore) {
// $arrVideo[] = [
// 'mingzi' => $Video->text(),
// 'url' => $Video->attr('href'),
// ];
if(!empty($Video->text()) && !empty($Video->attr('href'))){
$arrTask = [
'callback' => [self::class, 'getVideoInfo'],
'data' => [
'class_id' => $intClassId,
'video_href' => $Video->attr('href'),
'video_name' => $Video->text(),
]
];
$TaskCore->set($arrTask);
echo "视频投递成功 : " . $Video->text() . PHP_EOL;
}
});
// foreach ($arrVideo as $intVideoNum => $arrVideoOne) {
// $arrTask = [
// 'callback' => [self::class, 'getVideoInfo'],
// 'data' => [
// 'class_id' => $intClassId,
// 'video_href' => $arrVideoOne['url'],
// 'video_name' => $arrVideoOne['mingzi'],
// ]
// ];
// $TaskCore->set($arrTask);
// }
// 输出最终结果
// var_dump($arrVideo);
} else {
echo "No elements found for .stui-vodlist a" . PHP_EOL;
}
}
public function getVideoInfo($arrData)
{
$TaskCore = $this->TaskCore;
$Client = $this->Client;
$strSiteUrl = $this->getDadiDomain();
$strUrl = $strSiteUrl . $arrData['video_href'] ;
$intClassId = $arrData['class_id'];
$strVideoName = $arrData['video_name'];
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
$QueryList = QueryList::html($strResult);
$strCover = $QueryList->find('.stui-content__thumb img')->eq(0)->attr("data-original");
if (strpos($strCover, 'http') === false) {
$strCover = $strSiteUrl . $strCover;
}
$strCoverUri = $strCover;
// try {
// $strCoverUri = "";
// $strCoverUri = ImageProcessor::getInstance()->downloadImage( $intXiaoShuoSourceId,$strCover, 'dadi');
// } catch (\Throwable $t) {
// $strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
// echo $strErr;
// }
$arrCover = [
'code' => 'DADI_YINGSHI_COVER',
// 'uri' => getUriByUrl($strCover),
'uri' => $strCoverUri,
];
$strLanguage = "";
// if (strlen($strLanguage) > 0) {
// VideoLeixinModel::addVideoLeixin(['vc_id' => $intClassId, 'vla_name' => $strLanguage]);
// }
// 提取评分
$strPinfen = $QueryList->find('h1.title .score')->text();
// 提取类型-入库
$strLeixin = $QueryList->find('.col-pd a')->eq(1)->text();
if(strlen($strLeixin) > 0){
VideoLeixinModel::addVideoLeixin(['vc_id'=>$intClassId,'vl_name'=>$strLeixin]);
}
// 提取 HTML 的第一段 <p class="data">
$dataNode = $QueryList->find('p.data')->eq(0);
// 提取“类型”内容：确保只匹配 `href` 中包含 `/vodsearch/----`
$leixinNodes = $dataNode->find('a[href^="/vodsearch/----"]');
$strJuqing = $leixinNodes->map(function ($node) {
return $node->text();
})->all();
// 提取“地区”内容：确保只匹配 `href` 中包含 `/vodsearch/--`，但不包含 `/vodsearch/----`
$diquNodes = $dataNode->find('a[href^="/vodsearch/--"]')->not('a[href^="/vodsearch/----"]');
$strDiqu = $diquNodes->map(function ($node) {
return $node->text();
})->all();
// 提取“年份”内容：确保只匹配 `href` 中包含 `/vodsearch/-------------`
$yearsNode = $dataNode->find('a[href^="/vodsearch/-------------"]');
$strYears = $yearsNode->text();
if (!empty(trim($strYears))) {
$strYears = [trim($strYears)];
} else {
$strYears = [];
}
// 过滤掉“地区”和“年份”中的数据
$strJuqing = array_filter($strJuqing, function ($value) use ($strDiqu, $strYears) {
return !in_array($value, array_merge($strDiqu, $strYears));
});
// **在过滤后将“类型”数据入库**
foreach ($strJuqing as $value) {
if(strlen($value) > 0){
VideoJuqingModel::add(['vc_id' => $intClassId, 'vj_name' => $value]);
}
}
// 将“地区”数据入库
foreach ($strDiqu as $value) {
if(strlen($value) > 0){
VideoDiquModel::add(['vc_id' => $intClassId, 'vd_name' => $value]);
}
}
// 将“年份”数据入库
foreach ($strYears as $value) {
if(strlen($value) > 0){
VideoYearsModel::add(['vc_id' => $intClassId, 'vy_name' => $value]);
}
}
// 提取主演
$strZhuyan = $QueryList->find('p.data:contains("主演") a')->map(function ($item) {
return $item->text();
})->implode(', '); // 使用逗号分隔主演
// 提取导演
$strDaoyan = $QueryList->find('p.data:contains("导演") a')->map(function ($item) {
return $item->text();
})->implode(', '); // 使用逗号分隔主演
// 提取更新时间
$strLastDate = $QueryList->find('p.data.hidden-sm:contains("更新")')->text();
// 去掉“更新：”前缀
$strLastDate = str_replace('更新：', '', $strLastDate);
// 提取简介
$strDescription = $QueryList->find('#desc>.stui-pannel-box>.stui-pannel_bd>.col-pd')->text();
// 角标
$strJiaoBiao = $QueryList->find('.stui-content__thumb .text-right')->eq(0)->text();
$strPinyinId = extractFilename($strUrl);
$arrVideoInfo = [
'v_name' => $strVideoName,
'v_cover' => json_encode($arrCover),
'vc_id' => $intClassId,
'v_pinyin' => $strPinyinId,
'v_leixin' => $strLeixin ,
'v_juqing' => implode(', ', $strJuqing),
'v_diqu' => implode(', ', $strDiqu),
'v_years' => implode(', ', $strYears),
'v_language' => $strLanguage,
'v_daoyan' => $strZhuyan,
'v_zhuyan' => $strDaoyan,
'update_time' => $strLastDate,
'v_description'=>$strDescription,
'v_pingfen' => $strPinfen,
'v_orurl' => $strUrl,
'v_jiaobiao'=> $strJiaoBiao,
'v_md5' => md5($intClassId.$strVideoName.$strSiteUrl.$strUrl),
];
$VideoInfoModel = VideoInfoModel::addVideoXiangQing($arrVideoInfo);
$intVid = $VideoInfoModel->v_id;
$intChapterIndex = 0 ;
$arrZhangjie = [];
// // 检查是否找到元素
if ($QueryList->find('.stui-content__playlist a')->count() > 0) {
$QueryList->find('.stui-content__playlist a')->map(function ($ZhangJie) use (&$arrZhangjie, &$intChapterIndex ,&$intVid) {
//echo "Processing index $intChapterIndex: " . $ZhangJie->text() . PHP_EOL;
$arrZhangjie[] = [
'v_id' => $intVid,
'index' => $intChapterIndex,
'mingzi' => $ZhangJie->text(),
'url' => $ZhangJie->attr('href'),
];
$intChapterIndex++;
});
//var_dump($arrZhangjie);
foreach ($arrZhangjie as $intZhangJieNum => $arrZhangjieOne) {
var_dump($intZhangJieNum);
// 如果数据库里的章节最大索引大于当前索引，说明已经入库，不需要入队列了
// if($intMaxXiaoShuoZhangJiePaixu > $intZhangJieNum){
// var_dump("小说章节已经入库，不再推送任务");
// continue;// 如果任务已存在，直接跳过 break
// }
//if($intZhangJieNum<=3){
$arrTask = [
'callback' => [self::class, 'getNovelReader'],
'data' => [
'v_id' => $intVid,
'vp_sort' => $intZhangJieNum,
'vp_name' => $arrZhangjieOne['mingzi'],
'vp_url' => $strSiteUrl . $arrZhangjieOne['url']
]
];
$TaskCore->set($arrTask);
//}
}
} else {
echo "No elements found for .listmain a" . PHP_EOL;
}
} catch (\Throwable $T) {
throw $T;
}
// sleep(10);
}
public function getNovelFengMian($arrData)
{
//var_dump('getNovelFengMian');
//var_dump($arrData);
$Client = $this->Client;
$TaskCore = $this->TaskCore;
// 获取 Redis 实例
$redis = Cache::store('redis')->handler();
$intXiaoShuoSourceId = $arrData['xiaoshuo_id'];
$intXiaoShuoSourceSeoId = $arrData['xiaoshuo_seo_id'];
$redisKey = "task:novel_source_id:" . $intXiaoShuoSourceId;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($arrData['xiaoshuo_url']);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$strSiteUrl = $this->getAilisiDomain();
$strCover = $QueryList->find('#fmimg img')->eq(0)->attr("src");
if (strpos($strCover, 'http') === false) {
$strCover = $strSiteUrl . $strCover;
}
$strCoverUri = $strCover;
try {
$strCoverUri = "";
$strCoverUri = ImageProcessor::getInstance()->downloadAndEncryptImage($strCover, 'xs');
$arrCover = [
'code' => 'AI_LI_SI_COVER',
// 'uri' => getUriByUrl($strCover),
'uri' => $strCoverUri,
];
$arrXiaoShuoInfo = [
'xsxq_feng_mian' => json_encode($arrCover),
];
$XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
} catch (\Throwable $t) {
$strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
echo $strErr;
}
} catch (\Throwable $T) {
throw $T;
}
}
public function getNovelReader($arrData)
{
if (Cache::has('video_no_m3u8_' . $arrData['v_id'])) {
echo "跳过抓取，视频ID已在缓存中: " . $arrData['v_id'] . "\n";
return; // 跳过抓取逻辑
}
$strMuluHref = $arrData['vp_url'];
var_dump($strMuluHref);
$Client = $this->Client;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strMuluHref);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strMuluHref, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
unset($Response);
$QueryList = QueryList::html($strResult);
//提取 <script> 标签内容
$scriptContent = $QueryList->find('.vidoe-data-show')->text();
$strM3u8Url = "";
// 使用正则表达式提取 "url" 的值
if (preg_match('/"url":"(https:[^"]+\.m3u8)"/', $scriptContent, $matches)) {
$strM3u8Url = str_replace('\/', '/', $matches[1]); // 替换转义斜杠
echo "提取的 M3U8 地址: $strM3u8Url\n";
$arrM3u8Url = [
'code' => 'DADI_YINGSHI_M3U8',
'uri' => $strM3u8Url,
];
$arrVideoPlayurl = [
'v_id' => $arrData['v_id'],
'vp_sort' => $arrData['vp_sort'],
'vp_name' => $arrData['vp_name'],
'vp_url' => json_encode($arrM3u8Url),
'vp_md5' => md5($arrData['vp_url'] . $arrData['vp_url']),
];
VideoPlayurlModel::add($arrVideoPlayurl);
unset($QueryList);
} else {
echo "未找到 M3U8 地址\n";
// 将视频ID保存到缓存，设置一个过期时间，例如 1 小时
Cache::set('video_no_m3u8_' . $arrData['v_id'], true, 3600);
}
} catch (\Throwable $T) {
throw $T;
}
}
}

560
code/app/task/collection/heimuer/HeiMuErYingShiCol.php Normal file
View File

@@ -0,0 +1,560 @@
<?php
declare(strict_types=1);
namespace app\task\collection\heimuer;
use app\admin\model\VideoPlayurlModel;
use app\admin\model\VideoLeixinModel;
use app\admin\model\VideoDiquModel;
use app\admin\model\VideoYearsModel;
use app\admin\model\VideoJuqingModel;
use app\admin\model\XiaoShuoFenLeiModel;
use app\admin\model\XiaoShuoXiangQingModel;
use app\admin\model\VideoInfoModel;
use app\admin\model\XiaoShuoZhangJieModel;
use app\task\collection\PullDataColBase;
use GuzzleHttp\Cookie\CookieJar;
use processor\ImageProcessor;
use think\facade\Cache;
use QL\QueryList;
/**
* @mixin think\Model
*/
class HeiMuErYingShiCol extends PullDataColBase
{
public function pullDadiYingshi()
{
$strSiteUrl = $this->getHeiMuErDomain();
// 配置分类任务参数
$categories = [
['id' => 1, 'video_class_id' => 1, 'name' => '电影'],
['id' => 27, 'video_class_id' => 5, 'name' => '短剧'],
['id' => 2, 'video_class_id' => 2, 'name' => '电视剧'],
['id' => 4, 'video_class_id' => 3, 'name' => '综艺'],
['id' => 3, 'video_class_id' => 4, 'name' => '动漫'],
];
foreach ($categories as $category) {
$intClassId = $category['video_class_id'];
$intCategoryId = $category['id'];
// 先抓第一页
$strUrl = $strSiteUrl . '/index.php/vod/type/id/' . $intCategoryId . '/page/1.html';
$maxPage = $this->getMaxPage($strUrl,$intClassId);
// 首次抓取，全部抓取
if (config('task.task_is_first')) {
$this->createTasks($strSiteUrl, $category['id'], $category['video_class_id'], $maxPage);
}
}
}
/**
* 获取分类的最大页码且抓取第一页
*/
protected function getMaxPage($strUrl, $intClassId)
{
$TaskCore = $this->TaskCore;
$Client = $this->Client;
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$Response = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'Accept-Language' => 'en-US,en;q=0.5',
'cookie' => '__sk_Vm8nTqM7LqSCWmR6__=be3bd4f4e0483619e1a434ec1b18a938',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 300,
]);
$strResult = (string)$Response->getBody();
$QueryList = QueryList::html($strResult);
// 检查是否找到元素
if ($QueryList->find('.stui-vodlist li.clearfix h3 a')->count() > 0) {
$QueryList->find('.stui-vodlist li.clearfix h3 a ')->map(function ($Video) use ($intClassId, &$TaskCore) {
$strHref = $Video->attr('href');
$strName = $Video->text();
if (!empty($strName) && !empty($strHref) && strlen($strName) > 0 && strlen($strHref) > 0) {
$arrTask = [
'callback' => [self::class, 'getVideoInfo'],
'data' => [
'class_id' => $intClassId,
'video_href' => $strHref,
'video_name' => $strName,
]
];
// var_dump($arrTask);
$TaskCore->set($arrTask);
echo "视频投递成功 : " . $Video->text() . PHP_EOL;
}
});
} else {
echo "No elements found for .stui-vodlist a" . PHP_EOL;
}
// 获取分页部分最后一个页码链接
$lastPageHref = $QueryList->find('.stui-page li')->eq($QueryList->find('.stui-page li')->count() - 1)->find('a')->attr('href');
// 提取最大页码
preg_match('/page\/(\d+)\.html/', $lastPageHref, $matches);
$maxPage = isset($matches[1]) ? (int)$matches[1] : 1; // 默认返回 1
return $maxPage;
}
/**
* 创建任务
*/
protected function createTasks($strSiteUrl, $categoryId, $videoClassId, $maxPage)
{
$TaskCore = $this->TaskCore;
for ($i = $maxPage; $i >= 1; $i--) {
echo "当前分类: $categoryId, 当前页: $i\n";
$arrTask = [
'callback' => [self::class, 'getVideoPage'],
'data' => [
'video_page_url' => $strSiteUrl . '/index.php/vod/type/id/' . $categoryId . '/page/' . $i . '.html',
'video_class_id' => $videoClassId,
]
];
$TaskCore->set($arrTask);
}
}
# 获取分页里面的视频列表
public function getVideoPage($arrData)
{
$TaskCore = $this->TaskCore;
$Client = $this->Client;
$strUrl = $arrData['video_page_url'];
$intClassId = $arrData['video_class_id'];
$strSiteUrl = $this->getHeiMuErDomain();
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
var_dump($strUrl);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'Accept-Language' => 'en-US,en;q=0.5',
'cookie' => '__sk_Vm8nTqM7LqSCWmR6__=be3bd4f4e0483619e1a434ec1b18a938',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 300,
]);
$strResult = (string)$Response->getBody();
$QueryList = QueryList::html($strResult);
// 检查是否找到元素
if ($QueryList->find('.stui-vodlist li.clearfix h3 a')->count() > 0) {
$QueryList->find('.stui-vodlist li.clearfix h3 a ')->map(function ($Video) use (&$arrVideo, $intClassId, &$TaskCore, $strSiteUrl) {
$strHref = $Video->attr('href');
$strName = $Video->text();
if (!empty($strName) && !empty($strHref) && strlen($strName) > 0 && strlen($strHref) > 0) {
$arrTask = [
'callback' => [self::class, 'getVideoInfo'],
'data' => [
'class_id' => $intClassId,
'video_href' => $strHref,
'video_name' => $strName,
]
];
// var_dump($arrTask);
$TaskCore->set($arrTask);
echo "视频投递成功 : " . $Video->text() . PHP_EOL;
}
});
} else {
echo "No elements found for .stui-vodlist a" . PHP_EOL;
}
}
public function getVideoInfo($arrData)
{
$TaskCore = $this->TaskCore;
$Client = $this->Client;
$strSiteUrl = $this->getHeiMuErDomain();
$strUrl = $strSiteUrl . $arrData['video_href'];
$intClassId = $arrData['class_id'];
$strVideoName = $arrData['video_name'];
var_dump($strUrl);
if (!empty($strVideoName) && !empty($strUrl) && strlen($strVideoName) > 0 && strlen($strUrl) > 0) {
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
$QueryList = QueryList::html($strResult);
$strCover = $QueryList->find('.img-responsive')->eq(0)->attr("src");
if (strpos($strCover, 'http') === false) {
$strCover = $strSiteUrl . $strCover;
}
$strCoverUri = $strCover;
// try {
// $strCoverUri = "";
// $strCoverUri = ImageProcessor::getInstance()->downloadImage( $intXiaoShuoSourceId,$strCover, 'dadi');
// } catch (\Throwable $t) {
// $strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
// echo $strErr;
// }
$arrCover = [
'code' => 'HEIMUER_SOURCE_COVER',
'uri' => $strCoverUri,
];
$strLanguage = "";
// if (strlen($strLanguage) > 0) {
// VideoLeixinModel::addVideoLeixin(['vc_id' => $intClassId, 'vla_name' => $strLanguage]);
// }
// 提取别名
$strPinyinId = $QueryList->find('.stui-content__detail p.data')->eq(0)->text();
// 提取评分
$strPinfen = $QueryList->find('.stui-content__detail h1 .text-red')->text();
// 提取地区
$strDiqu = $QueryList->find('.stui-content__detail p.data')->eq(1)->text();
// 角标-状态
$strJiaoBiao = $QueryList->find('.stui-content__detail p.data')->eq(2)->find('.text-red')->eq(0)->text();
// 更新时间
$strLastDate = $QueryList->find('.stui-content__detail p.data')->eq(2)->find('.text-red')->eq(1)->text();
// 提取主演
$strZhuyan = $QueryList->find('.stui-content__detail p.data')->eq(3)->text();
// 提取导演
$strDaoyan = $QueryList->find('.stui-content__detail p.data')->eq(4)->text();
// 提取
$arrLeixinData = $QueryList->find('.stui-content__detail p.data')->eq(5)->text();
// 提取
$arrYuyanData = $QueryList->find('.stui-content__detail p.data')->eq(6)->text();
$strLeixin = '';
$strDiqu = '';
$strYears = '';
$strKuozhang = '';
// 使用正则表达式提取内容
$patternLeixin = '/类型：(.*?)扩展：(.*?)地区：(.*?)年份：(.*)/';
if (preg_match($patternLeixin, $arrLeixinData, $matches)) {
$strLeixin = trim($matches[1]);
$strKuozhang = trim($matches[2]);
$strDiqu = trim($matches[3]);
$strYears = trim($matches[4]);
}
$strLanguage = '';
// 使用正则表达式提取内容
$patternYuyan = '/语言：(.*?)集数：(.*?)时长：(.*?)点击：(.*)/';
if (preg_match($patternYuyan, $arrYuyanData, $matches)) {
$strLanguage = trim($matches[1]);
// $strKuozhang = trim($matches[2]);
// $strDiqu = trim($matches[3]);
// $strYears = trim($matches[4]);
}
if (strlen($strLeixin) > 0) {
VideoLeixinModel::addVideoLeixin(['vc_id' => $intClassId, 'vl_name' => $strLeixin]);
}
if (strlen($strDiqu) > 0) {
VideoDiquModel::add(['vc_id' => $intClassId, 'vd_name' => $strDiqu]);
}
if (strlen($strYears) > 0) {
VideoYearsModel::add(['vc_id' => $intClassId, 'vy_name' => $strYears]);
}
// 提取简介
$strDescription = $QueryList->find('.stui-content__desc')->text();
$arrVideoInfo = [
'v_name' => $strVideoName,
'v_cover' => json_encode($arrCover),
'vc_id' => $intClassId,
'v_pinyin' => trim(str_replace('别名：', '', $strPinyinId)),
'v_leixin' => $strLeixin,
'v_juqing' => '',
'v_diqu' => $strDiqu,
'v_years' => $strYears,
'v_language' => $strLanguage,
'v_daoyan' => trim(str_replace('主演：', '', $strZhuyan)),
'v_zhuyan' => trim(str_replace('导演：', '', $strDaoyan)),
'update_time' => $strLastDate,
'v_description' => $strDescription,
'v_pingfen' => $strPinfen,
'v_orurl' => $strUrl,
'v_jiaobiao' => $strJiaoBiao,
'v_md5' => md5($intClassId . $strVideoName . getUriByUrl($strUrl)),
];
//var_dump($arrVideoInfo);
// // 检查是否找到元素
if ($QueryList->find('.stui-content__playlist li')->count() > 0) {
// var_dump("视频入库成功--" . $strVideoName);
// $VideoInfoModel = VideoInfoModel::addVideoXiangQing($arrVideoInfo);
$resultVideoInfoModel = VideoInfoModel::addVideoXiangQing($arrVideoInfo);;
if ($resultVideoInfoModel['isNew']) {
var_dump("数据已新增：", $resultVideoInfoModel['model']->v_id."视频入库成功--" . $strVideoName);
} else {
echo "数据已存在：", $resultVideoInfoModel['model']->v_id;
}
$VideoInfoModel = $resultVideoInfoModel['model'];
$intVid = $VideoInfoModel->v_id;
$intChapterIndex = 0;
$QueryList->find('.stui-content__playlist li')->map(function ($ZhangJie) use (&$arrZhangjie, &$intChapterIndex, &$intVid) {
//echo "Processing index $intChapterIndex: " . $ZhangJie->text() . PHP_EOL;
$strM3u8 = $ZhangJie->find('a')->eq(1)->text();
$arrM3u8 = explode('$', $strM3u8);
$strM3u8Url = $arrM3u8[1];
$strM3u8Name = $arrM3u8[0];
$arrM3u8Url = [
'code' => 'HEIMUER_SOURCE_PLAY',
'uri' => $strM3u8Url,
];
$arrVideoPlayurl = [
'v_id' => $intVid,
'vp_sort' => $intChapterIndex,
'vp_name' => $strM3u8Name,
'vp_url' => json_encode($arrM3u8Url),
'vp_md5' => md5($intVid . getUriByUrl($strM3u8Url)),
];
$result = VideoPlayurlModel::add($arrVideoPlayurl);;
if ($result['isNew']) {
var_dump("数据已新增：", $result['model']->vp_id."播放线路入库成功--" . $strM3u8Url) ;
} else {
echo "数据已存在：", $result['model']->vp_id ;
}
$intChapterIndex++;
});
var_dump("--") ;
var_dump("一个视频任务完成------------------------------------------------------------------------------------") ;
} else {
echo "No elements found for .listmain a" . PHP_EOL;
}
} catch (\Throwable $T) {
throw $T;
}
}
// sleep(10);
}
public function getNovelFengMian($arrData)
{
//var_dump('getNovelFengMian');
//var_dump($arrData);
$Client = $this->Client;
$TaskCore = $this->TaskCore;
// 获取 Redis 实例
$redis = Cache::store('redis')->handler();
$intXiaoShuoSourceId = $arrData['xiaoshuo_id'];
$intXiaoShuoSourceSeoId = $arrData['xiaoshuo_seo_id'];
$redisKey = "task:novel_source_id:" . $intXiaoShuoSourceId;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($arrData['xiaoshuo_url']);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$strSiteUrl = $this->getAilisiDomain();
$strCover = $QueryList->find('#fmimg img')->eq(0)->attr("src");
if (strpos($strCover, 'http') === false) {
$strCover = $strSiteUrl . $strCover;
}
$strCoverUri = $strCover;
try {
$strCoverUri = "";
$strCoverUri = ImageProcessor::getInstance()->downloadAndEncryptImage($strCover, 'xs');
$arrCover = [
'code' => 'AI_LI_SI_COVER',
// 'uri' => getUriByUrl($strCover),
'uri' => $strCoverUri,
];
$arrXiaoShuoInfo = [
'xsxq_feng_mian' => json_encode($arrCover),
];
$XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
} catch (\Throwable $t) {
$strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
echo $strErr;
}
} catch (\Throwable $T) {
throw $T;
}
}
public function getNovelReader($arrData)
{
$strMuluHref = $arrData['vp_url'];
var_dump($strMuluHref);
$Client = $this->Client;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strMuluHref);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strMuluHref, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
unset($Response);
$QueryList = QueryList::html($strResult);
//提取 <script> 标签内容
$scriptContent = $QueryList->find('.vidoe-data-show')->text();
$strM3u8Url = "";
// 使用正则表达式提取 "url" 的值
if (preg_match('/"url":"(https:[^"]+\.m3u8)"/', $scriptContent, $matches)) {
$strM3u8Url = str_replace('\/', '/', $matches[1]); // 替换转义斜杠
echo "提取的 M3U8 地址: $strM3u8Url\n";
$arrM3u8Url = [
'code' => 'DADI_YINGSHI_M3U8',
'uri' => $strM3u8Url,
];
$arrVideoPlayurl = [
'v_id' => $arrData['v_id'],
'vp_sort' => $arrData['vp_sort'],
'vp_name' => $arrData['vp_name'],
'vp_url' => json_encode($arrM3u8Url),
'vp_md5' => md5($arrData['vp_url'] . $arrData['vp_url']),
];
VideoPlayurlModel::add($arrVideoPlayurl);
unset($QueryList);
} else {
echo "未找到 M3U8 地址\n";
}
} catch (\Throwable $T) {
throw $T;
}
}
public function pullDadiYingshi2()
{
$strSiteUrl = $this->getHeiMuErDomain();
$TaskCore = $this->TaskCore;
//电影 307 https://heimuer.tv/index.php/vod/type/id/1/page/307.html
for ($i = 307; $i >= 1; $i--) {
if (!config('task.task_is_first') && $i > 1) {
continue;
}
echo "当前值: $i\n";
$arrTask = [
'callback' => [self::class, 'getVideoPage'],
'data' => [
'video_page_url' => $strSiteUrl . '/index.php/vod/type/id/1/page/' . $i . '.html',
'video_class_id' => 1
]
];
$TaskCore->set($arrTask);
}
//5-87 短剧 https://heimuer.tv/index.php/vod/type/id/27/page/87.html
for ($i = 87; $i >= 1; $i--) {
if (!config('task.task_is_first') && $i > 1) {
continue;
}
echo "当前值: $i\n";
$arrTask = [
'callback' => [self::class, 'getVideoPage'],
'data' => [
'video_page_url' => $strSiteUrl . '/index.php/vod/type/id/27/page/' . $i . '.html',
'video_class_id' => 5
]
];
$TaskCore->set($arrTask);
}
//电视剧 2--225 https://heimuer.tv/index.php/vod/type/id/2/page/225.html
for ($i = 225; $i >= 1; $i--) {
if (!config('task.task_is_first') && $i > 1) {
continue;
}
echo "当前值: $i\n";
$arrTask = [
'callback' => [self::class, 'getVideoPage'],
'data' => [
'video_page_url' => $strSiteUrl . '/index.php/vod/type/id/2/page/' . $i . '.html',
'video_class_id' => 2
]
];
$TaskCore->set($arrTask);
}
//综艺 78 https://heimuer.tv/index.php/vod/type/id/4/page/78.html
for ($i = 78; $i >= 1; $i--) {
if (!config('task.task_is_first') && $i > 1) {
continue;
}
echo "当前值: $i\n";
$arrTask = [
'callback' => [self::class, 'getVideoPage'],
'data' => [
'video_page_url' => $strSiteUrl . '/index.php/vod/type/id/4/page/' . $i . '.html',
'video_class_id' => 3
]
];
$TaskCore->set($arrTask);
}
// 动漫--100 https://heimuer.tv/index.php/vod/type/id/3/page/1.html
for ($i = 100; $i >= 1; $i--) {
if (!config('task.task_is_first') && $i > 1) {
continue;
}
echo "当前值: $i\n";
$arrTask = [
'callback' => [self::class, 'getVideoPage'],
'data' => [
'video_page_url' => $strSiteUrl . '/index.php/vod/type/id/3/page/' . $i . '.html',
'video_class_id' => 4
]
];
$TaskCore->set($arrTask);
}
}
}

850
code/app/task/collection/novel/Zw630XiaoShuoCol.php Normal file
View File

@@ -0,0 +1,850 @@
<?php
declare(strict_types=1);
namespace app\task\collection\novel;
use app\admin\model\XiaoShuoFenLeiModel;
use app\admin\model\XiaoShuoXiangQingModel;
use app\admin\model\XiaoShuoZhangJieModel;
use app\admin\model\NovelInfoSeoModel;
use app\task\collection\PullDataColBase;
use GuzzleHttp\Cookie\CookieJar;
use processor\ImageProcessor;
use think\facade\Cache;
use QL\QueryList;
use app\common\Base64Helper as qsbs;
/**
* @mixin think\Model https://www.630zw.org/
*/
class Zw630XiaoShuoCol extends PullDataColBase
{
public function pullXiaoShuo()
{
// $decoded = '美女';
// $encoded = qsbs::encode($decoded);
// $decodedString = qsbs::decode('5oC75LmL77yM57uT5bGA5piv5aW955qE77yM5amJ5YS/5bqU6K+l5LiN5Lya5Zyo5LmO6L+Z5Lqb57uG6IqC44CCPC9wPg==');
// var_dump($encoded);
// var_dump($decodedString);
// 首次抓取，全部抓取
if (config('task.task_is_first')) {
$TaskCore = $this->TaskCore;
// 527797
for ($i = config('task.task_start_number'); $i <= config('task.task_end_number'); $i++) {
//echo "当前值: $i\n";
$arrTask = [
'callback' => [self::class, 'getNovelInfo'],
'data' => [
'xs_source_id' => $i, //528049
'xs_source_seo_id' => 1,
]
];
$TaskCore->set($arrTask);
// var_dump($arrTask);
}
}else{
$this->pullFenlei();
}
}
public function pullFenlei()
{
$TaskCore = $this->TaskCore;
for ($intFenleiId = 1; $intFenleiId <= 9; $intFenleiId++) {
for ($intPage = 1; $intPage <= 10; $intPage++) {
// https://www.630zw.org/list/1/10.html
$arrTask = [
'callback' => [self::class, 'getFenLei'],
'data' => [
'xs_fenlei_url' => 'https://www.630zw.org/list/'.$intFenleiId.'/'.$intPage.'.html',
]
];
$TaskCore->set($arrTask);
// var_dump($arrTask);
}
}
}
public function getFenLei($arrData)
{
$TaskCore = $this->TaskCore;
$Client = $this->Client;
$strSiteUrl = $this->get630zwDomain();
$strUrl = $arrData['xs_fenlei_url']; //1 ;
var_dump($strUrl);
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
$QueryList = QueryList::html($strResult);
$arrNovel = [];
// 检查是否找到元素
if ($QueryList->find('.layout2 .txt-list-row5')->eq(0)->find('li')->count() > 0) {
$QueryList->find('.layout2 .txt-list-row5')->eq(0)->find('li')->map(function ($Novel) use (&$arrNovel) {
$strNovelHref = $Novel->find('.s2 a')->attr('href') ;
$intNovelId = extractFilename($strNovelHref);
var_dump($strNovelHref);
$arrNovel[] = [
'xs_source_id' => $intNovelId, //528049
'xs_source_seo_id' => 1,
];
});
} else {
echo 'qie h5' . PHP_EOL;
if ($QueryList->find('.sort_list')->eq(0)->find('li')->count() > 0) {
$QueryList->find('.sort_list')->eq(0)->find('li')->map(function ($Novel) use (&$arrNovel) {
$strNovelHref = $Novel->find('.s2 a')->attr('href') ;
$intNovelId = extractFilename($strNovelHref);
var_dump($strNovelHref);
$arrNovel[] = [
'xs_source_id' => $intNovelId, //528049
'xs_source_seo_id' => 1,
];
});
}
}
foreach ($arrNovel as $intNum => $arrNovelOne) {
$arrTask = [
'callback' => [self::class, 'getNovelInfo'],
'data' => $arrNovelOne,
];
$TaskCore->set($arrTask);
}
} catch (\Throwable $T) {
$arrTask = [
'callback' => [self::class, 'getFenLei'],
'data' => [
'xs_fenlei_url' => $strUrl,
]
];
$TaskCore->set($arrTask);
throw $T;
}
}
public function getNovelInfo($arrData)
{
$TaskCore = $this->TaskCore;
$Client = $this->Client;
$strSiteUrl = $this->get630zwDomain();
$strUrl = 'https://630zw.org/shu/' . $arrData['xs_source_id'] . '.html'; //'https://630zw.org/shu/93716.html';
$intXiaoShuoSourceId = $arrData['xs_source_id']; //20053;
$intXiaoShuoSourceSeoId = $arrData['xs_source_seo_id']; //1 ;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
$QueryList = QueryList::html($strResult);
// var_dump($QueryList);
// 提取所有 <meta> 标签的属性
$result = $QueryList->find('meta[property^="og:"]')->map(function ($item) {
return [
'property' => $item->attr('property'),
'content' => $item->attr('content'),
];
});
// 打印结果
// print_r($result->all());
$strZuoZhe = '';
$strMingZi = '';
$strLastDate = '';
$strJieShao = '';
$strCover = '';
$strFenlei = '';
$strZiShu = 0;
$strZhuangTai = '';
foreach ($result as $item) {
switch ($item['property']) {
case 'og:title':
$strMingZi = $item['content'];
break;
case 'og:description':
$strJieShao = preg_split('/<br\s*\/?>/i', $item['content'])[0];
break;
case 'og:image':
$strCover = $item['content'];
break;
case 'og:novel:category':
$strFenlei = $item['content'];
break;
case 'og:novel:author':
$strZuoZhe = $item['content'];
break;
case 'og:novel:status':
$strZhuangTai = $item['content'];
break;
case 'og:novel:update_time':
$strLastDate = $item['content'];
break;
}
}
$strLastDate = trim((string)$strLastDate);
if (!isValidDateTime($strLastDate)) {
$strLastDate = date('Y-m-d H:i:s');
}
$arrTag = [];
if (strpos($strCover, 'http') === false) {
$strCover = $strSiteUrl . $strCover;
}
$strCoverUri = $strCover;
try {
$strCoverUri = "";
$strCoverUri = ImageProcessor::getInstance()->downloadImage($intXiaoShuoSourceId, $strCover, 'xs/630zw');
} catch (\Throwable $t) {
$strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
echo $strErr;
}
$arrCover = [
'code' => '630_ZW_XS_COVER',
// 'uri' => getUriByUrl($strCover),
'uri' => $strCoverUri,
];
$strFenleiIId = 1;
switch ($strFenlei) {
case '玄幻修真':
$strName = "玄幻";
$strFenleiIId = 1;
break;
case '重生穿越':
$strName = "修真";
$strFenleiIId = 2;
break;
case '都市小说':
$strName = "都市";
$strFenleiIId = 3;
break;
case '军史小说':
$strName = "历史";
$strFenleiIId = 4;
break;
case '网游小说':
$strName = "网游";
$strFenleiIId = 5;
break;
case '科幻小说':
$strName = "科幻";
$strFenleiIId = 6;
break;
case '灵异小说':
$strName = "女频";
$strFenleiIId = 7;
break;
case '言情小说':
$strName = "其它";
$strFenleiIId = 8;
break;
case '其他小说':
$strName = "其它";
$strFenleiIId = 9;
break;
default:
$strName = "其它";
$strFenleiIId = 9;
break;
}
$arrXiaoShuoInfo = [
'xsxq_ming_zi' => $strMingZi,
'xsxq_zuozhe' => $strZuoZhe, //$arrData['xiaoshuo_zuozhe'],
'xsfl_id' => $strFenleiIId,
'xsxq_zhuang_tai' => $strZhuangTai,
'xsxq_zi_shu' => $strZiShu,
'xsxq_jie_shao' => $strJieShao,
'xsxq_feng_mian' => json_encode($arrCover),
'xsxq_geng_xin_shi_jian' => $strLastDate,
'xsxq_tag' => json_encode($arrTag, JSON_UNESCAPED_UNICODE),
'xsxq_source_id' => $intXiaoShuoSourceId,
'xsxq_source_seo_id' => $intXiaoShuoSourceId . '1',
'xsxq_source_code' => 'ZW630_XIAOSHUO_' . $intXiaoShuoSourceId . '_' . $intXiaoShuoSourceSeoId,
];
// print_r($arrXiaoShuoInfo);
var_dump($arrXiaoShuoInfo['xsxq_source_code']);
$XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::where('xsxq_source_code', $arrXiaoShuoInfo['xsxq_source_code'])->find();
$arrZhangjie = [];
# seo 小说入库 - 只有新小说才会走这个步骤
if (!$XiaoShuoXiangQingModel) {
$XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
$intXiaoShuolId = $XiaoShuoXiangQingModel->xsxq_id;
// 检查是否找到元素-seo 小说入库
if ($QueryList->find('.first_txt a')->count() > 0 && $intXiaoShuolId) {
$QueryList->find('.first_txt a')->map(function ($ZhangJie) use ($intXiaoShuolId) {
$url = $ZhangJie->attr('href');
// 检查是否以 "/kan/" 开头
if (strpos($url, '/kan/') === 0) { // strpos 返回位置，0 表示是以 "/kan/" 开头
// 处理符合条件的链接
// echo "Processing index : " . $ZhangJie->text() . PHP_EOL;
$arrXiaoShuoSeoInfo = [
'nis_name' => $ZhangJie->text(),
'xsxq_id' => $intXiaoShuolId,
'nis_md5' => md5($intXiaoShuolId . $ZhangJie->text()),
];
NovelInfoSeoModel::addXiaoShuoXiangQingSeo($arrXiaoShuoSeoInfo);
}
});
} else {
echo "No elements found for .first_txt a" . PHP_EOL;
var_dump($strResult);
}
// # 新小说全部章节入库
// $intChapterIndex = 1; // 定义一个计数器变量
// // 检查是否找到元素
// if ($QueryList->find('.section-list')->eq(1)->find('li')->count() > 0) {
// $QueryList->find('.section-list')->eq(1)->find('li')->map(function ($ZhangJie) use (&$arrZhangjie, &$intChapterIndex) {
// echo "Processing index $intChapterIndex: " . $ZhangJie->text() . PHP_EOL;
// $arrZhangjie[] = [
// 'paixu' => $intChapterIndex,
// 'mingzi' => $ZhangJie->find('a')->text(),
// 'url' => $ZhangJie->find('a')->attr('href'),
// ];
// $intChapterIndex++;
// });
// } else {
// echo "No elements found for .listmain a" . PHP_EOL;
// }
//unset($QueryList);
}else{
$intXiaoShuolId = $XiaoShuoXiangQingModel->xsxq_id;
// 获取最大章节索引
$intMaxXiaoShuoZhangJiePaixu = XiaoShuoZhangJieModel::where('xsxq_id', $intXiaoShuolId)
->max('xszj_pai_xu'); // 获取最大值
// var_dump('获取小说章节最大值-'.$intMaxXiaoShuoZhangJiePaixu);
}
# 新小说全部章节入库
$intChapterIndex = 1; // 定义一个计数器变量
// 检查是否找到元素
if ($QueryList->find('.section-list')->eq(1)->find('li')->count() > 0) {
$QueryList->find('.section-list')->eq(1)->find('li')->map(function ($ZhangJie) use (&$arrZhangjie, &$intChapterIndex) {
// echo "Processing index $intChapterIndex: " . $ZhangJie->text() . PHP_EOL;
$arrZhangjie[] = [
'paixu' => $intChapterIndex,
'mingzi' => $ZhangJie->find('a')->text(),
'url' => $ZhangJie->find('a')->attr('href'),
];
$intChapterIndex++;
});
$this->eachNovelReader($arrZhangjie,$strSiteUrl,$intXiaoShuolId,$strFenleiIId);
} else {
echo "No elements found for .listmain a" . PHP_EOL;
}
# 章节处理
# 新小说全部章节入库
# 旧小说，只更新最新章节，获取数据库最大章节索引，对比源总章节
// // 如果没有结果，max 返回 null，可以设置默认值
$intMaxXiaoShuoZhangJiePaixu = $intMaxXiaoShuoZhangJiePaixu ?? 0;
# 目录页面处理
$strMuluHref = $QueryList->find('.btn-mulu')->eq(0)->attr('href');
$strMuluHref = $strSiteUrl . $strMuluHref ;
$ResponseMulu = $Client->request('GET', $strMuluHref, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResultMulu = (string)$ResponseMulu->getBody();
$QueryListMulu = QueryList::html($strResultMulu);
$intMuluTotal = $QueryListMulu->find('.page_num select')->eq(0)->find('option')->count();
$strMuluEndHref = $QueryListMulu->find('.page_num select')->eq(0)->find('option')->eq($intMuluTotal-1)->attr('value');
$strMuluEndText = $QueryListMulu->find('.page_num select')->eq(0)->find('option')->eq($intMuluTotal-1)->text();
// 使用正则表达式匹配
preg_match_all('/\.section-list\.ycxsid>li:nth-child\(/', $strResultMulu, $matches);
preg_match_all('/\.section-list\.ycxsid>li:nth-last-child\(/', $strResultMulu, $matchesLast);
$intProLiNum = count($matches[0]);
$intLastLiNum = count($matchesLast[0]);
$arrZhangjieMulu = [];
//if($intMaxXiaoShuoZhangJiePaixu < 200){
$intChapterPageIndex = 101; // 定义一个计数器变量
// 检查是否找到元素
if ($QueryListMulu->find('.chapter-list')->eq(0)->find('li')->count() > 0) {
$QueryListMulu->find('.chapter-list')->eq(0)->find('li')->map(function ($ZhangJie) use (&$arrZhangjieMulu, &$intChapterPageIndex) {
$arrZhangjieMulu[] = [
'paixu' => $intChapterPageIndex,
'mingzi' => $ZhangJie->find('a')->text(),
'url' => $ZhangJie->find('a')->attr('href'),
];
$intChapterPageIndex ++ ;
});
$arrZhangjieMulu = array_slice($arrZhangjieMulu, $intProLiNum);
// 删除后 11 条数据
$arrZhangjieMulu = array_slice($arrZhangjieMulu, 0, count($arrZhangjieMulu) - $intLastLiNum);
$this->eachNovelReader($arrZhangjieMulu,$strSiteUrl,$intXiaoShuolId,$strFenleiIId);
var_dump("chapter章节任务投递成功 $strMuluHref");
} else {
// var_dump($QueryListMulu->find('.section-list')->eq(0)->find('li')->count());
//echo "No elements found for .chapter-list $strMuluHref" . PHP_EOL;
// 检查是否找到元素
$intListIndex = 1;
if ($QueryListMulu->find('.section-list')->eq(0)->find('li')->count() > 0) {
$QueryListMulu->find('.section-list')->eq(0)->find('li')->map(function ($ZhangJie) use (&$arrZhangjieMulu, &$intChapterPageIndex,&$intListIndex) {
if($intListIndex >14){
$arrZhangjieMulu[] = [
'paixu' => $intChapterPageIndex,
'mingzi' => $ZhangJie->find('a')->text(),
'url' => $ZhangJie->find('a')->attr('href'),
];
$intChapterPageIndex ++ ;
};
$intListIndex++ ;
});
$arrZhangjieMulu = array_slice($arrZhangjieMulu, $intProLiNum);
// 删除后 11 条数据
$arrZhangjieMulu = array_slice($arrZhangjieMulu, 0, count($arrZhangjieMulu) - $intLastLiNum);
$this->eachNovelReader($arrZhangjieMulu,$strSiteUrl,$intXiaoShuolId,$strFenleiIId);
var_dump("section章节任务投递成功 $strMuluHref");
} else {
var_dump("No elements found for .section-list $strMuluHref") ;
}
}
//}
// 输出最终结果
// var_dump(count($arrZhangjie));
// var_dump($QueryListMulu->find('.chapter-list li')->count());
// var_dump($intMuluTotal);
// var_dump($strMuluEndHref);
// var_dump($strMuluEndText);
// var_dump($intMaxXiaoShuoZhangJiePaixu); // 1010
preg_match('/(\d+)-(\d+)章/', $strMuluEndText, $matches);
$intMaxPaixu = $matches[2]; // 最大章节
$intMaxMuluPage = floor($intMaxPaixu / 100); // 最大目录页面
// $intMaxMuluPage = 2;
// 所有目录分页处理
for ($i = 2; $i <= $intMaxMuluPage; $i++) {
//echo "当前值: $i\n";
$arrTask = [
'callback' => [self::class, 'getNovelMulu'],
'data' => [
'strMuluHref' => $strSiteUrl . '/shu/' . $XiaoShuoXiangQingModel->xsxq_source_id . '_' . $i . '.html' ,
'xsxq_id' => $XiaoShuoXiangQingModel->xsxq_id,
'xsfl_id' => $strFenleiIId,
'intStartIndex' => $i*100 + 1,
]
];
$TaskCore->set($arrTask);
// var_dump($arrTask);
}
unset($QueryList);
unset($QueryListMulu);
} catch (\Throwable $T) {
$arrTask = [
'callback' => [self::class, 'getNovelInfo'],
'data' => [
'xs_source_id' => $arrData['xs_source_id'],
'xs_source_seo_id' => $intXiaoShuoSourceSeoId,
]
];
$TaskCore->set($arrTask);
throw $T;
}
}
public function getNovelMulu($arrData)
{
$TaskCore = $this->TaskCore;
$Client = $this->Client;
$strSiteUrl = $this->get630zwDomain();
$strUrl = $arrData['strMuluHref'];
$intXiaoShuoId = $arrData['xsxq_id'];
$intXiaoShuoFenleiId = $arrData['xsfl_id'];
$intStartIndex = $arrData['intStartIndex'];
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
$QueryList = QueryList::html($strResult);
// 使用正则表达式匹配
preg_match_all('/\.section-list\.ycxsid>li:nth-child\(/', $strResult, $matches);
preg_match_all('/\.section-list\.ycxsid>li:nth-last-child\(/', $strResult, $matchesLast);
$intProLiNum = count($matches[0]);
$intLastLiNum = count($matchesLast[0]);
$arrZhangjie = [];
if ($QueryList->find('.chapter-list')->eq(0)->find('li')->count() > 0) {
$QueryList->find('.chapter-list')->eq(0)->find('li')->map(function ($ZhangJie) use (&$arrZhangjie, &$intStartIndex) {
$arrZhangjie[] = [
'paixu' => $intStartIndex,
'mingzi' => $ZhangJie->find('a')->text(),
'url' => $ZhangJie->find('a')->attr('href'),
];
$intStartIndex ++ ;
// var_dump($intStartIndex);
});
$arrZhangjie = array_slice($arrZhangjie, $intProLiNum);
// 删除后 11 条数据
$arrZhangjie = array_slice($arrZhangjie, 0, count($arrZhangjie) - $intLastLiNum);
$this->eachNovelReader($arrZhangjie,$strSiteUrl,$intXiaoShuoId,$intXiaoShuoFenleiId);
var_dump("chapter章节任务投递成功 $strUrl");
} else {
// 检查是否找到元素
if ($QueryList->find('.section-list')->eq(0)->find('li')->count() > 0) {
$QueryList->find('.section-list')->eq(0)->find('li')->map(function ($ZhangJie) use (&$arrZhangjie, &$intStartIndex) {
$arrZhangjie[] = [
'paixu' => $intStartIndex,
'mingzi' => $ZhangJie->find('a')->text(),
'url' => $ZhangJie->find('a')->attr('href'),
];
$intStartIndex ++ ;
// var_dump($intStartIndex);
});
$arrZhangjie = array_slice($arrZhangjie, $intProLiNum);
// 删除后 11 条数据
$arrZhangjie = array_slice($arrZhangjie, 0, count($arrZhangjie) - $intLastLiNum);
$this->eachNovelReader($arrZhangjie,$strSiteUrl,$intXiaoShuoId,$intXiaoShuoFenleiId);
var_dump("section章节任务投递成功 $strUrl");
} else {
var_dump("No elements found for .section-list $strUrl") ;
}
}
unset($QueryList);
} catch (\Throwable $T) {
$arrTask = [
'callback' => [self::class, 'getNovelMulu'],
'data' => [
'strMuluHref' => $strUrl ,
'xsxq_id' => $intXiaoShuoId,
'xsfl_id' => $intXiaoShuoFenleiId,
'intStartIndex' => $intStartIndex,
]
];
$TaskCore->set($arrTask);
throw $T;
}
}
public function eachNovelReader($arrZhangjie,$strSiteUrl,$intXiaoShuoId,$intXiaoShuoFenleiId)
{
$TaskCore = $this->TaskCore;
// var_dump($arrZhangjie);
foreach ($arrZhangjie as $intZhangJieNum => $arrZhangjieOne) {
// 如果数据库里的章节最大索引大于当前索引，说明已经入库，不需要入队列了
// if($intMaxXiaoShuoZhangJiePaixu > $intZhangJieNum){
// var_dump("小说章节已经入库，不再推送任务");
// continue;// 如果任务已存在，直接跳过 break
// }
$strZhangJieUrl = 'https://www.630zw.org' . $arrZhangjieOne['url'];
$strZhangJieSourceId = extractFilename($strZhangJieUrl);
# 'AILISI_XIAOSHUO_ZHANGJIE_小说源id_小说源章节id'
$strXszjSourceCode = '630ZW_XIAOSHUO_ZHANGJIE_'. $intXiaoShuoId . '_' . $strZhangJieSourceId . '_1' ;
$arrTask = [
'callback' => [self::class, 'getNovelReader'],
'data' => [
'xszj_source_id' => $strZhangJieSourceId,
'xszj_paixu' => $arrZhangjieOne['paixu'],
'xszj_name' => $arrZhangjieOne['mingzi'],
'xszj_url' => $strZhangJieUrl,
'xszj_source_code' => $strXszjSourceCode,
'xsxq_id' => $intXiaoShuoId,
'xsfl_id' => $intXiaoShuoFenleiId,
'intZhangJieSort' => 1,
]
];
$TaskCore->set($arrTask);
}
}
public function getNovelFengMian($arrData)
{
//var_dump('getNovelFengMian');
//var_dump($arrData);
$Client = $this->Client;
$TaskCore = $this->TaskCore;
// 获取 Redis 实例
$redis = Cache::store('redis')->handler();
$intXiaoShuoSourceId = $arrData['xiaoshuo_id'];
$intXiaoShuoSourceSeoId = $arrData['xiaoshuo_seo_id'];
$redisKey = "task:novel_source_id:" . $intXiaoShuoSourceId;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($arrData['xiaoshuo_url']);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$strSiteUrl = $this->getAilisiDomain();
$strCover = $QueryList->find('#fmimg img')->eq(0)->attr("src");
if (strpos($strCover, 'http') === false) {
$strCover = $strSiteUrl . $strCover;
}
$strCoverUri = $strCover;
try {
$strCoverUri = "";
$strCoverUri = ImageProcessor::getInstance()->downloadAndEncryptImage($strCover, 'xs');
$arrCover = [
'code' => 'AI_LI_SI_COVER',
// 'uri' => getUriByUrl($strCover),
'uri' => $strCoverUri,
];
$arrXiaoShuoInfo = [
'xsxq_feng_mian' => json_encode($arrCover),
];
$XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
} catch (\Throwable $t) {
$strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
echo $strErr;
}
} catch (\Throwable $T) {
throw $T;
}
}
//https://www.630zw.org/shu/438374/156339465.html
public function getNovelReader($arrData)
{
$TaskCore = $this->TaskCore;
$strSiteUrl = 'https://www.630zw.org';
var_dump('getNovelReader------------------------------------------------------------------------------------');
$Client = $this->Client;
$intZhangJieSort = $arrData['intZhangJieSort'];
$strXszjSourceCode = $arrData['xszj_source_code'];
$strZhangJieUrl = $arrData['xszj_url'];
$intZhangJiePaiXu = $arrData['xszj_paixu'];
$intFenleiId = $arrData['xsfl_id'];
$intXiaoShuoId = $arrData['xsxq_id'];
$strZhangJieSourceId = $arrData['xszj_source_id'];
$strZhangJieName = $arrData['xszj_name'];
$XiaoShuoZhangJieModel = XiaoShuoZhangJieModel::where('xszj_source_code', $strXszjSourceCode)->find();
if ($XiaoShuoZhangJieModel) {
var_dump($XiaoShuoZhangJieModel->xszj_source_code);
var_dump('章节已经存在 paixu: '.$intZhangJiePaiXu);
var_dump('章节已经存在strXszjSourceCode: '.$strXszjSourceCode);
var_dump('章节已经存在strZhangJieUrl: '.$strZhangJieUrl);
// return true;
}
var_dump($strZhangJieUrl);
var_dump('章节排序：'.$intZhangJiePaiXu);
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strZhangJieUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
if (XiaoShuoZhangJieModel::checkZhangJieExists($strXszjSourceCode)) {
return true;
}
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strZhangJieUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
unset($Response);
$ZhangJieQueryList = QueryList::html($strResult);
// 正则表达式，匹配 `_数字.html`
$pattern = '/_(\d+)\.html$/';
$patternHref = '/(\/shu\/\d+\/\d+_\d+\.html)/';
$strNeiRong = '';
$strNextHref = '';
// 使用 preg_match() 进行匹配
if (preg_match($patternHref, $strResult, $matches)) {
// 提取匹配到的部分
echo "提取到的 URL 是: " . $matches[1];
$strNextHref = $matches[1];
} else {
echo "没有匹配到 URL";
}
preg_match_all("/qsbs\.bb\('([^']+)'\)/", $strResult, $matches);
// 输出提取到的加密字符串
if (isset($matches[1])) {
$strNeiRong = implode(';;;', $matches[1]);
// var_dump($strNeiRong);
}
// 判断字符串是否匹配
if (preg_match($pattern, $strNextHref, $matches) ){
$intNextPage = 2;
// 确保匹配成功后再访问索引
if (isset($matches[1])) {
$intNextPage = (int)$matches[1];
$intNextPage +=1;
var_dump('提取到的数字是：' . $intNextPage);
}
var_dump('提取到的数字是-：' . $intNextPage);
$arrTask = [
'callback' => [self::class, 'getNovelReader'],
'data' => [
'xszj_source_id' => $strZhangJieSourceId,
'xszj_paixu' => $intZhangJiePaiXu,
'xszj_name' => $strZhangJieName,
'xszj_url' => $strSiteUrl . $strNextHref,
'xszj_source_code' => '630ZW_XIAOSHUO_ZHANGJIE_'. $intXiaoShuoId . '_' . $strZhangJieSourceId . '_' . $intNextPage,
'xsxq_id' => $intXiaoShuoId,
'xsfl_id' => $intFenleiId,
'intZhangJieSort' => $intNextPage,
]
];
$TaskCore->set($arrTask);
}
# '/novel/123yqw/分类id/小说源id/小说源章节id.txt'
$strNeiRongTxtPath = config("filesystem.novel") . '/novel/630zwxs/' . $intFenleiId . '/' . $intXiaoShuoId .'/' . $intZhangJiePaiXu .'/' . $intZhangJieSort . '.txt';
//$strNeiRongTxtPath = "/www/novel/123yqw/1/11/111.txt";
// 调用函数
$result = saveCompressAndDeleteTxt($strNeiRongTxtPath, $strNeiRong);
// 输出状态
if ($result['status']) {
var_dump("Success: " . $result['message']);
} else {
var_dump("Error: " . $result['message']);
}
$arrXiaoShuoZhangJie = [
'xsxq_id' => $intXiaoShuoId,
'xszj_pai_xu' => $intZhangJiePaiXu,
'xszj_ming_zi' => $strZhangJieName,
'xszj_nei_rong' => $strNeiRongTxtPath,
'xszj_source_id' => $strZhangJieSourceId,
'xszj_source_code' => $strXszjSourceCode,
'xszj_sort' => $intZhangJieSort,
];
// var_dump($arrXiaoShuoZhangJie);
if($strZhangJieName){
XiaoShuoZhangJieModel::addXiaoShuoZhangJie($arrXiaoShuoZhangJie);
}
unset($ZhangJieQueryList);
// // 延时 3 秒
//sleep(3);
} catch (\Throwable $T) {
$arrTask = [
'callback' => [self::class, 'getNovelReader'],
'data' => [
'xszj_source_id' => $strZhangJieSourceId,
'xszj_paixu' => $intZhangJiePaiXu,
'xszj_name' => $strZhangJieName,
'xszj_url' => $strZhangJieUrl,
'xszj_source_code' => $strXszjSourceCode,
'xsxq_id' => $intXiaoShuoId,
'xsfl_id' => $intFenleiId,
'intZhangJieSort' => $intZhangJieSort,
]
];
$TaskCore->set($arrTask);
throw $T;
}
}
}

426
code/app/task/collection/novelone/NovelJiuYuFanXianCol.php Normal file
View File

@@ -0,0 +1,426 @@
<?php
declare(strict_types=1);
namespace app\task\collection\novelone;
use app\admin\model\XiaoShuoXiangQingModel;
use app\admin\model\XiaoShuoZhangJieModel;
use app\task\collection\PullDataColBase;
use GuzzleHttp\Cookie\CookieJar;
use processor\ImageProcessor;
use QL\QueryList;
use app\common\Base64Helper;
/**
* @ 抓取 gdbzkz.info 站点相关小说
*/
class NovelJiuYuFanXianCol extends PullDataColBase
{
public function pullXiaoShuo()
{
$TaskCore = $this->TaskCore;
// http://www.gdbzkz.info/jiuyufanxian/
$arrNovel = [
['xs_source_id' => 'jiuyufanxian'],
];
foreach ($arrNovel as $Novel) {
$arrTask = [
'callback' => [self::class, 'getNovelInfo'],
'data' => [
'xs_source_id' => $Novel['xs_source_id'],
'xs_source_seo_id' => 1,
]
];
$TaskCore->set($arrTask);
var_dump($arrTask);
}
}
public function getNovelInfo($arrData)
{
$TaskCore = $this->TaskCore;
$Client = $this->Client;
$strSiteUrl = 'http://www.gdbzkz.info';
$intXiaoShuoSourceId = $arrData['xs_source_id'];
$intXiaoShuoSourceSeoId = $arrData['xs_source_seo_id'];
$strUrl = $strSiteUrl . '/' . $intXiaoShuoSourceId;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
$QueryList = QueryList::html($strResult);
// var_dump($QueryList);
// 提取所有 <meta> 标签的属性
$result = $QueryList->find('meta[property^="og:"]')->map(function ($item) {
return [
'property' => $item->attr('property'),
'content' => $item->attr('content'),
];
});
// 打印结果
// print_r($result->all());
$strZuoZhe = '';
$strMingZi = '';
$strLastDate = '';
$strJieShao = '';
$strCover = '';
$strFenlei = '';
$strZiShu = 0;
$strZhuangTai = '';
foreach ($result as $item) {
switch ($item['property']) {
case 'og:title':
$strMingZi = $item['content'];
break;
case 'og:description':
$strJieShao = $item['content'];
break;
case 'og:image':
$strCover = $item['content'];
break;
case 'og:novel:category':
$strFenlei = $item['content'];
break;
case 'og:novel:author':
$strZuoZhe = $item['content'];
break;
case 'og:novel:status':
$strZhuangTai = $item['content'];
break;
case 'og:novel:update_time':
$strLastDate = $item['content'];
break;
}
}
$strLastDate = trim((string)$strLastDate);
if (!isValidDateTime($strLastDate)) {
$strLastDate = date('Y-m-d H:i:s');
}
$arrTag = [];
if (strpos($strCover, 'http') === false) {
$strCover = $strSiteUrl . $strCover;
}
$strCoverUri = $strCover;
try {
$strCoverUri = "";
$strCoverUri = ImageProcessor::getInstance()->downloadImage($intXiaoShuoSourceId, $strCover, 'xs');
} catch (\Throwable $t) {
$strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
echo $strErr;
}
$arrCover = [
'code' => 'BI_QU_XS_COVER',
// 'uri' => getUriByUrl($strCover),
'uri' => $strCoverUri,
];
$strFenleiIId = 1;
switch ($strFenlei) {
case '玄幻小说':
$strName = "玄幻";
$strFenleiIId = 1;
break;
case '修真小说':
$strName = "修真";
$strFenleiIId = 2;
break;
case '都市小说':
$strName = "都市";
$strFenleiIId = 3;
break;
case '历史小说':
$strName = "历史";
$strFenleiIId = 4;
break;
case '网游小说':
$strName = "网游";
$strFenleiIId = 5;
break;
case '科幻小说':
$strName = "科幻";
$strFenleiIId = 6;
break;
case '女频小说':
$strName = "女频";
$strFenleiIId = 7;
break;
case '其它小说':
$strName = "其它";
$strFenleiIId = 8;
break;
default:
$strName = "其它";
$strFenleiIId = 8;
break;
}
$arrXiaoShuoInfo = [
'xsxq_ming_zi' => $strMingZi,
'xsxq_zuozhe' => $strZuoZhe, //$arrData['xiaoshuo_zuozhe'],
'xsfl_id' => $strFenleiIId,
'xsxq_zhuang_tai' => $strZhuangTai,
'xsxq_zi_shu' => $strZiShu,
'xsxq_jie_shao' => $strJieShao,
'xsxq_feng_mian' => json_encode($arrCover),
'xsxq_geng_xin_shi_jian' => $strLastDate,
'xsxq_tag' => json_encode($arrTag, JSON_UNESCAPED_UNICODE),
'xsxq_source_id' => $intXiaoShuoSourceId,
'xsxq_source_seo_id' => $intXiaoShuoSourceId . '1',
'xsxq_source_code' => 'BIQU_XIAOSHUO_' . $intXiaoShuoSourceId . '_' . $intXiaoShuoSourceSeoId,
];
// print_r($arrXiaoShuoInfo);
var_dump($arrXiaoShuoInfo['xsxq_source_code']);
$XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
$arrZhangjie = [];
$intChapterIndex = 0; // 定义一个计数器变量
// 检查是否找到元素
if ($QueryList->find('.listmain a')->count() > 0) {
$QueryList->find('.listmain a')->map(function ($ZhangJie) use (&$arrZhangjie, &$intChapterIndex) {
echo "Processing index $intChapterIndex: " . $ZhangJie->text() . PHP_EOL;
// 从索引6开始
if ($intChapterIndex >= 12) {
$arrZhangjie[] = [
'mingzi' => $ZhangJie->text(),
'url' => $ZhangJie->attr('href'),
];
}
$intChapterIndex++;
});
// 输出最终结果
// var_dump($arrZhangjie);
} else {
echo "No elements found for .listmain a" . PHP_EOL;
}
//unset($QueryList);
// 获取最大章节索引
$intMaxXiaoShuoZhangJiePaixu = XiaoShuoZhangJieModel::where('xsxq_id', $XiaoShuoXiangQingModel->xsxq_id)
->max('xszj_pai_xu'); // 获取最大值
var_dump('获取小说章节最大值-' . $intMaxXiaoShuoZhangJiePaixu);
// 如果没有结果，max 返回 null，可以设置默认值
$intMaxXiaoShuoZhangJiePaixu = $intMaxXiaoShuoZhangJiePaixu ?? 0;
foreach ($arrZhangjie as $intZhangJieNum => $arrZhangjieOne) {
var_dump($intZhangJieNum);
// 如果数据库里的章节最大索引大于当前索引，说明已经入库，不需要入队列了
// if ($intMaxXiaoShuoZhangJiePaixu > $intZhangJieNum) {
// var_dump("小说章节已经入库，不再推送任务");
// continue; // 如果任务已存在，直接跳过 break
// }
$strZhangJieUrl = $strSiteUrl . $arrZhangjieOne['url'];
$strZhangJieSourceId = extractFilename($strZhangJieUrl);
# 'AILISI_XIAOSHUO_ZHANGJIE_小说源id_小说源章节id'
$strXszjSourceCode = 'gdbzkz_XIAOSHUO_ZHANGJIE_' . $XiaoShuoXiangQingModel->xsxq_source_id . '_' . $strZhangJieSourceId . '_1';
$arrTask = [
'callback' => [self::class, 'getNovelReader'],
'data' => [
'xszj_source_id' => $strZhangJieSourceId,
'xszj_paixu' => $intZhangJieNum += 1,
'xszj_name' => $arrZhangjieOne['mingzi'],
'xszj_url' => $strZhangJieUrl,
'xszj_source_code' => $strXszjSourceCode,
'xsxq_id' => $XiaoShuoXiangQingModel->xsxq_id,
'xsfl_id' => $XiaoShuoXiangQingModel->xsfl_id,
'intZhangJieSort' => 1,
]
];
$TaskCore->set($arrTask);
}
var_dump("章节任务投递成功");
} catch (\Throwable $T) {
throw $T;
}
}
public function getNovelReader($arrData)
{
$TaskCore = $this->TaskCore;
$strSiteUrl = 'http://www.gdbzkz.info';
var_dump('getNovelReader------------------------------------------------------------------------------------');
$Client = $this->Client;
$intZhangJieSort = $arrData['intZhangJieSort'];
$strXszjSourceCode = $arrData['xszj_source_code'];
$strZhangJieUrl = $arrData['xszj_url'];
$intZhangJiePaiXu = $arrData['xszj_paixu'];
$intFenleiId = $arrData['xsfl_id'];
$intXiaoShuoId = $arrData['xsxq_id'];
$strZhangJieSourceId = $arrData['xszj_source_id'];
$strZhangJieName = $arrData['xszj_name'];
$XiaoShuoZhangJieModel = XiaoShuoZhangJieModel::where('xszj_source_code', $strXszjSourceCode)->find();
if ($XiaoShuoZhangJieModel) {
// var_dump($XiaoShuoZhangJieModel->xszj_source_code);
// var_dump('章节已经存在 paixu: ' . $intZhangJiePaiXu);
// var_dump('章节已经存在strXszjSourceCode: ' . $strXszjSourceCode);
// var_dump('章节已经存在strZhangJieUrl: ' . $strZhangJieUrl);
return true;
}
var_dump($strZhangJieUrl);
var_dump('章节排序：' . $intZhangJiePaiXu);
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strZhangJieUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
if (XiaoShuoZhangJieModel::checkZhangJieExists($strXszjSourceCode)) {
return true;
}
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strZhangJieUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
unset($Response);
$ZhangJieQueryList = QueryList::html($strResult);
$strNeiRong = '';
// 获取原始 HTML
$strNeiRong = $ZhangJieQueryList->find('#content')->html();
// 解码 HTML 实体
$strNeiRong = html_entity_decode($strNeiRong, ENT_QUOTES, 'UTF-8');
// 替换不间断空格 (\xc2\xa0) 和 &nbsp; 为普通空格
$strNeiRong = preg_replace('/\xc2\xa0/', ' ', $strNeiRong); // 替换 UTF-8 不间断空格
$strNeiRong = preg_replace('/&nbsp;/', ' ', $strNeiRong); // 替换 &nbsp;
// 清理多余空格和换行符
$strNeiRong = preg_replace('/\s+/', ' ', $strNeiRong);
// 1. 找到所有的 <br> 位置
$br_positions = [];
$offset = 0;
while (($pos = strpos($strNeiRong, "<br", $offset)) !== false) {
$br_positions[] = $pos;
$offset = $pos + 1;
}
// 2. 找到倒数第三个 <br> 的位置
if (count($br_positions) >= 3) {
$cut_position = $br_positions[count($br_positions) - 3]; // 倒数第三个 <br> 的索引
$strNeiRong = substr($strNeiRong, 0, $cut_position); // 截取前面部分，删除后面的内容
}
$arrNeiRong = explode('<br> <br>', $strNeiRong);
$arrEnCode = [];
foreach ($arrNeiRong as $intNeiRongNum => $strNeiRongOne) {
$strEnCode = Base64Helper::encode('<p>' . $strNeiRongOne . '</p>');
$arrEnCode[] = $strEnCode;
}
$strNeiRong = implode(';;;', $arrEnCode);
# '/novel/123yqw/分类id/小说源id/小说源章节id.txt'
$strNeiRongTxtPath = config("filesystem.novel") . '/novel/630zwxs/' . $intFenleiId . '/' . $intXiaoShuoId . '/' . $intZhangJiePaiXu . '/' . $intZhangJieSort . '.txt';
//$strNeiRongTxtPath = "/www/novel/123yqw/1/11/111.txt";
// 调用函数
$result = saveCompressAndDeleteTxt($strNeiRongTxtPath, $strNeiRong);
// 输出状态
if ($result['status']) {
var_dump("Success: " . $result['message']);
} else {
var_dump("Error: " . $result['message']);
}
$arrXiaoShuoZhangJie = [
'xsxq_id' => $intXiaoShuoId,
'xszj_pai_xu' => $intZhangJiePaiXu,
'xszj_ming_zi' => $strZhangJieName,
'xszj_nei_rong' => $strNeiRongTxtPath,
'xszj_source_id' => $strZhangJieSourceId,
'xszj_source_code' => $strXszjSourceCode,
'xszj_sort' => $intZhangJieSort,
];
var_dump($arrXiaoShuoZhangJie);
if ($strZhangJieName) {
XiaoShuoZhangJieModel::addXiaoShuoZhangJie($arrXiaoShuoZhangJie);
}
unset($ZhangJieQueryList);
// // 延时 3 秒
//sleep(3);
} catch (\Throwable $T) {
$arrTask = [
'callback' => [self::class, 'getNovelReader'],
'data' => [
'xszj_source_id' => $strZhangJieSourceId,
'xszj_paixu' => $intZhangJiePaiXu,
'xszj_name' => $strZhangJieName,
'xszj_url' => $strZhangJieUrl,
'xszj_source_code' => $strXszjSourceCode,
'xsxq_id' => $intXiaoShuoId,
'xsfl_id' => $intFenleiId,
'intZhangJieSort' => $intZhangJieSort,
]
];
$TaskCore->set($arrTask);
throw $T;
}
}
}

441
code/app/task/collection/novelone/NovelWenXueGuanCol.php Normal file
View File

@@ -0,0 +1,441 @@
<?php
declare(strict_types=1);
namespace app\task\collection\novelone;
use app\admin\model\XiaoShuoXiangQingModel;
use app\admin\model\XiaoShuoZhangJieModel;
use app\task\collection\PullDataColBase;
use GuzzleHttp\Cookie\CookieJar;
use processor\ImageProcessor;
use QL\QueryList;
use app\common\Base64Helper;
/**
* @ 抓取 bqvvxg8.cc 站点相关小说
*/
class NovelWenXueGuanCol extends PullDataColBase
{
public function pullXiaoShuo()
{
$TaskCore = $this->TaskCore;
// https://www.bqvvxg8.cc/wenzhang/86/86114/
$arrNovel = [
['xs_source_id' => 'wenxueguan86114', 'xs_url' => 'https://www.bqvvxg8.cc/wenzhang/86/86114/'],
['xs_source_id' => 'wenxueguan90205', 'xs_url' => 'https://www.bqvvxg8.cc/wenzhang/90/90205/'],
];
foreach ($arrNovel as $Novel) {
$arrTask = [
'callback' => [self::class, 'getNovelInfo'],
'data' => [
'xs_url' => $Novel['xs_url'],
'xs_source_id' => $Novel['xs_source_id'],
'xs_source_seo_id' => 1,
]
];
$TaskCore->set($arrTask);
var_dump($arrTask);
}
}
public function getNovelInfo($arrData)
{
$TaskCore = $this->TaskCore;
$Client = $this->Client;
$strSiteUrl = 'https://www.bqvvxg8.cc';
$intXiaoShuoSourceId = $arrData['xs_source_id'];
$intXiaoShuoSourceSeoId = $arrData['xs_source_seo_id'];
$strUrl = $arrData['xs_url'];
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
$QueryList = QueryList::html($strResult);
// var_dump($QueryList);
// 提取所有 <meta> 标签的属性
$result = $QueryList->find('meta[property^="og:"]')->map(function ($item) {
return [
'property' => $item->attr('property'),
'content' => $item->attr('content'),
];
});
// 打印结果
// print_r($result->all());
$strZuoZhe = '';
$strMingZi = '';
$strLastDate = '';
$strJieShao = '';
$strCover = '';
$strFenlei = '';
$strZiShu = 0;
$strZhuangTai = '';
foreach ($result as $item) {
switch ($item['property']) {
case 'og:title':
$strMingZi = $item['content'];
break;
case 'og:description':
$strJieShao = $item['content'];
break;
case 'og:image':
$strCover = $item['content'];
break;
case 'og:novel:category':
$strFenlei = $item['content'];
break;
case 'og:novel:author':
$strZuoZhe = $item['content'];
break;
case 'og:novel:status':
$strZhuangTai = $item['content'] . '中';
break;
case 'og:novel:update_time':
$strLastDate = $item['content'];
break;
}
}
$strLastDate = trim((string)$strLastDate);
if (!isValidDateTime($strLastDate)) {
$strLastDate = date('Y-m-d H:i:s');
}
$arrTag = [];
if (strpos($strCover, 'http') === false) {
$strCover = $strSiteUrl . $strCover;
}
$strCoverUri = $strCover;
try {
$strCoverUri = "";
$strCoverUri = ImageProcessor::getInstance()->downloadImage($intXiaoShuoSourceId, $strCover, 'xs');
} catch (\Throwable $t) {
$strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
echo $strErr;
}
$arrCover = [
'code' => 'BI_QU_XS_COVER',
// 'uri' => getUriByUrl($strCover),
'uri' => $strCoverUri,
];
$strFenleiIId = 1;
switch ($strFenlei) {
case '玄幻小说':
$strName = "玄幻";
$strFenleiIId = 1;
break;
case '修真小说':
$strName = "修真";
$strFenleiIId = 2;
break;
case '言情小说':
$strName = "都市";
$strFenleiIId = 3;
break;
case '历史小说':
$strName = "历史";
$strFenleiIId = 4;
break;
case '网游小说':
$strName = "网游";
$strFenleiIId = 5;
break;
case '科幻小说':
$strName = "科幻";
$strFenleiIId = 6;
break;
case '女频小说':
$strName = "女频";
$strFenleiIId = 7;
break;
case '其它小说':
$strName = "其它";
$strFenleiIId = 8;
break;
default:
$strName = "其它";
$strFenleiIId = 8;
break;
}
$arrXiaoShuoInfo = [
'xsxq_ming_zi' => $strMingZi,
'xsxq_zuozhe' => $strZuoZhe, //$arrData['xiaoshuo_zuozhe'],
'xsfl_id' => $strFenleiIId,
'xsxq_zhuang_tai' => $strZhuangTai,
'xsxq_zi_shu' => $strZiShu,
'xsxq_jie_shao' => $strJieShao,
'xsxq_feng_mian' => json_encode($arrCover),
'xsxq_geng_xin_shi_jian' => $strLastDate,
'xsxq_tag' => json_encode($arrTag, JSON_UNESCAPED_UNICODE),
'xsxq_source_id' => $intXiaoShuoSourceId,
'xsxq_source_seo_id' => $intXiaoShuoSourceId . '1',
'xsxq_source_code' => 'WENXUEGUANG_XIAOSHUO_' . $intXiaoShuoSourceId . '_' . $intXiaoShuoSourceSeoId,
];
// print_r($arrXiaoShuoInfo);
var_dump($arrXiaoShuoInfo['xsxq_source_code']);
$XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
$arrZhangjie = [];
$intChapterIndex = 0; // 定义一个计数器变量
// 检查是否找到元素
if ($QueryList->find('.listmain a')->count() > 0) {
$QueryList->find('.listmain a')->map(function ($ZhangJie) use (&$arrZhangjie, &$intChapterIndex) {
echo "Processing index $intChapterIndex: " . $ZhangJie->text() . PHP_EOL;
// 从索引6开始
if ($intChapterIndex >= 12) {
$arrZhangjie[] = [
'mingzi' => $ZhangJie->text(),
'url' => $ZhangJie->attr('href'),
];
}
$intChapterIndex++;
});
// 输出最终结果
var_dump($arrZhangjie);
} else {
echo "No elements found for .listmain a" . PHP_EOL;
}
//unset($QueryList);
// 获取最大章节索引
$intMaxXiaoShuoZhangJiePaixu = XiaoShuoZhangJieModel::where('xsxq_id', $XiaoShuoXiangQingModel->xsxq_id)
->max('xszj_pai_xu'); // 获取最大值
var_dump('获取小说章节最大值-' . $intMaxXiaoShuoZhangJiePaixu);
// 如果没有结果，max 返回 null，可以设置默认值
$intMaxXiaoShuoZhangJiePaixu = $intMaxXiaoShuoZhangJiePaixu ?? 0;
foreach ($arrZhangjie as $intZhangJieNum => $arrZhangjieOne) {
var_dump($intZhangJieNum);
// 如果数据库里的章节最大索引大于当前索引，说明已经入库，不需要入队列了
// if ($intMaxXiaoShuoZhangJiePaixu > $intZhangJieNum) {
// var_dump("小说章节已经入库，不再推送任务");
// continue; // 如果任务已存在，直接跳过 break
// }
$strZhangJieUrl = $strSiteUrl . $arrZhangjieOne['url'];
$strZhangJieSourceId = extractFilename($strZhangJieUrl);
# 'AILISI_XIAOSHUO_ZHANGJIE_小说源id_小说源章节id'
$strXszjSourceCode = 'gdbzkz_XIAOSHUO_ZHANGJIE_' . $XiaoShuoXiangQingModel->xsxq_source_id . '_' . $strZhangJieSourceId . '_1';
$arrTask = [
'callback' => [self::class, 'getNovelReader'],
'data' => [
'xszj_source_id' => $strZhangJieSourceId,
'xszj_paixu' => $intZhangJieNum += 1,
'xszj_name' => $arrZhangjieOne['mingzi'],
'xszj_url' => $strZhangJieUrl,
'xszj_source_code' => $strXszjSourceCode,
'xsxq_id' => $XiaoShuoXiangQingModel->xsxq_id,
'xsfl_id' => $XiaoShuoXiangQingModel->xsfl_id,
'intZhangJieSort' => 1,
]
];
$TaskCore->set($arrTask);
}
var_dump("章节任务投递成功");
} catch (\Throwable $T) {
throw $T;
}
}
public function getNovelReader($arrData)
{
$TaskCore = $this->TaskCore;
$strSiteUrl = 'https://www.bqvvxg8.cc';
var_dump('getNovelReader------------------------------------------------------------------------------------');
$Client = $this->Client;
$intZhangJieSort = $arrData['intZhangJieSort'];
$strXszjSourceCode = $arrData['xszj_source_code'];
$strZhangJieUrl = $arrData['xszj_url'];
$intZhangJiePaiXu = $arrData['xszj_paixu'];
$intFenleiId = $arrData['xsfl_id'];
$intXiaoShuoId = $arrData['xsxq_id'];
$strZhangJieSourceId = $arrData['xszj_source_id'];
$strZhangJieName = $arrData['xszj_name'];
$XiaoShuoZhangJieModel = XiaoShuoZhangJieModel::where('xszj_source_code', $strXszjSourceCode)->find();
if ($XiaoShuoZhangJieModel) {
// var_dump($XiaoShuoZhangJieModel->xszj_source_code);
// var_dump('章节已经存在 paixu: ' . $intZhangJiePaiXu);
// var_dump('章节已经存在strXszjSourceCode: ' . $strXszjSourceCode);
// var_dump('章节已经存在strZhangJieUrl: ' . $strZhangJieUrl);
return true;
}
var_dump($strZhangJieUrl);
var_dump('章节排序：' . $intZhangJiePaiXu);
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strZhangJieUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
if (XiaoShuoZhangJieModel::checkZhangJieExists($strXszjSourceCode)) {
return true;
}
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strZhangJieUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
'Accept-Charset' => 'GBK, UTF-8, *;q=0.8',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
// 先尝试自动检测编码
$strEncoding = mb_detect_encoding($strResult, ['GBK', 'GB2312', 'BIG5', 'UTF-8'], true);
// 如果不是 UTF-8，转换为 UTF-8
if ($strEncoding && $strEncoding !== 'UTF-8') {
$strResult = mb_convert_encoding($strResult, 'UTF-8', $strEncoding);
} else {
// 备用方案，强制转换
$strResult = @iconv('GBK', 'UTF-8//IGNORE', $strResult);
}
// **去掉可能影响解析的 meta 标签**
$strResult = preg_replace('/<meta[^>]+charset=[^>]+>/i', '', $strResult);
// **第一步：使用 QueryList**
$ZhangJieQueryList = QueryList::html($strResult);
$strNeiRong = '';
// 获取原始 HTML
$strNeiRong = $ZhangJieQueryList->find('#content')->html();
// 解码 HTML 实体
$strNeiRong = html_entity_decode($strNeiRong, ENT_QUOTES, 'UTF-8');
// 替换不间断空格 (\xc2\xa0) 和 &nbsp; 为普通空格
$strNeiRong = preg_replace('/\xc2\xa0/', ' ', $strNeiRong); // 替换 UTF-8 不间断空格
$strNeiRong = preg_replace('/&nbsp;/', ' ', $strNeiRong); // 替换 &nbsp;
// 清理多余空格和换行符
$strNeiRong = preg_replace('/\s+/', ' ', $strNeiRong);
// 1. 找到所有的 <br> 位置
$br_positions = [];
$offset = 0;
while (($pos = strpos($strNeiRong, "<br", $offset)) !== false) {
$br_positions[] = $pos;
$offset = $pos + 1;
}
// 2. 找到倒数第三个 <br> 的位置
if (count($br_positions) >= 3) {
$cut_position = $br_positions[count($br_positions) - 3]; // 倒数第三个 <br> 的索引
$strNeiRong = substr($strNeiRong, 0, $cut_position); // 截取前面部分，删除后面的内容
}
$arrNeiRong = explode('<script>app2();</script><br>', $strNeiRong);
$strNeiRong = $arrNeiRong[1];
$arrNeiRong = explode('<br><br>', $strNeiRong);
$arrEnCode = [];
foreach ($arrNeiRong as $intNeiRongNum => $strNeiRongOne) {
$strEnCode = Base64Helper::encode('<p>' . $strNeiRongOne . '</p>');
$arrEnCode[] = $strEnCode;
}
$strNeiRong = implode(';;;', $arrEnCode);
# '/novel/123yqw/分类id/小说源id/小说源章节id.txt'
$strNeiRongTxtPath = config("filesystem.novel") . '/novel/630zwxs/' . $intFenleiId . '/' . $intXiaoShuoId . '/' . $intZhangJiePaiXu . '/' . $intZhangJieSort . '.txt';
// 调用函数
$result = saveCompressAndDeleteTxt($strNeiRongTxtPath, $strNeiRong);
// 输出状态
if ($result['status']) {
var_dump("Success: " . $result['message']);
} else {
var_dump("Error: " . $result['message']);
}
$arrXiaoShuoZhangJie = [
'xsxq_id' => $intXiaoShuoId,
'xszj_pai_xu' => $intZhangJiePaiXu,
'xszj_ming_zi' => $strZhangJieName,
'xszj_nei_rong' => $strNeiRongTxtPath,
'xszj_source_id' => $strZhangJieSourceId,
'xszj_source_code' => $strXszjSourceCode,
'xszj_sort' => $intZhangJieSort,
];
if ($strZhangJieName) {
XiaoShuoZhangJieModel::addXiaoShuoZhangJie($arrXiaoShuoZhangJie);
}
unset($ZhangJieQueryList);
//sleep(3);
} catch (\Throwable $T) {
$arrTask = [
'callback' => [self::class, 'getNovelReader'],
'data' => [
'xszj_source_id' => $strZhangJieSourceId,
'xszj_paixu' => $intZhangJiePaiXu,
'xszj_name' => $strZhangJieName,
'xszj_url' => $strZhangJieUrl,
'xszj_source_code' => $strXszjSourceCode,
'xsxq_id' => $intXiaoShuoId,
'xsfl_id' => $intFenleiId,
'intZhangJieSort' => $intZhangJieSort,
]
];
$TaskCore->set($arrTask);
throw $T;
}
}
}

137
code/app/task/collection/seo/GenerateSiteMapCol.php Normal file
View File

@@ -0,0 +1,137 @@
<?php
declare(strict_types=1);
namespace app\task\collection\seo;
use app\task\collection\PullDataColBase;
use app\admin\model\VideoInfoModel;
use app\admin\model\SystemConfigModel;
use app\admin\model\VideoClassModel;
class GenerateSiteMapCol extends PullDataColBase
{
public function generate()
{
$strSiteDomain = SystemConfigModel::getValByCode('YONG_JIU_DOMAIN');
// 每个 Sitemap 包含的最大 URL 数量
$chunkSize = 40000;
// 从数据库获取所有视频总数
$totalVideoCount = VideoInfoModel::count();
// 获取分类信息
$categories = VideoClassModel::getDataByCache();
// 生成视频详情的 Sitemap
$videoSitemapFiles = $this->generateVideoSitemaps($chunkSize, $totalVideoCount, $strSiteDomain);
// 生成分类的 Sitemap
$categorySitemapFiles = $this->generateCategorySitemaps($categories, $strSiteDomain);
// 合并所有 Sitemap 文件路径
$allSitemapFiles = array_merge($videoSitemapFiles, $categorySitemapFiles);
// 生成 Sitemap Index 文件
$this->buildSitemapIndex($allSitemapFiles, $strSiteDomain);
var_dump(['message' => '所有 Sitemap 生成成功', 'files' => $allSitemapFiles]);
}
private function generateVideoSitemaps(int $chunkSize, int $totalCount, string $strSiteDomain): array
{
$totalFiles = ceil($totalCount / $chunkSize);
$sitemapFiles = [];
for ($i = 0; $i < $totalFiles; $i++) {
$videos = VideoInfoModel::field('v_pinyin')
->limit($i * $chunkSize, $chunkSize)
->select()
->toArray();
$sitemapXml = $this->buildSitemap($videos, $strSiteDomain);
$filePath = public_path() . "sitemap_video_{$i}.xml";
file_put_contents($filePath, $sitemapXml);
$sitemapFiles[] = $filePath;
}
return $sitemapFiles;
}
private function generateCategorySitemaps(array $categories, string $strSiteDomain): array
{
$sitemapFiles = [];
foreach ($categories as $category) {
$vcId = $category['vc_id'];
$vcNicheng = $category['vc_nicheng'];
$vcName = $category['vc_name'];
$videoCount = VideoInfoModel::where('vc_id', $vcId)->count();
$totalPages = ceil($videoCount / 30);
$categoryUrls = [];
$categoryUrls[] = [
'url' => "/vodtype/{$vcNicheng}.html",
'title' => "{$vcName} - 首页",
'is_home' => true,
];
for ($page = 2; $page <= $totalPages; $page++) {
$categoryUrls[] = [
'url' => "/vodshow/{$vcNicheng}--------{$page}---.html",
'title' => "{$vcName} - 第{$page}页",
'is_home' => false,
];
}
$sitemapXml = $this->buildSitemap($categoryUrls, $strSiteDomain, true);
$filePath = public_path() . "sitemap_category_{$vcNicheng}.xml";
file_put_contents($filePath, $sitemapXml);
$sitemapFiles[] = $filePath;
}
return $sitemapFiles;
}
private function buildSitemap(array $items, string $strSiteDomain, bool $isCategory = false): string
{
$xmlHeader = '<?xml version="1.0" encoding="UTF-8"?>';
$urlsetStart = '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">';
$urlsetEnd = '</urlset>';
$urlEntries = '';
foreach ($items as $item) {
$urlEntries .= '<url>';
$urlEntries .= '<loc>https://' . $strSiteDomain . ($isCategory ? $item['url'] : '/voddetail/' . htmlspecialchars($item['v_pinyin'] . '.html', ENT_QUOTES)) . '</loc>';
$urlEntries .= '<changefreq>daily</changefreq>';
$urlEntries .= '<priority>0.8</priority>';
$urlEntries .= '</url>';
}
return $xmlHeader . $urlsetStart . $urlEntries . $urlsetEnd;
}
private function buildSitemapIndex(array $sitemapFiles, string $strSiteDomain): void
{
$xmlHeader = '<?xml version="1.0" encoding="UTF-8"?>';
$sitemapIndexStart = '<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">';
$sitemapIndexEnd = '</sitemapindex>';
$sitemapEntries = '';
foreach ($sitemapFiles as $file) {
$url = 'https://' . $strSiteDomain . '/' . basename($file);
$sitemapEntries .= '<sitemap>';
$sitemapEntries .= '<loc>' . htmlspecialchars($url, ENT_QUOTES) . '</loc>';
$sitemapEntries .= '<lastmod>' . date('Y-m-d') . '</lastmod>';
$sitemapEntries .= '</sitemap>';
}
$sitemapIndexXml = $xmlHeader . $sitemapIndexStart . $sitemapEntries . $sitemapIndexEnd;
file_put_contents(public_path() . 'sitemap_index.xml', $sitemapIndexXml);
var_dump(['message' => 'Sitemap Index 生成成功', 'file' => 'sitemap_index.xml']);
}
}

188
code/app/task/collection/seo/GenerateSiteMapNovel.php Normal file
View File

@@ -0,0 +1,188 @@
<?php
declare(strict_types=1);
namespace app\task\collection\seo;
use app\task\collection\PullDataColBase;
use app\admin\model\XiaoShuoXiangQingModel;
use app\admin\model\XiaoShuoZhangJieModel;
use app\admin\model\SystemConfigModel;
use app\admin\model\SiteModel;
use app\admin\model\XiaoShuoFenLeiModel;
class GenerateSiteMapNovel extends PullDataColBase
{
public function generate()
{
$SiteModel = SiteModel::getValById(config('app.default_app_id'));
$strSiteDomain = $SiteModel['site_domain'];
$strFenleiPath = $SiteModel['site_fenlei_path'];
$strXqPath = $SiteModel['site_xiangqing_path'];
$strRankPath = $SiteModel['site_rank_path'];
$strTjPath = $SiteModel['site_tui_jian_path'];
$strSeoPath = $SiteModel['site_seo_path'];
$strZdyPath = $SiteModel['site_zdy_path'];
$strZjPath = $SiteModel['site_zhangjie_path'];
// 每个 Sitemap 包含的最大 URL 数量
$chunkSize = 40000;
// 从数据库获取所有总数
$totalNovelCount = XiaoShuoXiangQingModel::count();
// 从数据库获取所有总数
$totalNovelZjCount = XiaoShuoZhangJieModel::count();
// 获取分类信息
$categories = XiaoShuoFenLeiModel::getDataByCache();
// 生成详情的 Sitemap
$novelSitemapFiles = $this->generateNovelSitemaps($chunkSize, $totalNovelCount, $strSiteDomain);
// 生成章节的 Sitemap
$novelZjSitemapFiles = $this->generateNovelZjSitemaps($chunkSize, $totalNovelZjCount, $strSiteDomain);
// 生成分类的 Sitemap
$categorySitemapFiles = $this->generateCategorySitemaps($categories, $strSiteDomain);
// 合并所有 Sitemap 文件路径
$allSitemapFiles = array_merge($novelSitemapFiles, $categorySitemapFiles,$novelZjSitemapFiles);
// 生成 Sitemap Index 文件
$this->buildSitemapIndex($allSitemapFiles, $strSiteDomain);
var_dump(['message' => '所有 Sitemap 生成成功', 'files' => $allSitemapFiles]);
}
private function generateNovelZjSitemaps(int $chunkSize, int $totalCount, string $strSiteDomain): array
{
$totalFiles = ceil($totalCount / $chunkSize);
$sitemapFiles = [];
for ($i = 0; $i < $totalFiles; $i++) {
$Novels = XiaoShuoZhangJieModel::field(['xsxq_id','xszj_id']) //intNovelId-:intNovelSourceId-:intNovelSourceSeoId
->limit($i * $chunkSize, $chunkSize)
->select()
->toArray();
$sitemapXml = $this->buildZjSitemap($Novels, $strSiteDomain);
$filePath = public_path() . "sitemap_novel_zj_{$i}.xml";
file_put_contents($filePath, $sitemapXml);
$sitemapFiles[] = $filePath;
}
return $sitemapFiles;
}
private function generateNovelSitemaps(int $chunkSize, int $totalCount, string $strSiteDomain): array
{
$totalFiles = ceil($totalCount / $chunkSize);
$sitemapFiles = [];
for ($i = 0; $i < $totalFiles; $i++) {
$Novels = XiaoShuoXiangQingModel::field(['xsxq_id','xsxq_source_id','xsxq_source_seo_id']) //intNovelId-:intNovelSourceId-:intNovelSourceSeoId
->limit($i * $chunkSize, $chunkSize)
->select()
->toArray();
$sitemapXml = $this->buildSitemap($Novels, $strSiteDomain);
$filePath = public_path() . "sitemap_novel_{$i}.xml";
file_put_contents($filePath, $sitemapXml);
$sitemapFiles[] = $filePath;
}
return $sitemapFiles;
}
private function generateCategorySitemaps(array $categories, string $strSiteDomain): array
{
$sitemapFiles = [];
foreach ($categories as $category) {
$vcId = $category['xsfl_id'];
$vcName = $category['xsfl_name'];
$intNovelCount = XiaoShuoFenLeiModel::where('xsfl_id', $vcId)->count();
$totalPages = ceil($intNovelCount / 30);
$categoryUrls = [];
for ($type = 0; $type <= 3; $type++) {
for ($page = 1; $page <= $totalPages; $page++) {
$categoryUrls[] = [
// /fenlei/1-0-1.html
'url' => "/fenlei/{$vcId}-{$type}-{$page}.html",
'title' => "{$vcName} - 第{$page}页",
'is_home' => false,
];
}
}
$sitemapXml = $this->buildSitemap($categoryUrls, $strSiteDomain, true);
$filePath = public_path() . "sitemap_category_{$vcId}.xml";
file_put_contents($filePath, $sitemapXml);
$sitemapFiles[] = $filePath;
}
return $sitemapFiles;
}
private function buildZjSitemap(array $items, string $strSiteDomain, bool $isCategory = false): string
{
$xmlHeader = '<?xml version="1.0" encoding="UTF-8"?>';
$urlsetStart = '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">';
$urlsetEnd = '</urlset>';
// zj/18648-206.html
$urlEntries = '';
foreach ($items as $item) {
$urlEntries .= '<url>';
$urlEntries .= '<loc>https://' . $strSiteDomain . ($isCategory ? $item['url'] : '/zj/' . htmlspecialchars($item['xsxq_id'].'-'.$item['xszj_id']. '.html', ENT_QUOTES)) . '</loc>';
$urlEntries .= '<changefreq>daily</changefreq>';
$urlEntries .= '<priority>0.8</priority>';
$urlEntries .= '</url>';
}
return $xmlHeader . $urlsetStart . $urlEntries . $urlsetEnd;
}
private function buildSitemap(array $items, string $strSiteDomain, bool $isCategory = false): string
{
$xmlHeader = '<?xml version="1.0" encoding="UTF-8"?>';
$urlsetStart = '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">';
$urlsetEnd = '</urlset>';
// /xq/20052-16107-161071.html
$urlEntries = '';
foreach ($items as $item) {
$urlEntries .= '<url>';
$urlEntries .= '<loc>https://' . $strSiteDomain . ($isCategory ? $item['url'] : '/xq/' . htmlspecialchars($item['xsxq_id'].'-'.$item['xsxq_source_id'].'-'.$item['xsxq_source_seo_id']. '.html', ENT_QUOTES)) . '</loc>';
$urlEntries .= '<changefreq>daily</changefreq>';
$urlEntries .= '<priority>1.0</priority>';
$urlEntries .= '</url>';
}
return $xmlHeader . $urlsetStart . $urlEntries . $urlsetEnd;
}
private function buildSitemapIndex(array $sitemapFiles, string $strSiteDomain): void
{
$xmlHeader = '<?xml version="1.0" encoding="UTF-8"?>';
$sitemapIndexStart = '<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">';
$sitemapIndexEnd = '</sitemapindex>';
$sitemapEntries = '';
foreach ($sitemapFiles as $file) {
$url = 'https://' . $strSiteDomain . '/' . basename($file);
$sitemapEntries .= '<sitemap>';
$sitemapEntries .= '<loc>' . htmlspecialchars($url, ENT_QUOTES) . '</loc>';
$sitemapEntries .= '<lastmod>' . date('Y-m-d') . '</lastmod>';
$sitemapEntries .= '</sitemap>';
}
$sitemapIndexXml = $xmlHeader . $sitemapIndexStart . $sitemapEntries . $sitemapIndexEnd;
file_put_contents(public_path() . 'sitemap_index.xml', $sitemapIndexXml);
var_dump(['message' => 'Sitemap Index 生成成功', 'file' => 'sitemap_index.xml']);
}
}

242
code/app/task/collection/seo/GenerateSiteMapNovelSk.php Normal file
View File

@@ -0,0 +1,242 @@
<?php
declare(strict_types=1);
namespace app\task\collection\seo;
use app\task\collection\PullDataColBase;
use app\admin\model\XiaoShuoXiangQingModel;
use app\admin\model\NovelInfoSeoModel;
use app\admin\model\XiaoShuoZhangJieModel;
use app\admin\model\SiteModel;
use app\admin\model\XiaoShuoFenLeiModel;
class GenerateSiteMapNovelSk extends PullDataColBase
{
public function generate()
{
$SiteModel = SiteModel::getValById(config('app.default_app_id'));
// $strSiteDomain = SiteModel::getValBySiteIdByValCode('site_domain');
$strSiteDomain = $SiteModel['site_domain'];
$strFenleiPath = $SiteModel['site_fenlei_path'];
$strXqPath = $SiteModel['site_xiangqing_path'];
$strRankPath = $SiteModel['site_rank_path'];
$strTjPath = $SiteModel['site_tui_jian_path'];
$strSeoPath = $SiteModel['site_seo_path'];
$strZdyPath = $SiteModel['site_zdy_path'];
$strZjPath = $SiteModel['site_zhangjie_path'];
// 每个 Sitemap 包含的最大 URL 数量
$chunkSize = 40000;
// 从数据库获取所有总数
$totalNovelCount = XiaoShuoXiangQingModel::count();
// 从数据库获取所有总数
$totalNovelSeoCount = NovelInfoSeoModel::count();
// 从数据库获取所有总数
$totalNovelZjCount = XiaoShuoZhangJieModel::count();
// 获取分类信息
$categories = XiaoShuoFenLeiModel::getDataByCache();
// 生成详情的 Sitemap 小说详情
$novelSitemapFiles = $this->generateNovelSitemaps($chunkSize, $totalNovelCount, $strSiteDomain, $strXqPath);
// 生成详情的 Sitemap 小说seo
$novelSeoSitemapFiles = $this->generateNovelSeoSitemaps($chunkSize, $totalNovelSeoCount, $strSiteDomain, $strSeoPath);
// 生成章节的 Sitemap 小说章节
$novelZjSitemapFiles = $this->generateNovelZjSitemaps($chunkSize, $totalNovelZjCount, $strSiteDomain, $strXqPath);
// 生成分类的 Sitemap 小说分类
$categorySitemapFiles = $this->generateCategorySitemaps($categories, $strSiteDomain, $strFenleiPath, $strXqPath);
// 合并所有 Sitemap 文件路径
$allSitemapFiles = array_merge($novelSitemapFiles, $novelSeoSitemapFiles, $categorySitemapFiles, $novelZjSitemapFiles);
// 生成 Sitemap Index 文件
$this->buildSitemapIndex($allSitemapFiles, $strSiteDomain);
var_dump(['message' => '所有 Sitemap 生成成功', 'files' => $allSitemapFiles]);
}
# 生成小说章节 xml文件
private function generateNovelZjSitemaps(int $chunkSize, int $totalCount, string $strSiteDomain, string $strXqPath): array
{
$totalFiles = ceil($totalCount / $chunkSize);
$sitemapFiles = [];
for ($i = 0; $i < $totalFiles; $i++) {
$Novels = XiaoShuoZhangJieModel::field(['xsxq_id', 'xszj_id']) //intNovelId-:intNovelSourceId-:intNovelSourceSeoId
->limit($i * $chunkSize, $chunkSize)
->select()
->toArray();
$sitemapXml = $this->buildZjSitemap($Novels, $strSiteDomain, $strXqPath);
$filePath = public_path() . "sitemap_novel_zj_{$i}.xml";
file_put_contents($filePath, $sitemapXml);
$sitemapFiles[] = $filePath;
}
return $sitemapFiles;
}
# 生成小说详情 xml文件
private function generateNovelSitemaps(int $chunkSize, int $totalCount, string $strSiteDomain, string $strXqPath): array
{
$totalFiles = ceil($totalCount / $chunkSize);
$sitemapFiles = [];
for ($i = 0; $i < $totalFiles; $i++) {
$Novels = XiaoShuoXiangQingModel::field(['xsxq_id', 'xsxq_source_id', 'xsxq_source_seo_id']) //intNovelId-:intNovelSourceId-:intNovelSourceSeoId
->limit($i * $chunkSize, $chunkSize)
->select()
->toArray();
$sitemapXml = $this->buildSitemap($Novels, $strSiteDomain, $strXqPath);
$filePath = public_path() . "sitemap_novel_{$i}.xml";
file_put_contents($filePath, $sitemapXml);
$sitemapFiles[] = $filePath;
}
return $sitemapFiles;
}
# 生成小说详情 xml文件
private function generateNovelSeoSitemaps(int $chunkSize, int $totalCount, string $strSiteDomain, string $strSeoPath): array
{
$totalFiles = ceil($totalCount / $chunkSize);
$sitemapFiles = [];
for ($i = 0; $i < $totalFiles; $i++) {
$Novels = NovelInfoSeoModel::field(['xsxq_id', 'nis_id']) //show/nis_id
->limit($i * $chunkSize, $chunkSize)
->select()
->toArray();
$sitemapXml = $this->buildXiaoShuoSeoSitemap($Novels, $strSiteDomain, $strSeoPath);
$filePath = public_path() . "sitemap_novel_seo_{$i}.xml";
file_put_contents($filePath, $sitemapXml);
$sitemapFiles[] = $filePath;
}
return $sitemapFiles;
}
// 生成分类链接
private function generateCategorySitemaps(array $categories, string $strSiteDomain, string $strFenleiPath, string $strXqPath): array
{
$sitemapFiles = [];
foreach ($categories as $category) {
$vcId = $category['xsfl_id'];
$vcName = $category['xsfl_name'];
$intNovelCount = XiaoShuoFenLeiModel::where('xsfl_id', $vcId)->count();
$totalPages = ceil($intNovelCount / 30);
$categoryUrls = [];
for ($status = 0; $status <= 2; $status++) {
for ($sort = 0; $sort <= 4; $sort++) {
for ($page = 1; $page <= $totalPages; $page++) {
$categoryUrls[] = [
// /fl/:intCId-:intSort-[:intPage]
'url' => "/{$strFenleiPath}/{$vcId}-{$sort}-{$page}.html",
'title' => "{$vcName} - 第{$page}页",
'is_home' => false,
];
}
}
}
$sitemapXml = $this->buildSitemap($categoryUrls, $strSiteDomain, $strXqPath, true);
$filePath = public_path() . "sitemap_category_{$vcId}.xml";
file_put_contents($filePath, $sitemapXml);
$sitemapFiles[] = $filePath;
}
return $sitemapFiles;
}
//生成章节链接
private function buildZjSitemap(array $items, string $strSiteDomain, string $strXqPath, bool $isCategory = false): string
{
$xmlHeader = '<?xml version="1.0" encoding="UTF-8"?>';
$urlsetStart = '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">';
$urlsetEnd = '</urlset>';
// /xq/:intNovelId/:intZhangJiePaiXu-[:strSort]
$urlEntries = '';
foreach ($items as $item) {
$urlEntries .= '<url>';
$urlEntries .= '<loc>https://' . $strSiteDomain . ($isCategory ? $item['url'] : '/' . $strXqPath . '/' . htmlspecialchars($item['xsxq_id'] . '-' . $item['xszj_pai_xu']. '-' . $item['xszj_sort'] . '.html', ENT_QUOTES)) . '</loc>';
$urlEntries .= '<changefreq>daily</changefreq>';
$urlEntries .= '<priority>0.8</priority>';
$urlEntries .= '</url>';
}
return $xmlHeader . $urlsetStart . $urlEntries . $urlsetEnd;
}
// 生成 详情链接
private function buildSitemap(array $items, string $strSiteDomain, string $strXqPath, bool $isCategory = false): string
{
$xmlHeader = '<?xml version="1.0" encoding="UTF-8"?>';
$urlsetStart = '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">';
$urlsetEnd = '</urlset>';
// /xq/:intNovelId
$urlEntries = '';
foreach ($items as $item) {
$urlEntries .= '<url>';
$urlEntries .= '<loc>https://' . $strSiteDomain . ($isCategory ? $item['url'] : '/' . $strXqPath . '/' . htmlspecialchars($item['xsxq_id'] . '.html', ENT_QUOTES)) . '</loc>';
$urlEntries .= '<changefreq>daily</changefreq>';
$urlEntries .= '<priority>1.0</priority>';
$urlEntries .= '</url>';
}
return $xmlHeader . $urlsetStart . $urlEntries . $urlsetEnd;
}
// 生成详情 seo 链接
private function buildXiaoShuoSeoSitemap(array $items, string $strSiteDomain, string $strSeoPath, bool $isCategory = false): string
{
$xmlHeader = '<?xml version="1.0" encoding="UTF-8"?>';
$urlsetStart = '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">';
$urlsetEnd = '</urlset>';
// /show/:intNovelId
$urlEntries = '';
foreach ($items as $item) {
$urlEntries .= '<url>';
$urlEntries .= '<loc>https://' . $strSiteDomain . ($isCategory ? $item['url'] : '/' . $strSeoPath . '/' . htmlspecialchars($item['nis_id'] . '.html', ENT_QUOTES)) . '</loc>';
$urlEntries .= '<changefreq>daily</changefreq>';
$urlEntries .= '<priority>1.0</priority>';
$urlEntries .= '</url>';
}
return $xmlHeader . $urlsetStart . $urlEntries . $urlsetEnd;
}
// 生成 xml 文件
private function buildSitemapIndex(array $sitemapFiles, string $strSiteDomain): void
{
$xmlHeader = '<?xml version="1.0" encoding="UTF-8"?>';
$sitemapIndexStart = '<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">';
$sitemapIndexEnd = '</sitemapindex>';
$sitemapEntries = '';
foreach ($sitemapFiles as $file) {
$url = 'https://' . $strSiteDomain . '/' . basename($file);
$sitemapEntries .= '<sitemap>';
$sitemapEntries .= '<loc>' . htmlspecialchars($url, ENT_QUOTES) . '</loc>';
$sitemapEntries .= '<lastmod>' . date('Y-m-d') . '</lastmod>';
$sitemapEntries .= '</sitemap>';
}
$sitemapIndexXml = $xmlHeader . $sitemapIndexStart . $sitemapEntries . $sitemapIndexEnd;
file_put_contents(public_path() . 'sitemap_index.xml', $sitemapIndexXml);
var_dump(['message' => 'Sitemap Index 生成成功', 'file' => 'sitemap_index.xml']);
}
}

242
code/app/task/collection/seo/GenerateSiteMapNovelZw.php Normal file
View File

@@ -0,0 +1,242 @@
<?php
declare(strict_types=1);
namespace app\task\collection\seo;
use app\task\collection\PullDataColBase;
use app\admin\model\XiaoShuoXiangQingModel;
use app\admin\model\NovelInfoSeoModel;
use app\admin\model\XiaoShuoZhangJieModel;
use app\admin\model\SiteModel;
use app\admin\model\XiaoShuoFenLeiModel;
class GenerateSiteMapNovelZw extends PullDataColBase
{
public function generate()
{
$SiteModel = SiteModel::getValById(config('app.default_app_id'));
// $strSiteDomain = SiteModel::getValBySiteIdByValCode('site_domain');
$strSiteDomain = $SiteModel['site_domain'];
$strFenleiPath = $SiteModel['site_fenlei_path'];
$strXqPath = $SiteModel['site_xiangqing_path'];
$strRankPath = $SiteModel['site_rank_path'];
$strTjPath = $SiteModel['site_tui_jian_path'];
$strSeoPath = $SiteModel['site_seo_path'];
$strZdyPath = $SiteModel['site_zdy_path'];
$strZjPath = $SiteModel['site_zhangjie_path'];
// 每个 Sitemap 包含的最大 URL 数量
$chunkSize = 40000;
// 从数据库获取所有总数
$totalNovelCount = XiaoShuoXiangQingModel::count();
// 从数据库获取所有总数
$totalNovelSeoCount = NovelInfoSeoModel::count();
// 从数据库获取所有总数
$totalNovelZjCount = XiaoShuoZhangJieModel::count();
// 获取分类信息
$categories = XiaoShuoFenLeiModel::getDataByCache();
// 生成详情的 Sitemap 小说详情
$novelSitemapFiles = $this->generateNovelSitemaps($chunkSize, $totalNovelCount, $strSiteDomain, $strXqPath);
// 生成详情的 Sitemap 小说seo
$novelSeoSitemapFiles = $this->generateNovelSeoSitemaps($chunkSize, $totalNovelSeoCount, $strSiteDomain, $strSeoPath);
// 生成章节的 Sitemap 小说章节
$novelZjSitemapFiles = $this->generateNovelZjSitemaps($chunkSize, $totalNovelZjCount, $strSiteDomain, $strXqPath);
// 生成分类的 Sitemap 小说分类
$categorySitemapFiles = $this->generateCategorySitemaps($categories, $strSiteDomain, $strFenleiPath, $strXqPath);
// 合并所有 Sitemap 文件路径
$allSitemapFiles = array_merge($novelSitemapFiles, $novelSeoSitemapFiles, $categorySitemapFiles, $novelZjSitemapFiles);
// 生成 Sitemap Index 文件
$this->buildSitemapIndex($allSitemapFiles, $strSiteDomain);
var_dump(['message' => '所有 Sitemap 生成成功', 'files' => $allSitemapFiles]);
}
# 生成小说章节 xml文件
private function generateNovelZjSitemaps(int $chunkSize, int $totalCount, string $strSiteDomain, string $strXqPath): array
{
$totalFiles = ceil($totalCount / $chunkSize);
$sitemapFiles = [];
for ($i = 0; $i < $totalFiles; $i++) {
$Novels = XiaoShuoZhangJieModel::field(['xsxq_id', 'xszj_id']) //intNovelId-:intNovelSourceId-:intNovelSourceSeoId
->limit($i * $chunkSize, $chunkSize)
->select()
->toArray();
$sitemapXml = $this->buildZjSitemap($Novels, $strSiteDomain, $strXqPath);
$filePath = public_path() . "sitemap_novel_zj_{$i}.xml";
file_put_contents($filePath, $sitemapXml);
$sitemapFiles[] = $filePath;
}
return $sitemapFiles;
}
# 生成小说详情 xml文件
private function generateNovelSitemaps(int $chunkSize, int $totalCount, string $strSiteDomain, string $strXqPath): array
{
$totalFiles = ceil($totalCount / $chunkSize);
$sitemapFiles = [];
for ($i = 0; $i < $totalFiles; $i++) {
$Novels = XiaoShuoXiangQingModel::field(['xsxq_id', 'xsxq_source_id', 'xsxq_source_seo_id']) //intNovelId-:intNovelSourceId-:intNovelSourceSeoId
->limit($i * $chunkSize, $chunkSize)
->select()
->toArray();
$sitemapXml = $this->buildSitemap($Novels, $strSiteDomain, $strXqPath);
$filePath = public_path() . "sitemap_novel_{$i}.xml";
file_put_contents($filePath, $sitemapXml);
$sitemapFiles[] = $filePath;
}
return $sitemapFiles;
}
# 生成小说详情 xml文件
private function generateNovelSeoSitemaps(int $chunkSize, int $totalCount, string $strSiteDomain, string $strSeoPath): array
{
$totalFiles = ceil($totalCount / $chunkSize);
$sitemapFiles = [];
for ($i = 0; $i < $totalFiles; $i++) {
$Novels = NovelInfoSeoModel::field(['xsxq_id', 'nis_id']) //show/nis_id
->limit($i * $chunkSize, $chunkSize)
->select()
->toArray();
$sitemapXml = $this->buildXiaoShuoSeoSitemap($Novels, $strSiteDomain, $strSeoPath);
$filePath = public_path() . "sitemap_novel_seo_{$i}.xml";
file_put_contents($filePath, $sitemapXml);
$sitemapFiles[] = $filePath;
}
return $sitemapFiles;
}
// 生成分类链接
private function generateCategorySitemaps(array $categories, string $strSiteDomain, string $strFenleiPath, string $strXqPath): array
{
$sitemapFiles = [];
foreach ($categories as $category) {
$vcId = $category['xsfl_id'];
$vcName = $category['xsfl_name'];
$intNovelCount = XiaoShuoFenLeiModel::where('xsfl_id', $vcId)->count();
$totalPages = ceil($intNovelCount / 30);
$categoryUrls = [];
for ($status = 0; $status <= 2; $status++) {
for ($sort = 0; $sort <= 4; $sort++) {
for ($page = 1; $page <= $totalPages; $page++) {
$categoryUrls[] = [
// /fl/:intCId-:intSort-[:intPage]
'url' => "/{$strFenleiPath}/{$vcId}-{$sort}-{$page}.html",
'title' => "{$vcName} - 第{$page}页",
'is_home' => false,
];
}
}
}
$sitemapXml = $this->buildSitemap($categoryUrls, $strSiteDomain, $strXqPath, true);
$filePath = public_path() . "sitemap_category_{$vcId}.xml";
file_put_contents($filePath, $sitemapXml);
$sitemapFiles[] = $filePath;
}
return $sitemapFiles;
}
//生成章节链接
private function buildZjSitemap(array $items, string $strSiteDomain, string $strXqPath, bool $isCategory = false): string
{
$xmlHeader = '<?xml version="1.0" encoding="UTF-8"?>';
$urlsetStart = '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">';
$urlsetEnd = '</urlset>';
// /xq/:intNovelId/:intZhangJiePaiXu-[:strSort]
$urlEntries = '';
foreach ($items as $item) {
$urlEntries .= '<url>';
$urlEntries .= '<loc>https://' . $strSiteDomain . ($isCategory ? $item['url'] : '/' . $strXqPath . '/' . htmlspecialchars($item['xsxq_id'] . '-' . $item['xszj_pai_xu']. '-' . $item['xszj_sort'] . '.html', ENT_QUOTES)) . '</loc>';
$urlEntries .= '<changefreq>daily</changefreq>';
$urlEntries .= '<priority>0.8</priority>';
$urlEntries .= '</url>';
}
return $xmlHeader . $urlsetStart . $urlEntries . $urlsetEnd;
}
// 生成 详情链接
private function buildSitemap(array $items, string $strSiteDomain, string $strXqPath, bool $isCategory = false): string
{
$xmlHeader = '<?xml version="1.0" encoding="UTF-8"?>';
$urlsetStart = '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">';
$urlsetEnd = '</urlset>';
// /xq/:intNovelId
$urlEntries = '';
foreach ($items as $item) {
$urlEntries .= '<url>';
$urlEntries .= '<loc>https://' . $strSiteDomain . ($isCategory ? $item['url'] : '/' . $strXqPath . '/' . htmlspecialchars($item['xsxq_id'] . '.html', ENT_QUOTES)) . '</loc>';
$urlEntries .= '<changefreq>daily</changefreq>';
$urlEntries .= '<priority>1.0</priority>';
$urlEntries .= '</url>';
}
return $xmlHeader . $urlsetStart . $urlEntries . $urlsetEnd;
}
// 生成详情 seo 链接
private function buildXiaoShuoSeoSitemap(array $items, string $strSiteDomain, string $strSeoPath, bool $isCategory = false): string
{
$xmlHeader = '<?xml version="1.0" encoding="UTF-8"?>';
$urlsetStart = '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">';
$urlsetEnd = '</urlset>';
// /show/:intNovelId
$urlEntries = '';
foreach ($items as $item) {
$urlEntries .= '<url>';
$urlEntries .= '<loc>https://' . $strSiteDomain . ($isCategory ? $item['url'] : '/' . $strSeoPath . '/' . htmlspecialchars($item['nis_id'] . '.html', ENT_QUOTES)) . '</loc>';
$urlEntries .= '<changefreq>daily</changefreq>';
$urlEntries .= '<priority>1.0</priority>';
$urlEntries .= '</url>';
}
return $xmlHeader . $urlsetStart . $urlEntries . $urlsetEnd;
}
// 生成 xml 文件
private function buildSitemapIndex(array $sitemapFiles, string $strSiteDomain): void
{
$xmlHeader = '<?xml version="1.0" encoding="UTF-8"?>';
$sitemapIndexStart = '<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">';
$sitemapIndexEnd = '</sitemapindex>';
$sitemapEntries = '';
foreach ($sitemapFiles as $file) {
$url = 'https://' . $strSiteDomain . '/' . basename($file);
$sitemapEntries .= '<sitemap>';
$sitemapEntries .= '<loc>' . htmlspecialchars($url, ENT_QUOTES) . '</loc>';
$sitemapEntries .= '<lastmod>' . date('Y-m-d') . '</lastmod>';
$sitemapEntries .= '</sitemap>';
}
$sitemapIndexXml = $xmlHeader . $sitemapIndexStart . $sitemapEntries . $sitemapIndexEnd;
file_put_contents(public_path() . 'sitemap_index.xml', $sitemapIndexXml);
var_dump(['message' => 'Sitemap Index 生成成功', 'file' => 'sitemap_index.xml']);
}
}

538
code/app/task/collection/tianlai/TianLaiXiaoShuoCol.php Normal file
View File

@@ -0,0 +1,538 @@
<?php
declare(strict_types=1);
namespace app\task\collection\tianlai;
use app\admin\model\XiaoShuoFenLeiModel;
use app\admin\model\XiaoShuoXiangQingModel;
use app\admin\model\XiaoShuoZhangJieModel;
use app\task\collection\PullDataColBase;
use GuzzleHttp\Cookie\CookieJar;
use processor\ImageProcessor;
use think\facade\Cache;
use QL\QueryList;
/**
* @mixin think\Model
*/
class TianLaiXiaoShuoCol extends PullDataColBase
{
public function pullXiaoShuo()
{
$TaskCore = $this->TaskCore;
$strUrl = $this->getTianLaiDomain();
var_dump($strUrl);
$Client = $this->Client;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
// $proxy = '127.0.0.1:8080'; // 代理地址
// $response = fetchContentWithCurl($strUrl, [], 10, $proxy);
$strResult = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$strResult->getBody();
var_dump($strResult);
// exit;
// $strResult = $response['data'];
$QueryList = QueryList::html($strResult);
$QueryList->find('.nav a')->map(function ($FenLeiQueryList) use ($strUrl, $TaskCore) {
$strName = trim($FenLeiQueryList->text());
switch ($strName) {
case '都市言情':
$strName = "都市言情";
break;
case '历史军事':
$strName = "历史军事";
break;
case '玄幻奇幻':
$strName = "玄幻奇幻";
break;
case '武侠修真':
$strName = "武侠修真";
break;
case '女生纯爱':
$strName = "女频言情";
break;
case '科幻网游':
$strName = "科幻网游";
break;
case '悬疑灵异':
$strName = "悬疑灵异";
break;
default:
$strName = "其它";
break;
}
$strFenLeiUrl = $strUrl . $FenLeiQueryList->attr('href');
// $intFenLeiId = extractFilename($strFenLeiUrl."index.html");
$intFenLeiId = extractNumberFromUrl($strFenLeiUrl);
if($intFenLeiId !== null){
$arrClass = [
'xsfl_source_id' => $intFenLeiId,
'xsfl_source_code' => 'TIANLAI_FEN_LEI_ID_' . $intFenLeiId,
'xsfl_name' => $strName,
];
var_dump($arrClass);
$XiaoShuoFenLeiModel = XiaoShuoFenLeiModel::addXiaoShuoFenLei($arrClass);
$arrTask = [
'callback' => [self::class, 'getNovelPage'],
'data' => [
'url' => $strFenLeiUrl,
'xsfl_source_id' => $intFenLeiId,
'xsfl_name' => $strName,
'xsfl_id' => $XiaoShuoFenLeiModel->xsfl_id,
'xsfl_page' => 1,
'xsfl_max_page' => 3000,
]
];
$TaskCore->set($arrTask);
}
});
} catch (\Throwable $T) {
//var_dump($T);
throw $T;
}
}
public function getNovelPage($arrData)
{
var_dump('getNovelPage');
var_dump($arrData);
$strBasePage = $arrData['url'];
$Client = $this->Client;
$TaskCore = $this->TaskCore;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strBasePage);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$strSiteUrl = $this->getTianLaiDomain();
$intPage = $arrData['xsfl_page'];
$intMaxPage = $arrData['xsfl_max_page'];
$strUrl = $strBasePage;
var_dump($strUrl);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$strResult = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => $userAgent
],
'cookies' => $CookieJar,
#'proxy' => $proxy,
'timeout' => 10,
]);
$strResult = (string)$strResult->getBody();
$QueryList = QueryList::html($strResult);
$intMaxPage = $QueryList->find('#newscontent h2 span')->eq(1)->text();
$intMaxPage = ceil($intMaxPage / 100);
var_dump($intMaxPage);
$intMaxPage = 2;
var_dump('$strUrl');
// 如果是首页 则根据最大页面吧所有分页 追加到任务
if($intPage == 1){
for ($intCurrentPage = 2; $intCurrentPage <= $intMaxPage; $intCurrentPage++) {
$strNewsFenleiUrl = replaceLastPageNumber($strBasePage, $intCurrentPage);
var_dump($strNewsFenleiUrl);
$arrTask = [
'callback' => [self::class, 'getNovelPage'],
'data' => [
'url' => $strNewsFenleiUrl,
'xsfl_source_id' => $arrData['xsfl_source_id'],
'xsfl_name' => $arrData['xsfl_name'],
'xsfl_id' => $arrData['xsfl_id'],
'xsfl_page' => $intCurrentPage,
'xsfl_max_page' => $intMaxPage,
]
];
$TaskCore->set($arrTask);
}
}
var_dump('提取小说');
$XiaoShuoListQueryList = $QueryList->find('#newscontent li');
if (!$XiaoShuoListQueryList || $XiaoShuoListQueryList->count() <= 0) {
var_dump('这个分类已经市最后一页');
return true;
}
var_dump($XiaoShuoListQueryList->count());
var_dump('提取小说2');
// var_dump($XiaoShuoListQueryList);
$XiaoShuoListQueryList->map(function ($XiaoShuoQueryList) use ($strSiteUrl, $arrData, $TaskCore, $strUrl) {
// 提取小说链接
// var_dump($XiaoShuoQueryList->html());exit;
var_dump('提取小说链接');
$A = $XiaoShuoQueryList->find('a')->eq(0);
$strXiaoShuoUrl = $A->attr('href');
$strXiaoShuoUrl = strpos($strXiaoShuoUrl, 'https:') === 0 ? $strXiaoShuoUrl : 'https:' . $strXiaoShuoUrl;
var_dump('$A');
var_dump($A);
var_dump($strXiaoShuoUrl);
exit;
// var_dump($A);
// 提取作者信息
$strZuoZhe = trim($XiaoShuoQueryList->find('.s4')->eq(0)->text());
// 提取小说 ID
$arrXiaoShuoId = extractNumbersFromUrl($strXiaoShuoUrl);
if (!isset($arrXiaoShuoId['first'], $arrXiaoShuoId['second'])) {
return; // 如果 ID 提取失败，直接跳过
}
// 准备任务数据
$arrXiaoShuo = array_merge($arrData, [
'xiaoshuo_url' => $strXiaoShuoUrl,
'xiaoshuo_id' => $arrXiaoShuoId['first'],
'xiaoshuo_seo_id' => $arrXiaoShuoId['second'],
'xiaoshuo_zuozhe' => $strZuoZhe,
'url' => $strUrl
]);
var_dump($arrXiaoShuo);
$arrTask = [
'callback' => [self::class, 'getNovelInfo'],
'data' => $arrXiaoShuo,
];
// 将任务加入队列
$TaskCore->set($arrTask);
var_dump("将任务加入队列getNovelInfo-{$strXiaoShuoUrl}");
});
var_dump('提取小说end');
} catch (\Throwable $T) {
throw $T;
}
}
public function getNovelInfo($arrData)
{
var_dump('getNovelInfo');
var_dump($arrData);
$Client = $this->Client;
$TaskCore = $this->TaskCore;
// 获取 Redis 实例
$redis = Cache::store('redis')->handler();
$intXiaoShuoSourceId = $arrData['xiaoshuo_id'];
$intXiaoShuoSourceSeoId = $arrData['xiaoshuo_seo_id'];
$redisKey = "task:novel_source_id:" . $intXiaoShuoSourceId;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($arrData['xiaoshuo_url']);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$strSiteUrl = $this->getTianLaiDomain();
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $arrData['xiaoshuo_url'], [
'headers' => [
'User-Agent' => $userAgent
],
'cookies' => $CookieJar,
#'proxy' => $proxy,
'timeout' => 10,
]);
$strResult = (string)$Response->getBody();
unset($Response);
$QueryList = QueryList::html($strResult);
$strMingZi = $QueryList->find('#info h1')->eq(0)->text();
$strZhuangTai = "状态：连载中";
$strZiShu = 0;
$strZhuangTai = getStringAfterWord($strZhuangTai, '状态：');
// 提取作者信息
$strZuoZhe = trim($QueryList->find('p a')->eq(0)->text());
// $strLastDate = $QueryList->find('meta[property="og:novel:update_time"]')->attr('content');
$strLastDate = $QueryList->find('#info p')->eq(2)->text();
$strLastDate = getStringAfterWord($strLastDate, '最后更新：');
$strLastDate = trim((string)$strLastDate);
if (!isValidDateTime($strLastDate)) {
$strLastDate = date('Y-m-d H:i:s');
}
$arrTag = [];
// $QueryList->find('.tg span')->map(function ($TagQuery) use (&$arrTag) {
// $arrTag[] = trim($TagQuery->text());
// });
$strJieShao = $QueryList->find('#intro')->eq(0)->text();
$strJieShao = removeScriptTags('/<script>.*?<\/script>/is',$strJieShao) ;
$strJieShao = removeScriptTags('/myJs\.bookJs\(\);/',$strJieShao) ;
$strCover = $QueryList->find('#fmimg img')->eq(0)->attr("src");
// if (strpos($strCover, 'http') === false) {
// $strCover = $strSiteUrl . $strCover;
// }
$strCoverUri = $strCover;
// try {
// $strCoverUri = "";
// $strCoverUri = ImageProcessor::getInstance()->downloadAndEncryptImage($strCover, 'xs');
// } catch (\Throwable $t) {
// $strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
// echo $strErr;
// }
$arrCover = [
'code' => 'AI_LI_SI_COVER',
// 'uri' => getUriByUrl($strCover),
'uri' => $strCoverUri,
];
$arrXiaoShuoInfo = [
'xsxq_ming_zi' => $strMingZi,
'xsxq_zuozhe' => $strZuoZhe , //$arrData['xiaoshuo_zuozhe'],
'xsfl_id' => $arrData['xsfl_id'],
'xsxq_zhuang_tai' => $strZhuangTai,
'xsxq_zi_shu' => $strZiShu,
'xsxq_jie_shao' => $strJieShao,
'xsxq_feng_mian' => json_encode($arrCover),
'xsxq_geng_xin_shi_jian' => $strLastDate,
'xsxq_tag' => json_encode($arrTag, JSON_UNESCAPED_UNICODE),
'xsxq_source_id' => $intXiaoShuoSourceId,
'xsxq_source_seo_id' => $intXiaoShuoSourceSeoId,
'xsxq_source_code' => 'AILISI_XIAOSHUO_' . $intXiaoShuoSourceId . '_' . $intXiaoShuoSourceSeoId,
];
// print_r($arrXiaoShuoInfo);
var_dump($arrXiaoShuoInfo['xsxq_source_code']);
$XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
/*
if ($redis->exists($redisKey)) {
var_dump('小说任务已存在，直接跳过');
return true; // 如果任务已存在，直接跳过
}
// 在 Redis 中标记任务为已存在
$redis->set($redisKey, 1);
$arrZhangjie = [];
$QueryList->find('#list a')->map(function ($ZhangJie) use (&$arrZhangjie) {
$arrZhangjie[] = [
'mingzi' => $ZhangJie->text(),
'url' => $ZhangJie->attr('href'),
];
});
unset($QueryList);
// 获取最大章节索引
$intMaxXiaoShuoZhangJiePaixu = XiaoShuoZhangJieModel::where('xsxq_id', $intXiaoShuoSourceId)
->max('xszj_pai_xu'); // 获取最大值
// 如果没有结果，max 返回 null，可以设置默认值
$intMaxXiaoShuoZhangJiePaixu = $intMaxXiaoShuoZhangJiePaixu ?? 0;
foreach ($arrZhangjie as $intZhangJieNum => $arrZhangjieOne) {
// 如果数据库里的章节最大索引大于当前索引，说明已经入库，不需要入队列了
if($intMaxXiaoShuoZhangJiePaixu > $intZhangJieNum){
var_dump("小说章节已经入库，不再推送任务");
continue;// 如果任务已存在，直接跳过 break
}
$strZhangJieUrl = "https:" . $arrZhangjieOne['url'];
$strZhangJieId = extractFilename($strZhangJieUrl);
# 'AILISI_XIAOSHUO_ZHANGJIE_小说源id_小说源章节id'
$strXszjSourceCode = 'AILISI_XIAOSHUO_ZHANGJIE_'. $XiaoShuoXiangQingModel->xsxq_source_id . '_' . $strZhangJieId;
$arrTask = [
'callback' => [self::class, 'getNovelReader'],
'data' => [
'xszj_source_id' => $strZhangJieId,
'xszj_paixu' => $intZhangJieNum,
'xszj_url' => $strZhangJieUrl,
'xszj_source_code' => $strXszjSourceCode,
'xsxq_source_id' => $XiaoShuoXiangQingModel->xsxq_source_id,
'xsfl_id' => $arrData['xsfl_id'],
]
];
$TaskCore->set($arrTask);
//var_dump("章节任务投递成功".$strXszjSourceCode);
}
// 任务成功后删除 Redis 标记
$redis->del($redisKey);
*/
} catch (\Throwable $T) {
throw $T;
}
}
public function getNovelFengMian($arrData)
{
//var_dump('getNovelFengMian');
//var_dump($arrData);
$Client = $this->Client;
$TaskCore = $this->TaskCore;
// 获取 Redis 实例
$redis = Cache::store('redis')->handler();
$intXiaoShuoSourceId = $arrData['xiaoshuo_id'];
$intXiaoShuoSourceSeoId = $arrData['xiaoshuo_seo_id'];
$redisKey = "task:novel_source_id:" . $intXiaoShuoSourceId;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($arrData['xiaoshuo_url']);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$strSiteUrl = $this->getTianLaiDomain();
$strCover = $QueryList->find('#fmimg img')->eq(0)->attr("src");
if (strpos($strCover, 'http') === false) {
$strCover = $strSiteUrl . $strCover;
}
$strCoverUri = $strCover;
try {
$strCoverUri = "";
$strCoverUri = ImageProcessor::getInstance()->downloadAndEncryptImage($strCover, 'xs');
$arrCover = [
'code' => 'AI_LI_SI_COVER',
// 'uri' => getUriByUrl($strCover),
'uri' => $strCoverUri,
];
$arrXiaoShuoInfo = [
'xsxq_feng_mian' => json_encode($arrCover),
];
$XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
} catch (\Throwable $t) {
$strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
echo $strErr;
}
} catch (\Throwable $T) {
throw $T;
}
}
public function getNovelReader($arrData)
{
//var_dump('getNovelReader');
//var_dump($arrData);
$Client = $this->Client;
$strXszjSourceCode = $arrData['xszj_source_code'];
$strZhangJieUrl = $arrData['xszj_url'];
$intZhangJiePaiXu = $arrData['xszj_paixu'];
$intFenleiId = $arrData['xsfl_id'];
$intXiaoShuoSourceID = $arrData['xsxq_source_id'];
$strZhangJieSourceId = $arrData['xszj_source_id'];
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strZhangJieUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
if (XiaoShuoZhangJieModel::checkZhangJieExists($strXszjSourceCode)) {
return true;
}
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strZhangJieUrl, [
'headers' => [
'User-Agent' => $userAgent
],
'cookies' => $CookieJar,
#'proxy' => $proxy,
'timeout' => 10,
]);
$strResult = (string)$Response->getBody();
unset($Response);
$ZhangJieQueryList = QueryList::html($strResult);
$strZhangJieName = $ZhangJieQueryList->find('h1')->text();
$strNeiRong = $ZhangJieQueryList->find('#content')->eq(0)->text();
# '/novel/123yqw/分类id/小说源id/小说源章节id.txt'
$strNeiRongTxtPath = config("filesystem.novel").'/novel/123yqw/'.$intFenleiId.'/'.$intXiaoShuoSourceID.'/'.$strZhangJieSourceId.'.txt';
//$strNeiRongTxtPath = "/www/novel/123yqw/1/11/111.txt";
// 调用函数
$result = saveCompressAndDeleteTxt($strNeiRongTxtPath, $strNeiRong);
// 输出状态
if ($result['status']) {
var_dump("Success: " . $result['message']) ;
} else {
var_dump("Error: " . $result['message']) ;
}
$arrXiaoShuoZhangJie = [
'xsxq_id' => $intXiaoShuoSourceID,
'xszj_pai_xu' => $intZhangJiePaiXu,
'xszj_ming_zi' => $strZhangJieName,
'xszj_nei_rong' => $strNeiRongTxtPath,
'xszj_source_id' => $strZhangJieSourceId,
'xszj_source_code' => $strXszjSourceCode,
];
XiaoShuoZhangJieModel::addXiaoShuoZhangJie($arrXiaoShuoZhangJie);
unset($ZhangJieQueryList);
} catch (\Throwable $T) {
throw $T;
}
}
}

90
code/app/task/collection/visitlogs/ContentStatsCol.php Normal file
View File

@@ -0,0 +1,90 @@
<?php
declare(strict_types=1);
namespace app\task\collection\visitlogs;
use app\admin\model\XiaoShuoFenLeiModel;
use app\admin\model\XiaoShuoXiangQingModel;
use app\admin\model\XiaoShuoZhangJieModel;
use app\task\collection\PullDataColBase;
use GuzzleHttp\Cookie\CookieJar;
use processor\ImageProcessor;
use think\facade\Cache;
use QL\QueryList;
use app\admin\model\VisitLogsModel;
use app\admin\model\ContentStatsModel;
/**
* @mixin think\Model
*/
class ContentStatsCol extends PullDataColBase
{
public function run()
{
var_dump('ContentStatsCol');
try {
// 定义时间范围统计的周期类型和 SQL 表达式
$arrPeriods = [
'day' => "DATE(vil_timestamp)", // 按日统计
'week' => "YEARWEEK(vil_timestamp)", // 按周统计
'month' => "DATE_FORMAT(vil_timestamp, '%Y-%m')", // 按月统计
'quarter' => "CONCAT(YEAR(vil_timestamp), '-', QUARTER(vil_timestamp))", // 按季度统计
'year' => "YEAR(vil_timestamp)" // 按年统计
];
foreach ($arrPeriods as $periodName => $groupExpression) {
var_dump($groupExpression);
$results = VisitLogsModel::where('vil_is_spider', 0) // 排除爬虫访问
->field([
'vil_content_type',
'vil_content_id',
"$groupExpression" => 'stat_date', // 根据周期分组
'COUNT(*)' => 'pv', // PV
'COUNT(DISTINCT vil_ip)' => 'uv' // UV
])
->group('vil_content_type, vil_content_id, stat_date')
->select()->toArray();
// var_dump($results);
foreach ($results as $result) {
$ContentStatsModel = [
'cs_content_type' => $result['vil_content_type'],
'cs_content_id' => $result['vil_content_id'],
'cs_stat_date' => $result['stat_date'],
'cs_stat_period' => $periodName, // 周期类型（day, week, month, quarter, year）
'cs_pv' => $result['pv'],
'cs_uv' => $result['uv'],
'cs_md5' => md5($result['vil_content_type'].$result['vil_content_id'].$result['stat_date'].$periodName),
'updated_at' => date('Y-m-d H:i:s'),
];
var_dump($ContentStatsModel);
ContentStatsModel::add($ContentStatsModel);
// var_dump($r);
// 插入或更新到统计表
// ContentStatsModel::insertOrUpdate([
// 'cs_content_type' => $result['vil_content_type'],
// 'cs_content_id' => $result['vil_content_id'],
// 'cs_stat_date' => $result['stat_date'],
// 'cs_stat_period' => $periodName, // 周期类型（day, week, month, quarter, year）
// 'cs_pv' => $result['pv'],
// 'cs_uv' => $result['uv'],
// 'updated_at' => date('Y-m-d H:i:s'),
// ], [
// 'cs_content_type' => $result['vil_content_type'],
// 'cs_content_id' => $result['vil_content_id'],
// 'cs_stat_date' => $result['stat_date'],
// 'cs_stat_period' => $periodName,
// ]);
}
}
} catch (\Exception $e) {
// 记录异常日志
//\think\facade\Log::error('ContentStatsTask error: ' . $e->getMessage());
}
}
}

47
code/app/task/collection/visitlogs/VisitLogsCol.php Normal file
View File

@@ -0,0 +1,47 @@
<?php
declare(strict_types=1);
namespace app\task\collection\visitlogs;
use app\admin\model\XiaoShuoFenLeiModel;
use app\admin\model\XiaoShuoXiangQingModel;
use app\admin\model\XiaoShuoZhangJieModel;
use app\task\collection\PullDataColBase;
use GuzzleHttp\Cookie\CookieJar;
use processor\ImageProcessor;
use think\facade\Cache;
use QL\QueryList;
use app\admin\model\VisitLogsModel;
/**
* @mixin think\Model
*/
class VisitLogsCol extends PullDataColBase
{
public function add($arrData)
{
try {
// 'vil_ip', 'vil_url', 'vil_content_id', 'vil_content_type', 'vil_method'
if (empty($arrData['vil_ip']) || empty($arrData['vil_url']) || empty($arrData['vil_content_id']) || empty($arrData['vil_content_type']) || empty($arrData['vil_method'])) {
var_dump("信息不全!");
}else{
$strLocation = '';
// if(!empty($arrData['vil_ip'])){
// $strLocation = getLocationByIp($arrData['vil_ip']);
// }
$arrData['vil_location'] = $strLocation;
VisitLogsModel::add($arrData);
}
} catch (\Throwable $T) {
//var_dump($T);
throw $T;
}
}
}

375
code/app/task/collection/yzzy/YouZhiYingShiCol.php Normal file
View File

@@ -0,0 +1,375 @@
<?php
declare(strict_types=1);
namespace app\task\collection\yzzy;
use app\admin\model\VideoYuyanModel;
use app\admin\model\VideoPlayurlModel;
use app\admin\model\VideoLeixinModel;
use app\admin\model\VideoDiquModel;
use app\admin\model\VideoYearsModel;
use app\admin\model\VideoJuqingModel;
use app\admin\model\XiaoShuoFenLeiModel;
use app\admin\model\XiaoShuoXiangQingModel;
use app\admin\model\VideoInfoModel;
use app\admin\model\XiaoShuoZhangJieModel;
use app\task\collection\PullDataColBase;
use GuzzleHttp\Cookie\CookieJar;
use processor\ImageProcessor;
use think\facade\Cache;
use QL\QueryList;
/**
* @mixin think\Model 优质资源
*/
class YouZhiYingShiCol extends PullDataColBase
{
public function pullYouZhiYingshi()
{
// https://1080zyk1.com/?m=vod-type-id-1-pg-1.html
// https://1080zyk1.com/?m=vod-type-id-2-pg-2.html
// https://1080zyk1.com/?m=vod-type-id-3-pg-2.html
// https://1080zyk1.com/?m=vod-type-id-4-pg-2.html
// https://1080zyk1.com/?m=vod-type-id-83-pg-2.html
// https://1080zyk1.com/?m=vod-type-id-94-pg-2.html
$strSiteUrl = $this->YouZhiZiYuanDomain();
// 配置分类任务参数
$categories = [
['id' => 1, 'video_class_id' => 1, 'name' => '电影'],
['id' => 83, 'video_class_id' => 5, 'name' => '短剧'],
['id' => 2, 'video_class_id' => 2, 'name' => '电视剧'],
['id' => 3, 'video_class_id' => 3, 'name' => '综艺'],
['id' => 4, 'video_class_id' => 4, 'name' => '动漫'],
['id' => 94, 'video_class_id' => 6, 'name' => '体育'],
];
foreach ($categories as $category) {
$intClassId = $category['video_class_id'];
$intCategoryId = $category['id'];
// 先抓第一页
$strUrl = $strSiteUrl . '/?m=vod-type-id-' . $intCategoryId . '-pg-1.html';
var_dump($strUrl);
$maxPage = $this->getMaxPage($strUrl,$intClassId);
// 首次抓取，全部抓取
if (config('task.task_is_first')) {
$this->createTasks($strSiteUrl, $intCategoryId, $intClassId, $maxPage);
}
}
}
/**
* 获取分类的最大页码且抓取第一页
*/
protected function getMaxPage($strUrl, $intClassId)
{
$TaskCore = $this->TaskCore;
$Client = $this->Client;
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$Response = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'Accept-Language' => 'en-US,en;q=0.5',
'cookie' => '__sk_Vm8nTqM7LqSCWmR6__=be3bd4f4e0483619e1a434ec1b18a938',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 300,
]);
$strResult = (string)$Response->getBody();
$QueryList = QueryList::html($strResult);
// 检查是否找到元素
$intVideoListIndex = 0;
if ($QueryList->find('.xing_vb ul')->count() > 1) {
$QueryList->find('.xing_vb ul')->map(function ($Video) use ($intClassId, &$TaskCore,&$intVideoListIndex) {
if($intVideoListIndex >=1){
$strHref = $Video->find('.xing_vb4 a')->attr('href');
$strName = '';
if (!empty($strHref) && strlen($strHref) > 0) {
$arrTask = [
'callback' => [self::class, 'getVideoInfo'],
'data' => [
'class_id' => $intClassId,
'video_href' => $strHref,
'video_name' => $strName,
]
];
// var_dump($arrTask);
$TaskCore->set($arrTask);
echo "视频投递成功 : " . $Video->text() . PHP_EOL;
}
}
$intVideoListIndex ++ ;
});
} else {
echo "No elements found for .xing_vb ul" . PHP_EOL;
}
// 获取分页部分最后一个页码链接
$lastPageHref = $QueryList->find('.pages a')->eq($QueryList->find('.pages a')->count() - 1)->attr('href');
// 提取最大页码
preg_match('/-pg-(\d+)\.html/', $lastPageHref, $matches);
$maxPage = isset($matches[1]) ? (int)$matches[1] : 1; // 默认返回 1
return $maxPage;
}
/**
* 创建任务
*/
protected function createTasks($strSiteUrl, $intCategoryId, $videoClassId, $maxPage)
{
$TaskCore = $this->TaskCore;
for ($i = $maxPage; $i >= 1; $i--) {
echo "当前分类: $intCategoryId, 当前页: $i\n";
$arrTask = [
'callback' => [self::class, 'getVideoPage'],
'data' => [
'video_page_url' => $strSiteUrl . '/?m=vod-type-id-' . $intCategoryId . '-pg-' . $i . '.html',
'video_class_id' => $videoClassId,
]
];
$TaskCore->set($arrTask);
}
}
# 获取分页里面的视频列表
public function getVideoPage($arrData)
{
$TaskCore = $this->TaskCore;
$Client = $this->Client;
$strUrl = $arrData['video_page_url'];
$intClassId = $arrData['video_class_id'];
$strSiteUrl = $this->YouZhiZiYuanDomain();
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
var_dump($strUrl);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'Accept-Language' => 'en-US,en;q=0.5',
'cookie' => '__sk_Vm8nTqM7LqSCWmR6__=be3bd4f4e0483619e1a434ec1b18a938',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 300,
]);
$strResult = (string)$Response->getBody();
$QueryList = QueryList::html($strResult);
$intVideoListIndex = 0;
if ($QueryList->find('.xing_vb ul')->count() > 1) {
$QueryList->find('.xing_vb ul')->map(function ($Video) use ($intClassId, &$TaskCore,&$intVideoListIndex) {
if($intVideoListIndex >=1){
$strHref = $Video->find('.xing_vb4 a')->attr('href');
$strName = '';
if (!empty($strHref) && strlen($strHref) > 0) {
$arrTask = [
'callback' => [self::class, 'getVideoInfo'],
'data' => [
'class_id' => $intClassId,
'video_href' => $strHref,
'video_name' => $strName,
]
];
// var_dump($arrTask);
$TaskCore->set($arrTask);
echo "视频投递成功 : " . $strHref . PHP_EOL;
}
}
$intVideoListIndex ++ ;
});
} else {
echo "No elements found for .xing_vb ul" . PHP_EOL;
}
}
public function getVideoInfo($arrData)
{
$TaskCore = $this->TaskCore;
$Client = $this->Client;
$strSiteUrl = $this->YouZhiZiYuanDomain();
$strUrl = $strSiteUrl . $arrData['video_href'];
$intClassId = $arrData['class_id'];
$strVideoName = $arrData['video_name'] ?? '';
var_dump($strUrl);
if (!empty($strUrl) && strlen($strUrl) > 0) {
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
$QueryList = QueryList::html($strResult);
// 提取别名
$strPinyinId = 'yzzy'.extractIdFromUrl($strUrl);
$strVideoName = $QueryList->find('.vodh h2')->text();
$strJiaoBiao = $QueryList->find('.vodh span')->text();
$strPinfen = $QueryList->find('.vodh label')->text();
$strLeixin = $QueryList->find('.nvc dd a')->eq(1)->text();
$strCover = $QueryList->find('.vodImg img')->eq(0)->attr("src");
if (strpos($strCover, 'http') === false) {
$strCover = $strSiteUrl . $strCover;
}
$strCoverUri = $strCover;
$arrCover = [
'code' => 'YOUZHI_SOURCE_COVER',
'uri' => $strCoverUri,
];
// 提取导演
$strDaoyan = $QueryList->find('.vodInfo .vodinfobox ul li')->eq(1)->find('span')->text();
// 提取主演 多个逗号分割
$strZhuyan = $QueryList->find('.vodInfo .vodinfobox li')->eq(2)->find('span')->text();
// 提取类型
$arrLeixinData = $QueryList->find('.vodInfo .vodinfobox li')->eq(3)->find('span')->text();
// 切割后移除空元素
$arrJuqing = preg_split('/\s+/u', trim($arrLeixinData));
array_shift($arrJuqing);
// **在过滤后将“类型”数据入库**
foreach ($arrJuqing as $value) {
if(strlen($value) > 0){
VideoJuqingModel::add(['vc_id' => $intClassId, 'vj_name' => $value]);
}
}
// 提取地区
$strDiqu = $QueryList->find('.vodinfobox li span')->eq(4)->text();
// 提取语言
$strLanguage = $QueryList->find('.vodinfobox li span')->eq(5)->text();
// 提取年份
$strYears = $QueryList->find('.vodinfobox li span')->eq(6)->text();
// 提取更新时间
$strLastDate = $QueryList->find('.vodinfobox li span')->eq(8)->text();
if (strlen($strLeixin) > 0) {
VideoLeixinModel::addVideoLeixin(['vc_id' => $intClassId, 'vl_name' => $strLeixin]);
}
if (strlen($strDiqu) > 0) {
VideoDiquModel::add(['vc_id' => $intClassId, 'vd_name' => $strDiqu]);
}
if (strlen($strYears) > 0) {
VideoYearsModel::add(['vc_id' => $intClassId, 'vy_name' => $strYears]);
}
if (strlen($strLanguage) > 0) {
VideoYuyanModel::add(['vc_id' => $intClassId, 'vla_name' => $strLanguage]);
}
// 提取简介
$strDescription = $QueryList->find('.vodplayinfo')->eq(0)->text();
if(empty($strVideoName)){
var_dump('$strVideoName------');
var_dump($strVideoName);
return true;
}
$arrVideoInfo = [
'v_name' => $strVideoName,
'v_cover' => json_encode($arrCover),
'vc_id' => $intClassId,
'v_pinyin' => $strPinyinId,
'v_leixin' => $strLeixin,
'v_juqing' => implode(',', $arrJuqing),
'v_diqu' => $strDiqu,
'v_years' => $strYears,
'v_language' => $strLanguage,
'v_daoyan' => trim(str_replace('主演：', '', $strZhuyan)),
'v_zhuyan' => trim(str_replace('导演：', '', $strDaoyan)),
'update_time' => $strLastDate,
'v_description' => $strDescription,
'v_pingfen' => $strPinfen,
'v_orurl' => $strUrl,
'v_jiaobiao' => $strJiaoBiao,
'v_md5' => md5($intClassId .$strPinyinId. $strVideoName . getUriByUrl($strUrl)),
];
// var_dump($arrVideoInfo);
// // 检查是否找到元素
if ($QueryList->find('#play_2 ul li')->count() > 0) {
// var_dump("视频入库成功--" . $strVideoName);
$resultVideoInfoModel = VideoInfoModel::addVideoXiangQing($arrVideoInfo);;
if ($resultVideoInfoModel['isNew']) {
var_dump("视频详情数据已新增：", $resultVideoInfoModel['model']->v_id."视频入库成功--" . $strVideoName);
} else {
echo "视频详情数据已存在：", $resultVideoInfoModel['model']->v_id;
}
$VideoInfoModel = $resultVideoInfoModel['model'];
$intVid = $VideoInfoModel->v_id;
$intChapterIndex = 0;
$QueryList->find('#play_2 ul li')->map(function ($ZhangJie) use (&$arrZhangjie, &$intChapterIndex, &$intVid,&$intClassId) {
//echo "Processing index $intChapterIndex: " . $ZhangJie->text() . PHP_EOL;
$strM3u8 = $ZhangJie->find('a')->text();
$arrM3u8 = explode('$', $strM3u8);
$strM3u8Url = $arrM3u8[1];
$strM3u8Name = $arrM3u8[0];
$arrM3u8Url = [
'code' => 'YOUZHI_SOURCE_PLAY',
'uri' => $strM3u8Url,
];
$arrVideoPlayurl = [
'v_id' => $intVid,
'vp_sort' => $intChapterIndex,
'vp_name' => $strM3u8Name,
'vp_url' => json_encode($arrM3u8Url),
'vp_md5' => md5($intClassId.$intVid . getUriByUrl($strM3u8Url)),
];
$result = VideoPlayurlModel::add($arrVideoPlayurl);
if ($result['isNew']) {
var_dump("播放地址已新增：", $result['model']->vp_id."播放线路入库成功--" . $strM3u8Url) ;
} else {
echo "播放地址已存在：", $result['model']->vp_id ;
}
$intChapterIndex++;
});
var_dump("--") ;
var_dump("一个视频任务完成------------------------------------------------------------------------------------") ;
} else {
echo "No elements found for .listmain a" . PHP_EOL;
}
} catch (\Throwable $T) {
throw $T;
}
}
// sleep(10);
}
}

115
code/app/task/command/DatabaseCommand.php Normal file
View File

@@ -0,0 +1,115 @@
<?php
declare(strict_types=1);
namespace app\task\command;
use database\seeders\SeedManager;
use think\console\Command;
use think\console\Input;
use think\console\input\Argument;
use think\console\Output;
use think\console\input\Option;
use think\facade\Db;
class DatabaseCommand extends Command
{
protected function configure()
{
$this->setName('db:manage')
->addArgument('action', \think\console\input\Argument::OPTIONAL, 'The action to perform: backup or init or seed')
->setDescription('Manage the database: backup or initialize');
}
protected function execute(Input $input, Output $Output)
{
$strAction = $input->getArgument('action');
if ($strAction === 'backup') {
$this->backupDatabase($Output);
} elseif ($strAction === 'init') {
$this->initializeDatabase($Output);
} elseif ($strAction === 'seed') {
$this->seed($Output);
} else {
$Output->writeln("Invalid action. Use 'backup' or 'init'. or 'seed'");
}
}
private function seed(Output $Output)
{
$SeedManager = new SeedManager;
$Output->writeln("开始数据播种" . PHP_EOL);
$SeedManager->handle();
$Output->writeln("数据播种结束");
}
private function backupDatabase(Output $Output)
{
$arrConfig = config('database.connections.mysql');
$strDb = $arrConfig['database'];
$strUserName = $arrConfig['username'];
$strPwd = $arrConfig['password'];
$strHost = $arrConfig['hostname'];
$strBackupFile = app()->getRootPath() . 'database/schema/mysql-schema.dump';
$strTempFile = tempnam(sys_get_temp_dir(), 'mysql-schema-') . '.dump';
$strCmd = "mysqldump -h{$strHost} -u{$strUserName} -p{$strPwd} --no-data {$strDb} > {$strTempFile}";
system($strCmd, $intResultCode);
if ($intResultCode === 0) {
if (file_exists($strBackupFile)) {
unlink($strBackupFile);
}
if (rename($strTempFile, $strBackupFile)) {
$Output->writeln("Database backup successful: {$strBackupFile}");
} else {
$Output->writeln("Failed to move backup file to destination.");
}
} else {
$Output->writeln("Database backup failed.");
unlink($strTempFile);
}
}
private function initializeDatabase(Output $Output)
{
$arrConfig = config('database.connections.mysql');
$strDb = $arrConfig['database'];
$strUserName = $arrConfig['username'];
$strPwd = $arrConfig['password'];
$strHost = $arrConfig['hostname'];
$strPort = $arrConfig['hostport'];
$strBackupFile = app()->getRootPath() . 'database/schema/mysql-schema.dump';
$checkTableQuery = "SHOW TABLES LIKE 'migrations'";
$boolMigrateExists = Db::query($checkTableQuery);
if (!empty($boolMigrateExists)) {
$Output->writeln("数据库已存在，跳过初始化导入");
return;
}
$strTempFile = tempnam(sys_get_temp_dir(), 'mysql-schema-') . '.dump';
if (!copy($strBackupFile, $strTempFile)) {
$Output->writeln("创建临时导入文件失败.");
return;
}
$strCmd = "mysql -h{$strHost} -P{$strPort} -u{$strUserName} -p{$strPwd} {$strDb} < {$strTempFile}";
system($strCmd, $intResultCode);
if ($intResultCode === 0) {
$Output->writeln("成功导入初始化文件");
} else {
$Output->writeln("初始化文件导入失败");
}
unlink($strTempFile);
}
}

50
code/app/task/command/InsertSeoWordsCommand.php Normal file
View File

@@ -0,0 +1,50 @@
<?php
namespace app\task\command;
use think\console\Command;
use think\console\Input;
use think\console\Output;
use think\console\input\Argument;
use think\facade\Db;
class InsertSeoWordsCommand extends Command
{
protected function configure()
{
// 配置命令的名称和参数
$this->setName('insert:seowords')
->setDescription('Insert SEO words from a text file into the seo_keywords table')
->addArgument('file', Argument::REQUIRED, 'The path of the text file');
}
protected function execute(Input $input, Output $output)
{
// 获取传递的文件路径参数
$file = $input->getArgument('file');
// 检查文件是否存在
if (!file_exists($file)) {
$output->writeln("The file does not exist.");
return;
}
// 读取文件内容
$lines = file($file, FILE_IGNORE_NEW_LINES | FILE_SKIP_EMPTY_LINES);
// 遍历每一行，插入数据库
foreach ($lines as $line) {
// 假设每一行是以逗号分隔的字段值，例如：关键词标题,状态,HTML路径
// list($title, $status, $htmlPath) = explode(',', $line);
$title = $line;
// 插入数据到 seo_keywords 表
Db::name('seo_keywords')->insert([
'sw_title' => trim($title),
]);
$output->writeln("Inserted: $title");
}
$output->writeln("Data insertion completed.");
}
}

89
code/app/task/command/Task.php Normal file
View File

@@ -0,0 +1,89 @@
<?php
declare(strict_types=1);
namespace app\task\command;
use think\console\Command;
use think\console\Input;
use think\console\input\Argument;
use think\console\Output;
use think\console\input\Option;
class Task extends Command
{
private $arrArguments = [
'action' => ['start', 'stop', 'restart', 'status',],
'daemonize' => ['d', "false",],
'forcibly' => ['f', "false",],
];
private $arrReflection = [
\app\task\core\Core::class,
];
protected function configure()
{
$this->setName('php think task')
->addArgument('action', Argument::REQUIRED, "action: (" . implode('|', $this->arrArguments['action']) . ')')
->addOption('daemonize', 'd', Option::VALUE_NONE, "daemonize start service")
->addOption('forcibly', 'f', Option::VALUE_NONE, "ignore user forcibly start service")
->setDescription('swoole task server')
->setHelp('php think task <action> [-d] [-f] run server');
}
protected function checkUser(Input $Input)
{
if (!$Input->getOption('forcibly')) {
$strUser = get_current_user();
if ($strUser == 'root') {
$strError = sprintf("Sorry , Can't start service by [%s] user!,\n but you can add option '-f' forcibly start on you debug ,\n Do not use in production environment !!!!!!!!!",$strUser);
throw new \InvalidArgumentException($strError);
}
if ($strUser != 'www') {
throw new \InvalidArgumentException("Sorry , Can't start service by [{$strUser}] user!, please use [www] user !");
}
}
}
protected function execute(Input $Input, Output $Output)
{
$this->checkArguments(['action'], $Input, $Output);
$strAction = trim($Input->getArgument('action'));
if ($strAction == 'start' || $strAction == 'restart') {
$this->checkUser($Input);
}
$arrArgs = array_merge($Input->getArguments(), $Input->getOptions());
$this->{$strAction}($arrArgs);
}
private function checkArguments($arrArguments, Input $Input)
{
foreach ($arrArguments as $strArguments) {
$strVal = trim($Input->getArgument($strArguments));
if (!in_array($strVal, $this->arrArguments[$strArguments])) {
throw new \InvalidArgumentException("Arguments [{$strVal}] is not defined! Optional parameters [" . implode(' ', $this->arrArguments[$strArguments]) . "] !");
}
}
}
public function __call($strAction, $arrArguments)
{
foreach ($this->arrReflection as $strClass) {
if (method_exists($strClass, $strAction)) {
return call_user_func([new $strClass, $strAction], $arrArguments);
}
}
throw new \InvalidArgumentException("Action [{$strAction}] no method available! ");
}
}

129
code/app/task/core/Core.php Normal file
View File

@@ -0,0 +1,129 @@
<?php
declare(strict_types=1);
namespace app\task\core;
class Core
{
static $HttpService;
static $arrDynamicConfig = [];
public function __init($arrArgvs = [])
{
$this->setConfigDaemonize($arrArgvs[0]['daemonize'] ?? 'false');
}
private function setConfigDaemonize($strMode)
{
self::$arrDynamicConfig['daemonize'] = (bool)$strMode;
}
private function getPid()
{
return file_exists(config('task.service.pid_file')) ? file_get_contents(config('task.service.pid_file')) : NULL;
}
public function start($arrArgvs = [])
{
$this->__init($arrArgvs);
$intPid = (int)$this->getPid();
if ($intPid > 0 && \Swoole\Process::kill($intPid, SIG_DFL)) {
echo " Service is \e[0;32mRunning\e[0m ! " . PHP_EOL;
exit;
}
echo " Execute the \e[0;32m start\e[0m command ...... " . PHP_EOL;
$this->createService();
$this->addProcessList();
$this->addScheduledTasks();
self::$HttpService->getServer()->start();
}
public function stop($arrArgvs = [])
{
$this->__init($arrArgvs);
$intPid = (int)$this->getPid();
if ($intPid <= 0) {
echo " Service is \e[0;31mStop\e[0m ! " . PHP_EOL;
} else {
echo " Execute the \e[0;32m stop\e[0m command ...... " . PHP_EOL;
\Swoole\Process::kill($intPid, SIGTERM);
echo " Service is \e[0;31mStop\e[0m ! " . PHP_EOL;
}
}
public function status($arrArgvs = [])
{
$this->__init($arrArgvs);
$intPid = (int)$this->getPid();
if ($intPid <= 0) {
echo " Service is \e[0;31mStop\e[0m ! " . PHP_EOL;
} else if (\Swoole\Process::kill($intPid, SIG_DFL)) {
echo " Service is \e[0;32mRunning\e[0m ! " . PHP_EOL;
} else {
echo " Service is \e[0;31mStop\e[0m ! " . PHP_EOL;
}
}
public function restart($arrArgvs = [])
{
$this->__init($arrArgvs);
$intPid = (int)$this->getPid();
if ($intPid <= 0) {
echo " \e[0;31m Get Pid Error\e[0m !" . PHP_EOL;
echo " Execute the \e[0;32m start\e[0m command ...... " . PHP_EOL;
$this->start();
} else {
echo " Execute the \e[0;32m restart\e[0m command ...... " . PHP_EOL;
\Swoole\Process::kill($intPid, SIGUSR1);
}
}
private function addScheduledTasks()
{
if (config('task.scheduled_tasks_pool')) {
foreach (config('task.scheduled_tasks_pool') as $arrProcessTask) {
if ($arrProcessTask['status']) {
$Process = new \Swoole\Process(function () use ($arrProcessTask) {
call_user_func([\app\task\core\ScheduledTasks::class, 'hander'], $arrProcessTask);
});
self::$HttpService->getServer()->addProcess($Process);
}
}
}
}
private function addProcessList()
{
if (config('task.process_pool')) {
foreach (config('task.process_pool') as $v) {
for ($i = 0; $i < $v['Num']; $i++) {
$Process = new \Swoole\Process($v['callback']);
self::$HttpService->getServer()->addProcess($Process);
}
}
}
}
private function createService()
{
$arrHttpServiceConfig = array_merge(config('task.service'), self::$arrDynamicConfig);
self::$HttpService = Service::instance($arrHttpServiceConfig);
}
}

90
code/app/task/core/Listen.php Normal file
View File

@@ -0,0 +1,90 @@
<?php
declare(strict_types=1);
namespace app\task\core;
use app\task\logic\SystemLogic;
class Listen
{
/**
* 监听队列1代理
*
* @param \Swoole\Process $Process
* @return void
*/
static public function innerConsume(\Swoole\Process $Process)
{
return self::innerConsumeCore($Process, 1);
}
/**
* 监听队列2代理
*
* @param \Swoole\Process $Process
* @return void
*/
static public function innerConsume02(\Swoole\Process $Process)
{
return self::innerConsumeCore($Process, 2);
}
/**
* 监听队列
*
* @param \Swoole\Process $Process
* @return void
*/
static private function innerConsumeCore(\Swoole\Process $Process, int $intIndex = 1)
{
Service::$Process = $Process;
SystemLogic::destructConnectSource();
$arrConfig = config('task.queue');
$intLimit = $arrConfig[$intIndex]['exec_num'];
$TaskCore = new TaskCore($intIndex);
return self::excuteTask($TaskCore, $intLimit);
}
/**
* 队列任务消费
*
* @param \Swoole\Process $Process
* @return void
*/
static private function excuteTask(TaskCore $TaskCore, $intLimit)
{
while ($intLimit > 0) {
$intLimit--;
try {
$strData = $TaskCore->get();
if (empty($strData)) {
sleep(1);
continue;
}
$arrData = json_decode($strData, true);
if (!is_array($arrData)) {
echo '监听Redis队列读取到异常无法解析的数据[' . $strData . ']' . PHP_EOL;
continue;
}
$Class = new $arrData['callback'][0];
$Class->{$arrData['callback'][1]}($arrData['data']);
unset($Class);
} catch (\Throwable $t) {
$strData = $strData ?? '';
echo date('Y-m-d H:i:s') . ':出现致命错误需要处理' . PHP_EOL .
' 队列数据:' . $strData . PHP_EOL .
' 文件:' . $t->getFile() . PHP_EOL .
' 行数:' . $t->getLine() . PHP_EOL .
' 错误描述:' . $t->getMessage() . PHP_EOL .
' 堆栈跟踪:' . $t->getTraceAsString() . PHP_EOL;
}
}
}
}

31
code/app/task/core/Rout.php Normal file
View File

@@ -0,0 +1,31 @@
<?php
declare (strict_types = 1);
namespace app\task\core;
class Rout
{
static public function httpDispense(\Swoole\Http\Request $Request, \Swoole\Http\Response $Response)
{
// print_r($Request->getData());
// print_r($Request->server);
# 简单的HTTP服务
$Response->status(999,'Hei Guys ~');
$Response->header("Content-Type", "text/html; charset=utf-8");
$Response->end("<h1>Hello reptile~. #".rand(1000, 9999)."</h1>");
}
static public function tcpDispense(\Swoole\Server $server, $fd, $reactor_id, $mixedData)
{
# 简单的HTTP服务
print_r($mixedData);
}
static public function test()
{
var_dump(date('Y-m-d H:i:s'));
return;
}
}

24
code/app/task/core/ScheduledTasks.php Normal file
View File

@@ -0,0 +1,24 @@
<?php
declare(strict_types=1);
namespace app\task\core;
use app\task\logic\SystemLogic;
class ScheduledTasks
{
static public function hander($arrProcessTask = [])
{
SystemLogic::destructConnectSource();
// while(true){
call_user_func($arrProcessTask['callback'], $arrProcessTask['param']);
sleep($arrProcessTask['execution_interval']);
// }
// \Swoole\Timer::tick($arrProcessTask['execution_interval']*1000, $arrProcessTask['callback'], $arrProcessTask['param']);
// \Swoole\Event::wait();
}
}

87
code/app/task/core/Service.php Normal file
View File

@@ -0,0 +1,87 @@
<?php
declare(strict_types=1);
namespace app\task\core;
class Service
{
private $arrConfig;
private static $obj;
private $Server;
static $Process = NULL;
static function instance($arrConfig)
{
if (self::$obj == null) {
self::$obj = new self($arrConfig);
}
return self::$obj;
}
public function __construct($arrConfig)
{
$this->arrConfig = $arrConfig;
$this->createServer();
}
private function serHttpConfig()
{
$this->Server->set($this->filterConfig());
$this->Server->on('request', [\app\task\core\Rout::class, 'httpDispense']);
}
private function filterConfig()
{
$arrConfig = $this->arrConfig;
unset($arrConfig['host']);
unset($arrConfig['port']);
unset($arrConfig['mode']);
unset($arrConfig['sockType']);
unset($arrConfig['server_type']);
return $arrConfig;
}
private function serTcpConfig()
{
$this->Server->set($this->filterConfig());
$this->Server->on('receive', [\app\task\core\Rout::class, 'tcpDispense']);
}
private function createServer()
{
switch ($this->arrConfig['server_type']) {
case 'HTTP':
$this->Server = new \Swoole\Http\Server($this->arrConfig['host'], $this->arrConfig['port']);
$this->serHttpConfig();
break;
default:
$this->Server = new \Swoole\Server($this->arrConfig['host'], $this->arrConfig['port'], $this->arrConfig['mode'], $this->arrConfig['sockType']);
$this->serTcpConfig();
break;
}
$this->Server->on('task', [$this, 'onTask']);
$this->Server->on('finish', [$this, 'onFinish']);
}
public function getServer()
{
return $this->Server;
}
public function onTask($Http, $task_id, $from_id, $arrData)
{
$mixedResult = call_user_func_array($arrData[0], $arrData[1]);
$Http->finish($mixedResult);
}
public function onFinish($Http, $task_id, $mixedData)
{
}
}

35
code/app/task/core/TaskCore.php Normal file
View File

@@ -0,0 +1,35 @@
<?php
declare(strict_types=1);
namespace app\task\core;
use think\facade\Cache;
class TaskCore
{
public $strQueueName = '';
public function __construct($intIndex = 1, $strQueueName = '')
{
if ($strQueueName == '') {
$arrConfig = config('task.queue');
$strQueueName = $arrConfig[$intIndex]['name'];
}
$this->strQueueName = $strQueueName;
}
public function get()
{
return Cache::store('redis')->lpop($this->strQueueName);
}
public function set($arrData)
{
$strData = json_encode($arrData);
// 打印调试信息
//\think\facade\Log::info('TaskCore set: '.$strData );
return Cache::store('redis')->rpush($this->strQueueName, $strData);
}
}

70
code/app/task/logic/AdTask.php Normal file
View File

@@ -0,0 +1,70 @@
<?php
declare(strict_types=1);
namespace app\task\logic;
use app\admin\model\GuangGaoFenZuModel;
use app\admin\model\GuangGaoModel;
use app\admin\model\GuangGaoPeiZhiModel;
use GuzzleHttp\Client;
/**
* @mixin think\Model
*/
class AdTask
{
/**
* Undocumented variable
*
* @var \GuzzleHttp\Client
*/
protected $Client = NULL;
/**
* Undocumented variable
*
* @var \app\task\core\TaskCore
*/
protected $TaskCore = NULL;
protected $arrCaiJiPeiZhi = [];
public function __construct()
{
$this->Client = new Client();
}
public function pullAdBySourceSite()
{
$intOpen = GuangGaoPeiZhiModel::getValByCode('GUANG_GAO_TONG_BU_KAI_GUAN');
if ($intOpen != 1) {
return false;
}
$strUrl = GuangGaoPeiZhiModel::getValByCode('GUANG_GAO_TONG_BU_DI_ZHI');
$strResult = $this->Client->request('GET', $strUrl);
$strResult = (string)$strResult->getBody();
$arrData = json_decode($strResult, true);
if (!is_array($arrData) || empty($arrData)) {
return false;
}
if ($arrData['code'] != 1000) {
return false;
}
GuangGaoModel::where('gg_id', '>', 0)->delete();
GuangGaoFenZuModel::where('ggfz_id', '>', 0)->delete();
GuangGaoFenZuModel::insertAll($arrData['data']['ggfz']);
GuangGaoModel::insertAll($arrData['data']['gg']);
GuangGaoModel::flushCache();
}
}

30
code/app/task/logic/SystemLogic.php Normal file
View File

@@ -0,0 +1,30 @@
<?php
declare (strict_types = 1);
namespace app\task\logic;
use think\Container;
use think\facade\Db;
/**
* @mixin think\Model
*/
class SystemLogic
{
static public function reloadRedis()
{
Container::getInstance()->delete('cache');
}
static public function reloadDb()
{
Db::close();
Container::getInstance()->delete('think\DbManager');
}
static public function destructConnectSource()
{
self::reloadDb();
self::reloadRedis();
}
}

358
code/app/task/module/PlanTask.php Normal file
View File

@@ -0,0 +1,358 @@
<?php
declare(strict_types=1);
namespace app\task\module;
use app\admin\model\PlanTaskModel;
use app\task\collection\ailisi\XiaoShuoCol;
use app\task\collection\biquxs\BiquxsXiaoShuoCol;
use app\task\collection\ccc5\XiaoShuo5cccCol;
use app\task\collection\novel\Zw630XiaoShuoCol;
use app\task\collection\novelone\NovelJiuYuFanXianCol;
use app\task\collection\novelone\NovelWenXueGuanCol;
use app\task\collection\tianlai\TianLaiXiaoShuoCol;
use app\task\collection\seo\GenerateSiteMapCol;
use app\task\collection\seo\GenerateSiteMapNovel;
use app\task\collection\seo\GenerateSiteMapNovelZw;
use app\task\collection\seo\GenerateSiteMapNovelSk;
use app\task\collection\visitlogs\ContentStatsCol;
use app\task\collection\dadi\DaDiDianyingCol;
use app\task\collection\heimuer\HeiMuErYingShiCol;
use app\task\collection\yzzy\YouZhiYingShiCol;
use app\task\core\TaskCore;
use app\task\logic\SystemLogic;
class PlanTask
{
public static function scanPlanTask($arrData)
{
SystemLogic::destructConnectSource();
$PlanTaskAll = PlanTaskModel::where('pt_enable', 1)->where('site_id', config('app.default_app_id'))->select();
foreach ($PlanTaskAll as $PlanTask) {
if ($PlanTask->pt_last_exec != 0 && ($PlanTask->pt_last_exec + $PlanTask->pt_limit) > time()) continue;
$PlanTask->pt_last_exec = time();
$PlanTask->save();
switch ($PlanTask->pt_code) {
case 'PULL_XIAOSHUO':
self::pullXiaoShuo();
break;
case 'PULL_TIANLAIXIAOSHUO':
self::pullTianLaiXiaoShuo();
break;
case 'PULL_BIQUXSXIAOSHUO':
self::pullBiquxsXiaoShuo();
break;
case 'PULL_5CCCXIAOSHUO':
self::pull5cccXiaoShuo();
break;
case 'PULL_DADI_DIANYHHING':
self::pullDadiYingshi();
break;
case 'PULL_HEI_MUER':
self::pullHeiMuEr();
break;
case 'PULL_YOUZHI':
self::pullYouZhiZiYuan();
break;
case 'PULL_TONJI_HUIZON':
self::AddTongJiHuizon();
break;
case 'GENERATE_SITEMAP':
self::generateSiteMap();
break;
case 'NOVEL_GENERATE_SITEMAP':
self::generateNovelSiteMap();
break;
case 'NOVELZW_GENERATE_SITEMAP':
self::generateSiteMapNovelZw();
break;
case 'NOVEL_SK_GENERATE_SITEMAP':
self::generateSiteMapNovelSk();
break;
case 'PULL_630ZWXIAOSHUO':
self::pull630ZwXiaoShuo();
break;
case 'PULL_GUICHUIDENG_XIAOSHUO':
self::pullGuiChuiDengJiuYuFanXianXiaoShuo();
break;
case 'PULL_WENXUEGUAN_XIAOSHUO':
self::pullWenXueGuangXianXiaoShuo();
break;
default:
break;
}
}
}
/**
* 拉取文学馆小说-都市极品医神
*
* @return void
*/
public static function pullWenXueGuangXianXiaoShuo()
{
$arrTask = [
'callback' => [NovelWenXueGuanCol::class, 'pullXiaoShuo'],
'data' => []
];
$TaskCore = new TaskCore();
$TaskCore->set($arrTask);
}
/**
* 拉取鬼吹灯小说-九域凡仙
*
* @return void
*/
public static function pullGuiChuiDengJiuYuFanXianXiaoShuo()
{
$arrTask = [
'callback' => [NovelJiuYuFanXianCol::class, 'pullXiaoShuo'],
'data' => []
];
$TaskCore = new TaskCore();
$TaskCore->set($arrTask);
}
/**
* 拉取小说
*
* @return void
*/
public static function pull630ZwXiaoShuo()
{
$arrTask = [
'callback' => [Zw630XiaoShuoCol::class, 'pullXiaoShuo'],
'data' => []
];
$TaskCore = new TaskCore();
$TaskCore->set($arrTask);
}
/**
* 生成地图-小说
*
* @return void
*/
public static function generateNovelSiteMap()
{
$arrTask = [
'callback' => [GenerateSiteMapNovel::class, 'generate'],
'data' => []
];
$TaskCore = new TaskCore();
$TaskCore->set($arrTask);
}
/**
* 生成地图-小说 novelzw 模板
*
* @return void
*/
public static function generateSiteMapNovelZw()
{
$arrTask = [
'callback' => [GenerateSiteMapNovelZw::class, 'generate'],
'data' => []
];
$TaskCore = new TaskCore();
$TaskCore->set($arrTask);
}
/**
* 生成地图-小说 novelzw 模板
*
* @return void
*/
public static function generateSiteMapNovelSk()
{
$arrTask = [
'callback' => [GenerateSiteMapNovelSk::class, 'generate'],
'data' => []
];
$TaskCore = new TaskCore();
$TaskCore->set($arrTask);
}
/**
* 生成地图-视频
*
* @return void
*/
public static function generateSiteMap()
{
$arrTask = [
'callback' => [GenerateSiteMapCol::class, 'generate'],
'data' => []
];
$TaskCore = new TaskCore();
$TaskCore->set($arrTask);
}
/**
* 拉取优质资源
*
* @return void
*/
public static function AddTongJiHuizon()
{
$arrTask = [
'callback' => [ContentStatsCol::class, 'run'],
'data' => []
];
$TaskCore = new TaskCore();
$TaskCore->set($arrTask);
}
/**
* 拉取优质资源
*
* @return void
*/
public static function AddVisitLogs()
{
$arrTask = [
'callback' => [YouZhiYingShiCol::class, 'add'],
'data' => []
];
$TaskCore = new TaskCore();
$TaskCore->set($arrTask);
}
/**
* 拉取优质资源
*
* @return void
*/
public static function pullYouZhiZiYuan()
{
$arrTask = [
'callback' => [YouZhiYingShiCol::class, 'pullYouZhiYingshi'],
'data' => []
];
$TaskCore = new TaskCore();
$TaskCore->set($arrTask);
}
/**
* 拉取黑木耳
*
* @return void
*/
public static function pullHeiMuEr()
{
$arrTask = [
'callback' => [HeiMuErYingShiCol::class, 'pullDadiYingshi'],
'data' => []
];
$TaskCore = new TaskCore();
$TaskCore->set($arrTask);
}
/**
* 拉取大地影视
*
* @return void
*/
public static function pullDadiYingshi()
{
$arrTask = [
'callback' => [DaDiDianyingCol::class, 'pullDadiYingshi'],
'data' => []
];
$TaskCore = new TaskCore();
$TaskCore->set($arrTask);
}
/**
* 拉取小说
*
* @return void
*/
public static function pull5cccXiaoShuo()
{
$arrTask = [
'callback' => [XiaoShuo5cccCol::class, 'pullXiaoShuo'],
'data' => []
];
$TaskCore = new TaskCore();
$TaskCore->set($arrTask);
}
/**
* 拉取小说
*
* @return void
*/
public static function pullXiaoShuo()
{
$arrTask = [
'callback' => [XiaoShuoCol::class, 'pullXiaoShuo'],
'data' => []
];
$TaskCore = new TaskCore();
$TaskCore->set($arrTask);
}
/**
* 拉取小说
*
* @return void
*/
public static function pullBiquxsXiaoShuo()
{
$arrTask = [
'callback' => [BiquxsXiaoShuoCol::class, 'pullXiaoShuo'],
'data' => []
];
$TaskCore = new TaskCore();
$TaskCore->set($arrTask);
}
/**
* 拉取小说
*
* @return void
*/
public static function pullTianLaiXiaoShuo()
{
$arrTask = [
'callback' => [TianLaiXiaoShuoCol::class, 'pullXiaoShuo'],
'data' => []
];
$TaskCore = new TaskCore();
$TaskCore->set($arrTask);
}
}