admin/SEONexus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

debug

Browse Source
This commit is contained in:
Default
2025-03-15 20:12:21 +08:00
parent 105ffe11c7
commit eb9035a7ae
134 changed files with 15450 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files

560
code/app/task/collection/heimuer/HeiMuErYingShiCol.php Normal file
View File

@@ -0,0 +1,560 @@
<?php
declare(strict_types=1);
namespace app\task\collection\heimuer;
use app\admin\model\VideoPlayurlModel;
use app\admin\model\VideoLeixinModel;
use app\admin\model\VideoDiquModel;
use app\admin\model\VideoYearsModel;
use app\admin\model\VideoJuqingModel;
use app\admin\model\XiaoShuoFenLeiModel;
use app\admin\model\XiaoShuoXiangQingModel;
use app\admin\model\VideoInfoModel;
use app\admin\model\XiaoShuoZhangJieModel;
use app\task\collection\PullDataColBase;
use GuzzleHttp\Cookie\CookieJar;
use processor\ImageProcessor;
use think\facade\Cache;
use QL\QueryList;
/**
* @mixin think\Model
*/
class HeiMuErYingShiCol extends PullDataColBase
{
public function pullDadiYingshi()
{
$strSiteUrl = $this->getHeiMuErDomain();
// 配置分类任务参数
$categories = [
['id' => 1, 'video_class_id' => 1, 'name' => '电影'],
['id' => 27, 'video_class_id' => 5, 'name' => '短剧'],
['id' => 2, 'video_class_id' => 2, 'name' => '电视剧'],
['id' => 4, 'video_class_id' => 3, 'name' => '综艺'],
['id' => 3, 'video_class_id' => 4, 'name' => '动漫'],
];
foreach ($categories as $category) {
$intClassId = $category['video_class_id'];
$intCategoryId = $category['id'];
// 先抓第一页
$strUrl = $strSiteUrl . '/index.php/vod/type/id/' . $intCategoryId . '/page/1.html';
$maxPage = $this->getMaxPage($strUrl,$intClassId);
// 首次抓取，全部抓取
if (config('task.task_is_first')) {
$this->createTasks($strSiteUrl, $category['id'], $category['video_class_id'], $maxPage);
}
}
}
/**
* 获取分类的最大页码且抓取第一页
*/
protected function getMaxPage($strUrl, $intClassId)
{
$TaskCore = $this->TaskCore;
$Client = $this->Client;
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$Response = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'Accept-Language' => 'en-US,en;q=0.5',
'cookie' => '__sk_Vm8nTqM7LqSCWmR6__=be3bd4f4e0483619e1a434ec1b18a938',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 300,
]);
$strResult = (string)$Response->getBody();
$QueryList = QueryList::html($strResult);
// 检查是否找到元素
if ($QueryList->find('.stui-vodlist li.clearfix h3 a')->count() > 0) {
$QueryList->find('.stui-vodlist li.clearfix h3 a ')->map(function ($Video) use ($intClassId, &$TaskCore) {
$strHref = $Video->attr('href');
$strName = $Video->text();
if (!empty($strName) && !empty($strHref) && strlen($strName) > 0 && strlen($strHref) > 0) {
$arrTask = [
'callback' => [self::class, 'getVideoInfo'],
'data' => [
'class_id' => $intClassId,
'video_href' => $strHref,
'video_name' => $strName,
]
];
// var_dump($arrTask);
$TaskCore->set($arrTask);
echo "视频投递成功 : " . $Video->text() . PHP_EOL;
}
});
} else {
echo "No elements found for .stui-vodlist a" . PHP_EOL;
}
// 获取分页部分最后一个页码链接
$lastPageHref = $QueryList->find('.stui-page li')->eq($QueryList->find('.stui-page li')->count() - 1)->find('a')->attr('href');
// 提取最大页码
preg_match('/page\/(\d+)\.html/', $lastPageHref, $matches);
$maxPage = isset($matches[1]) ? (int)$matches[1] : 1; // 默认返回 1
return $maxPage;
}
/**
* 创建任务
*/
protected function createTasks($strSiteUrl, $categoryId, $videoClassId, $maxPage)
{
$TaskCore = $this->TaskCore;
for ($i = $maxPage; $i >= 1; $i--) {
echo "当前分类: $categoryId, 当前页: $i\n";
$arrTask = [
'callback' => [self::class, 'getVideoPage'],
'data' => [
'video_page_url' => $strSiteUrl . '/index.php/vod/type/id/' . $categoryId . '/page/' . $i . '.html',
'video_class_id' => $videoClassId,
]
];
$TaskCore->set($arrTask);
}
}
# 获取分页里面的视频列表
public function getVideoPage($arrData)
{
$TaskCore = $this->TaskCore;
$Client = $this->Client;
$strUrl = $arrData['video_page_url'];
$intClassId = $arrData['video_class_id'];
$strSiteUrl = $this->getHeiMuErDomain();
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
var_dump($strUrl);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'Accept-Language' => 'en-US,en;q=0.5',
'cookie' => '__sk_Vm8nTqM7LqSCWmR6__=be3bd4f4e0483619e1a434ec1b18a938',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 300,
]);
$strResult = (string)$Response->getBody();
$QueryList = QueryList::html($strResult);
// 检查是否找到元素
if ($QueryList->find('.stui-vodlist li.clearfix h3 a')->count() > 0) {
$QueryList->find('.stui-vodlist li.clearfix h3 a ')->map(function ($Video) use (&$arrVideo, $intClassId, &$TaskCore, $strSiteUrl) {
$strHref = $Video->attr('href');
$strName = $Video->text();
if (!empty($strName) && !empty($strHref) && strlen($strName) > 0 && strlen($strHref) > 0) {
$arrTask = [
'callback' => [self::class, 'getVideoInfo'],
'data' => [
'class_id' => $intClassId,
'video_href' => $strHref,
'video_name' => $strName,
]
];
// var_dump($arrTask);
$TaskCore->set($arrTask);
echo "视频投递成功 : " . $Video->text() . PHP_EOL;
}
});
} else {
echo "No elements found for .stui-vodlist a" . PHP_EOL;
}
}
public function getVideoInfo($arrData)
{
$TaskCore = $this->TaskCore;
$Client = $this->Client;
$strSiteUrl = $this->getHeiMuErDomain();
$strUrl = $strSiteUrl . $arrData['video_href'];
$intClassId = $arrData['class_id'];
$strVideoName = $arrData['video_name'];
var_dump($strUrl);
if (!empty($strVideoName) && !empty($strUrl) && strlen($strVideoName) > 0 && strlen($strUrl) > 0) {
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
$QueryList = QueryList::html($strResult);
$strCover = $QueryList->find('.img-responsive')->eq(0)->attr("src");
if (strpos($strCover, 'http') === false) {
$strCover = $strSiteUrl . $strCover;
}
$strCoverUri = $strCover;
// try {
// $strCoverUri = "";
// $strCoverUri = ImageProcessor::getInstance()->downloadImage( $intXiaoShuoSourceId,$strCover, 'dadi');
// } catch (\Throwable $t) {
// $strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
// echo $strErr;
// }
$arrCover = [
'code' => 'HEIMUER_SOURCE_COVER',
'uri' => $strCoverUri,
];
$strLanguage = "";
// if (strlen($strLanguage) > 0) {
// VideoLeixinModel::addVideoLeixin(['vc_id' => $intClassId, 'vla_name' => $strLanguage]);
// }
// 提取别名
$strPinyinId = $QueryList->find('.stui-content__detail p.data')->eq(0)->text();
// 提取评分
$strPinfen = $QueryList->find('.stui-content__detail h1 .text-red')->text();
// 提取地区
$strDiqu = $QueryList->find('.stui-content__detail p.data')->eq(1)->text();
// 角标-状态
$strJiaoBiao = $QueryList->find('.stui-content__detail p.data')->eq(2)->find('.text-red')->eq(0)->text();
// 更新时间
$strLastDate = $QueryList->find('.stui-content__detail p.data')->eq(2)->find('.text-red')->eq(1)->text();
// 提取主演
$strZhuyan = $QueryList->find('.stui-content__detail p.data')->eq(3)->text();
// 提取导演
$strDaoyan = $QueryList->find('.stui-content__detail p.data')->eq(4)->text();
// 提取
$arrLeixinData = $QueryList->find('.stui-content__detail p.data')->eq(5)->text();
// 提取
$arrYuyanData = $QueryList->find('.stui-content__detail p.data')->eq(6)->text();
$strLeixin = '';
$strDiqu = '';
$strYears = '';
$strKuozhang = '';
// 使用正则表达式提取内容
$patternLeixin = '/类型：(.*?)扩展：(.*?)地区：(.*?)年份：(.*)/';
if (preg_match($patternLeixin, $arrLeixinData, $matches)) {
$strLeixin = trim($matches[1]);
$strKuozhang = trim($matches[2]);
$strDiqu = trim($matches[3]);
$strYears = trim($matches[4]);
}
$strLanguage = '';
// 使用正则表达式提取内容
$patternYuyan = '/语言：(.*?)集数：(.*?)时长：(.*?)点击：(.*)/';
if (preg_match($patternYuyan, $arrYuyanData, $matches)) {
$strLanguage = trim($matches[1]);
// $strKuozhang = trim($matches[2]);
// $strDiqu = trim($matches[3]);
// $strYears = trim($matches[4]);
}
if (strlen($strLeixin) > 0) {
VideoLeixinModel::addVideoLeixin(['vc_id' => $intClassId, 'vl_name' => $strLeixin]);
}
if (strlen($strDiqu) > 0) {
VideoDiquModel::add(['vc_id' => $intClassId, 'vd_name' => $strDiqu]);
}
if (strlen($strYears) > 0) {
VideoYearsModel::add(['vc_id' => $intClassId, 'vy_name' => $strYears]);
}
// 提取简介
$strDescription = $QueryList->find('.stui-content__desc')->text();
$arrVideoInfo = [
'v_name' => $strVideoName,
'v_cover' => json_encode($arrCover),
'vc_id' => $intClassId,
'v_pinyin' => trim(str_replace('别名：', '', $strPinyinId)),
'v_leixin' => $strLeixin,
'v_juqing' => '',
'v_diqu' => $strDiqu,
'v_years' => $strYears,
'v_language' => $strLanguage,
'v_daoyan' => trim(str_replace('主演：', '', $strZhuyan)),
'v_zhuyan' => trim(str_replace('导演：', '', $strDaoyan)),
'update_time' => $strLastDate,
'v_description' => $strDescription,
'v_pingfen' => $strPinfen,
'v_orurl' => $strUrl,
'v_jiaobiao' => $strJiaoBiao,
'v_md5' => md5($intClassId . $strVideoName . getUriByUrl($strUrl)),
];
//var_dump($arrVideoInfo);
// // 检查是否找到元素
if ($QueryList->find('.stui-content__playlist li')->count() > 0) {
// var_dump("视频入库成功--" . $strVideoName);
// $VideoInfoModel = VideoInfoModel::addVideoXiangQing($arrVideoInfo);
$resultVideoInfoModel = VideoInfoModel::addVideoXiangQing($arrVideoInfo);;
if ($resultVideoInfoModel['isNew']) {
var_dump("数据已新增：", $resultVideoInfoModel['model']->v_id."视频入库成功--" . $strVideoName);
} else {
echo "数据已存在：", $resultVideoInfoModel['model']->v_id;
}
$VideoInfoModel = $resultVideoInfoModel['model'];
$intVid = $VideoInfoModel->v_id;
$intChapterIndex = 0;
$QueryList->find('.stui-content__playlist li')->map(function ($ZhangJie) use (&$arrZhangjie, &$intChapterIndex, &$intVid) {
//echo "Processing index $intChapterIndex: " . $ZhangJie->text() . PHP_EOL;
$strM3u8 = $ZhangJie->find('a')->eq(1)->text();
$arrM3u8 = explode('$', $strM3u8);
$strM3u8Url = $arrM3u8[1];
$strM3u8Name = $arrM3u8[0];
$arrM3u8Url = [
'code' => 'HEIMUER_SOURCE_PLAY',
'uri' => $strM3u8Url,
];
$arrVideoPlayurl = [
'v_id' => $intVid,
'vp_sort' => $intChapterIndex,
'vp_name' => $strM3u8Name,
'vp_url' => json_encode($arrM3u8Url),
'vp_md5' => md5($intVid . getUriByUrl($strM3u8Url)),
];
$result = VideoPlayurlModel::add($arrVideoPlayurl);;
if ($result['isNew']) {
var_dump("数据已新增：", $result['model']->vp_id."播放线路入库成功--" . $strM3u8Url) ;
} else {
echo "数据已存在：", $result['model']->vp_id ;
}
$intChapterIndex++;
});
var_dump("--") ;
var_dump("一个视频任务完成------------------------------------------------------------------------------------") ;
} else {
echo "No elements found for .listmain a" . PHP_EOL;
}
} catch (\Throwable $T) {
throw $T;
}
}
// sleep(10);
}
public function getNovelFengMian($arrData)
{
//var_dump('getNovelFengMian');
//var_dump($arrData);
$Client = $this->Client;
$TaskCore = $this->TaskCore;
// 获取 Redis 实例
$redis = Cache::store('redis')->handler();
$intXiaoShuoSourceId = $arrData['xiaoshuo_id'];
$intXiaoShuoSourceSeoId = $arrData['xiaoshuo_seo_id'];
$redisKey = "task:novel_source_id:" . $intXiaoShuoSourceId;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($arrData['xiaoshuo_url']);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$strSiteUrl = $this->getAilisiDomain();
$strCover = $QueryList->find('#fmimg img')->eq(0)->attr("src");
if (strpos($strCover, 'http') === false) {
$strCover = $strSiteUrl . $strCover;
}
$strCoverUri = $strCover;
try {
$strCoverUri = "";
$strCoverUri = ImageProcessor::getInstance()->downloadAndEncryptImage($strCover, 'xs');
$arrCover = [
'code' => 'AI_LI_SI_COVER',
// 'uri' => getUriByUrl($strCover),
'uri' => $strCoverUri,
];
$arrXiaoShuoInfo = [
'xsxq_feng_mian' => json_encode($arrCover),
];
$XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
} catch (\Throwable $t) {
$strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
echo $strErr;
}
} catch (\Throwable $T) {
throw $T;
}
}
public function getNovelReader($arrData)
{
$strMuluHref = $arrData['vp_url'];
var_dump($strMuluHref);
$Client = $this->Client;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strMuluHref);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strMuluHref, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
unset($Response);
$QueryList = QueryList::html($strResult);
//提取 <script> 标签内容
$scriptContent = $QueryList->find('.vidoe-data-show')->text();
$strM3u8Url = "";
// 使用正则表达式提取 "url" 的值
if (preg_match('/"url":"(https:[^"]+\.m3u8)"/', $scriptContent, $matches)) {
$strM3u8Url = str_replace('\/', '/', $matches[1]); // 替换转义斜杠
echo "提取的 M3U8 地址: $strM3u8Url\n";
$arrM3u8Url = [
'code' => 'DADI_YINGSHI_M3U8',
'uri' => $strM3u8Url,
];
$arrVideoPlayurl = [
'v_id' => $arrData['v_id'],
'vp_sort' => $arrData['vp_sort'],
'vp_name' => $arrData['vp_name'],
'vp_url' => json_encode($arrM3u8Url),
'vp_md5' => md5($arrData['vp_url'] . $arrData['vp_url']),
];
VideoPlayurlModel::add($arrVideoPlayurl);
unset($QueryList);
} else {
echo "未找到 M3U8 地址\n";
}
} catch (\Throwable $T) {
throw $T;
}
}
public function pullDadiYingshi2()
{
$strSiteUrl = $this->getHeiMuErDomain();
$TaskCore = $this->TaskCore;
//电影 307 https://heimuer.tv/index.php/vod/type/id/1/page/307.html
for ($i = 307; $i >= 1; $i--) {
if (!config('task.task_is_first') && $i > 1) {
continue;
}
echo "当前值: $i\n";
$arrTask = [
'callback' => [self::class, 'getVideoPage'],
'data' => [
'video_page_url' => $strSiteUrl . '/index.php/vod/type/id/1/page/' . $i . '.html',
'video_class_id' => 1
]
];
$TaskCore->set($arrTask);
}
//5-87 短剧 https://heimuer.tv/index.php/vod/type/id/27/page/87.html
for ($i = 87; $i >= 1; $i--) {
if (!config('task.task_is_first') && $i > 1) {
continue;
}
echo "当前值: $i\n";
$arrTask = [
'callback' => [self::class, 'getVideoPage'],
'data' => [
'video_page_url' => $strSiteUrl . '/index.php/vod/type/id/27/page/' . $i . '.html',
'video_class_id' => 5
]
];
$TaskCore->set($arrTask);
}
//电视剧 2--225 https://heimuer.tv/index.php/vod/type/id/2/page/225.html
for ($i = 225; $i >= 1; $i--) {
if (!config('task.task_is_first') && $i > 1) {
continue;
}
echo "当前值: $i\n";
$arrTask = [
'callback' => [self::class, 'getVideoPage'],
'data' => [
'video_page_url' => $strSiteUrl . '/index.php/vod/type/id/2/page/' . $i . '.html',
'video_class_id' => 2
]
];
$TaskCore->set($arrTask);
}
//综艺 78 https://heimuer.tv/index.php/vod/type/id/4/page/78.html
for ($i = 78; $i >= 1; $i--) {
if (!config('task.task_is_first') && $i > 1) {
continue;
}
echo "当前值: $i\n";
$arrTask = [
'callback' => [self::class, 'getVideoPage'],
'data' => [
'video_page_url' => $strSiteUrl . '/index.php/vod/type/id/4/page/' . $i . '.html',
'video_class_id' => 3
]
];
$TaskCore->set($arrTask);
}
// 动漫--100 https://heimuer.tv/index.php/vod/type/id/3/page/1.html
for ($i = 100; $i >= 1; $i--) {
if (!config('task.task_is_first') && $i > 1) {
continue;
}
echo "当前值: $i\n";
$arrTask = [
'callback' => [self::class, 'getVideoPage'],
'data' => [
'video_page_url' => $strSiteUrl . '/index.php/vod/type/id/3/page/' . $i . '.html',
'video_class_id' => 4
]
];
$TaskCore->set($arrTask);
}
}
}