admin/SEONexus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

debug

Browse Source
This commit is contained in:
Default
2025-03-15 20:12:21 +08:00
parent 105ffe11c7
commit eb9035a7ae
134 changed files with 15450 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files

375
code/app/task/collection/yzzy/YouZhiYingShiCol.php Normal file
View File

@@ -0,0 +1,375 @@
<?php
declare(strict_types=1);
namespace app\task\collection\yzzy;
use app\admin\model\VideoYuyanModel;
use app\admin\model\VideoPlayurlModel;
use app\admin\model\VideoLeixinModel;
use app\admin\model\VideoDiquModel;
use app\admin\model\VideoYearsModel;
use app\admin\model\VideoJuqingModel;
use app\admin\model\XiaoShuoFenLeiModel;
use app\admin\model\XiaoShuoXiangQingModel;
use app\admin\model\VideoInfoModel;
use app\admin\model\XiaoShuoZhangJieModel;
use app\task\collection\PullDataColBase;
use GuzzleHttp\Cookie\CookieJar;
use processor\ImageProcessor;
use think\facade\Cache;
use QL\QueryList;
/**
* @mixin think\Model 优质资源
*/
class YouZhiYingShiCol extends PullDataColBase
{
public function pullYouZhiYingshi()
{
// https://1080zyk1.com/?m=vod-type-id-1-pg-1.html
// https://1080zyk1.com/?m=vod-type-id-2-pg-2.html
// https://1080zyk1.com/?m=vod-type-id-3-pg-2.html
// https://1080zyk1.com/?m=vod-type-id-4-pg-2.html
// https://1080zyk1.com/?m=vod-type-id-83-pg-2.html
// https://1080zyk1.com/?m=vod-type-id-94-pg-2.html
$strSiteUrl = $this->YouZhiZiYuanDomain();
// 配置分类任务参数
$categories = [
['id' => 1, 'video_class_id' => 1, 'name' => '电影'],
['id' => 83, 'video_class_id' => 5, 'name' => '短剧'],
['id' => 2, 'video_class_id' => 2, 'name' => '电视剧'],
['id' => 3, 'video_class_id' => 3, 'name' => '综艺'],
['id' => 4, 'video_class_id' => 4, 'name' => '动漫'],
['id' => 94, 'video_class_id' => 6, 'name' => '体育'],
];
foreach ($categories as $category) {
$intClassId = $category['video_class_id'];
$intCategoryId = $category['id'];
// 先抓第一页
$strUrl = $strSiteUrl . '/?m=vod-type-id-' . $intCategoryId . '-pg-1.html';
var_dump($strUrl);
$maxPage = $this->getMaxPage($strUrl,$intClassId);
// 首次抓取，全部抓取
if (config('task.task_is_first')) {
$this->createTasks($strSiteUrl, $intCategoryId, $intClassId, $maxPage);
}
}
}
/**
* 获取分类的最大页码且抓取第一页
*/
protected function getMaxPage($strUrl, $intClassId)
{
$TaskCore = $this->TaskCore;
$Client = $this->Client;
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$Response = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'Accept-Language' => 'en-US,en;q=0.5',
'cookie' => '__sk_Vm8nTqM7LqSCWmR6__=be3bd4f4e0483619e1a434ec1b18a938',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 300,
]);
$strResult = (string)$Response->getBody();
$QueryList = QueryList::html($strResult);
// 检查是否找到元素
$intVideoListIndex = 0;
if ($QueryList->find('.xing_vb ul')->count() > 1) {
$QueryList->find('.xing_vb ul')->map(function ($Video) use ($intClassId, &$TaskCore,&$intVideoListIndex) {
if($intVideoListIndex >=1){
$strHref = $Video->find('.xing_vb4 a')->attr('href');
$strName = '';
if (!empty($strHref) && strlen($strHref) > 0) {
$arrTask = [
'callback' => [self::class, 'getVideoInfo'],
'data' => [
'class_id' => $intClassId,
'video_href' => $strHref,
'video_name' => $strName,
]
];
// var_dump($arrTask);
$TaskCore->set($arrTask);
echo "视频投递成功 : " . $Video->text() . PHP_EOL;
}
}
$intVideoListIndex ++ ;
});
} else {
echo "No elements found for .xing_vb ul" . PHP_EOL;
}
// 获取分页部分最后一个页码链接
$lastPageHref = $QueryList->find('.pages a')->eq($QueryList->find('.pages a')->count() - 1)->attr('href');
// 提取最大页码
preg_match('/-pg-(\d+)\.html/', $lastPageHref, $matches);
$maxPage = isset($matches[1]) ? (int)$matches[1] : 1; // 默认返回 1
return $maxPage;
}
/**
* 创建任务
*/
protected function createTasks($strSiteUrl, $intCategoryId, $videoClassId, $maxPage)
{
$TaskCore = $this->TaskCore;
for ($i = $maxPage; $i >= 1; $i--) {
echo "当前分类: $intCategoryId, 当前页: $i\n";
$arrTask = [
'callback' => [self::class, 'getVideoPage'],
'data' => [
'video_page_url' => $strSiteUrl . '/?m=vod-type-id-' . $intCategoryId . '-pg-' . $i . '.html',
'video_class_id' => $videoClassId,
]
];
$TaskCore->set($arrTask);
}
}
# 获取分页里面的视频列表
public function getVideoPage($arrData)
{
$TaskCore = $this->TaskCore;
$Client = $this->Client;
$strUrl = $arrData['video_page_url'];
$intClassId = $arrData['video_class_id'];
$strSiteUrl = $this->YouZhiZiYuanDomain();
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
var_dump($strUrl);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'Accept-Language' => 'en-US,en;q=0.5',
'cookie' => '__sk_Vm8nTqM7LqSCWmR6__=be3bd4f4e0483619e1a434ec1b18a938',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 300,
]);
$strResult = (string)$Response->getBody();
$QueryList = QueryList::html($strResult);
$intVideoListIndex = 0;
if ($QueryList->find('.xing_vb ul')->count() > 1) {
$QueryList->find('.xing_vb ul')->map(function ($Video) use ($intClassId, &$TaskCore,&$intVideoListIndex) {
if($intVideoListIndex >=1){
$strHref = $Video->find('.xing_vb4 a')->attr('href');
$strName = '';
if (!empty($strHref) && strlen($strHref) > 0) {
$arrTask = [
'callback' => [self::class, 'getVideoInfo'],
'data' => [
'class_id' => $intClassId,
'video_href' => $strHref,
'video_name' => $strName,
]
];
// var_dump($arrTask);
$TaskCore->set($arrTask);
echo "视频投递成功 : " . $strHref . PHP_EOL;
}
}
$intVideoListIndex ++ ;
});
} else {
echo "No elements found for .xing_vb ul" . PHP_EOL;
}
}
public function getVideoInfo($arrData)
{
$TaskCore = $this->TaskCore;
$Client = $this->Client;
$strSiteUrl = $this->YouZhiZiYuanDomain();
$strUrl = $strSiteUrl . $arrData['video_href'];
$intClassId = $arrData['class_id'];
$strVideoName = $arrData['video_name'] ?? '';
var_dump($strUrl);
if (!empty($strUrl) && strlen($strUrl) > 0) {
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
$QueryList = QueryList::html($strResult);
// 提取别名
$strPinyinId = 'yzzy'.extractIdFromUrl($strUrl);
$strVideoName = $QueryList->find('.vodh h2')->text();
$strJiaoBiao = $QueryList->find('.vodh span')->text();
$strPinfen = $QueryList->find('.vodh label')->text();
$strLeixin = $QueryList->find('.nvc dd a')->eq(1)->text();
$strCover = $QueryList->find('.vodImg img')->eq(0)->attr("src");
if (strpos($strCover, 'http') === false) {
$strCover = $strSiteUrl . $strCover;
}
$strCoverUri = $strCover;
$arrCover = [
'code' => 'YOUZHI_SOURCE_COVER',
'uri' => $strCoverUri,
];
// 提取导演
$strDaoyan = $QueryList->find('.vodInfo .vodinfobox ul li')->eq(1)->find('span')->text();
// 提取主演 多个逗号分割
$strZhuyan = $QueryList->find('.vodInfo .vodinfobox li')->eq(2)->find('span')->text();
// 提取类型
$arrLeixinData = $QueryList->find('.vodInfo .vodinfobox li')->eq(3)->find('span')->text();
// 切割后移除空元素
$arrJuqing = preg_split('/\s+/u', trim($arrLeixinData));
array_shift($arrJuqing);
// **在过滤后将“类型”数据入库**
foreach ($arrJuqing as $value) {
if(strlen($value) > 0){
VideoJuqingModel::add(['vc_id' => $intClassId, 'vj_name' => $value]);
}
}
// 提取地区
$strDiqu = $QueryList->find('.vodinfobox li span')->eq(4)->text();
// 提取语言
$strLanguage = $QueryList->find('.vodinfobox li span')->eq(5)->text();
// 提取年份
$strYears = $QueryList->find('.vodinfobox li span')->eq(6)->text();
// 提取更新时间
$strLastDate = $QueryList->find('.vodinfobox li span')->eq(8)->text();
if (strlen($strLeixin) > 0) {
VideoLeixinModel::addVideoLeixin(['vc_id' => $intClassId, 'vl_name' => $strLeixin]);
}
if (strlen($strDiqu) > 0) {
VideoDiquModel::add(['vc_id' => $intClassId, 'vd_name' => $strDiqu]);
}
if (strlen($strYears) > 0) {
VideoYearsModel::add(['vc_id' => $intClassId, 'vy_name' => $strYears]);
}
if (strlen($strLanguage) > 0) {
VideoYuyanModel::add(['vc_id' => $intClassId, 'vla_name' => $strLanguage]);
}
// 提取简介
$strDescription = $QueryList->find('.vodplayinfo')->eq(0)->text();
if(empty($strVideoName)){
var_dump('$strVideoName------');
var_dump($strVideoName);
return true;
}
$arrVideoInfo = [
'v_name' => $strVideoName,
'v_cover' => json_encode($arrCover),
'vc_id' => $intClassId,
'v_pinyin' => $strPinyinId,
'v_leixin' => $strLeixin,
'v_juqing' => implode(',', $arrJuqing),
'v_diqu' => $strDiqu,
'v_years' => $strYears,
'v_language' => $strLanguage,
'v_daoyan' => trim(str_replace('主演：', '', $strZhuyan)),
'v_zhuyan' => trim(str_replace('导演：', '', $strDaoyan)),
'update_time' => $strLastDate,
'v_description' => $strDescription,
'v_pingfen' => $strPinfen,
'v_orurl' => $strUrl,
'v_jiaobiao' => $strJiaoBiao,
'v_md5' => md5($intClassId .$strPinyinId. $strVideoName . getUriByUrl($strUrl)),
];
// var_dump($arrVideoInfo);
// // 检查是否找到元素
if ($QueryList->find('#play_2 ul li')->count() > 0) {
// var_dump("视频入库成功--" . $strVideoName);
$resultVideoInfoModel = VideoInfoModel::addVideoXiangQing($arrVideoInfo);;
if ($resultVideoInfoModel['isNew']) {
var_dump("视频详情数据已新增：", $resultVideoInfoModel['model']->v_id."视频入库成功--" . $strVideoName);
} else {
echo "视频详情数据已存在：", $resultVideoInfoModel['model']->v_id;
}
$VideoInfoModel = $resultVideoInfoModel['model'];
$intVid = $VideoInfoModel->v_id;
$intChapterIndex = 0;
$QueryList->find('#play_2 ul li')->map(function ($ZhangJie) use (&$arrZhangjie, &$intChapterIndex, &$intVid,&$intClassId) {
//echo "Processing index $intChapterIndex: " . $ZhangJie->text() . PHP_EOL;
$strM3u8 = $ZhangJie->find('a')->text();
$arrM3u8 = explode('$', $strM3u8);
$strM3u8Url = $arrM3u8[1];
$strM3u8Name = $arrM3u8[0];
$arrM3u8Url = [
'code' => 'YOUZHI_SOURCE_PLAY',
'uri' => $strM3u8Url,
];
$arrVideoPlayurl = [
'v_id' => $intVid,
'vp_sort' => $intChapterIndex,
'vp_name' => $strM3u8Name,
'vp_url' => json_encode($arrM3u8Url),
'vp_md5' => md5($intClassId.$intVid . getUriByUrl($strM3u8Url)),
];
$result = VideoPlayurlModel::add($arrVideoPlayurl);
if ($result['isNew']) {
var_dump("播放地址已新增：", $result['model']->vp_id."播放线路入库成功--" . $strM3u8Url) ;
} else {
echo "播放地址已存在：", $result['model']->vp_id ;
}
$intChapterIndex++;
});
var_dump("--") ;
var_dump("一个视频任务完成------------------------------------------------------------------------------------") ;
} else {
echo "No elements found for .listmain a" . PHP_EOL;
}
} catch (\Throwable $T) {
throw $T;
}
}
// sleep(10);
}
}