admin/SEONexus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

debug

Browse Source
This commit is contained in:
Default
2025-03-15 20:12:21 +08:00
parent 105ffe11c7
commit eb9035a7ae
134 changed files with 15450 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files

498
code/app/task/collection/dadi/DaDiDianyingCol.php Normal file
View File

@@ -0,0 +1,498 @@
<?php
declare(strict_types=1);
namespace app\task\collection\dadi;
use app\admin\model\VideoPlayurlModel;
use app\admin\model\VideoLeixinModel;
use app\admin\model\VideoDiquModel;
use app\admin\model\VideoYearsModel;
use app\admin\model\VideoJuqingModel;
use app\admin\model\XiaoShuoFenLeiModel;
use app\admin\model\XiaoShuoXiangQingModel;
use app\admin\model\VideoInfoModel;
use app\admin\model\XiaoShuoZhangJieModel;
use app\task\collection\PullDataColBase;
use GuzzleHttp\Cookie\CookieJar;
use processor\ImageProcessor;
use think\facade\Cache;
use QL\QueryList;
/**
* @mixin think\Model
*/
class DaDiDianyingCol extends PullDataColBase
{
public function pullDadiYingshi()
{
$TaskCore = $this->TaskCore;
// 1-75
for ($i = 1; $i <= 75; $i++) {
echo "当前值: $i\n";
$arrTask = [
'callback' => [self::class, 'getVideoPage'],
'data' => [
'video_page_url' => 'https://www.czdadi.net/vodshow/dy--------'.$i.'---.html',
'video_class_id' => 1
]
];
$TaskCore->set($arrTask);
}
//5-25 https://www.czdadi.net/vodshow/duanju-----------.html
for ($i = 1; $i <= 25; $i++) {
echo "当前值: $i\n";
$arrTask = [
'callback' => [self::class, 'getVideoPage'],
'data' => [
'video_page_url' => 'https://www.czdadi.net/vodshow/duanju--------'.$i.'---.html',
'video_class_id' => 5
]
];
$TaskCore->set($arrTask);
}
// 2--18 https://www.czdadi.net/vodshow/dsj--------18---.html
for ($i = 1; $i <= 18; $i++) {
echo "当前值: $i\n";
$arrTask = [
'callback' => [self::class, 'getVideoPage'],
'data' => [
'video_page_url' => 'https://www.czdadi.net/vodshow/dsj--------'.$i.'---.html',
'video_class_id' => 2
]
];
$TaskCore->set($arrTask);
}
// 3--11 https://www.czdadi.net/vodshow/zy--------11---.html
for ($i = 1; $i <= 11; $i++) {
echo "当前值: $i\n";
$arrTask = [
'callback' => [self::class, 'getVideoPage'],
'data' => [
'video_page_url' => 'https://www.czdadi.net/vodshow/zy--------'.$i.'---.html',
'video_class_id' => 3
]
];
$TaskCore->set($arrTask);
}
// 4--29 https://www.czdadi.net/vodshow/dm-----------.html
for ($i = 1; $i <= 29; $i++) {
echo "当前值: $i\n";
$arrTask = [
'callback' => [self::class, 'getVideoPage'],
'data' => [
'video_page_url' => 'https://www.czdadi.net/vodshow/dm--------'.$i.'---.html',
'video_class_id' => 4
]
];
$TaskCore->set($arrTask);
}
}
# 获取分页里面的视频列表
public function getVideoPage($arrData)
{
$TaskCore = $this->TaskCore;
$Client = $this->Client;
$strUrl= $arrData['video_page_url'];
$intClassId = $arrData['video_class_id'];
$strSiteUrl = $this->get5cccDomain();
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
$QueryList = QueryList::html($strResult);
//$arrVideo = [];
// 检查是否找到元素
if ($QueryList->find('.stui-vodlist li .stui-vodlist__detail a')->count() > 0) {
$QueryList->find('.stui-vodlist li .stui-vodlist__detail a ')->map(function ($Video) use (&$arrVideo,$intClassId,&$TaskCore) {
// $arrVideo[] = [
// 'mingzi' => $Video->text(),
// 'url' => $Video->attr('href'),
// ];
if(!empty($Video->text()) && !empty($Video->attr('href'))){
$arrTask = [
'callback' => [self::class, 'getVideoInfo'],
'data' => [
'class_id' => $intClassId,
'video_href' => $Video->attr('href'),
'video_name' => $Video->text(),
]
];
$TaskCore->set($arrTask);
echo "视频投递成功 : " . $Video->text() . PHP_EOL;
}
});
// foreach ($arrVideo as $intVideoNum => $arrVideoOne) {
// $arrTask = [
// 'callback' => [self::class, 'getVideoInfo'],
// 'data' => [
// 'class_id' => $intClassId,
// 'video_href' => $arrVideoOne['url'],
// 'video_name' => $arrVideoOne['mingzi'],
// ]
// ];
// $TaskCore->set($arrTask);
// }
// 输出最终结果
// var_dump($arrVideo);
} else {
echo "No elements found for .stui-vodlist a" . PHP_EOL;
}
}
public function getVideoInfo($arrData)
{
$TaskCore = $this->TaskCore;
$Client = $this->Client;
$strSiteUrl = $this->getDadiDomain();
$strUrl = $strSiteUrl . $arrData['video_href'] ;
$intClassId = $arrData['class_id'];
$strVideoName = $arrData['video_name'];
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strUrl);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strUrl, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
$QueryList = QueryList::html($strResult);
$strCover = $QueryList->find('.stui-content__thumb img')->eq(0)->attr("data-original");
if (strpos($strCover, 'http') === false) {
$strCover = $strSiteUrl . $strCover;
}
$strCoverUri = $strCover;
// try {
// $strCoverUri = "";
// $strCoverUri = ImageProcessor::getInstance()->downloadImage( $intXiaoShuoSourceId,$strCover, 'dadi');
// } catch (\Throwable $t) {
// $strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
// echo $strErr;
// }
$arrCover = [
'code' => 'DADI_YINGSHI_COVER',
// 'uri' => getUriByUrl($strCover),
'uri' => $strCoverUri,
];
$strLanguage = "";
// if (strlen($strLanguage) > 0) {
// VideoLeixinModel::addVideoLeixin(['vc_id' => $intClassId, 'vla_name' => $strLanguage]);
// }
// 提取评分
$strPinfen = $QueryList->find('h1.title .score')->text();
// 提取类型-入库
$strLeixin = $QueryList->find('.col-pd a')->eq(1)->text();
if(strlen($strLeixin) > 0){
VideoLeixinModel::addVideoLeixin(['vc_id'=>$intClassId,'vl_name'=>$strLeixin]);
}
// 提取 HTML 的第一段 <p class="data">
$dataNode = $QueryList->find('p.data')->eq(0);
// 提取“类型”内容：确保只匹配 `href` 中包含 `/vodsearch/----`
$leixinNodes = $dataNode->find('a[href^="/vodsearch/----"]');
$strJuqing = $leixinNodes->map(function ($node) {
return $node->text();
})->all();
// 提取“地区”内容：确保只匹配 `href` 中包含 `/vodsearch/--`，但不包含 `/vodsearch/----`
$diquNodes = $dataNode->find('a[href^="/vodsearch/--"]')->not('a[href^="/vodsearch/----"]');
$strDiqu = $diquNodes->map(function ($node) {
return $node->text();
})->all();
// 提取“年份”内容：确保只匹配 `href` 中包含 `/vodsearch/-------------`
$yearsNode = $dataNode->find('a[href^="/vodsearch/-------------"]');
$strYears = $yearsNode->text();
if (!empty(trim($strYears))) {
$strYears = [trim($strYears)];
} else {
$strYears = [];
}
// 过滤掉“地区”和“年份”中的数据
$strJuqing = array_filter($strJuqing, function ($value) use ($strDiqu, $strYears) {
return !in_array($value, array_merge($strDiqu, $strYears));
});
// **在过滤后将“类型”数据入库**
foreach ($strJuqing as $value) {
if(strlen($value) > 0){
VideoJuqingModel::add(['vc_id' => $intClassId, 'vj_name' => $value]);
}
}
// 将“地区”数据入库
foreach ($strDiqu as $value) {
if(strlen($value) > 0){
VideoDiquModel::add(['vc_id' => $intClassId, 'vd_name' => $value]);
}
}
// 将“年份”数据入库
foreach ($strYears as $value) {
if(strlen($value) > 0){
VideoYearsModel::add(['vc_id' => $intClassId, 'vy_name' => $value]);
}
}
// 提取主演
$strZhuyan = $QueryList->find('p.data:contains("主演") a')->map(function ($item) {
return $item->text();
})->implode(', '); // 使用逗号分隔主演
// 提取导演
$strDaoyan = $QueryList->find('p.data:contains("导演") a')->map(function ($item) {
return $item->text();
})->implode(', '); // 使用逗号分隔主演
// 提取更新时间
$strLastDate = $QueryList->find('p.data.hidden-sm:contains("更新")')->text();
// 去掉“更新：”前缀
$strLastDate = str_replace('更新：', '', $strLastDate);
// 提取简介
$strDescription = $QueryList->find('#desc>.stui-pannel-box>.stui-pannel_bd>.col-pd')->text();
// 角标
$strJiaoBiao = $QueryList->find('.stui-content__thumb .text-right')->eq(0)->text();
$strPinyinId = extractFilename($strUrl);
$arrVideoInfo = [
'v_name' => $strVideoName,
'v_cover' => json_encode($arrCover),
'vc_id' => $intClassId,
'v_pinyin' => $strPinyinId,
'v_leixin' => $strLeixin ,
'v_juqing' => implode(', ', $strJuqing),
'v_diqu' => implode(', ', $strDiqu),
'v_years' => implode(', ', $strYears),
'v_language' => $strLanguage,
'v_daoyan' => $strZhuyan,
'v_zhuyan' => $strDaoyan,
'update_time' => $strLastDate,
'v_description'=>$strDescription,
'v_pingfen' => $strPinfen,
'v_orurl' => $strUrl,
'v_jiaobiao'=> $strJiaoBiao,
'v_md5' => md5($intClassId.$strVideoName.$strSiteUrl.$strUrl),
];
$VideoInfoModel = VideoInfoModel::addVideoXiangQing($arrVideoInfo);
$intVid = $VideoInfoModel->v_id;
$intChapterIndex = 0 ;
$arrZhangjie = [];
// // 检查是否找到元素
if ($QueryList->find('.stui-content__playlist a')->count() > 0) {
$QueryList->find('.stui-content__playlist a')->map(function ($ZhangJie) use (&$arrZhangjie, &$intChapterIndex ,&$intVid) {
//echo "Processing index $intChapterIndex: " . $ZhangJie->text() . PHP_EOL;
$arrZhangjie[] = [
'v_id' => $intVid,
'index' => $intChapterIndex,
'mingzi' => $ZhangJie->text(),
'url' => $ZhangJie->attr('href'),
];
$intChapterIndex++;
});
//var_dump($arrZhangjie);
foreach ($arrZhangjie as $intZhangJieNum => $arrZhangjieOne) {
var_dump($intZhangJieNum);
// 如果数据库里的章节最大索引大于当前索引，说明已经入库，不需要入队列了
// if($intMaxXiaoShuoZhangJiePaixu > $intZhangJieNum){
// var_dump("小说章节已经入库，不再推送任务");
// continue;// 如果任务已存在，直接跳过 break
// }
//if($intZhangJieNum<=3){
$arrTask = [
'callback' => [self::class, 'getNovelReader'],
'data' => [
'v_id' => $intVid,
'vp_sort' => $intZhangJieNum,
'vp_name' => $arrZhangjieOne['mingzi'],
'vp_url' => $strSiteUrl . $arrZhangjieOne['url']
]
];
$TaskCore->set($arrTask);
//}
}
} else {
echo "No elements found for .listmain a" . PHP_EOL;
}
} catch (\Throwable $T) {
throw $T;
}
// sleep(10);
}
public function getNovelFengMian($arrData)
{
//var_dump('getNovelFengMian');
//var_dump($arrData);
$Client = $this->Client;
$TaskCore = $this->TaskCore;
// 获取 Redis 实例
$redis = Cache::store('redis')->handler();
$intXiaoShuoSourceId = $arrData['xiaoshuo_id'];
$intXiaoShuoSourceSeoId = $arrData['xiaoshuo_seo_id'];
$redisKey = "task:novel_source_id:" . $intXiaoShuoSourceId;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($arrData['xiaoshuo_url']);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$strSiteUrl = $this->getAilisiDomain();
$strCover = $QueryList->find('#fmimg img')->eq(0)->attr("src");
if (strpos($strCover, 'http') === false) {
$strCover = $strSiteUrl . $strCover;
}
$strCoverUri = $strCover;
try {
$strCoverUri = "";
$strCoverUri = ImageProcessor::getInstance()->downloadAndEncryptImage($strCover, 'xs');
$arrCover = [
'code' => 'AI_LI_SI_COVER',
// 'uri' => getUriByUrl($strCover),
'uri' => $strCoverUri,
];
$arrXiaoShuoInfo = [
'xsxq_feng_mian' => json_encode($arrCover),
];
$XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
} catch (\Throwable $t) {
$strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
echo $strErr;
}
} catch (\Throwable $T) {
throw $T;
}
}
public function getNovelReader($arrData)
{
if (Cache::has('video_no_m3u8_' . $arrData['v_id'])) {
echo "跳过抓取，视频ID已在缓存中: " . $arrData['v_id'] . "\n";
return; // 跳过抓取逻辑
}
$strMuluHref = $arrData['vp_url'];
var_dump($strMuluHref);
$Client = $this->Client;
try {
$arrCookie = $this->getAilisiCookie();
$strHost = getDomainFromUrl($strMuluHref);
$CookieJar = CookieJar::fromArray($arrCookie, $strHost);
$proxy = getRandomProxies();
$userAgent = getRandomUserAgent();
$Response = $Client->request('GET', $strMuluHref, [
'headers' => [
'User-Agent' => $userAgent,
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language' => 'en-US,en;q=0.5',
],
'cookies' => $CookieJar,
// 'proxy' => $proxy,
'timeout' => 30,
]);
$strResult = (string)$Response->getBody();
unset($Response);
$QueryList = QueryList::html($strResult);
//提取 <script> 标签内容
$scriptContent = $QueryList->find('.vidoe-data-show')->text();
$strM3u8Url = "";
// 使用正则表达式提取 "url" 的值
if (preg_match('/"url":"(https:[^"]+\.m3u8)"/', $scriptContent, $matches)) {
$strM3u8Url = str_replace('\/', '/', $matches[1]); // 替换转义斜杠
echo "提取的 M3U8 地址: $strM3u8Url\n";
$arrM3u8Url = [
'code' => 'DADI_YINGSHI_M3U8',
'uri' => $strM3u8Url,
];
$arrVideoPlayurl = [
'v_id' => $arrData['v_id'],
'vp_sort' => $arrData['vp_sort'],
'vp_name' => $arrData['vp_name'],
'vp_url' => json_encode($arrM3u8Url),
'vp_md5' => md5($arrData['vp_url'] . $arrData['vp_url']),
];
VideoPlayurlModel::add($arrVideoPlayurl);
unset($QueryList);
} else {
echo "未找到 M3U8 地址\n";
// 将视频ID保存到缓存，设置一个过期时间，例如 1 小时
Cache::set('video_no_m3u8_' . $arrData['v_id'], true, 3600);
}
} catch (\Throwable $T) {
throw $T;
}
}
}