');
-
- // 替换不间断空格 (\xc2\xa0) 和 为普通空格
- $strNeiRong = preg_replace('/\xc2\xa0/', ' ', $strNeiRong); // 替换 UTF-8 不间断空格
- $strNeiRong = preg_replace('/ /', ' ', $strNeiRong); // 替换
-
- // 清理多余空格和换行符
- $strNeiRong = preg_replace('/\s+/', ' ', $strNeiRong);
- $strNeiRong = preg_replace('/\s+/', '
', $strNeiRong); // 清理
标签内部多余空格
- $strNeiRong = preg_replace('/\s+<\/p>/', '
', $strNeiRong); // 清理 标签结尾多余空格
-
- // 输出结果
- // var_dump($strNeiRong);
-
-
- # '/novel/123yqw/分类id/小说源id/小说源章节id.txt'
- $strNeiRongTxtPath = config("filesystem.novel").'/novel/biquxs/'.$intFenleiId.'/'.$intXiaoShuoSourceID.'/'.$strZhangJieSourceId.'.txt';
- //$strNeiRongTxtPath = "/www/novel/123yqw/1/11/111.txt";
- // 调用函数
- $result = saveCompressAndDeleteTxt($strNeiRongTxtPath, $strNeiRong);
-
- // 输出状态
- if ($result['status']) {
- var_dump("Success: " . $result['message']) ;
- } else {
- var_dump("Error: " . $result['message']) ;
- }
-
- $arrXiaoShuoZhangJie = [
- 'xsxq_id' => $intXiaoShuoSourceID,
- 'xszj_pai_xu' => $intZhangJiePaiXu,
- 'xszj_ming_zi' => $strZhangJieName,
- 'xszj_nei_rong' => $strNeiRongTxtPath,
- 'xszj_source_id' => $strZhangJieSourceId,
- 'xszj_source_code' => $strXszjSourceCode,
- ];
-
- XiaoShuoZhangJieModel::addXiaoShuoZhangJie($arrXiaoShuoZhangJie);
- unset($ZhangJieQueryList);
- // 延时 3 秒
- //sleep(3);
-
- } catch (\Throwable $T) {
- throw $T;
- }
- }
-}
diff --git a/code/app/task/collection/ccc5/XiaoShuo5cccCol.php b/code/app/task/collection/ccc5/XiaoShuo5cccCol.php
deleted file mode 100644
index 34d7e1a..0000000
--- a/code/app/task/collection/ccc5/XiaoShuo5cccCol.php
+++ /dev/null
@@ -1,545 +0,0 @@
-TaskCore;
- // 20054
- for ($i = 3; $i <= 3; $i++) {
- echo "当前值: $i\n";
- $arrTask = [
- 'callback' => [self::class, 'getNovelInfo'],
- 'data' => [
- 'xs_source_id' => $i,
- 'xs_source_seo_id' => 1,
- ]
- ];
- $TaskCore->set($arrTask);
- var_dump($arrTask);
- }
-
- }
-
- public function getNovelInfo($arrData)
- {
- $TaskCore = $this->TaskCore;
- $Client = $this->Client;
- $strSiteUrl = $this->get5cccDomain();
-
- $strUrl = 'https://www.5ccc.org/chapter/' . $arrData['xs_source_id'].'.html' ; //'http://www.biquxs.com/book/20053';
- $intXiaoShuoSourceId = $arrData['xs_source_id']; //20053;
- $intXiaoShuoSourceSeoId = $arrData['xs_source_seo_id']; //1 ;
- try {
- $arrCookie = $this->getAilisiCookie();
- $strHost = getDomainFromUrl($strUrl);
-
- $CookieJar = CookieJar::fromArray($arrCookie, $strHost);
-
- $proxy = getRandomProxies();
- $userAgent = getRandomUserAgent();
- $Response = $Client->request('GET', $strUrl, [
- 'headers' => [
- 'User-Agent' => $userAgent,
- 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
- 'Accept-Language' => 'en-US,en;q=0.5',
- ],
- 'cookies' => $CookieJar,
- // 'proxy' => $proxy,
- 'timeout' => 30,
- ]);
-
- $strResult = (string)$Response->getBody();
-
- $QueryList = QueryList::html($strResult);
- // var_dump($QueryList);
- // 提取所有 标签的属性
- $result = $QueryList->find('meta[property^="og:"]')->map(function ($item) {
- return [
- 'property' => $item->attr('property'),
- 'content' => $item->attr('content'),
- ];
- });
-
- // 打印结果
- print_r($result->all());
- $strZuoZhe = '';
- $strMingZi = '';
- $strLastDate = '';
- $strJieShao = '';
- $strCover = '';
- $strFenlei = '';
- $strZiShu = 0;
- $strZhuangTai = "";
-
- foreach ($result as $item) {
- switch ($item['property']) {
- case 'og:title':
- $strMingZi = $item['content'];
- break;
- case 'og:description':
- $strJieShao = $item['content'];
- break;
- case 'og:image':
- $strCover = $item['content'];
- break;
- case 'og:novel:category':
- $strFenlei = $item['content'];
- break;
- case 'og:novel:author':
- $strZuoZhe = $item['content'];
- break;
- case 'og:novel:update_time':
- $strLastDate = $item['content'];
- break;
- case 'og:novel:status':
- $strZhuangTai = $item['content'];
- break;
- }
- }
-
-
- $strLastDate = trim((string)$strLastDate);
- if (!isValidDateTime($strLastDate)) {
- $strLastDate = date('Y-m-d H:i:s');
- }
-
- $arrTag = [];
-
- $strCover = $QueryList->find('.imgbox img')->eq(0)->attr("src");
- if (strpos($strCover, 'http') === false) {
- $strCover = $strSiteUrl . $strCover;
- }
- $strCoverUri = $strCover;
-
- try {
- $strCoverUri = "";
- $strCoverUri = ImageProcessor::getInstance()->downloadImage( $intXiaoShuoSourceId,$strCover, 'xs');
- } catch (\Throwable $t) {
- $strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
- echo $strErr;
- }
-
- $arrCover = [
- 'code' => '5CCC_XS_COVER',
- // 'uri' => getUriByUrl($strCover),
- 'uri' => $strCoverUri,
-
- ];
-
- $strFenleiIId = 1 ;
- switch ($strFenlei) {
- case '玄幻修真':
- $strName = "玄幻修真";
- $strFenleiIId = 1;
- break;
- case '重生穿越':
- $strName = "重生穿越";
- $strFenleiIId = 2;
- break;
- case '都市小说':
- $strName = "都市小说";
- $strFenleiIId = 3;
- break;
- case '军史小说':
- $strName = "军史小说";
- $strFenleiIId = 4;
- break;
- case '网游小说':
- $strName = "网游小说";
- $strFenleiIId = 5;
- break;
- case '科幻小说':
- $strName = "科幻小说";
- $strFenleiIId = 6;
- break;
- case '灵异小说':
- $strName = "灵异小说";
- $strFenleiIId = 7;
- break;
- case '言情小说':
- $strName = "言情小说";
- $strFenleiIId = 8;
- break;
- case '其他小说':
- $strName = "其他小说";
- $strFenleiIId = 9;
- break;
- }
-
- $arrXiaoShuoInfo = [
- 'xsxq_ming_zi' => $strMingZi,
- 'xsxq_zuozhe' => $strZuoZhe , //$arrData['xiaoshuo_zuozhe'],
- 'xsfl_id' => $strFenleiIId,
- 'xsxq_zhuang_tai' => $strZhuangTai,
- 'xsxq_zi_shu' => $strZiShu,
- 'xsxq_jie_shao' => $strJieShao,
- 'xsxq_feng_mian' => json_encode($arrCover),
- 'xsxq_geng_xin_shi_jian' => $strLastDate,
- 'xsxq_tag' => json_encode($arrTag, JSON_UNESCAPED_UNICODE),
- 'xsxq_source_id' => $intXiaoShuoSourceId,
- 'xsxq_source_seo_id' => $intXiaoShuoSourceId.'1',
- 'xsxq_source_code' => 'BIQU_XIAOSHUO_' . $intXiaoShuoSourceId . '_' . $intXiaoShuoSourceSeoId,
- ];
- // print_r($arrXiaoShuoInfo);
- var_dump($arrXiaoShuoInfo['xsxq_source_code']);
-
- $XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
-
- $strMuluHref = $QueryList->find('.btn-mulu')->eq(0)->attr("href");
-
- $arrTask = [
- 'callback' => [self::class, 'getNovelReaderAll'],
- 'data' => [
- 'xszj_href' => $strMuluHref,
- ]
- ];
- $TaskCore->set($arrTask);
-/*
-
- $arrZhangjie = [];
- $intChapterIndex = 0; // 定义一个计数器变量
-
- // 检查是否找到元素
- if ($QueryList->find('.section-list a')->count() > 0) {
- $QueryList->find('.section-list a')->map(function ($ZhangJie) use (&$arrZhangjie, &$intChapterIndex) {
- echo "Processing index $intChapterIndex: " . $ZhangJie->text() . PHP_EOL;
-
- // 从索引6开始
- //if ($intChapterIndex >= 5) {
- $arrZhangjie[] = [
- 'mingzi' => $ZhangJie->text(),
- 'url' => $ZhangJie->attr('href'),
- ];
- //}
- $intChapterIndex++;
- });
-
- // 输出最终结果
- var_dump($arrZhangjie);
- } else {
- echo "No elements found for .section-list a" . PHP_EOL;
- }
-
- //unset($QueryList);
- var_dump($arrZhangjie);
-
-
- // 获取最大章节索引
- $intMaxXiaoShuoZhangJiePaixu = XiaoShuoZhangJieModel::where('xsxq_id', $intXiaoShuoSourceId)
- ->max('xszj_pai_xu'); // 获取最大值
- var_dump($intMaxXiaoShuoZhangJiePaixu);
-
- // 如果没有结果，max 返回 null，可以设置默认值
- $intMaxXiaoShuoZhangJiePaixu = $intMaxXiaoShuoZhangJiePaixu ?? 0;
-
- foreach ($arrZhangjie as $intZhangJieNum => $arrZhangjieOne) {
- var_dump($intZhangJieNum);
-
- // 如果数据库里的章节最大索引大于当前索引，说明已经入库，不需要入队列了
- if($intMaxXiaoShuoZhangJiePaixu > $intZhangJieNum){
- var_dump("小说章节已经入库，不再推送任务");
- continue;// 如果任务已存在，直接跳过 break
- }
-
- $strZhangJieUrl = $strSiteUrl . $arrZhangjieOne['url'];
-
- $strZhangJieId = extractFilename($strZhangJieUrl);
-
- # 'AILISI_XIAOSHUO_ZHANGJIE_小说源id_小说源章节id'
- $strXszjSourceCode = '5CCC_XIAOSHUO_ZHANGJIE_'. $XiaoShuoXiangQingModel->xsxq_source_id . '_' . $strZhangJieId;
-
- $arrTask = [
- 'callback' => [self::class, 'getNovelReader'],
- 'data' => [
- 'xszj_source_id' => $strZhangJieId,
- 'xszj_paixu' => $intZhangJieNum,
- 'xszj_name' => $arrZhangjieOne['mingzi'],
- 'xszj_url' => $strZhangJieUrl,
- 'xszj_source_code' => $strXszjSourceCode,
- 'xsxq_source_id' => $XiaoShuoXiangQingModel->xsxq_source_id,
- 'xsfl_id' => $strFenleiIId,
- ]
- ];
- $TaskCore->set($arrTask);
- var_dump("章节任务投递成功".$strXszjSourceCode);
- }
- */
-
- } catch (\Throwable $T) {
- throw $T;
- }
- }
-
- public function getNovelFengMian($arrData)
- {
- //var_dump('getNovelFengMian');
- //var_dump($arrData);
- $Client = $this->Client;
- $TaskCore = $this->TaskCore;
-
- // 获取 Redis 实例
- $redis = Cache::store('redis')->handler();
- $intXiaoShuoSourceId = $arrData['xiaoshuo_id'];
- $intXiaoShuoSourceSeoId = $arrData['xiaoshuo_seo_id'];
- $redisKey = "task:novel_source_id:" . $intXiaoShuoSourceId;
-
- try {
- $arrCookie = $this->getAilisiCookie();
- $strHost = getDomainFromUrl($arrData['xiaoshuo_url']);
- $CookieJar = CookieJar::fromArray($arrCookie, $strHost);
-
- $strSiteUrl = $this->getAilisiDomain();
-
-
- $strCover = $QueryList->find('#fmimg img')->eq(0)->attr("src");
-
- if (strpos($strCover, 'http') === false) {
- $strCover = $strSiteUrl . $strCover;
- }
- $strCoverUri = $strCover;
-
- try {
- $strCoverUri = "";
- $strCoverUri = ImageProcessor::getInstance()->downloadAndEncryptImage($strCover, 'xs');
- $arrCover = [
- 'code' => 'AI_LI_SI_COVER',
- // 'uri' => getUriByUrl($strCover),
- 'uri' => $strCoverUri,
-
- ];
-
- $arrXiaoShuoInfo = [
- 'xsxq_feng_mian' => json_encode($arrCover),
-
- ];
- $XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
- } catch (\Throwable $t) {
- $strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
- echo $strErr;
- }
-
-
-
- } catch (\Throwable $T) {
- throw $T;
- }
- }
-
- public function getNovelReaderAll($arrData)
- {
- var_dump('getNovelReaderAll');
- $strMuluHref = $arrData['xszj_href'];
- var_dump($strMuluHref);
- $Client = $this->Client;
- try {
- $arrCookie = $this->getAilisiCookie();
- $strHost = getDomainFromUrl($strMuluHref);
- $CookieJar = CookieJar::fromArray($arrCookie, $strHost);
-
- $proxy = getRandomProxies();
- $userAgent = getRandomUserAgent();
- $Response = $Client->request('GET', $strMuluHref, [
- 'headers' => [
- 'User-Agent' => $userAgent,
- 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
- 'Accept-Language' => 'en-US,en;q=0.5',
- ],
- 'cookies' => $CookieJar,
- // 'proxy' => $proxy,
- 'timeout' => 30,
- ]);
-
- $strResult = (string)$Response->getBody();
- unset($Response);
- $QueryList = QueryList::html($strResult);
-
-
- $intChapterIndex = 0; // 定义一个计数器变量
- var_dump($QueryList->find('.section-list a')->count());
- // 检查是否找到元素
- if ($QueryList->find('.section-list a')->count() > 0) {
- $QueryList->find('.section-list a')->map(function ($ZhangJie) use (&$arrZhangjie, &$intChapterIndex) {
- echo "Processing index $intChapterIndex: " . $ZhangJie->text() . PHP_EOL;
-
- // 从索引6开始
- //if ($intChapterIndex >= 5) {
- $arrZhangjie[] = [
- 'mingzi' => $ZhangJie->text(),
- 'url' => $ZhangJie->attr('href'),
- ];
- //}
- $intChapterIndex++;
- });
-
- // 输出最终结果
- var_dump($arrZhangjie);
- } else {
- echo "No elements found for .section-list a" . PHP_EOL;
- }
-
- //unset($QueryList);
- var_dump($arrZhangjie);
-/*
-
- // 获取最大章节索引
- $intMaxXiaoShuoZhangJiePaixu = XiaoShuoZhangJieModel::where('xsxq_id', $intXiaoShuoSourceId)
- ->max('xszj_pai_xu'); // 获取最大值
- var_dump($intMaxXiaoShuoZhangJiePaixu);
-
- // 如果没有结果，max 返回 null，可以设置默认值
- $intMaxXiaoShuoZhangJiePaixu = $intMaxXiaoShuoZhangJiePaixu ?? 0;
-
- foreach ($arrZhangjie as $intZhangJieNum => $arrZhangjieOne) {
- var_dump($intZhangJieNum);
-
- // 如果数据库里的章节最大索引大于当前索引，说明已经入库，不需要入队列了
- if($intMaxXiaoShuoZhangJiePaixu > $intZhangJieNum){
- var_dump("小说章节已经入库，不再推送任务");
- continue;// 如果任务已存在，直接跳过 break
- }
-
- $strZhangJieUrl = $strSiteUrl . $arrZhangjieOne['url'];
-
- $strZhangJieId = extractFilename($strZhangJieUrl);
-
- # 'AILISI_XIAOSHUO_ZHANGJIE_小说源id_小说源章节id'
- $strXszjSourceCode = '5CCC_XIAOSHUO_ZHANGJIE_'. $XiaoShuoXiangQingModel->xsxq_source_id . '_' . $strZhangJieId;
-
- $arrTask = [
- 'callback' => [self::class, 'getNovelReader'],
- 'data' => [
- 'xszj_source_id' => $strZhangJieId,
- 'xszj_paixu' => $intZhangJieNum,
- 'xszj_name' => $arrZhangjieOne['mingzi'],
- 'xszj_url' => $strZhangJieUrl,
- 'xszj_source_code' => $strXszjSourceCode,
- 'xsxq_source_id' => $XiaoShuoXiangQingModel->xsxq_source_id,
- 'xsfl_id' => $strFenleiIId,
- ]
- ];
- $TaskCore->set($arrTask);
- var_dump("章节任务投递成功".$strXszjSourceCode);
- }
- */
-
- } catch (\Throwable $T) {
- throw $T;
- }
- }
-
- public function getNovelReader($arrData)
- {
- var_dump('getNovelReader');
- var_dump($arrData['xszj_url']);
- $Client = $this->Client;
- $strXszjSourceCode = $arrData['xszj_source_code'];
- $strZhangJieUrl = $arrData['xszj_url'];
- $intZhangJiePaiXu = $arrData['xszj_paixu'];
- $intFenleiId = $arrData['xsfl_id'];
- $intXiaoShuoSourceID = $arrData['xsxq_source_id'];
- $strZhangJieSourceId = $arrData['xszj_source_id'];
- $strZhangJieName = $arrData['xszj_name'];
- try {
- $arrCookie = $this->getAilisiCookie();
- $strHost = getDomainFromUrl($strZhangJieUrl);
- $CookieJar = CookieJar::fromArray($arrCookie, $strHost);
-
- if (XiaoShuoZhangJieModel::checkZhangJieExists($strXszjSourceCode)) {
- return true;
- }
-
- $proxy = getRandomProxies();
- $userAgent = getRandomUserAgent();
- $Response = $Client->request('GET', $strZhangJieUrl, [
- 'headers' => [
- 'User-Agent' => $userAgent,
- 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
- 'Accept-Language' => 'en-US,en;q=0.5',
- ],
- 'cookies' => $CookieJar,
- // 'proxy' => $proxy,
- 'timeout' => 30,
- ]);
-
- $strResult = (string)$Response->getBody();
- unset($Response);
- $ZhangJieQueryList = QueryList::html($strResult);
-
-
- // 获取原始 HTML
- $strNeiRong = $ZhangJieQueryList->find('#content')->eq(0)->html();
-
- // 解码 HTML 实体
- $strNeiRong = html_entity_decode($strNeiRong, ENT_QUOTES, 'UTF-8');
-
- // 去掉
标签中的 class="content_detail" 属性
- $strNeiRong = preg_replace('/');
-
- // 替换不间断空格 (\xc2\xa0) 和 为普通空格
- $strNeiRong = preg_replace('/\xc2\xa0/', ' ', $strNeiRong); // 替换 UTF-8 不间断空格
- $strNeiRong = preg_replace('/ /', ' ', $strNeiRong); // 替换
-
- // 清理多余空格和换行符
- $strNeiRong = preg_replace('/\s+/', ' ', $strNeiRong);
- $strNeiRong = preg_replace('/\s+/', '
', $strNeiRong); // 清理
标签内部多余空格
- $strNeiRong = preg_replace('/\s+<\/p>/', '
', $strNeiRong); // 清理 标签结尾多余空格
-
- // 输出结果
- // var_dump($strNeiRong);
-
-
- # '/novel/123yqw/分类id/小说源id/小说源章节id.txt'
- $strNeiRongTxtPath = config("filesystem.novel").'/novel/biquxs/'.$intFenleiId.'/'.$intXiaoShuoSourceID.'/'.$strZhangJieSourceId.'.txt';
- //$strNeiRongTxtPath = "/www/novel/123yqw/1/11/111.txt";
- // 调用函数
- $result = saveCompressAndDeleteTxt($strNeiRongTxtPath, $strNeiRong);
-
- // 输出状态
- if ($result['status']) {
- var_dump("Success: " . $result['message']) ;
- } else {
- var_dump("Error: " . $result['message']) ;
- }
-
- $arrXiaoShuoZhangJie = [
- 'xsxq_id' => $intXiaoShuoSourceID,
- 'xszj_pai_xu' => $intZhangJiePaiXu,
- 'xszj_ming_zi' => $strZhangJieName,
- 'xszj_nei_rong' => $strNeiRongTxtPath,
- 'xszj_source_id' => $strZhangJieSourceId,
- 'xszj_source_code' => $strXszjSourceCode,
- ];
-
- XiaoShuoZhangJieModel::addXiaoShuoZhangJie($arrXiaoShuoZhangJie);
- unset($ZhangJieQueryList);
- // 延时 3 秒
- //sleep(3);
-
- } catch (\Throwable $T) {
- throw $T;
- }
- }
-}
diff --git a/code/app/task/collection/dadi/DaDiDianyingCol.php b/code/app/task/collection/dadi/DaDiDianyingCol.php
deleted file mode 100644
index 521a264..0000000
--- a/code/app/task/collection/dadi/DaDiDianyingCol.php
+++ /dev/null
@@ -1,498 +0,0 @@
-TaskCore;
- // 1-75
- for ($i = 1; $i <= 75; $i++) {
- echo "当前值: $i\n";
- $arrTask = [
- 'callback' => [self::class, 'getVideoPage'],
- 'data' => [
- 'video_page_url' => 'https://www.czdadi.net/vodshow/dy--------'.$i.'---.html',
- 'video_class_id' => 1
- ]
- ];
- $TaskCore->set($arrTask);
- }
- //5-25 https://www.czdadi.net/vodshow/duanju-----------.html
- for ($i = 1; $i <= 25; $i++) {
- echo "当前值: $i\n";
- $arrTask = [
- 'callback' => [self::class, 'getVideoPage'],
- 'data' => [
- 'video_page_url' => 'https://www.czdadi.net/vodshow/duanju--------'.$i.'---.html',
- 'video_class_id' => 5
- ]
- ];
- $TaskCore->set($arrTask);
- }
-
- // 2--18 https://www.czdadi.net/vodshow/dsj--------18---.html
- for ($i = 1; $i <= 18; $i++) {
- echo "当前值: $i\n";
- $arrTask = [
- 'callback' => [self::class, 'getVideoPage'],
- 'data' => [
- 'video_page_url' => 'https://www.czdadi.net/vodshow/dsj--------'.$i.'---.html',
- 'video_class_id' => 2
- ]
- ];
- $TaskCore->set($arrTask);
- }
- // 3--11 https://www.czdadi.net/vodshow/zy--------11---.html
- for ($i = 1; $i <= 11; $i++) {
- echo "当前值: $i\n";
- $arrTask = [
- 'callback' => [self::class, 'getVideoPage'],
- 'data' => [
- 'video_page_url' => 'https://www.czdadi.net/vodshow/zy--------'.$i.'---.html',
- 'video_class_id' => 3
- ]
- ];
- $TaskCore->set($arrTask);
- }
- // 4--29 https://www.czdadi.net/vodshow/dm-----------.html
- for ($i = 1; $i <= 29; $i++) {
- echo "当前值: $i\n";
- $arrTask = [
- 'callback' => [self::class, 'getVideoPage'],
- 'data' => [
- 'video_page_url' => 'https://www.czdadi.net/vodshow/dm--------'.$i.'---.html',
- 'video_class_id' => 4
- ]
- ];
- $TaskCore->set($arrTask);
- }
-
-
-
- }
-
- # 获取分页里面的视频列表
- public function getVideoPage($arrData)
- {
- $TaskCore = $this->TaskCore;
- $Client = $this->Client;
- $strUrl= $arrData['video_page_url'];
- $intClassId = $arrData['video_class_id'];
- $strSiteUrl = $this->get5cccDomain();
- $arrCookie = $this->getAilisiCookie();
- $strHost = getDomainFromUrl($strUrl);
- $CookieJar = CookieJar::fromArray($arrCookie, $strHost);
-
- $proxy = getRandomProxies();
- $userAgent = getRandomUserAgent();
- $Response = $Client->request('GET', $strUrl, [
- 'headers' => [
- 'User-Agent' => $userAgent,
- 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
- 'Accept-Language' => 'en-US,en;q=0.5',
- ],
- 'cookies' => $CookieJar,
- // 'proxy' => $proxy,
- 'timeout' => 30,
- ]);
-
- $strResult = (string)$Response->getBody();
-
- $QueryList = QueryList::html($strResult);
-
- //$arrVideo = [];
- // 检查是否找到元素
- if ($QueryList->find('.stui-vodlist li .stui-vodlist__detail a')->count() > 0) {
- $QueryList->find('.stui-vodlist li .stui-vodlist__detail a ')->map(function ($Video) use (&$arrVideo,$intClassId,&$TaskCore) {
-
- // $arrVideo[] = [
- // 'mingzi' => $Video->text(),
- // 'url' => $Video->attr('href'),
- // ];
- if(!empty($Video->text()) && !empty($Video->attr('href'))){
- $arrTask = [
- 'callback' => [self::class, 'getVideoInfo'],
- 'data' => [
- 'class_id' => $intClassId,
- 'video_href' => $Video->attr('href'),
- 'video_name' => $Video->text(),
- ]
- ];
- $TaskCore->set($arrTask);
- echo "视频投递成功 : " . $Video->text() . PHP_EOL;
- }
- });
-
-
- // foreach ($arrVideo as $intVideoNum => $arrVideoOne) {
- // $arrTask = [
- // 'callback' => [self::class, 'getVideoInfo'],
- // 'data' => [
- // 'class_id' => $intClassId,
- // 'video_href' => $arrVideoOne['url'],
- // 'video_name' => $arrVideoOne['mingzi'],
- // ]
- // ];
- // $TaskCore->set($arrTask);
-
- // }
-
- // 输出最终结果
- // var_dump($arrVideo);
- } else {
- echo "No elements found for .stui-vodlist a" . PHP_EOL;
- }
- }
-
- public function getVideoInfo($arrData)
- {
- $TaskCore = $this->TaskCore;
- $Client = $this->Client;
- $strSiteUrl = $this->getDadiDomain();
- $strUrl = $strSiteUrl . $arrData['video_href'] ;
- $intClassId = $arrData['class_id'];
- $strVideoName = $arrData['video_name'];
-
- try {
- $arrCookie = $this->getAilisiCookie();
- $strHost = getDomainFromUrl($strUrl);
-
- $CookieJar = CookieJar::fromArray($arrCookie, $strHost);
-
- $proxy = getRandomProxies();
- $userAgent = getRandomUserAgent();
- $Response = $Client->request('GET', $strUrl, [
- 'headers' => [
- 'User-Agent' => $userAgent,
- 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
- 'Accept-Language' => 'en-US,en;q=0.5',
- ],
- 'cookies' => $CookieJar,
- // 'proxy' => $proxy,
- 'timeout' => 30,
- ]);
-
- $strResult = (string)$Response->getBody();
-
- $QueryList = QueryList::html($strResult);
-
- $strCover = $QueryList->find('.stui-content__thumb img')->eq(0)->attr("data-original");
- if (strpos($strCover, 'http') === false) {
- $strCover = $strSiteUrl . $strCover;
- }
- $strCoverUri = $strCover;
- // try {
- // $strCoverUri = "";
- // $strCoverUri = ImageProcessor::getInstance()->downloadImage( $intXiaoShuoSourceId,$strCover, 'dadi');
- // } catch (\Throwable $t) {
- // $strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
- // echo $strErr;
- // }
- $arrCover = [
- 'code' => 'DADI_YINGSHI_COVER',
- // 'uri' => getUriByUrl($strCover),
- 'uri' => $strCoverUri,
- ];
-
- $strLanguage = "";
- // if (strlen($strLanguage) > 0) {
- // VideoLeixinModel::addVideoLeixin(['vc_id' => $intClassId, 'vla_name' => $strLanguage]);
- // }
-
- // 提取评分
- $strPinfen = $QueryList->find('h1.title .score')->text();
- // 提取类型-入库
- $strLeixin = $QueryList->find('.col-pd a')->eq(1)->text();
-
- if(strlen($strLeixin) > 0){
- VideoLeixinModel::addVideoLeixin(['vc_id'=>$intClassId,'vl_name'=>$strLeixin]);
- }
-
- // 提取 HTML 的第一段
- $dataNode = $QueryList->find('p.data')->eq(0);
-
- // 提取“类型”内容：确保只匹配 `href` 中包含 `/vodsearch/----`
- $leixinNodes = $dataNode->find('a[href^="/vodsearch/----"]');
- $strJuqing = $leixinNodes->map(function ($node) {
- return $node->text();
- })->all();
-
- // 提取“地区”内容：确保只匹配 `href` 中包含 `/vodsearch/--`，但不包含 `/vodsearch/----`
- $diquNodes = $dataNode->find('a[href^="/vodsearch/--"]')->not('a[href^="/vodsearch/----"]');
- $strDiqu = $diquNodes->map(function ($node) {
- return $node->text();
- })->all();
-
- // 提取“年份”内容：确保只匹配 `href` 中包含 `/vodsearch/-------------`
- $yearsNode = $dataNode->find('a[href^="/vodsearch/-------------"]');
- $strYears = $yearsNode->text();
- if (!empty(trim($strYears))) {
- $strYears = [trim($strYears)];
- } else {
- $strYears = [];
- }
-
- // 过滤掉“地区”和“年份”中的数据
- $strJuqing = array_filter($strJuqing, function ($value) use ($strDiqu, $strYears) {
- return !in_array($value, array_merge($strDiqu, $strYears));
- });
-
- // **在过滤后将“类型”数据入库**
- foreach ($strJuqing as $value) {
- if(strlen($value) > 0){
- VideoJuqingModel::add(['vc_id' => $intClassId, 'vj_name' => $value]);
- }
- }
-
- // 将“地区”数据入库
- foreach ($strDiqu as $value) {
- if(strlen($value) > 0){
- VideoDiquModel::add(['vc_id' => $intClassId, 'vd_name' => $value]);
- }
- }
-
- // 将“年份”数据入库
- foreach ($strYears as $value) {
- if(strlen($value) > 0){
- VideoYearsModel::add(['vc_id' => $intClassId, 'vy_name' => $value]);
- }
-
- }
-
- // 提取主演
- $strZhuyan = $QueryList->find('p.data:contains("主演") a')->map(function ($item) {
- return $item->text();
- })->implode(', '); // 使用逗号分隔主演
-
- // 提取导演
- $strDaoyan = $QueryList->find('p.data:contains("导演") a')->map(function ($item) {
- return $item->text();
- })->implode(', '); // 使用逗号分隔主演
-
-
- // 提取更新时间
- $strLastDate = $QueryList->find('p.data.hidden-sm:contains("更新")')->text();
- // 去掉“更新：”前缀
- $strLastDate = str_replace('更新：', '', $strLastDate);
-
- // 提取简介
- $strDescription = $QueryList->find('#desc>.stui-pannel-box>.stui-pannel_bd>.col-pd')->text();
-
- // 角标
- $strJiaoBiao = $QueryList->find('.stui-content__thumb .text-right')->eq(0)->text();
-
- $strPinyinId = extractFilename($strUrl);
-
- $arrVideoInfo = [
- 'v_name' => $strVideoName,
- 'v_cover' => json_encode($arrCover),
- 'vc_id' => $intClassId,
- 'v_pinyin' => $strPinyinId,
- 'v_leixin' => $strLeixin ,
- 'v_juqing' => implode(', ', $strJuqing),
- 'v_diqu' => implode(', ', $strDiqu),
- 'v_years' => implode(', ', $strYears),
- 'v_language' => $strLanguage,
- 'v_daoyan' => $strZhuyan,
- 'v_zhuyan' => $strDaoyan,
- 'update_time' => $strLastDate,
- 'v_description'=>$strDescription,
- 'v_pingfen' => $strPinfen,
- 'v_orurl' => $strUrl,
- 'v_jiaobiao'=> $strJiaoBiao,
- 'v_md5' => md5($intClassId.$strVideoName.$strSiteUrl.$strUrl),
- ];
-
- $VideoInfoModel = VideoInfoModel::addVideoXiangQing($arrVideoInfo);
-
- $intVid = $VideoInfoModel->v_id;
- $intChapterIndex = 0 ;
- $arrZhangjie = [];
- // // 检查是否找到元素
- if ($QueryList->find('.stui-content__playlist a')->count() > 0) {
- $QueryList->find('.stui-content__playlist a')->map(function ($ZhangJie) use (&$arrZhangjie, &$intChapterIndex ,&$intVid) {
- //echo "Processing index $intChapterIndex: " . $ZhangJie->text() . PHP_EOL;
-
- $arrZhangjie[] = [
- 'v_id' => $intVid,
- 'index' => $intChapterIndex,
- 'mingzi' => $ZhangJie->text(),
- 'url' => $ZhangJie->attr('href'),
- ];
- $intChapterIndex++;
- });
-
- //var_dump($arrZhangjie);
- foreach ($arrZhangjie as $intZhangJieNum => $arrZhangjieOne) {
- var_dump($intZhangJieNum);
-
- // 如果数据库里的章节最大索引大于当前索引，说明已经入库，不需要入队列了
- // if($intMaxXiaoShuoZhangJiePaixu > $intZhangJieNum){
- // var_dump("小说章节已经入库，不再推送任务");
- // continue;// 如果任务已存在，直接跳过 break
- // }
- //if($intZhangJieNum<=3){
- $arrTask = [
- 'callback' => [self::class, 'getNovelReader'],
- 'data' => [
- 'v_id' => $intVid,
- 'vp_sort' => $intZhangJieNum,
- 'vp_name' => $arrZhangjieOne['mingzi'],
- 'vp_url' => $strSiteUrl . $arrZhangjieOne['url']
- ]
- ];
- $TaskCore->set($arrTask);
- //}
-
- }
- } else {
- echo "No elements found for .listmain a" . PHP_EOL;
- }
-
- } catch (\Throwable $T) {
- throw $T;
- }
- // sleep(10);
- }
-
- public function getNovelFengMian($arrData)
- {
- //var_dump('getNovelFengMian');
- //var_dump($arrData);
- $Client = $this->Client;
- $TaskCore = $this->TaskCore;
-
- // 获取 Redis 实例
- $redis = Cache::store('redis')->handler();
- $intXiaoShuoSourceId = $arrData['xiaoshuo_id'];
- $intXiaoShuoSourceSeoId = $arrData['xiaoshuo_seo_id'];
- $redisKey = "task:novel_source_id:" . $intXiaoShuoSourceId;
-
- try {
- $arrCookie = $this->getAilisiCookie();
- $strHost = getDomainFromUrl($arrData['xiaoshuo_url']);
- $CookieJar = CookieJar::fromArray($arrCookie, $strHost);
-
- $strSiteUrl = $this->getAilisiDomain();
-
-
- $strCover = $QueryList->find('#fmimg img')->eq(0)->attr("src");
-
- if (strpos($strCover, 'http') === false) {
- $strCover = $strSiteUrl . $strCover;
- }
- $strCoverUri = $strCover;
-
- try {
- $strCoverUri = "";
- $strCoverUri = ImageProcessor::getInstance()->downloadAndEncryptImage($strCover, 'xs');
- $arrCover = [
- 'code' => 'AI_LI_SI_COVER',
- // 'uri' => getUriByUrl($strCover),
- 'uri' => $strCoverUri,
-
- ];
-
- $arrXiaoShuoInfo = [
- 'xsxq_feng_mian' => json_encode($arrCover),
-
- ];
- $XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo);
- } catch (\Throwable $t) {
- $strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage());
- echo $strErr;
- }
-
-
-
- } catch (\Throwable $T) {
- throw $T;
- }
- }
-
- public function getNovelReader($arrData)
- {
- if (Cache::has('video_no_m3u8_' . $arrData['v_id'])) {
- echo "跳过抓取，视频ID已在缓存中: " . $arrData['v_id'] . "\n";
- return; // 跳过抓取逻辑
- }
-
- $strMuluHref = $arrData['vp_url'];
- var_dump($strMuluHref);
- $Client = $this->Client;
- try {
- $arrCookie = $this->getAilisiCookie();
- $strHost = getDomainFromUrl($strMuluHref);
- $CookieJar = CookieJar::fromArray($arrCookie, $strHost);
-
- $proxy = getRandomProxies();
- $userAgent = getRandomUserAgent();
- $Response = $Client->request('GET', $strMuluHref, [
- 'headers' => [
- 'User-Agent' => $userAgent,
- 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
- 'Accept-Language' => 'en-US,en;q=0.5',
- ],
- 'cookies' => $CookieJar,
- // 'proxy' => $proxy,
- 'timeout' => 30,
- ]);
-
- $strResult = (string)$Response->getBody();
- unset($Response);
- $QueryList = QueryList::html($strResult);
- //提取
', $strNeiRong);
-
- $strNeiRong = $arrNeiRong[1];
-
- $arrNeiRong = explode('
', $strNeiRong);
-
- $arrEnCode = [];
- foreach ($arrNeiRong as $intNeiRongNum => $strNeiRongOne) {
- $strEnCode = Base64Helper::encode('
' . $strNeiRongOne . '
');
- $arrEnCode[] = $strEnCode;
- }
-
- $strNeiRong = implode(';;;', $arrEnCode);
-
- # '/novel/123yqw/分类id/小说源id/小说源章节id.txt'
- $strNeiRongTxtPath = config("filesystem.novel") . '/novel/630zwxs/' . $intFenleiId . '/' . $intXiaoShuoId . '/' . $intZhangJiePaiXu . '/' . $intZhangJieSort . '.txt';
- // 调用函数
- $result = saveCompressAndDeleteTxt($strNeiRongTxtPath, $strNeiRong);
-
- // 输出状态
- if ($result['status']) {
- var_dump("Success: " . $result['message']);
- } else {
- var_dump("Error: " . $result['message']);
- }
-
- $arrXiaoShuoZhangJie = [
- 'xsxq_id' => $intXiaoShuoId,
- 'xszj_pai_xu' => $intZhangJiePaiXu,
- 'xszj_ming_zi' => $strZhangJieName,
- 'xszj_nei_rong' => $strNeiRongTxtPath,
- 'xszj_source_id' => $strZhangJieSourceId,
- 'xszj_source_code' => $strXszjSourceCode,
- 'xszj_sort' => $intZhangJieSort,
-
- ];
- if ($strZhangJieName) {
- XiaoShuoZhangJieModel::addXiaoShuoZhangJie($arrXiaoShuoZhangJie);
- }
-
- unset($ZhangJieQueryList);
- //sleep(3);
-
-
- } catch (\Throwable $T) {
- $arrTask = [
- 'callback' => [self::class, 'getNovelReader'],
- 'data' => [
- 'xszj_source_id' => $strZhangJieSourceId,
- 'xszj_paixu' => $intZhangJiePaiXu,
- 'xszj_name' => $strZhangJieName,
- 'xszj_url' => $strZhangJieUrl,
- 'xszj_source_code' => $strXszjSourceCode,
- 'xsxq_id' => $intXiaoShuoId,
- 'xsfl_id' => $intFenleiId,
- 'intZhangJieSort' => $intZhangJieSort,
- ]
- ];
-
- $TaskCore->set($arrTask);
- throw $T;
- }
- }
-}
diff --git a/code/app/task/collection/seo/GenerateSiteMapCol.php b/code/app/task/collection/seo/GenerateSiteMapCol.php
deleted file mode 100644
index 4c3dfd6..0000000
--- a/code/app/task/collection/seo/GenerateSiteMapCol.php
+++ /dev/null
@@ -1,137 +0,0 @@
-generateVideoSitemaps($chunkSize, $totalVideoCount, $strSiteDomain);
-
- // 生成分类的 Sitemap
- $categorySitemapFiles = $this->generateCategorySitemaps($categories, $strSiteDomain);
-
- // 合并所有 Sitemap 文件路径
- $allSitemapFiles = array_merge($videoSitemapFiles, $categorySitemapFiles);
-
- // 生成 Sitemap Index 文件
- $this->buildSitemapIndex($allSitemapFiles, $strSiteDomain);
-
- var_dump(['message' => '所有 Sitemap 生成成功', 'files' => $allSitemapFiles]);
- }
-
- private function generateVideoSitemaps(int $chunkSize, int $totalCount, string $strSiteDomain): array
- {
- $totalFiles = ceil($totalCount / $chunkSize);
- $sitemapFiles = [];
-
- for ($i = 0; $i < $totalFiles; $i++) {
- $videos = VideoInfoModel::field('v_pinyin')
- ->limit($i * $chunkSize, $chunkSize)
- ->select()
- ->toArray();
-
- $sitemapXml = $this->buildSitemap($videos, $strSiteDomain);
- $filePath = public_path() . "sitemap_video_{$i}.xml";
- file_put_contents($filePath, $sitemapXml);
- $sitemapFiles[] = $filePath;
- }
-
- return $sitemapFiles;
- }
-
- private function generateCategorySitemaps(array $categories, string $strSiteDomain): array
- {
- $sitemapFiles = [];
-
- foreach ($categories as $category) {
- $vcId = $category['vc_id'];
- $vcNicheng = $category['vc_nicheng'];
- $vcName = $category['vc_name'];
-
- $videoCount = VideoInfoModel::where('vc_id', $vcId)->count();
- $totalPages = ceil($videoCount / 30);
-
- $categoryUrls = [];
- $categoryUrls[] = [
- 'url' => "/vodtype/{$vcNicheng}.html",
- 'title' => "{$vcName} - 首页",
- 'is_home' => true,
- ];
-
- for ($page = 2; $page <= $totalPages; $page++) {
- $categoryUrls[] = [
- 'url' => "/vodshow/{$vcNicheng}--------{$page}---.html",
- 'title' => "{$vcName} - 第{$page}页",
- 'is_home' => false,
- ];
- }
-
- $sitemapXml = $this->buildSitemap($categoryUrls, $strSiteDomain, true);
- $filePath = public_path() . "sitemap_category_{$vcNicheng}.xml";
- file_put_contents($filePath, $sitemapXml);
- $sitemapFiles[] = $filePath;
- }
-
- return $sitemapFiles;
- }
-
- private function buildSitemap(array $items, string $strSiteDomain, bool $isCategory = false): string
- {
- $xmlHeader = '';
- $urlsetStart = '';
- $urlsetEnd = '';
-
- $urlEntries = '';
- foreach ($items as $item) {
- $urlEntries .= '';
- $urlEntries .= 'https://' . $strSiteDomain . ($isCategory ? $item['url'] : '/voddetail/' . htmlspecialchars($item['v_pinyin'] . '.html', ENT_QUOTES)) . '';
- $urlEntries .= 'daily';
- $urlEntries .= '0.8';
- $urlEntries .= '';
- }
-
- return $xmlHeader . $urlsetStart . $urlEntries . $urlsetEnd;
- }
-
- private function buildSitemapIndex(array $sitemapFiles, string $strSiteDomain): void
- {
- $xmlHeader = '';
- $sitemapIndexStart = '';
- $sitemapIndexEnd = '';
-
- $sitemapEntries = '';
- foreach ($sitemapFiles as $file) {
- $url = 'https://' . $strSiteDomain . '/' . basename($file);
- $sitemapEntries .= '';
- $sitemapEntries .= '' . htmlspecialchars($url, ENT_QUOTES) . '';
- $sitemapEntries .= '' . date('Y-m-d') . '';
- $sitemapEntries .= '';
- }
-
- $sitemapIndexXml = $xmlHeader . $sitemapIndexStart . $sitemapEntries . $sitemapIndexEnd;
- file_put_contents(public_path() . 'sitemap_index.xml', $sitemapIndexXml);
-
- var_dump(['message' => 'Sitemap Index 生成成功', 'file' => 'sitemap_index.xml']);
- }
-}
diff --git a/code/app/task/collection/seo/GenerateSiteMapNovel.php b/code/app/task/collection/seo/GenerateSiteMapNovel.php
deleted file mode 100644
index 73a524c..0000000
--- a/code/app/task/collection/seo/GenerateSiteMapNovel.php
+++ /dev/null
@@ -1,188 +0,0 @@
-generateNovelSitemaps($chunkSize, $totalNovelCount, $strSiteDomain);
-
- // 生成章节的 Sitemap
- $novelZjSitemapFiles = $this->generateNovelZjSitemaps($chunkSize, $totalNovelZjCount, $strSiteDomain);
-
- // 生成分类的 Sitemap
- $categorySitemapFiles = $this->generateCategorySitemaps($categories, $strSiteDomain);
-
- // 合并所有 Sitemap 文件路径
- $allSitemapFiles = array_merge($novelSitemapFiles, $categorySitemapFiles,$novelZjSitemapFiles);
-
- // 生成 Sitemap Index 文件
- $this->buildSitemapIndex($allSitemapFiles, $strSiteDomain);
-
- var_dump(['message' => '所有 Sitemap 生成成功', 'files' => $allSitemapFiles]);
- }
-
- private function generateNovelZjSitemaps(int $chunkSize, int $totalCount, string $strSiteDomain): array
- {
- $totalFiles = ceil($totalCount / $chunkSize);
- $sitemapFiles = [];
-
- for ($i = 0; $i < $totalFiles; $i++) {
- $Novels = XiaoShuoZhangJieModel::field(['xsxq_id','xszj_id']) //intNovelId-:intNovelSourceId-:intNovelSourceSeoId
- ->limit($i * $chunkSize, $chunkSize)
- ->select()
- ->toArray();
-
- $sitemapXml = $this->buildZjSitemap($Novels, $strSiteDomain);
- $filePath = public_path() . "sitemap_novel_zj_{$i}.xml";
- file_put_contents($filePath, $sitemapXml);
- $sitemapFiles[] = $filePath;
- }
-
- return $sitemapFiles;
- }
-
-
- private function generateNovelSitemaps(int $chunkSize, int $totalCount, string $strSiteDomain): array
- {
- $totalFiles = ceil($totalCount / $chunkSize);
- $sitemapFiles = [];
-
- for ($i = 0; $i < $totalFiles; $i++) {
- $Novels = XiaoShuoXiangQingModel::field(['xsxq_id','xsxq_source_id','xsxq_source_seo_id']) //intNovelId-:intNovelSourceId-:intNovelSourceSeoId
- ->limit($i * $chunkSize, $chunkSize)
- ->select()
- ->toArray();
-
- $sitemapXml = $this->buildSitemap($Novels, $strSiteDomain);
- $filePath = public_path() . "sitemap_novel_{$i}.xml";
- file_put_contents($filePath, $sitemapXml);
- $sitemapFiles[] = $filePath;
- }
-
- return $sitemapFiles;
- }
-
- private function generateCategorySitemaps(array $categories, string $strSiteDomain): array
- {
- $sitemapFiles = [];
-
- foreach ($categories as $category) {
- $vcId = $category['xsfl_id'];
- $vcName = $category['xsfl_name'];
-
- $intNovelCount = XiaoShuoFenLeiModel::where('xsfl_id', $vcId)->count();
- $totalPages = ceil($intNovelCount / 30);
-
- $categoryUrls = [];
- for ($type = 0; $type <= 3; $type++) {
- for ($page = 1; $page <= $totalPages; $page++) {
- $categoryUrls[] = [
- // /fenlei/1-0-1.html
- 'url' => "/fenlei/{$vcId}-{$type}-{$page}.html",
- 'title' => "{$vcName} - 第{$page}页",
- 'is_home' => false,
- ];
- }
- }
-
- $sitemapXml = $this->buildSitemap($categoryUrls, $strSiteDomain, true);
- $filePath = public_path() . "sitemap_category_{$vcId}.xml";
- file_put_contents($filePath, $sitemapXml);
- $sitemapFiles[] = $filePath;
- }
-
- return $sitemapFiles;
- }
-
- private function buildZjSitemap(array $items, string $strSiteDomain, bool $isCategory = false): string
- {
- $xmlHeader = '';
- $urlsetStart = '';
- $urlsetEnd = '';
- // zj/18648-206.html
- $urlEntries = '';
- foreach ($items as $item) {
- $urlEntries .= '';
- $urlEntries .= 'https://' . $strSiteDomain . ($isCategory ? $item['url'] : '/zj/' . htmlspecialchars($item['xsxq_id'].'-'.$item['xszj_id']. '.html', ENT_QUOTES)) . '';
- $urlEntries .= 'daily';
- $urlEntries .= '0.8';
- $urlEntries .= '';
- }
-
- return $xmlHeader . $urlsetStart . $urlEntries . $urlsetEnd;
- }
-
- private function buildSitemap(array $items, string $strSiteDomain, bool $isCategory = false): string
- {
- $xmlHeader = '';
- $urlsetStart = '';
- $urlsetEnd = '';
- // /xq/20052-16107-161071.html
- $urlEntries = '';
- foreach ($items as $item) {
- $urlEntries .= '';
- $urlEntries .= 'https://' . $strSiteDomain . ($isCategory ? $item['url'] : '/xq/' . htmlspecialchars($item['xsxq_id'].'-'.$item['xsxq_source_id'].'-'.$item['xsxq_source_seo_id']. '.html', ENT_QUOTES)) . '';
- $urlEntries .= 'daily';
- $urlEntries .= '1.0';
- $urlEntries .= '';
- }
-
- return $xmlHeader . $urlsetStart . $urlEntries . $urlsetEnd;
- }
-
- private function buildSitemapIndex(array $sitemapFiles, string $strSiteDomain): void
- {
- $xmlHeader = '';
- $sitemapIndexStart = '';
- $sitemapIndexEnd = '';
-
- $sitemapEntries = '';
- foreach ($sitemapFiles as $file) {
- $url = 'https://' . $strSiteDomain . '/' . basename($file);
- $sitemapEntries .= '';
- $sitemapEntries .= '' . htmlspecialchars($url, ENT_QUOTES) . '';
- $sitemapEntries .= '' . date('Y-m-d') . '';
- $sitemapEntries .= '';
- }
-
- $sitemapIndexXml = $xmlHeader . $sitemapIndexStart . $sitemapEntries . $sitemapIndexEnd;
- file_put_contents(public_path() . 'sitemap_index.xml', $sitemapIndexXml);
-
- var_dump(['message' => 'Sitemap Index 生成成功', 'file' => 'sitemap_index.xml']);
- }
-}
diff --git a/code/app/task/collection/seo/GenerateSiteMapNovelSk.php b/code/app/task/collection/seo/GenerateSiteMapNovelSk.php
deleted file mode 100644
index 1d1cbc0..0000000
--- a/code/app/task/collection/seo/GenerateSiteMapNovelSk.php
+++ /dev/null
@@ -1,242 +0,0 @@
-generateNovelSitemaps($chunkSize, $totalNovelCount, $strSiteDomain, $strXqPath);
-
- // 生成详情的 Sitemap 小说seo
- $novelSeoSitemapFiles = $this->generateNovelSeoSitemaps($chunkSize, $totalNovelSeoCount, $strSiteDomain, $strSeoPath);
-
- // 生成章节的 Sitemap 小说章节
- $novelZjSitemapFiles = $this->generateNovelZjSitemaps($chunkSize, $totalNovelZjCount, $strSiteDomain, $strXqPath);
-
- // 生成分类的 Sitemap 小说分类
- $categorySitemapFiles = $this->generateCategorySitemaps($categories, $strSiteDomain, $strFenleiPath, $strXqPath);
-
- // 合并所有 Sitemap 文件路径
- $allSitemapFiles = array_merge($novelSitemapFiles, $novelSeoSitemapFiles, $categorySitemapFiles, $novelZjSitemapFiles);
-
- // 生成 Sitemap Index 文件
- $this->buildSitemapIndex($allSitemapFiles, $strSiteDomain);
-
- var_dump(['message' => '所有 Sitemap 生成成功', 'files' => $allSitemapFiles]);
- }
-
- # 生成小说章节 xml文件
- private function generateNovelZjSitemaps(int $chunkSize, int $totalCount, string $strSiteDomain, string $strXqPath): array
- {
- $totalFiles = ceil($totalCount / $chunkSize);
- $sitemapFiles = [];
-
- for ($i = 0; $i < $totalFiles; $i++) {
- $Novels = XiaoShuoZhangJieModel::field(['xsxq_id', 'xszj_id']) //intNovelId-:intNovelSourceId-:intNovelSourceSeoId
- ->limit($i * $chunkSize, $chunkSize)
- ->select()
- ->toArray();
-
- $sitemapXml = $this->buildZjSitemap($Novels, $strSiteDomain, $strXqPath);
- $filePath = public_path() . "sitemap_novel_zj_{$i}.xml";
- file_put_contents($filePath, $sitemapXml);
- $sitemapFiles[] = $filePath;
- }
-
- return $sitemapFiles;
- }
-
- # 生成小说详情 xml文件
- private function generateNovelSitemaps(int $chunkSize, int $totalCount, string $strSiteDomain, string $strXqPath): array
- {
- $totalFiles = ceil($totalCount / $chunkSize);
- $sitemapFiles = [];
-
- for ($i = 0; $i < $totalFiles; $i++) {
- $Novels = XiaoShuoXiangQingModel::field(['xsxq_id', 'xsxq_source_id', 'xsxq_source_seo_id']) //intNovelId-:intNovelSourceId-:intNovelSourceSeoId
- ->limit($i * $chunkSize, $chunkSize)
- ->select()
- ->toArray();
-
- $sitemapXml = $this->buildSitemap($Novels, $strSiteDomain, $strXqPath);
- $filePath = public_path() . "sitemap_novel_{$i}.xml";
- file_put_contents($filePath, $sitemapXml);
- $sitemapFiles[] = $filePath;
- }
-
- return $sitemapFiles;
- }
-
- # 生成小说详情 xml文件
- private function generateNovelSeoSitemaps(int $chunkSize, int $totalCount, string $strSiteDomain, string $strSeoPath): array
- {
- $totalFiles = ceil($totalCount / $chunkSize);
- $sitemapFiles = [];
-
- for ($i = 0; $i < $totalFiles; $i++) {
- $Novels = NovelInfoSeoModel::field(['xsxq_id', 'nis_id']) //show/nis_id
- ->limit($i * $chunkSize, $chunkSize)
- ->select()
- ->toArray();
-
- $sitemapXml = $this->buildXiaoShuoSeoSitemap($Novels, $strSiteDomain, $strSeoPath);
- $filePath = public_path() . "sitemap_novel_seo_{$i}.xml";
- file_put_contents($filePath, $sitemapXml);
- $sitemapFiles[] = $filePath;
- }
-
- return $sitemapFiles;
- }
-
- // 生成分类链接
- private function generateCategorySitemaps(array $categories, string $strSiteDomain, string $strFenleiPath, string $strXqPath): array
- {
- $sitemapFiles = [];
-
- foreach ($categories as $category) {
- $vcId = $category['xsfl_id'];
- $vcName = $category['xsfl_name'];
-
- $intNovelCount = XiaoShuoFenLeiModel::where('xsfl_id', $vcId)->count();
- $totalPages = ceil($intNovelCount / 30);
-
- $categoryUrls = [];
- for ($status = 0; $status <= 2; $status++) {
- for ($sort = 0; $sort <= 4; $sort++) {
- for ($page = 1; $page <= $totalPages; $page++) {
- $categoryUrls[] = [
- // /fl/:intCId-:intSort-[:intPage]
- 'url' => "/{$strFenleiPath}/{$vcId}-{$sort}-{$page}.html",
- 'title' => "{$vcName} - 第{$page}页",
- 'is_home' => false,
- ];
- }
- }
- }
-
- $sitemapXml = $this->buildSitemap($categoryUrls, $strSiteDomain, $strXqPath, true);
- $filePath = public_path() . "sitemap_category_{$vcId}.xml";
- file_put_contents($filePath, $sitemapXml);
- $sitemapFiles[] = $filePath;
- }
-
- return $sitemapFiles;
- }
-
- //生成章节链接
- private function buildZjSitemap(array $items, string $strSiteDomain, string $strXqPath, bool $isCategory = false): string
- {
- $xmlHeader = '';
- $urlsetStart = '';
- $urlsetEnd = '';
- // /xq/:intNovelId/:intZhangJiePaiXu-[:strSort]
- $urlEntries = '';
- foreach ($items as $item) {
- $urlEntries .= '';
- $urlEntries .= 'https://' . $strSiteDomain . ($isCategory ? $item['url'] : '/' . $strXqPath . '/' . htmlspecialchars($item['xsxq_id'] . '-' . $item['xszj_pai_xu']. '-' . $item['xszj_sort'] . '.html', ENT_QUOTES)) . '';
- $urlEntries .= 'daily';
- $urlEntries .= '0.8';
- $urlEntries .= '';
- }
-
- return $xmlHeader . $urlsetStart . $urlEntries . $urlsetEnd;
- }
-
- // 生成 详情链接
- private function buildSitemap(array $items, string $strSiteDomain, string $strXqPath, bool $isCategory = false): string
- {
- $xmlHeader = '';
- $urlsetStart = '';
- $urlsetEnd = '';
- // /xq/:intNovelId
- $urlEntries = '';
- foreach ($items as $item) {
- $urlEntries .= '';
- $urlEntries .= 'https://' . $strSiteDomain . ($isCategory ? $item['url'] : '/' . $strXqPath . '/' . htmlspecialchars($item['xsxq_id'] . '.html', ENT_QUOTES)) . '';
- $urlEntries .= 'daily';
- $urlEntries .= '1.0';
- $urlEntries .= '';
- }
-
- return $xmlHeader . $urlsetStart . $urlEntries . $urlsetEnd;
- }
-
- // 生成详情 seo 链接
- private function buildXiaoShuoSeoSitemap(array $items, string $strSiteDomain, string $strSeoPath, bool $isCategory = false): string
- {
- $xmlHeader = '';
- $urlsetStart = '';
- $urlsetEnd = '';
- // /show/:intNovelId
- $urlEntries = '';
- foreach ($items as $item) {
- $urlEntries .= '';
- $urlEntries .= 'https://' . $strSiteDomain . ($isCategory ? $item['url'] : '/' . $strSeoPath . '/' . htmlspecialchars($item['nis_id'] . '.html', ENT_QUOTES)) . '';
- $urlEntries .= 'daily';
- $urlEntries .= '1.0';
- $urlEntries .= '';
- }
-
- return $xmlHeader . $urlsetStart . $urlEntries . $urlsetEnd;
- }
-
- // 生成 xml 文件
- private function buildSitemapIndex(array $sitemapFiles, string $strSiteDomain): void
- {
- $xmlHeader = '';
- $sitemapIndexStart = '';
- $sitemapIndexEnd = '';
-
- $sitemapEntries = '';
- foreach ($sitemapFiles as $file) {
- $url = 'https://' . $strSiteDomain . '/' . basename($file);
- $sitemapEntries .= '';
- $sitemapEntries .= '' . htmlspecialchars($url, ENT_QUOTES) . '';
- $sitemapEntries .= '' . date('Y-m-d') . '';
- $sitemapEntries .= '';
- }
-
- $sitemapIndexXml = $xmlHeader . $sitemapIndexStart . $sitemapEntries . $sitemapIndexEnd;
- file_put_contents(public_path() . 'sitemap_index.xml', $sitemapIndexXml);
-
- var_dump(['message' => 'Sitemap Index 生成成功', 'file' => 'sitemap_index.xml']);
- }
-}
diff --git a/code/app/task/collection/seo/GenerateSiteMapNovelZw.php b/code/app/task/collection/seo/GenerateSiteMapNovelZw.php
deleted file mode 100644
index dc28062..0000000
--- a/code/app/task/collection/seo/GenerateSiteMapNovelZw.php
+++ /dev/null
@@ -1,242 +0,0 @@
-generateNovelSitemaps($chunkSize, $totalNovelCount, $strSiteDomain, $strXqPath);
-
- // 生成详情的 Sitemap 小说seo
- $novelSeoSitemapFiles = $this->generateNovelSeoSitemaps($chunkSize, $totalNovelSeoCount, $strSiteDomain, $strSeoPath);
-
- // 生成章节的 Sitemap 小说章节
- $novelZjSitemapFiles = $this->generateNovelZjSitemaps($chunkSize, $totalNovelZjCount, $strSiteDomain, $strXqPath);
-
- // 生成分类的 Sitemap 小说分类
- $categorySitemapFiles = $this->generateCategorySitemaps($categories, $strSiteDomain, $strFenleiPath, $strXqPath);
-
- // 合并所有 Sitemap 文件路径
- $allSitemapFiles = array_merge($novelSitemapFiles, $novelSeoSitemapFiles, $categorySitemapFiles, $novelZjSitemapFiles);
-
- // 生成 Sitemap Index 文件
- $this->buildSitemapIndex($allSitemapFiles, $strSiteDomain);
-
- var_dump(['message' => '所有 Sitemap 生成成功', 'files' => $allSitemapFiles]);
- }
-
- # 生成小说章节 xml文件
- private function generateNovelZjSitemaps(int $chunkSize, int $totalCount, string $strSiteDomain, string $strXqPath): array
- {
- $totalFiles = ceil($totalCount / $chunkSize);
- $sitemapFiles = [];
-
- for ($i = 0; $i < $totalFiles; $i++) {
- $Novels = XiaoShuoZhangJieModel::field(['xsxq_id', 'xszj_id']) //intNovelId-:intNovelSourceId-:intNovelSourceSeoId
- ->limit($i * $chunkSize, $chunkSize)
- ->select()
- ->toArray();
-
- $sitemapXml = $this->buildZjSitemap($Novels, $strSiteDomain, $strXqPath);
- $filePath = public_path() . "sitemap_novel_zj_{$i}.xml";
- file_put_contents($filePath, $sitemapXml);
- $sitemapFiles[] = $filePath;
- }
-
- return $sitemapFiles;
- }
-
- # 生成小说详情 xml文件
- private function generateNovelSitemaps(int $chunkSize, int $totalCount, string $strSiteDomain, string $strXqPath): array
- {
- $totalFiles = ceil($totalCount / $chunkSize);
- $sitemapFiles = [];
-
- for ($i = 0; $i < $totalFiles; $i++) {
- $Novels = XiaoShuoXiangQingModel::field(['xsxq_id', 'xsxq_source_id', 'xsxq_source_seo_id']) //intNovelId-:intNovelSourceId-:intNovelSourceSeoId
- ->limit($i * $chunkSize, $chunkSize)
- ->select()
- ->toArray();
-
- $sitemapXml = $this->buildSitemap($Novels, $strSiteDomain, $strXqPath);
- $filePath = public_path() . "sitemap_novel_{$i}.xml";
- file_put_contents($filePath, $sitemapXml);
- $sitemapFiles[] = $filePath;
- }
-
- return $sitemapFiles;
- }
-
- # 生成小说详情 xml文件
- private function generateNovelSeoSitemaps(int $chunkSize, int $totalCount, string $strSiteDomain, string $strSeoPath): array
- {
- $totalFiles = ceil($totalCount / $chunkSize);
- $sitemapFiles = [];
-
- for ($i = 0; $i < $totalFiles; $i++) {
- $Novels = NovelInfoSeoModel::field(['xsxq_id', 'nis_id']) //show/nis_id
- ->limit($i * $chunkSize, $chunkSize)
- ->select()
- ->toArray();
-
- $sitemapXml = $this->buildXiaoShuoSeoSitemap($Novels, $strSiteDomain, $strSeoPath);
- $filePath = public_path() . "sitemap_novel_seo_{$i}.xml";
- file_put_contents($filePath, $sitemapXml);
- $sitemapFiles[] = $filePath;
- }
-
- return $sitemapFiles;
- }
-
- // 生成分类链接
- private function generateCategorySitemaps(array $categories, string $strSiteDomain, string $strFenleiPath, string $strXqPath): array
- {
- $sitemapFiles = [];
-
- foreach ($categories as $category) {
- $vcId = $category['xsfl_id'];
- $vcName = $category['xsfl_name'];
-
- $intNovelCount = XiaoShuoFenLeiModel::where('xsfl_id', $vcId)->count();
- $totalPages = ceil($intNovelCount / 30);
-
- $categoryUrls = [];
- for ($status = 0; $status <= 2; $status++) {
- for ($sort = 0; $sort <= 4; $sort++) {
- for ($page = 1; $page <= $totalPages; $page++) {
- $categoryUrls[] = [
- // /fl/:intCId-:intSort-[:intPage]
- 'url' => "/{$strFenleiPath}/{$vcId}-{$sort}-{$page}.html",
- 'title' => "{$vcName} - 第{$page}页",
- 'is_home' => false,
- ];
- }
- }
- }
-
- $sitemapXml = $this->buildSitemap($categoryUrls, $strSiteDomain, $strXqPath, true);
- $filePath = public_path() . "sitemap_category_{$vcId}.xml";
- file_put_contents($filePath, $sitemapXml);
- $sitemapFiles[] = $filePath;
- }
-
- return $sitemapFiles;
- }
-
- //生成章节链接
- private function buildZjSitemap(array $items, string $strSiteDomain, string $strXqPath, bool $isCategory = false): string
- {
- $xmlHeader = '';
- $urlsetStart = '';
- $urlsetEnd = '';
- // /xq/:intNovelId/:intZhangJiePaiXu-[:strSort]
- $urlEntries = '';
- foreach ($items as $item) {
- $urlEntries .= '';
- $urlEntries .= 'https://' . $strSiteDomain . ($isCategory ? $item['url'] : '/' . $strXqPath . '/' . htmlspecialchars($item['xsxq_id'] . '-' . $item['xszj_pai_xu']. '-' . $item['xszj_sort'] . '.html', ENT_QUOTES)) . '';
- $urlEntries .= 'daily';
- $urlEntries .= '0.8';
- $urlEntries .= '';
- }
-
- return $xmlHeader . $urlsetStart . $urlEntries . $urlsetEnd;
- }
-
- // 生成 详情链接
- private function buildSitemap(array $items, string $strSiteDomain, string $strXqPath, bool $isCategory = false): string
- {
- $xmlHeader = '';
- $urlsetStart = '';
- $urlsetEnd = '';
- // /xq/:intNovelId
- $urlEntries = '';
- foreach ($items as $item) {
- $urlEntries .= '';
- $urlEntries .= 'https://' . $strSiteDomain . ($isCategory ? $item['url'] : '/' . $strXqPath . '/' . htmlspecialchars($item['xsxq_id'] . '.html', ENT_QUOTES)) . '';
- $urlEntries .= 'daily';
- $urlEntries .= '1.0';
- $urlEntries .= '';
- }
-
- return $xmlHeader . $urlsetStart . $urlEntries . $urlsetEnd;
- }
-
- // 生成详情 seo 链接
- private function buildXiaoShuoSeoSitemap(array $items, string $strSiteDomain, string $strSeoPath, bool $isCategory = false): string
- {
- $xmlHeader = '';
- $urlsetStart = '';
- $urlsetEnd = '';
- // /show/:intNovelId
- $urlEntries = '';
- foreach ($items as $item) {
- $urlEntries .= '';
- $urlEntries .= 'https://' . $strSiteDomain . ($isCategory ? $item['url'] : '/' . $strSeoPath . '/' . htmlspecialchars($item['nis_id'] . '.html', ENT_QUOTES)) . '';
- $urlEntries .= 'daily';
- $urlEntries .= '1.0';
- $urlEntries .= '';
- }
-
- return $xmlHeader . $urlsetStart . $urlEntries . $urlsetEnd;
- }
-
- // 生成 xml 文件
- private function buildSitemapIndex(array $sitemapFiles, string $strSiteDomain): void
- {
- $xmlHeader = '';
- $sitemapIndexStart = '';
- $sitemapIndexEnd = '';
-
- $sitemapEntries = '';
- foreach ($sitemapFiles as $file) {
- $url = 'https://' . $strSiteDomain . '/' . basename($file);
- $sitemapEntries .= '';
- $sitemapEntries .= '' . htmlspecialchars($url, ENT_QUOTES) . '';
- $sitemapEntries .= '' . date('Y-m-d') . '';
- $sitemapEntries .= '';
- }
-
- $sitemapIndexXml = $xmlHeader . $sitemapIndexStart . $sitemapEntries . $sitemapIndexEnd;
- file_put_contents(public_path() . 'sitemap_index.xml', $sitemapIndexXml);
-
- var_dump(['message' => 'Sitemap Index 生成成功', 'file' => 'sitemap_index.xml']);
- }
-}
diff --git a/code/app/task/collection/tianlai/TianLaiXiaoShuoCol.php b/code/app/task/collection/tianlai/TianLaiXiaoShuoCol.php
deleted file mode 100644
index d49c299..0000000
--- a/code/app/task/collection/tianlai/TianLaiXiaoShuoCol.php
+++ /dev/null
@@ -1,538 +0,0 @@
-TaskCore;
- $strUrl = $this->getTianLaiDomain();
- var_dump($strUrl);
- $Client = $this->Client;
-
- try {
- $arrCookie = $this->getAilisiCookie();
- $strHost = getDomainFromUrl($strUrl);
-
- $CookieJar = CookieJar::fromArray($arrCookie, $strHost);
- $proxy = getRandomProxies();
- $userAgent = getRandomUserAgent();
-
-
- // $proxy = '127.0.0.1:8080'; // 代理地址
- // $response = fetchContentWithCurl($strUrl, [], 10, $proxy);
-
- $strResult = $Client->request('GET', $strUrl, [
- 'headers' => [
- 'User-Agent' => $userAgent,
- 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
- 'Accept-Language' => 'en-US,en;q=0.5',
- ],
- 'cookies' => $CookieJar,
- 'proxy' => $proxy,
- 'timeout' => 30,
- ]);
- $strResult = (string)$strResult->getBody();
- var_dump($strResult);
- // exit;
- // $strResult = $response['data'];
- $QueryList = QueryList::html($strResult);
-
-
- $QueryList->find('.nav a')->map(function ($FenLeiQueryList) use ($strUrl, $TaskCore) {
- $strName = trim($FenLeiQueryList->text());
- switch ($strName) {
-
- case '都市言情':
- $strName = "都市言情";
- break;
- case '历史军事':
- $strName = "历史军事";
- break;
- case '玄幻奇幻':
- $strName = "玄幻奇幻";
- break;
- case '武侠修真':
- $strName = "武侠修真";
- break;
- case '女生纯爱':
- $strName = "女频言情";
- break;
- case '科幻网游':
- $strName = "科幻网游";
- break;
- case '悬疑灵异':
- $strName = "悬疑灵异";
- break;
- default:
- $strName = "其它";
- break;
- }
- $strFenLeiUrl = $strUrl . $FenLeiQueryList->attr('href');
- // $intFenLeiId = extractFilename($strFenLeiUrl."index.html");
- $intFenLeiId = extractNumberFromUrl($strFenLeiUrl);
- if($intFenLeiId !== null){
-
- $arrClass = [
- 'xsfl_source_id' => $intFenLeiId,
- 'xsfl_source_code' => 'TIANLAI_FEN_LEI_ID_' . $intFenLeiId,
- 'xsfl_name' => $strName,
- ];
- var_dump($arrClass);
-
- $XiaoShuoFenLeiModel = XiaoShuoFenLeiModel::addXiaoShuoFenLei($arrClass);
-
- $arrTask = [
- 'callback' => [self::class, 'getNovelPage'],
- 'data' => [
- 'url' => $strFenLeiUrl,
- 'xsfl_source_id' => $intFenLeiId,
- 'xsfl_name' => $strName,
- 'xsfl_id' => $XiaoShuoFenLeiModel->xsfl_id,
- 'xsfl_page' => 1,
- 'xsfl_max_page' => 3000,
- ]
- ];
-
- $TaskCore->set($arrTask);
- }
-
- });
- } catch (\Throwable $T) {
- //var_dump($T);
- throw $T;
- }
- }
-
- public function getNovelPage($arrData)
- {
-
-
- var_dump('getNovelPage');
- var_dump($arrData);
-
- $strBasePage = $arrData['url'];
- $Client = $this->Client;
-
- $TaskCore = $this->TaskCore;
-
- try {
- $arrCookie = $this->getAilisiCookie();
- $strHost = getDomainFromUrl($strBasePage);
- $CookieJar = CookieJar::fromArray($arrCookie, $strHost);
-
- $strSiteUrl = $this->getTianLaiDomain();
-
- $intPage = $arrData['xsfl_page'];
- $intMaxPage = $arrData['xsfl_max_page'];
-
- $strUrl = $strBasePage;
-
- var_dump($strUrl);
-
-
- $proxy = getRandomProxies();
- $userAgent = getRandomUserAgent();
- $strResult = $Client->request('GET', $strUrl, [
- 'headers' => [
- 'User-Agent' => $userAgent
- ],
- 'cookies' => $CookieJar,
- #'proxy' => $proxy,
- 'timeout' => 10,
- ]);
- $strResult = (string)$strResult->getBody();
-
- $QueryList = QueryList::html($strResult);
-
-
- $intMaxPage = $QueryList->find('#newscontent h2 span')->eq(1)->text();
- $intMaxPage = ceil($intMaxPage / 100);
- var_dump($intMaxPage);
- $intMaxPage = 2;
-
-
- var_dump('$strUrl');
- // 如果是首页 则根据最大页面吧所有分页 追加到任务
- if($intPage == 1){
- for ($intCurrentPage = 2; $intCurrentPage <= $intMaxPage; $intCurrentPage++) {
- $strNewsFenleiUrl = replaceLastPageNumber($strBasePage, $intCurrentPage);
- var_dump($strNewsFenleiUrl);
- $arrTask = [
- 'callback' => [self::class, 'getNovelPage'],
- 'data' => [
- 'url' => $strNewsFenleiUrl,
- 'xsfl_source_id' => $arrData['xsfl_source_id'],
- 'xsfl_name' => $arrData['xsfl_name'],
- 'xsfl_id' => $arrData['xsfl_id'],
- 'xsfl_page' => $intCurrentPage,
- 'xsfl_max_page' => $intMaxPage,
- ]
- ];
- $TaskCore->set($arrTask);
- }
- }
- var_dump('提取小说');
- $XiaoShuoListQueryList = $QueryList->find('#newscontent li');
-
- if (!$XiaoShuoListQueryList || $XiaoShuoListQueryList->count() <= 0) {
- var_dump('这个分类已经市最后一页');
- return true;
- }
- var_dump($XiaoShuoListQueryList->count());
- var_dump('提取小说2');
- // var_dump($XiaoShuoListQueryList);
- $XiaoShuoListQueryList->map(function ($XiaoShuoQueryList) use ($strSiteUrl, $arrData, $TaskCore, $strUrl) {
- // 提取小说链接
- // var_dump($XiaoShuoQueryList->html());exit;
- var_dump('提取小说链接');
- $A = $XiaoShuoQueryList->find('a')->eq(0);
-
- $strXiaoShuoUrl = $A->attr('href');
- $strXiaoShuoUrl = strpos($strXiaoShuoUrl, 'https:') === 0 ? $strXiaoShuoUrl : 'https:' . $strXiaoShuoUrl;
- var_dump('$A');
- var_dump($A);
- var_dump($strXiaoShuoUrl);
- exit;
- // var_dump($A);
- // 提取作者信息
- $strZuoZhe = trim($XiaoShuoQueryList->find('.s4')->eq(0)->text());
-
- // 提取小说 ID
- $arrXiaoShuoId = extractNumbersFromUrl($strXiaoShuoUrl);
- if (!isset($arrXiaoShuoId['first'], $arrXiaoShuoId['second'])) {
- return; // 如果 ID 提取失败，直接跳过
- }
-
- // 准备任务数据
- $arrXiaoShuo = array_merge($arrData, [
- 'xiaoshuo_url' => $strXiaoShuoUrl,
- 'xiaoshuo_id' => $arrXiaoShuoId['first'],
- 'xiaoshuo_seo_id' => $arrXiaoShuoId['second'],
- 'xiaoshuo_zuozhe' => $strZuoZhe,
- 'url' => $strUrl
- ]);
- var_dump($arrXiaoShuo);
- $arrTask = [
- 'callback' => [self::class, 'getNovelInfo'],
- 'data' => $arrXiaoShuo,
- ];
-
- // 将任务加入队列
- $TaskCore->set($arrTask);
- var_dump("将任务加入队列getNovelInfo-{$strXiaoShuoUrl}");
- });
- var_dump('提取小说end');
-
- } catch (\Throwable $T) {
- throw $T;
- }
- }
-
-
-
- public function getNovelInfo($arrData)
- {
- var_dump('getNovelInfo');
- var_dump($arrData);
- $Client = $this->Client;
- $TaskCore = $this->TaskCore;
-
- // 获取 Redis 实例
- $redis = Cache::store('redis')->handler();
- $intXiaoShuoSourceId = $arrData['xiaoshuo_id'];
- $intXiaoShuoSourceSeoId = $arrData['xiaoshuo_seo_id'];
- $redisKey = "task:novel_source_id:" . $intXiaoShuoSourceId;
-
- try {
- $arrCookie = $this->getAilisiCookie();
- $strHost = getDomainFromUrl($arrData['xiaoshuo_url']);
- $CookieJar = CookieJar::fromArray($arrCookie, $strHost);
-
- $strSiteUrl = $this->getTianLaiDomain();
-
- $proxy = getRandomProxies();
- $userAgent = getRandomUserAgent();
- $Response = $Client->request('GET', $arrData['xiaoshuo_url'], [
- 'headers' => [
- 'User-Agent' => $userAgent
- ],
- 'cookies' => $CookieJar,
- #'proxy' => $proxy,
- 'timeout' => 10,
- ]);
- $strResult = (string)$Response->getBody();
- unset($Response);
-
- $QueryList = QueryList::html($strResult);
-
- $strMingZi = $QueryList->find('#info h1')->eq(0)->text();
- $strZhuangTai = "状态：连载中";
- $strZiShu = 0;
- $strZhuangTai = getStringAfterWord($strZhuangTai, '状态：');
-
- // 提取作者信息
- $strZuoZhe = trim($QueryList->find('p a')->eq(0)->text());
-
- // $strLastDate = $QueryList->find('meta[property="og:novel:update_time"]')->attr('content');
- $strLastDate = $QueryList->find('#info p')->eq(2)->text();
- $strLastDate = getStringAfterWord($strLastDate, '最后更新：');
- $strLastDate = trim((string)$strLastDate);
- if (!isValidDateTime($strLastDate)) {
- $strLastDate = date('Y-m-d H:i:s');
- }
-
- $arrTag = [];
- // $QueryList->find('.tg span')->map(function ($TagQuery) use (&$arrTag) {
- // $arrTag[] = trim($TagQuery->text());
- // });
-
- $strJieShao = $QueryList->find('#intro')->eq(0)->text();
- $strJieShao = removeScriptTags('/
+ 
+
+ 
+
+ 
+
+ 
+
+ 
+
+ 