From b09e4fca115afbbdf3cf1b273bcba6e9dcc4e729 Mon Sep 17 00:00:00 2001 From: Default Date: Thu, 20 Mar 2025 21:10:55 +0800 Subject: [PATCH] debug --- code/123.html | 387 ++++++++ code/333.php | 72 ++ code/444.php | 85 ++ code/app/task/collection/PullDataColBase.php | 8 +- .../collection/biquxs/BiquxsXiaoShuoCol.php | 427 --------- .../task/collection/ccc5/XiaoShuo5cccCol.php | 545 ----------- .../task/collection/dadi/DaDiDianyingCol.php | 498 ---------- .../collection/heimuer/HeiMuErYingShiCol.php | 560 ------------ .../collection/novel/Zw630XiaoShuoCol.php | 850 ------------------ .../novelone/NovelJiuYuFanXianCol.php | 426 --------- .../novelone/NovelWenXueGuanCol.php | 441 --------- .../collection/seo/GenerateSiteMapCol.php | 137 --- .../collection/seo/GenerateSiteMapNovel.php | 188 ---- .../collection/seo/GenerateSiteMapNovelSk.php | 242 ----- .../collection/seo/GenerateSiteMapNovelZw.php | 242 ----- .../collection/tianlai/TianLaiXiaoShuoCol.php | 538 ----------- .../collection/visitlogs/ContentStatsCol.php | 90 -- .../collection/visitlogs/VisitLogsCol.php | 47 - .../task/collection/yzzy/YouZhiYingShiCol.php | 375 -------- code/app/task/collection/zw630/Zw630Col.php | 477 ++++++++++ .../task/command/InsertSeoWordsCommand.php | 50 -- code/app/task/core/Rout.php | 31 - code/app/task/core/ScheduledTasks.php | 24 - code/app/task/logic/AdTask.php | 70 -- code/app/task/{core => module}/Listen.php | 22 +- code/app/task/module/PlanTask.php | 328 +------ code/composer.json | 3 +- code/config/console.php | 4 +- code/config/database.php | 1 + code/config/mongodb.php | 19 + code/config/task.php | 4 +- code/database/schema/mysql-schema.dump | 178 ++-- .../db/DatabaseManage.php} | 4 +- code/extend/db/mongo/MongoBase.php | 183 ++++ .../minserver/ProcessSanitizer.php} | 7 +- .../minserver/QueueManage.php} | 6 +- code/extend/minserver/Rout.php | 17 + code/extend/minserver/ScheduledTasks.php | 19 + .../minserver/ServerConsole.php} | 6 +- .../minserver/ServerCore.php} | 8 +- .../minserver/ServerManage.php} | 8 +- 41 files changed, 1369 insertions(+), 6258 deletions(-) create mode 100644 code/123.html create mode 100644 code/333.php create mode 100644 code/444.php delete mode 100644 code/app/task/collection/biquxs/BiquxsXiaoShuoCol.php delete mode 100644 code/app/task/collection/ccc5/XiaoShuo5cccCol.php delete mode 100644 code/app/task/collection/dadi/DaDiDianyingCol.php delete mode 100644 code/app/task/collection/heimuer/HeiMuErYingShiCol.php delete mode 100644 code/app/task/collection/novel/Zw630XiaoShuoCol.php delete mode 100644 code/app/task/collection/novelone/NovelJiuYuFanXianCol.php delete mode 100644 code/app/task/collection/novelone/NovelWenXueGuanCol.php delete mode 100644 code/app/task/collection/seo/GenerateSiteMapCol.php delete mode 100644 code/app/task/collection/seo/GenerateSiteMapNovel.php delete mode 100644 code/app/task/collection/seo/GenerateSiteMapNovelSk.php delete mode 100644 code/app/task/collection/seo/GenerateSiteMapNovelZw.php delete mode 100644 code/app/task/collection/tianlai/TianLaiXiaoShuoCol.php delete mode 100644 code/app/task/collection/visitlogs/ContentStatsCol.php delete mode 100644 code/app/task/collection/visitlogs/VisitLogsCol.php delete mode 100644 code/app/task/collection/yzzy/YouZhiYingShiCol.php create mode 100644 code/app/task/collection/zw630/Zw630Col.php delete mode 100644 code/app/task/command/InsertSeoWordsCommand.php delete mode 100644 code/app/task/core/Rout.php delete mode 100644 code/app/task/core/ScheduledTasks.php delete mode 100644 code/app/task/logic/AdTask.php rename code/app/task/{core => module}/Listen.php (77%) create mode 100644 code/config/mongodb.php rename code/{app/task/command/DatabaseCommand.php => extend/db/DatabaseManage.php} (98%) create mode 100644 code/extend/db/mongo/MongoBase.php rename code/{app/task/logic/SystemLogic.php => extend/minserver/ProcessSanitizer.php} (85%) rename code/{app/task/core/TaskCore.php => extend/minserver/QueueManage.php} (82%) create mode 100644 code/extend/minserver/Rout.php create mode 100644 code/extend/minserver/ScheduledTasks.php rename code/{app/task/command/Task.php => extend/minserver/ServerConsole.php} (96%) rename code/{app/task/core/Service.php => extend/minserver/ServerCore.php} (90%) rename code/{app/task/core/Core.php => extend/minserver/ServerManage.php} (94%) diff --git a/code/123.html b/code/123.html new file mode 100644 index 0000000..e34d10d --- /dev/null +++ b/code/123.html @@ -0,0 +1,387 @@ + + + + + + + + + + + 御兽：我的御主是位恶役大小姐最新章节_御兽：我的御主是位恶役大小姐白色的竹篱笆丶_御兽：我的御主是位恶役大小姐小说免费全文阅读_630阅读网 + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
将本站设为首页
+
收藏630阅读网
+ +
+
+
+

630阅读网

+ +
+ +
+
+
+

630阅读网>御兽：我的御主是位恶役大小姐 +

+
+
御兽：我的御主是位恶役大小姐
+
+
+

御兽：我的御主是位恶役大小姐

+
+

作  者：白色的竹篱笆丶

+

类  别：玄幻修真

+

状  态：连载中

+

动  作：加入书架、章节目录、开始阅读

+

最后更新：2025-03-18 14:49:04

+

最新章节：第262章 你不喜欢最好了

+
+
+
+ 关于御兽：我的御主是位恶役大小姐：恶役大小姐伸出手想要推开面前巨龙的脑袋：“混蛋巨龙，不要再吃了！最后一点也没有了！”这里是御兽的世界，这里是御兽师的世界，每个人将会在十五岁时召唤出属于自己的御兽。觉醒，契约，收服，修炼，进化，异化，战斗……是这片大陆的主旋律，山野，城市，战场，秘境，遗迹都有他们的痕迹，御兽资质：甲乙丙丁四等当异世界传奇生物巨龙融合英雄联盟中科加斯能够无限成长的大招被一位恶役，跋扈，淡漠的大小姐召唤觉醒出丁级资质，期间会发生什么故事。 + 御兽：我的御主是位恶役大小姐
+
+
+
+
+

《御兽：我的御主是位恶役大小姐》第262章 你不喜欢最好了

+
+

铭城主连连点头，补充说：“不止是位女生，还是个烈女，扬言即便是傲厉殿下在她面前都不敢欺辱她。” 傲慧勾起一个笑容：“有趣。” + 跟在傲慧身后的铭城主打磨他这句话半晌，带着上不了台面的权色交易意味说：“调教一个烈女的感觉一定很舒服。” 皇室嘛~，都是滥交之辈，不然道乾那个皇帝也不会有十几位皇子， + “呵呵，本皇子不喜欢烈女，你要是喜欢的话，还是自己留着吧！” 铭城主愣了片刻，想起刚才地牢的发生，那双眼神太可怕了， 他今天必须把这个女人的翅膀折断，不然就只能想办法借助慧殿下的手，把她斩杀， + 不然想起她的话，自己指定睡不着觉。 打着这样的算盘，他同傲慧皇子进入了铁砧堡的地牢， ...

+

相邻推荐：咒术燃冬？？？  春潮泛滥  穿越后我成了鱼玄机的老师  四合院：绝世家族  成为耀眼的一颗星星吧  大明：捡到朱雄英，调教成恶霸  重生罪女嫁王爷，渣男全家破大防  戏神：我不是死神  三国之风起扬州  为什么？重生后一切都变啦  美人很风骚  明末崛起：打造一个崭新华夏帝国  见燕台  一米八的柳同学  陶李恋：宫廷异闻录  原神：美露莘的我不想当抱枕  在灾难世界收房租  层层迷雾之都市神探  群星：我的领袖不太对劲  闪婚老伴是富豪  御兽我的主人实在太稳健了  御兽我的御主是位恶役大小姐正版阅读  御兽我的御主是位恶役大小姐  御兽我的御主是位恶役大小姐免费阅读全文最新章节  

+
+
+
+
+
+

《御兽：我的御主是位恶役大小姐》最新章节

+ +

《御兽：我的御主是位恶役大小姐》章节列表

+ 查看更多章节... +
+
+
+

已完结热门小说推荐

+
+
+
建造帝国
+
+
九十年代01建造帝国
+
一梦倒流30年，来到90年代的杨大利看着包裹里帝国时代的原始驴车城镇中心，表示有个卵用！然而没多久，被生活所迫的他，本想着建造城镇中心招募农夫种田搞点... +
+
+
+
+
韩娱之单身爸爸
+
+
夜的七宗罪韩娱之单身爸爸
+
2014年1月1日，悲剧的李胜林因为和身为李胜基狂粉的女友吵架而被车撞死，而附身穿越到了2006年一个同名的韩国人身上。然后居然一个不小心成为了一个单身爸爸，接下来就让我们看这个单身爸爸能不能在韩国娱乐圈给自己的宝贝儿子找... +
+
+
+
+
花式作死的位面商人
+
+
张扬的五月花式作死的位面商人
+
想要成为一个大商人的叶俊文突然得到了一个跨位面的交易系统，突然间好像无敌了？然而事情没这么简单。这个系统有这三条使用原则一只有对你产生强烈恨意的人才能杀死你，死亡后进行位面跳跃。二同一个人无法杀死你第二次。... +
+
+
+
+
仙寇
+
+
罗玛作品集仙寇
+
小说分类其他类型首发状态他站首发小说状态连载中小说作者罗玛作品集授权级别暂未授权全文字数2590448字收藏总数2更新时间20151020... +
+
+
+
+
星际烧包女王
+
+
鱼香蹂丝星际烧包女王
+
重生后武力值爆表突破天际，怀揣四颗黑芝麻馅的包子，姐十分无奈，智商极高的小包子们也十分无奈，谁能体谅本女王压力山大？！复兴虫族，多子多孙什么的，千万别来找我啊啊啊啊啊啊人类战斗英雄重生成虫族女王，姐死... +
+
+
+
+
全能天才
+
+
温东篱全能天才
+
东篱新书玩美人生，求收藏17kcombook6212o6htm1红鸾星动，天降桃劫穷小子苏杰在美女董事长肚子里播了个种从此，他的人生开始变得与众不同桃劫之下，美女纷纷来... +
+
+
+
+
+
+
+
+

最新标签

+ +
+
+ +
+ + + + + \ No newline at end of file diff --git a/code/333.php b/code/333.php new file mode 100644 index 0000000..4331037 --- /dev/null +++ b/code/333.php @@ -0,0 +1,72 @@ + 'https://www.630zw.org', + 'http_errors' => false, + 'version' => 2.0, // 强制使用 HTTP/2 + 'timeout' => 160, + 'connect_timeout' => 30, // 连接超时 10 秒 + + 'curl' => [ + CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_2_0, // 强制 HTTP/2 + CURLOPT_TCP_KEEPALIVE => 1, // 启用 TCP Keep-Alive + CURLOPT_SSL_VERIFYPEER => true, // 验证 SSL + ], + 'headers' => [ + 'User-Agent' => 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Mobile Safari/537.36 Edg/134.0.0.0', + 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7', + 'Accept-Language' => 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6', + 'Referer' => 'https://www.630zw.org/', + + 'sec-ch-ua-mobile' => '?1', + 'sec-ch-ua-platform' => '"Android"', + 'sec-fetch-dest' => 'document', + 'sec-fetch-mode' => 'navigate', + 'sec-fetch-site' => 'same-origin', + 'sec-fetch-user' => '?1', + 'upgrade-insecure-requests' => '1', + ], +]); + +$i = 20284; +while (true) { + + $strStartDate = date('Y-m-d H:i:s') . PHP_EOL; + + + $promises = []; + + $j = $i + 15; + for (; $i <= $j; $i++) { + $promises['res' . $i] = $client->getAsync('/shu/' . $i . '.html'); + } + + + // 等待所有请求完成 + $responses = Promise\Utils::unwrap($promises); + + foreach ($responses as $key => $response) { + // echo "$key: " . $response->getBody() . "\n"; + + $html = (string) $response->getBody(); + $dom = new DOMDocument(); + @$dom->loadHTML($html); + $title = $dom->getElementsByTagName('title')->item(0); + $titleText = $title ? $title->textContent : 'No title found'; + echo "$key: Title: $titleText\n"; + } + + + $strEndDate = date('Y-m-d H:i:s') . PHP_EOL; + + echo $strStartDate; + echo $strEndDate; +} diff --git a/code/444.php b/code/444.php new file mode 100644 index 0000000..c724a18 --- /dev/null +++ b/code/444.php @@ -0,0 +1,85 @@ +client = new Client($uri); + $this->db = $this->client->selectDatabase($dbName); + } catch (MongoDBException $e) { + die("MongoDB connection failed: " . $e->getMessage()); + } + } + + // 获取下一个自增序列值 + public function getNextSequence($sequenceName) { + try { + $counters = $this->db->selectCollection('counters'); + $result = $counters->findOneAndUpdate( + ['_id' => $sequenceName], + ['$inc' => ['sequence_value' => 1]], + [ + 'upsert' => true, // 如果不存在则创建 + 'returnDocument' => MongoDB\Operation\FindOneAndUpdate::RETURN_DOCUMENT_AFTER // 返回更新后的文档 + ] + ); + + // 如果是第一次创建，sequence_value 可能为 null，初始化为 1 + if ($result === null || !isset($result->sequence_value)) { + return 1; + } + + return $result->sequence_value; + } catch (MongoDBException $e) { + throw new Exception("Failed to generate sequence: " . $e->getMessage()); + } + } + + // 示例：插入用户数据 + public function insertUser($name, $age) { + try { + $userId = $this->getNextSequence('userId'); + $users = $this->db->selectCollection('users'); + $result = $users->insertOne([ + '_id' => $userId, + 'name' => $name, + 'age' => $age, + 'created_at' => new MongoDB\BSON\UTCDateTime() + ]); + + return $result->getInsertedId(); + } catch (MongoDBException $e) { + throw new Exception("Failed to insert user: " . $e->getMessage()); + } + } +} + +// 使用示例 +try { + // 设置你的 MongoDB 连接信息 + $mongoUri = 'mongodb://admin:qq123123@127.0.0.1:27017'; // 替换为你的 IP、用户名和密码 + $dbName = 'test001'; // 替换为你的数据库名 + + $generator = new MongoSequenceGenerator($mongoUri, $dbName); + + // 插入几个测试用户 + $id1 = $generator->insertUser('Alice', 25); + echo "Inserted user with ID: " . $id1 . "\n"; + + $id2 = $generator->insertUser('Bob', 30); + echo "Inserted user with ID: " . $id2 . "\n"; + + $id3 = $generator->insertUser('Charlie', 35); + echo "Inserted user with ID: " . $id3 . "\n"; + +} catch (Exception $e) { + echo "Error: " . $e->getMessage() . "\n"; +} +?> \ No newline at end of file diff --git a/code/app/task/collection/PullDataColBase.php b/code/app/task/collection/PullDataColBase.php index fe9254a..50f76e2 100644 --- a/code/app/task/collection/PullDataColBase.php +++ b/code/app/task/collection/PullDataColBase.php @@ -6,7 +6,7 @@ namespace app\task\collection; use app\model\CaiJiPeiZhiModel; use app\model\SystemConfigModel; -use app\task\core\TaskCore; +use minserver\QueueManage; use GuzzleHttp\Client; /** @@ -24,9 +24,9 @@ class PullDataColBase /** * Undocumented variable * - * @var \app\task\core\TaskCore + * @var \minserver\QueueManage */ - protected $TaskCore = NULL; + protected $QueueManage = NULL; protected $arrCaiJiPeiZhi = []; @@ -34,7 +34,7 @@ class PullDataColBase { $this->Client = new Client(); - $this->TaskCore = new TaskCore(); + $this->QueueManage = new QueueManage(); } diff --git a/code/app/task/collection/biquxs/BiquxsXiaoShuoCol.php b/code/app/task/collection/biquxs/BiquxsXiaoShuoCol.php deleted file mode 100644 index e2887ff..0000000 --- a/code/app/task/collection/biquxs/BiquxsXiaoShuoCol.php +++ /dev/null @@ -1,427 +0,0 @@ -TaskCore; - // 20054 - for ($i = 1; $i <= 20054; $i++) { - echo "当前值: $i\n"; - $arrTask = [ - 'callback' => [self::class, 'getNovelInfo'], - 'data' => [ - 'xs_source_id' => $i, - 'xs_source_seo_id' => 1, - ] - ]; - $TaskCore->set($arrTask); - var_dump($arrTask); - } - - } - - public function getNovelInfo($arrData) - { - $TaskCore = $this->TaskCore; - $Client = $this->Client; - $strSiteUrl = $this->getBiquxsDomain(); - - $strUrl = 'http://www.biquxs.com/book/' . $arrData['xs_source_id'] ; //'http://www.biquxs.com/book/20053'; - $intXiaoShuoSourceId = $arrData['xs_source_id']; //20053; - $intXiaoShuoSourceSeoId = $arrData['xs_source_seo_id']; //1 ; - try { - $arrCookie = $this->getAilisiCookie(); - $strHost = getDomainFromUrl($strUrl); - - $CookieJar = CookieJar::fromArray($arrCookie, $strHost); - - $proxy = getRandomProxies(); - $userAgent = getRandomUserAgent(); - $Response = $Client->request('GET', $strUrl, [ - 'headers' => [ - 'User-Agent' => $userAgent, - 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', - 'Accept-Language' => 'en-US,en;q=0.5', - ], - 'cookies' => $CookieJar, - // 'proxy' => $proxy, - 'timeout' => 30, - ]); - - $strResult = (string)$Response->getBody(); - - $QueryList = QueryList::html($strResult); - // var_dump($QueryList); - // 提取所有 标签的属性 - $result = $QueryList->find('meta[property^="og:"]')->map(function ($item) { - return [ - 'property' => $item->attr('property'), - 'content' => $item->attr('content'), - ]; - }); - - // 打印结果 - // print_r($result->all()); - $strZuoZhe = ''; - $strMingZi = ''; - $strLastDate = ''; - $strJieShao = ''; - $strCover = ''; - $strFenlei = ''; - $strZiShu = 0; - $strZhuangTai = $QueryList->find('#info p')->eq(1)->text(); - $strZhuangTai = getStringAfterWord($strZhuangTai, '状态：'); - - foreach ($result as $item) { - switch ($item['property']) { - case 'og:title': - $strMingZi = $item['content']; - break; - case 'og:description': - $strJieShao = $item['content']; - break; - case 'og:image': - $strCover = $item['content']; - break; - case 'og:book:category': - $strFenlei = $item['content']; - break; - case 'og:book:author': - $strZuoZhe = $item['content']; - break; - case 'og:book:update_time': - $strLastDate = $item['content']; - break; - } - } - - - $strLastDate = trim((string)$strLastDate); - if (!isValidDateTime($strLastDate)) { - $strLastDate = date('Y-m-d H:i:s'); - } - - $arrTag = []; - - if (strpos($strCover, 'http') === false) { - $strCover = $strSiteUrl . $strCover; - } - $strCoverUri = $strCover; - - try { - $strCoverUri = ""; - $strCoverUri = ImageProcessor::getInstance()->downloadImage( $intXiaoShuoSourceId,$strCover, 'xs'); - } catch (\Throwable $t) { - $strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage()); - echo $strErr; - } - - $arrCover = [ - 'code' => 'BI_QU_XS_COVER', - // 'uri' => getUriByUrl($strCover), - 'uri' => $strCoverUri, - - ]; - - $strFenleiIId = 1 ; - switch ($strFenlei) { - case '玄幻小说': - $strName = "玄幻"; - $strFenleiIId = 1; - break; - case '修真小说': - $strName = "修真"; - $strFenleiIId = 2; - break; - case '都市小说': - $strName = "都市"; - $strFenleiIId = 3; - break; - case '历史小说': - $strName = "历史"; - $strFenleiIId = 4; - break; - case '网游小说': - $strName = "网游"; - $strFenleiIId = 5; - break; - case '科幻小说': - $strName = "科幻"; - $strFenleiIId = 6; - break; - case '女频小说': - $strName = "女频"; - $strFenleiIId = 7; - break; - case '其它小说': - $strName = "其它"; - $strFenleiIId = 8; - break; - default: - $strName = "其它"; - $strFenleiIId = 8; - break; - } - - $arrXiaoShuoInfo = [ - 'xsxq_ming_zi' => $strMingZi, - 'xsxq_zuozhe' => $strZuoZhe , //$arrData['xiaoshuo_zuozhe'], - 'xsfl_id' => $strFenleiIId, - 'xsxq_zhuang_tai' => $strZhuangTai, - 'xsxq_zi_shu' => $strZiShu, - 'xsxq_jie_shao' => $strJieShao, - 'xsxq_feng_mian' => json_encode($arrCover), - 'xsxq_geng_xin_shi_jian' => $strLastDate, - 'xsxq_tag' => json_encode($arrTag, JSON_UNESCAPED_UNICODE), - 'xsxq_source_id' => $intXiaoShuoSourceId, - 'xsxq_source_seo_id' => $intXiaoShuoSourceId.'1', - 'xsxq_source_code' => 'BIQU_XIAOSHUO_' . $intXiaoShuoSourceId . '_' . $intXiaoShuoSourceSeoId, - ]; - // print_r($arrXiaoShuoInfo); - var_dump($arrXiaoShuoInfo['xsxq_source_code']); - - $XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo); - - $arrZhangjie = []; - $intChapterIndex = 0; // 定义一个计数器变量 - - // 检查是否找到元素 - if ($QueryList->find('.listmain a')->count() > 0) { - $QueryList->find('.listmain a')->map(function ($ZhangJie) use (&$arrZhangjie, &$intChapterIndex) { - echo "Processing index $intChapterIndex: " . $ZhangJie->text() . PHP_EOL; - - // 从索引6开始 - if ($intChapterIndex >= 5) { - $arrZhangjie[] = [ - 'mingzi' => $ZhangJie->text(), - 'url' => $ZhangJie->attr('href'), - ]; - } - $intChapterIndex++; - }); - - // 输出最终结果 - var_dump($arrZhangjie); - } else { - echo "No elements found for .listmain a" . PHP_EOL; - } - - //unset($QueryList); - - // 获取最大章节索引 - $intMaxXiaoShuoZhangJiePaixu = XiaoShuoZhangJieModel::where('xsxq_id', $intXiaoShuoSourceId) - ->max('xszj_pai_xu'); // 获取最大值 - var_dump('获取小说章节最大值-'.$intMaxXiaoShuoZhangJiePaixu); - - // 如果没有结果，max 返回 null，可以设置默认值 - $intMaxXiaoShuoZhangJiePaixu = $intMaxXiaoShuoZhangJiePaixu ?? 0; - - foreach ($arrZhangjie as $intZhangJieNum => $arrZhangjieOne) { - var_dump($intZhangJieNum); - - // 如果数据库里的章节最大索引大于当前索引，说明已经入库，不需要入队列了 - if($intMaxXiaoShuoZhangJiePaixu > $intZhangJieNum){ - var_dump("小说章节已经入库，不再推送任务"); - continue;// 如果任务已存在，直接跳过 break - } - - $strZhangJieUrl = $strSiteUrl . $arrZhangjieOne['url']; - - $strZhangJieId = extractFilename($strZhangJieUrl); - - # 'AILISI_XIAOSHUO_ZHANGJIE_小说源id_小说源章节id' - $strXszjSourceCode = 'AILISI_XIAOSHUO_ZHANGJIE_'. $XiaoShuoXiangQingModel->xsxq_source_id . '_' . $strZhangJieId; - - $arrTask = [ - 'callback' => [self::class, 'getNovelReader'], - 'data' => [ - 'xszj_source_id' => $strZhangJieId, - 'xszj_paixu' => $intZhangJieNum, - 'xszj_name' => $arrZhangjieOne['mingzi'], - 'xszj_url' => $strZhangJieUrl, - 'xszj_source_code' => $strXszjSourceCode, - 'xsxq_source_id' => $XiaoShuoXiangQingModel->xsxq_source_id, - 'xsfl_id' => $strFenleiIId, - ] - ]; - $TaskCore->set($arrTask); - - } - var_dump("章节任务投递成功"); - - } catch (\Throwable $T) { - throw $T; - } - } - - public function getNovelFengMian($arrData) - { - //var_dump('getNovelFengMian'); - //var_dump($arrData); - $Client = $this->Client; - $TaskCore = $this->TaskCore; - - // 获取 Redis 实例 - $redis = Cache::store('redis')->handler(); - $intXiaoShuoSourceId = $arrData['xiaoshuo_id']; - $intXiaoShuoSourceSeoId = $arrData['xiaoshuo_seo_id']; - $redisKey = "task:novel_source_id:" . $intXiaoShuoSourceId; - - try { - $arrCookie = $this->getAilisiCookie(); - $strHost = getDomainFromUrl($arrData['xiaoshuo_url']); - $CookieJar = CookieJar::fromArray($arrCookie, $strHost); - - $strSiteUrl = $this->getAilisiDomain(); - - - $strCover = $QueryList->find('#fmimg img')->eq(0)->attr("src"); - - if (strpos($strCover, 'http') === false) { - $strCover = $strSiteUrl . $strCover; - } - $strCoverUri = $strCover; - - try { - $strCoverUri = ""; - $strCoverUri = ImageProcessor::getInstance()->downloadAndEncryptImage($strCover, 'xs'); - $arrCover = [ - 'code' => 'AI_LI_SI_COVER', - // 'uri' => getUriByUrl($strCover), - 'uri' => $strCoverUri, - - ]; - - $arrXiaoShuoInfo = [ - 'xsxq_feng_mian' => json_encode($arrCover), - - ]; - $XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo); - } catch (\Throwable $t) { - $strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage()); - echo $strErr; - } - - - - } catch (\Throwable $T) { - throw $T; - } - } - - public function getNovelReader($arrData) - { - var_dump('getNovelReader'); - var_dump($arrData['xszj_url']); - $Client = $this->Client; - $strXszjSourceCode = $arrData['xszj_source_code']; - $strZhangJieUrl = $arrData['xszj_url']; - $intZhangJiePaiXu = $arrData['xszj_paixu']; - $intFenleiId = $arrData['xsfl_id']; - $intXiaoShuoSourceID = $arrData['xsxq_source_id']; - $strZhangJieSourceId = $arrData['xszj_source_id']; - $strZhangJieName = $arrData['xszj_name']; - try { - $arrCookie = $this->getAilisiCookie(); - $strHost = getDomainFromUrl($strZhangJieUrl); - $CookieJar = CookieJar::fromArray($arrCookie, $strHost); - - if (XiaoShuoZhangJieModel::checkZhangJieExists($strXszjSourceCode)) { - return true; - } - - $proxy = getRandomProxies(); - $userAgent = getRandomUserAgent(); - $Response = $Client->request('GET', $strZhangJieUrl, [ - 'headers' => [ - 'User-Agent' => $userAgent, - 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', - 'Accept-Language' => 'en-US,en;q=0.5', - ], - 'cookies' => $CookieJar, - // 'proxy' => $proxy, - 'timeout' => 30, - ]); - - $strResult = (string)$Response->getBody(); - unset($Response); - $ZhangJieQueryList = QueryList::html($strResult); - - - // 获取原始 HTML - $strNeiRong = $ZhangJieQueryList->find('#content')->eq(0)->html(); - - // 解码 HTML 实体 - $strNeiRong = html_entity_decode($strNeiRong, ENT_QUOTES, 'UTF-8'); - - // 去掉

标签中的 class="content_detail" 属性 - $strNeiRong = preg_replace('/'); - - // 替换不间断空格 (\xc2\xa0) 和   为普通空格 - $strNeiRong = preg_replace('/\xc2\xa0/', ' ', $strNeiRong); // 替换 UTF-8 不间断空格 - $strNeiRong = preg_replace('/ /', ' ', $strNeiRong); // 替换   - - // 清理多余空格和换行符 - $strNeiRong = preg_replace('/\s+/', ' ', $strNeiRong); - $strNeiRong = preg_replace('/

\s+/', '

', $strNeiRong); // 清理

标签内部多余空格 - $strNeiRong = preg_replace('/\s+<\/p>/', '

', $strNeiRong); // 清理

标签结尾多余空格 - - // 输出结果 - // var_dump($strNeiRong); - - - # '/novel/123yqw/分类id/小说源id/小说源章节id.txt' - $strNeiRongTxtPath = config("filesystem.novel").'/novel/biquxs/'.$intFenleiId.'/'.$intXiaoShuoSourceID.'/'.$strZhangJieSourceId.'.txt'; - //$strNeiRongTxtPath = "/www/novel/123yqw/1/11/111.txt"; - // 调用函数 - $result = saveCompressAndDeleteTxt($strNeiRongTxtPath, $strNeiRong); - - // 输出状态 - if ($result['status']) { - var_dump("Success: " . $result['message']) ; - } else { - var_dump("Error: " . $result['message']) ; - } - - $arrXiaoShuoZhangJie = [ - 'xsxq_id' => $intXiaoShuoSourceID, - 'xszj_pai_xu' => $intZhangJiePaiXu, - 'xszj_ming_zi' => $strZhangJieName, - 'xszj_nei_rong' => $strNeiRongTxtPath, - 'xszj_source_id' => $strZhangJieSourceId, - 'xszj_source_code' => $strXszjSourceCode, - ]; - - XiaoShuoZhangJieModel::addXiaoShuoZhangJie($arrXiaoShuoZhangJie); - unset($ZhangJieQueryList); - // 延时 3 秒 - //sleep(3); - - } catch (\Throwable $T) { - throw $T; - } - } -} diff --git a/code/app/task/collection/ccc5/XiaoShuo5cccCol.php b/code/app/task/collection/ccc5/XiaoShuo5cccCol.php deleted file mode 100644 index 34d7e1a..0000000 --- a/code/app/task/collection/ccc5/XiaoShuo5cccCol.php +++ /dev/null @@ -1,545 +0,0 @@ -TaskCore; - // 20054 - for ($i = 3; $i <= 3; $i++) { - echo "当前值: $i\n"; - $arrTask = [ - 'callback' => [self::class, 'getNovelInfo'], - 'data' => [ - 'xs_source_id' => $i, - 'xs_source_seo_id' => 1, - ] - ]; - $TaskCore->set($arrTask); - var_dump($arrTask); - } - - } - - public function getNovelInfo($arrData) - { - $TaskCore = $this->TaskCore; - $Client = $this->Client; - $strSiteUrl = $this->get5cccDomain(); - - $strUrl = 'https://www.5ccc.org/chapter/' . $arrData['xs_source_id'].'.html' ; //'http://www.biquxs.com/book/20053'; - $intXiaoShuoSourceId = $arrData['xs_source_id']; //20053; - $intXiaoShuoSourceSeoId = $arrData['xs_source_seo_id']; //1 ; - try { - $arrCookie = $this->getAilisiCookie(); - $strHost = getDomainFromUrl($strUrl); - - $CookieJar = CookieJar::fromArray($arrCookie, $strHost); - - $proxy = getRandomProxies(); - $userAgent = getRandomUserAgent(); - $Response = $Client->request('GET', $strUrl, [ - 'headers' => [ - 'User-Agent' => $userAgent, - 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', - 'Accept-Language' => 'en-US,en;q=0.5', - ], - 'cookies' => $CookieJar, - // 'proxy' => $proxy, - 'timeout' => 30, - ]); - - $strResult = (string)$Response->getBody(); - - $QueryList = QueryList::html($strResult); - // var_dump($QueryList); - // 提取所有 标签的属性 - $result = $QueryList->find('meta[property^="og:"]')->map(function ($item) { - return [ - 'property' => $item->attr('property'), - 'content' => $item->attr('content'), - ]; - }); - - // 打印结果 - print_r($result->all()); - $strZuoZhe = ''; - $strMingZi = ''; - $strLastDate = ''; - $strJieShao = ''; - $strCover = ''; - $strFenlei = ''; - $strZiShu = 0; - $strZhuangTai = ""; - - foreach ($result as $item) { - switch ($item['property']) { - case 'og:title': - $strMingZi = $item['content']; - break; - case 'og:description': - $strJieShao = $item['content']; - break; - case 'og:image': - $strCover = $item['content']; - break; - case 'og:novel:category': - $strFenlei = $item['content']; - break; - case 'og:novel:author': - $strZuoZhe = $item['content']; - break; - case 'og:novel:update_time': - $strLastDate = $item['content']; - break; - case 'og:novel:status': - $strZhuangTai = $item['content']; - break; - } - } - - - $strLastDate = trim((string)$strLastDate); - if (!isValidDateTime($strLastDate)) { - $strLastDate = date('Y-m-d H:i:s'); - } - - $arrTag = []; - - $strCover = $QueryList->find('.imgbox img')->eq(0)->attr("src"); - if (strpos($strCover, 'http') === false) { - $strCover = $strSiteUrl . $strCover; - } - $strCoverUri = $strCover; - - try { - $strCoverUri = ""; - $strCoverUri = ImageProcessor::getInstance()->downloadImage( $intXiaoShuoSourceId,$strCover, 'xs'); - } catch (\Throwable $t) { - $strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage()); - echo $strErr; - } - - $arrCover = [ - 'code' => '5CCC_XS_COVER', - // 'uri' => getUriByUrl($strCover), - 'uri' => $strCoverUri, - - ]; - - $strFenleiIId = 1 ; - switch ($strFenlei) { - case '玄幻修真': - $strName = "玄幻修真"; - $strFenleiIId = 1; - break; - case '重生穿越': - $strName = "重生穿越"; - $strFenleiIId = 2; - break; - case '都市小说': - $strName = "都市小说"; - $strFenleiIId = 3; - break; - case '军史小说': - $strName = "军史小说"; - $strFenleiIId = 4; - break; - case '网游小说': - $strName = "网游小说"; - $strFenleiIId = 5; - break; - case '科幻小说': - $strName = "科幻小说"; - $strFenleiIId = 6; - break; - case '灵异小说': - $strName = "灵异小说"; - $strFenleiIId = 7; - break; - case '言情小说': - $strName = "言情小说"; - $strFenleiIId = 8; - break; - case '其他小说': - $strName = "其他小说"; - $strFenleiIId = 9; - break; - } - - $arrXiaoShuoInfo = [ - 'xsxq_ming_zi' => $strMingZi, - 'xsxq_zuozhe' => $strZuoZhe , //$arrData['xiaoshuo_zuozhe'], - 'xsfl_id' => $strFenleiIId, - 'xsxq_zhuang_tai' => $strZhuangTai, - 'xsxq_zi_shu' => $strZiShu, - 'xsxq_jie_shao' => $strJieShao, - 'xsxq_feng_mian' => json_encode($arrCover), - 'xsxq_geng_xin_shi_jian' => $strLastDate, - 'xsxq_tag' => json_encode($arrTag, JSON_UNESCAPED_UNICODE), - 'xsxq_source_id' => $intXiaoShuoSourceId, - 'xsxq_source_seo_id' => $intXiaoShuoSourceId.'1', - 'xsxq_source_code' => 'BIQU_XIAOSHUO_' . $intXiaoShuoSourceId . '_' . $intXiaoShuoSourceSeoId, - ]; - // print_r($arrXiaoShuoInfo); - var_dump($arrXiaoShuoInfo['xsxq_source_code']); - - $XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo); - - $strMuluHref = $QueryList->find('.btn-mulu')->eq(0)->attr("href"); - - $arrTask = [ - 'callback' => [self::class, 'getNovelReaderAll'], - 'data' => [ - 'xszj_href' => $strMuluHref, - ] - ]; - $TaskCore->set($arrTask); -/* - - $arrZhangjie = []; - $intChapterIndex = 0; // 定义一个计数器变量 - - // 检查是否找到元素 - if ($QueryList->find('.section-list a')->count() > 0) { - $QueryList->find('.section-list a')->map(function ($ZhangJie) use (&$arrZhangjie, &$intChapterIndex) { - echo "Processing index $intChapterIndex: " . $ZhangJie->text() . PHP_EOL; - - // 从索引6开始 - //if ($intChapterIndex >= 5) { - $arrZhangjie[] = [ - 'mingzi' => $ZhangJie->text(), - 'url' => $ZhangJie->attr('href'), - ]; - //} - $intChapterIndex++; - }); - - // 输出最终结果 - var_dump($arrZhangjie); - } else { - echo "No elements found for .section-list a" . PHP_EOL; - } - - //unset($QueryList); - var_dump($arrZhangjie); - - - // 获取最大章节索引 - $intMaxXiaoShuoZhangJiePaixu = XiaoShuoZhangJieModel::where('xsxq_id', $intXiaoShuoSourceId) - ->max('xszj_pai_xu'); // 获取最大值 - var_dump($intMaxXiaoShuoZhangJiePaixu); - - // 如果没有结果，max 返回 null，可以设置默认值 - $intMaxXiaoShuoZhangJiePaixu = $intMaxXiaoShuoZhangJiePaixu ?? 0; - - foreach ($arrZhangjie as $intZhangJieNum => $arrZhangjieOne) { - var_dump($intZhangJieNum); - - // 如果数据库里的章节最大索引大于当前索引，说明已经入库，不需要入队列了 - if($intMaxXiaoShuoZhangJiePaixu > $intZhangJieNum){ - var_dump("小说章节已经入库，不再推送任务"); - continue;// 如果任务已存在，直接跳过 break - } - - $strZhangJieUrl = $strSiteUrl . $arrZhangjieOne['url']; - - $strZhangJieId = extractFilename($strZhangJieUrl); - - # 'AILISI_XIAOSHUO_ZHANGJIE_小说源id_小说源章节id' - $strXszjSourceCode = '5CCC_XIAOSHUO_ZHANGJIE_'. $XiaoShuoXiangQingModel->xsxq_source_id . '_' . $strZhangJieId; - - $arrTask = [ - 'callback' => [self::class, 'getNovelReader'], - 'data' => [ - 'xszj_source_id' => $strZhangJieId, - 'xszj_paixu' => $intZhangJieNum, - 'xszj_name' => $arrZhangjieOne['mingzi'], - 'xszj_url' => $strZhangJieUrl, - 'xszj_source_code' => $strXszjSourceCode, - 'xsxq_source_id' => $XiaoShuoXiangQingModel->xsxq_source_id, - 'xsfl_id' => $strFenleiIId, - ] - ]; - $TaskCore->set($arrTask); - var_dump("章节任务投递成功".$strXszjSourceCode); - } - */ - - } catch (\Throwable $T) { - throw $T; - } - } - - public function getNovelFengMian($arrData) - { - //var_dump('getNovelFengMian'); - //var_dump($arrData); - $Client = $this->Client; - $TaskCore = $this->TaskCore; - - // 获取 Redis 实例 - $redis = Cache::store('redis')->handler(); - $intXiaoShuoSourceId = $arrData['xiaoshuo_id']; - $intXiaoShuoSourceSeoId = $arrData['xiaoshuo_seo_id']; - $redisKey = "task:novel_source_id:" . $intXiaoShuoSourceId; - - try { - $arrCookie = $this->getAilisiCookie(); - $strHost = getDomainFromUrl($arrData['xiaoshuo_url']); - $CookieJar = CookieJar::fromArray($arrCookie, $strHost); - - $strSiteUrl = $this->getAilisiDomain(); - - - $strCover = $QueryList->find('#fmimg img')->eq(0)->attr("src"); - - if (strpos($strCover, 'http') === false) { - $strCover = $strSiteUrl . $strCover; - } - $strCoverUri = $strCover; - - try { - $strCoverUri = ""; - $strCoverUri = ImageProcessor::getInstance()->downloadAndEncryptImage($strCover, 'xs'); - $arrCover = [ - 'code' => 'AI_LI_SI_COVER', - // 'uri' => getUriByUrl($strCover), - 'uri' => $strCoverUri, - - ]; - - $arrXiaoShuoInfo = [ - 'xsxq_feng_mian' => json_encode($arrCover), - - ]; - $XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo); - } catch (\Throwable $t) { - $strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage()); - echo $strErr; - } - - - - } catch (\Throwable $T) { - throw $T; - } - } - - public function getNovelReaderAll($arrData) - { - var_dump('getNovelReaderAll'); - $strMuluHref = $arrData['xszj_href']; - var_dump($strMuluHref); - $Client = $this->Client; - try { - $arrCookie = $this->getAilisiCookie(); - $strHost = getDomainFromUrl($strMuluHref); - $CookieJar = CookieJar::fromArray($arrCookie, $strHost); - - $proxy = getRandomProxies(); - $userAgent = getRandomUserAgent(); - $Response = $Client->request('GET', $strMuluHref, [ - 'headers' => [ - 'User-Agent' => $userAgent, - 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', - 'Accept-Language' => 'en-US,en;q=0.5', - ], - 'cookies' => $CookieJar, - // 'proxy' => $proxy, - 'timeout' => 30, - ]); - - $strResult = (string)$Response->getBody(); - unset($Response); - $QueryList = QueryList::html($strResult); - - - $intChapterIndex = 0; // 定义一个计数器变量 - var_dump($QueryList->find('.section-list a')->count()); - // 检查是否找到元素 - if ($QueryList->find('.section-list a')->count() > 0) { - $QueryList->find('.section-list a')->map(function ($ZhangJie) use (&$arrZhangjie, &$intChapterIndex) { - echo "Processing index $intChapterIndex: " . $ZhangJie->text() . PHP_EOL; - - // 从索引6开始 - //if ($intChapterIndex >= 5) { - $arrZhangjie[] = [ - 'mingzi' => $ZhangJie->text(), - 'url' => $ZhangJie->attr('href'), - ]; - //} - $intChapterIndex++; - }); - - // 输出最终结果 - var_dump($arrZhangjie); - } else { - echo "No elements found for .section-list a" . PHP_EOL; - } - - //unset($QueryList); - var_dump($arrZhangjie); -/* - - // 获取最大章节索引 - $intMaxXiaoShuoZhangJiePaixu = XiaoShuoZhangJieModel::where('xsxq_id', $intXiaoShuoSourceId) - ->max('xszj_pai_xu'); // 获取最大值 - var_dump($intMaxXiaoShuoZhangJiePaixu); - - // 如果没有结果，max 返回 null，可以设置默认值 - $intMaxXiaoShuoZhangJiePaixu = $intMaxXiaoShuoZhangJiePaixu ?? 0; - - foreach ($arrZhangjie as $intZhangJieNum => $arrZhangjieOne) { - var_dump($intZhangJieNum); - - // 如果数据库里的章节最大索引大于当前索引，说明已经入库，不需要入队列了 - if($intMaxXiaoShuoZhangJiePaixu > $intZhangJieNum){ - var_dump("小说章节已经入库，不再推送任务"); - continue;// 如果任务已存在，直接跳过 break - } - - $strZhangJieUrl = $strSiteUrl . $arrZhangjieOne['url']; - - $strZhangJieId = extractFilename($strZhangJieUrl); - - # 'AILISI_XIAOSHUO_ZHANGJIE_小说源id_小说源章节id' - $strXszjSourceCode = '5CCC_XIAOSHUO_ZHANGJIE_'. $XiaoShuoXiangQingModel->xsxq_source_id . '_' . $strZhangJieId; - - $arrTask = [ - 'callback' => [self::class, 'getNovelReader'], - 'data' => [ - 'xszj_source_id' => $strZhangJieId, - 'xszj_paixu' => $intZhangJieNum, - 'xszj_name' => $arrZhangjieOne['mingzi'], - 'xszj_url' => $strZhangJieUrl, - 'xszj_source_code' => $strXszjSourceCode, - 'xsxq_source_id' => $XiaoShuoXiangQingModel->xsxq_source_id, - 'xsfl_id' => $strFenleiIId, - ] - ]; - $TaskCore->set($arrTask); - var_dump("章节任务投递成功".$strXszjSourceCode); - } - */ - - } catch (\Throwable $T) { - throw $T; - } - } - - public function getNovelReader($arrData) - { - var_dump('getNovelReader'); - var_dump($arrData['xszj_url']); - $Client = $this->Client; - $strXszjSourceCode = $arrData['xszj_source_code']; - $strZhangJieUrl = $arrData['xszj_url']; - $intZhangJiePaiXu = $arrData['xszj_paixu']; - $intFenleiId = $arrData['xsfl_id']; - $intXiaoShuoSourceID = $arrData['xsxq_source_id']; - $strZhangJieSourceId = $arrData['xszj_source_id']; - $strZhangJieName = $arrData['xszj_name']; - try { - $arrCookie = $this->getAilisiCookie(); - $strHost = getDomainFromUrl($strZhangJieUrl); - $CookieJar = CookieJar::fromArray($arrCookie, $strHost); - - if (XiaoShuoZhangJieModel::checkZhangJieExists($strXszjSourceCode)) { - return true; - } - - $proxy = getRandomProxies(); - $userAgent = getRandomUserAgent(); - $Response = $Client->request('GET', $strZhangJieUrl, [ - 'headers' => [ - 'User-Agent' => $userAgent, - 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', - 'Accept-Language' => 'en-US,en;q=0.5', - ], - 'cookies' => $CookieJar, - // 'proxy' => $proxy, - 'timeout' => 30, - ]); - - $strResult = (string)$Response->getBody(); - unset($Response); - $ZhangJieQueryList = QueryList::html($strResult); - - - // 获取原始 HTML - $strNeiRong = $ZhangJieQueryList->find('#content')->eq(0)->html(); - - // 解码 HTML 实体 - $strNeiRong = html_entity_decode($strNeiRong, ENT_QUOTES, 'UTF-8'); - - // 去掉

标签中的 class="content_detail" 属性 - $strNeiRong = preg_replace('/'); - - // 替换不间断空格 (\xc2\xa0) 和   为普通空格 - $strNeiRong = preg_replace('/\xc2\xa0/', ' ', $strNeiRong); // 替换 UTF-8 不间断空格 - $strNeiRong = preg_replace('/ /', ' ', $strNeiRong); // 替换   - - // 清理多余空格和换行符 - $strNeiRong = preg_replace('/\s+/', ' ', $strNeiRong); - $strNeiRong = preg_replace('/

\s+/', '

', $strNeiRong); // 清理

标签内部多余空格 - $strNeiRong = preg_replace('/\s+<\/p>/', '

', $strNeiRong); // 清理

标签结尾多余空格 - - // 输出结果 - // var_dump($strNeiRong); - - - # '/novel/123yqw/分类id/小说源id/小说源章节id.txt' - $strNeiRongTxtPath = config("filesystem.novel").'/novel/biquxs/'.$intFenleiId.'/'.$intXiaoShuoSourceID.'/'.$strZhangJieSourceId.'.txt'; - //$strNeiRongTxtPath = "/www/novel/123yqw/1/11/111.txt"; - // 调用函数 - $result = saveCompressAndDeleteTxt($strNeiRongTxtPath, $strNeiRong); - - // 输出状态 - if ($result['status']) { - var_dump("Success: " . $result['message']) ; - } else { - var_dump("Error: " . $result['message']) ; - } - - $arrXiaoShuoZhangJie = [ - 'xsxq_id' => $intXiaoShuoSourceID, - 'xszj_pai_xu' => $intZhangJiePaiXu, - 'xszj_ming_zi' => $strZhangJieName, - 'xszj_nei_rong' => $strNeiRongTxtPath, - 'xszj_source_id' => $strZhangJieSourceId, - 'xszj_source_code' => $strXszjSourceCode, - ]; - - XiaoShuoZhangJieModel::addXiaoShuoZhangJie($arrXiaoShuoZhangJie); - unset($ZhangJieQueryList); - // 延时 3 秒 - //sleep(3); - - } catch (\Throwable $T) { - throw $T; - } - } -} diff --git a/code/app/task/collection/dadi/DaDiDianyingCol.php b/code/app/task/collection/dadi/DaDiDianyingCol.php deleted file mode 100644 index 521a264..0000000 --- a/code/app/task/collection/dadi/DaDiDianyingCol.php +++ /dev/null @@ -1,498 +0,0 @@ -TaskCore; - // 1-75 - for ($i = 1; $i <= 75; $i++) { - echo "当前值: $i\n"; - $arrTask = [ - 'callback' => [self::class, 'getVideoPage'], - 'data' => [ - 'video_page_url' => 'https://www.czdadi.net/vodshow/dy--------'.$i.'---.html', - 'video_class_id' => 1 - ] - ]; - $TaskCore->set($arrTask); - } - //5-25 https://www.czdadi.net/vodshow/duanju-----------.html - for ($i = 1; $i <= 25; $i++) { - echo "当前值: $i\n"; - $arrTask = [ - 'callback' => [self::class, 'getVideoPage'], - 'data' => [ - 'video_page_url' => 'https://www.czdadi.net/vodshow/duanju--------'.$i.'---.html', - 'video_class_id' => 5 - ] - ]; - $TaskCore->set($arrTask); - } - - // 2--18 https://www.czdadi.net/vodshow/dsj--------18---.html - for ($i = 1; $i <= 18; $i++) { - echo "当前值: $i\n"; - $arrTask = [ - 'callback' => [self::class, 'getVideoPage'], - 'data' => [ - 'video_page_url' => 'https://www.czdadi.net/vodshow/dsj--------'.$i.'---.html', - 'video_class_id' => 2 - ] - ]; - $TaskCore->set($arrTask); - } - // 3--11 https://www.czdadi.net/vodshow/zy--------11---.html - for ($i = 1; $i <= 11; $i++) { - echo "当前值: $i\n"; - $arrTask = [ - 'callback' => [self::class, 'getVideoPage'], - 'data' => [ - 'video_page_url' => 'https://www.czdadi.net/vodshow/zy--------'.$i.'---.html', - 'video_class_id' => 3 - ] - ]; - $TaskCore->set($arrTask); - } - // 4--29 https://www.czdadi.net/vodshow/dm-----------.html - for ($i = 1; $i <= 29; $i++) { - echo "当前值: $i\n"; - $arrTask = [ - 'callback' => [self::class, 'getVideoPage'], - 'data' => [ - 'video_page_url' => 'https://www.czdadi.net/vodshow/dm--------'.$i.'---.html', - 'video_class_id' => 4 - ] - ]; - $TaskCore->set($arrTask); - } - - - - } - - # 获取分页里面的视频列表 - public function getVideoPage($arrData) - { - $TaskCore = $this->TaskCore; - $Client = $this->Client; - $strUrl= $arrData['video_page_url']; - $intClassId = $arrData['video_class_id']; - $strSiteUrl = $this->get5cccDomain(); - $arrCookie = $this->getAilisiCookie(); - $strHost = getDomainFromUrl($strUrl); - $CookieJar = CookieJar::fromArray($arrCookie, $strHost); - - $proxy = getRandomProxies(); - $userAgent = getRandomUserAgent(); - $Response = $Client->request('GET', $strUrl, [ - 'headers' => [ - 'User-Agent' => $userAgent, - 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', - 'Accept-Language' => 'en-US,en;q=0.5', - ], - 'cookies' => $CookieJar, - // 'proxy' => $proxy, - 'timeout' => 30, - ]); - - $strResult = (string)$Response->getBody(); - - $QueryList = QueryList::html($strResult); - - //$arrVideo = []; - // 检查是否找到元素 - if ($QueryList->find('.stui-vodlist li .stui-vodlist__detail a')->count() > 0) { - $QueryList->find('.stui-vodlist li .stui-vodlist__detail a ')->map(function ($Video) use (&$arrVideo,$intClassId,&$TaskCore) { - - // $arrVideo[] = [ - // 'mingzi' => $Video->text(), - // 'url' => $Video->attr('href'), - // ]; - if(!empty($Video->text()) && !empty($Video->attr('href'))){ - $arrTask = [ - 'callback' => [self::class, 'getVideoInfo'], - 'data' => [ - 'class_id' => $intClassId, - 'video_href' => $Video->attr('href'), - 'video_name' => $Video->text(), - ] - ]; - $TaskCore->set($arrTask); - echo "视频投递成功 : " . $Video->text() . PHP_EOL; - } - }); - - - // foreach ($arrVideo as $intVideoNum => $arrVideoOne) { - // $arrTask = [ - // 'callback' => [self::class, 'getVideoInfo'], - // 'data' => [ - // 'class_id' => $intClassId, - // 'video_href' => $arrVideoOne['url'], - // 'video_name' => $arrVideoOne['mingzi'], - // ] - // ]; - // $TaskCore->set($arrTask); - - // } - - // 输出最终结果 - // var_dump($arrVideo); - } else { - echo "No elements found for .stui-vodlist a" . PHP_EOL; - } - } - - public function getVideoInfo($arrData) - { - $TaskCore = $this->TaskCore; - $Client = $this->Client; - $strSiteUrl = $this->getDadiDomain(); - $strUrl = $strSiteUrl . $arrData['video_href'] ; - $intClassId = $arrData['class_id']; - $strVideoName = $arrData['video_name']; - - try { - $arrCookie = $this->getAilisiCookie(); - $strHost = getDomainFromUrl($strUrl); - - $CookieJar = CookieJar::fromArray($arrCookie, $strHost); - - $proxy = getRandomProxies(); - $userAgent = getRandomUserAgent(); - $Response = $Client->request('GET', $strUrl, [ - 'headers' => [ - 'User-Agent' => $userAgent, - 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', - 'Accept-Language' => 'en-US,en;q=0.5', - ], - 'cookies' => $CookieJar, - // 'proxy' => $proxy, - 'timeout' => 30, - ]); - - $strResult = (string)$Response->getBody(); - - $QueryList = QueryList::html($strResult); - - $strCover = $QueryList->find('.stui-content__thumb img')->eq(0)->attr("data-original"); - if (strpos($strCover, 'http') === false) { - $strCover = $strSiteUrl . $strCover; - } - $strCoverUri = $strCover; - // try { - // $strCoverUri = ""; - // $strCoverUri = ImageProcessor::getInstance()->downloadImage( $intXiaoShuoSourceId,$strCover, 'dadi'); - // } catch (\Throwable $t) { - // $strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage()); - // echo $strErr; - // } - $arrCover = [ - 'code' => 'DADI_YINGSHI_COVER', - // 'uri' => getUriByUrl($strCover), - 'uri' => $strCoverUri, - ]; - - $strLanguage = ""; - // if (strlen($strLanguage) > 0) { - // VideoLeixinModel::addVideoLeixin(['vc_id' => $intClassId, 'vla_name' => $strLanguage]); - // } - - // 提取评分 - $strPinfen = $QueryList->find('h1.title .score')->text(); - // 提取类型-入库 - $strLeixin = $QueryList->find('.col-pd a')->eq(1)->text(); - - if(strlen($strLeixin) > 0){ - VideoLeixinModel::addVideoLeixin(['vc_id'=>$intClassId,'vl_name'=>$strLeixin]); - } - - // 提取 HTML 的第一段

- $dataNode = $QueryList->find('p.data')->eq(0); - - // 提取“类型”内容：确保只匹配 `href` 中包含 `/vodsearch/----` - $leixinNodes = $dataNode->find('a[href^="/vodsearch/----"]'); - $strJuqing = $leixinNodes->map(function ($node) { - return $node->text(); - })->all(); - - // 提取“地区”内容：确保只匹配 `href` 中包含 `/vodsearch/--`，但不包含 `/vodsearch/----` - $diquNodes = $dataNode->find('a[href^="/vodsearch/--"]')->not('a[href^="/vodsearch/----"]'); - $strDiqu = $diquNodes->map(function ($node) { - return $node->text(); - })->all(); - - // 提取“年份”内容：确保只匹配 `href` 中包含 `/vodsearch/-------------` - $yearsNode = $dataNode->find('a[href^="/vodsearch/-------------"]'); - $strYears = $yearsNode->text(); - if (!empty(trim($strYears))) { - $strYears = [trim($strYears)]; - } else { - $strYears = []; - } - - // 过滤掉“地区”和“年份”中的数据 - $strJuqing = array_filter($strJuqing, function ($value) use ($strDiqu, $strYears) { - return !in_array($value, array_merge($strDiqu, $strYears)); - }); - - // **在过滤后将“类型”数据入库** - foreach ($strJuqing as $value) { - if(strlen($value) > 0){ - VideoJuqingModel::add(['vc_id' => $intClassId, 'vj_name' => $value]); - } - } - - // 将“地区”数据入库 - foreach ($strDiqu as $value) { - if(strlen($value) > 0){ - VideoDiquModel::add(['vc_id' => $intClassId, 'vd_name' => $value]); - } - } - - // 将“年份”数据入库 - foreach ($strYears as $value) { - if(strlen($value) > 0){ - VideoYearsModel::add(['vc_id' => $intClassId, 'vy_name' => $value]); - } - - } - - // 提取主演 - $strZhuyan = $QueryList->find('p.data:contains("主演") a')->map(function ($item) { - return $item->text(); - })->implode(', '); // 使用逗号分隔主演 - - // 提取导演 - $strDaoyan = $QueryList->find('p.data:contains("导演") a')->map(function ($item) { - return $item->text(); - })->implode(', '); // 使用逗号分隔主演 - - - // 提取更新时间 - $strLastDate = $QueryList->find('p.data.hidden-sm:contains("更新")')->text(); - // 去掉“更新：”前缀 - $strLastDate = str_replace('更新：', '', $strLastDate); - - // 提取简介 - $strDescription = $QueryList->find('#desc>.stui-pannel-box>.stui-pannel_bd>.col-pd')->text(); - - // 角标 - $strJiaoBiao = $QueryList->find('.stui-content__thumb .text-right')->eq(0)->text(); - - $strPinyinId = extractFilename($strUrl); - - $arrVideoInfo = [ - 'v_name' => $strVideoName, - 'v_cover' => json_encode($arrCover), - 'vc_id' => $intClassId, - 'v_pinyin' => $strPinyinId, - 'v_leixin' => $strLeixin , - 'v_juqing' => implode(', ', $strJuqing), - 'v_diqu' => implode(', ', $strDiqu), - 'v_years' => implode(', ', $strYears), - 'v_language' => $strLanguage, - 'v_daoyan' => $strZhuyan, - 'v_zhuyan' => $strDaoyan, - 'update_time' => $strLastDate, - 'v_description'=>$strDescription, - 'v_pingfen' => $strPinfen, - 'v_orurl' => $strUrl, - 'v_jiaobiao'=> $strJiaoBiao, - 'v_md5' => md5($intClassId.$strVideoName.$strSiteUrl.$strUrl), - ]; - - $VideoInfoModel = VideoInfoModel::addVideoXiangQing($arrVideoInfo); - - $intVid = $VideoInfoModel->v_id; - $intChapterIndex = 0 ; - $arrZhangjie = []; - // // 检查是否找到元素 - if ($QueryList->find('.stui-content__playlist a')->count() > 0) { - $QueryList->find('.stui-content__playlist a')->map(function ($ZhangJie) use (&$arrZhangjie, &$intChapterIndex ,&$intVid) { - //echo "Processing index $intChapterIndex: " . $ZhangJie->text() . PHP_EOL; - - $arrZhangjie[] = [ - 'v_id' => $intVid, - 'index' => $intChapterIndex, - 'mingzi' => $ZhangJie->text(), - 'url' => $ZhangJie->attr('href'), - ]; - $intChapterIndex++; - }); - - //var_dump($arrZhangjie); - foreach ($arrZhangjie as $intZhangJieNum => $arrZhangjieOne) { - var_dump($intZhangJieNum); - - // 如果数据库里的章节最大索引大于当前索引，说明已经入库，不需要入队列了 - // if($intMaxXiaoShuoZhangJiePaixu > $intZhangJieNum){ - // var_dump("小说章节已经入库，不再推送任务"); - // continue;// 如果任务已存在，直接跳过 break - // } - //if($intZhangJieNum<=3){ - $arrTask = [ - 'callback' => [self::class, 'getNovelReader'], - 'data' => [ - 'v_id' => $intVid, - 'vp_sort' => $intZhangJieNum, - 'vp_name' => $arrZhangjieOne['mingzi'], - 'vp_url' => $strSiteUrl . $arrZhangjieOne['url'] - ] - ]; - $TaskCore->set($arrTask); - //} - - } - } else { - echo "No elements found for .listmain a" . PHP_EOL; - } - - } catch (\Throwable $T) { - throw $T; - } - // sleep(10); - } - - public function getNovelFengMian($arrData) - { - //var_dump('getNovelFengMian'); - //var_dump($arrData); - $Client = $this->Client; - $TaskCore = $this->TaskCore; - - // 获取 Redis 实例 - $redis = Cache::store('redis')->handler(); - $intXiaoShuoSourceId = $arrData['xiaoshuo_id']; - $intXiaoShuoSourceSeoId = $arrData['xiaoshuo_seo_id']; - $redisKey = "task:novel_source_id:" . $intXiaoShuoSourceId; - - try { - $arrCookie = $this->getAilisiCookie(); - $strHost = getDomainFromUrl($arrData['xiaoshuo_url']); - $CookieJar = CookieJar::fromArray($arrCookie, $strHost); - - $strSiteUrl = $this->getAilisiDomain(); - - - $strCover = $QueryList->find('#fmimg img')->eq(0)->attr("src"); - - if (strpos($strCover, 'http') === false) { - $strCover = $strSiteUrl . $strCover; - } - $strCoverUri = $strCover; - - try { - $strCoverUri = ""; - $strCoverUri = ImageProcessor::getInstance()->downloadAndEncryptImage($strCover, 'xs'); - $arrCover = [ - 'code' => 'AI_LI_SI_COVER', - // 'uri' => getUriByUrl($strCover), - 'uri' => $strCoverUri, - - ]; - - $arrXiaoShuoInfo = [ - 'xsxq_feng_mian' => json_encode($arrCover), - - ]; - $XiaoShuoXiangQingModel = XiaoShuoXiangQingModel::addXiaoShuoXiangQing($arrXiaoShuoInfo); - } catch (\Throwable $t) { - $strErr = sprintf("采集[%s]小说图片错误，异常：%s", $strCover, $t->getMessage()); - echo $strErr; - } - - - - } catch (\Throwable $T) { - throw $T; - } - } - - public function getNovelReader($arrData) - { - if (Cache::has('video_no_m3u8_' . $arrData['v_id'])) { - echo "跳过抓取，视频ID已在缓存中: " . $arrData['v_id'] . "\n"; - return; // 跳过抓取逻辑 - } - - $strMuluHref = $arrData['vp_url']; - var_dump($strMuluHref); - $Client = $this->Client; - try { - $arrCookie = $this->getAilisiCookie(); - $strHost = getDomainFromUrl($strMuluHref); - $CookieJar = CookieJar::fromArray($arrCookie, $strHost); - - $proxy = getRandomProxies(); - $userAgent = getRandomUserAgent(); - $Response = $Client->request('GET', $strMuluHref, [ - 'headers' => [ - 'User-Agent' => $userAgent, - 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', - 'Accept-Language' => 'en-US,en;q=0.5', - ], - 'cookies' => $CookieJar, - // 'proxy' => $proxy, - 'timeout' => 30, - ]); - - $strResult = (string)$Response->getBody(); - unset($Response); - $QueryList = QueryList::html($strResult); - //提取
', $strNeiRong); - - $strNeiRong = $arrNeiRong[1]; - - $arrNeiRong = explode('

', $strNeiRong); - - $arrEnCode = []; - foreach ($arrNeiRong as $intNeiRongNum => $strNeiRongOne) { - $strEnCode = Base64Helper::encode('

' . $strNeiRongOne . '

'); - $arrEnCode[] = $strEnCode; - } - - $strNeiRong = implode(';;;', $arrEnCode); - - # '/novel/123yqw/分类id/小说源id/小说源章节id.txt' - $strNeiRongTxtPath = config("filesystem.novel") . '/novel/630zwxs/' . $intFenleiId . '/' . $intXiaoShuoId . '/' . $intZhangJiePaiXu . '/' . $intZhangJieSort . '.txt'; - // 调用函数 - $result = saveCompressAndDeleteTxt($strNeiRongTxtPath, $strNeiRong); - - // 输出状态 - if ($result['status']) { - var_dump("Success: " . $result['message']); - } else { - var_dump("Error: " . $result['message']); - } - - $arrXiaoShuoZhangJie = [ - 'xsxq_id' => $intXiaoShuoId, - 'xszj_pai_xu' => $intZhangJiePaiXu, - 'xszj_ming_zi' => $strZhangJieName, - 'xszj_nei_rong' => $strNeiRongTxtPath, - 'xszj_source_id' => $strZhangJieSourceId, - 'xszj_source_code' => $strXszjSourceCode, - 'xszj_sort' => $intZhangJieSort, - - ]; - if ($strZhangJieName) { - XiaoShuoZhangJieModel::addXiaoShuoZhangJie($arrXiaoShuoZhangJie); - } - - unset($ZhangJieQueryList); - //sleep(3); - - - } catch (\Throwable $T) { - $arrTask = [ - 'callback' => [self::class, 'getNovelReader'], - 'data' => [ - 'xszj_source_id' => $strZhangJieSourceId, - 'xszj_paixu' => $intZhangJiePaiXu, - 'xszj_name' => $strZhangJieName, - 'xszj_url' => $strZhangJieUrl, - 'xszj_source_code' => $strXszjSourceCode, - 'xsxq_id' => $intXiaoShuoId, - 'xsfl_id' => $intFenleiId, - 'intZhangJieSort' => $intZhangJieSort, - ] - ]; - - $TaskCore->set($arrTask); - throw $T; - } - } -} diff --git a/code/app/task/collection/seo/GenerateSiteMapCol.php b/code/app/task/collection/seo/GenerateSiteMapCol.php deleted file mode 100644 index 4c3dfd6..0000000 --- a/code/app/task/collection/seo/GenerateSiteMapCol.php +++ /dev/null @@ -1,137 +0,0 @@ -generateVideoSitemaps($chunkSize, $totalVideoCount, $strSiteDomain); - - // 生成分类的 Sitemap - $categorySitemapFiles = $this->generateCategorySitemaps($categories, $strSiteDomain); - - // 合并所有 Sitemap 文件路径 - $allSitemapFiles = array_merge($videoSitemapFiles, $categorySitemapFiles); - - // 生成 Sitemap Index 文件 - $this->buildSitemapIndex($allSitemapFiles, $strSiteDomain); - - var_dump(['message' => '所有 Sitemap 生成成功', 'files' => $allSitemapFiles]); - } - - private function generateVideoSitemaps(int $chunkSize, int $totalCount, string $strSiteDomain): array - { - $totalFiles = ceil($totalCount / $chunkSize); - $sitemapFiles = []; - - for ($i = 0; $i < $totalFiles; $i++) { - $videos = VideoInfoModel::field('v_pinyin') - ->limit($i * $chunkSize, $chunkSize) - ->select() - ->toArray(); - - $sitemapXml = $this->buildSitemap($videos, $strSiteDomain); - $filePath = public_path() . "sitemap_video_{$i}.xml"; - file_put_contents($filePath, $sitemapXml); - $sitemapFiles[] = $filePath; - } - - return $sitemapFiles; - } - - private function generateCategorySitemaps(array $categories, string $strSiteDomain): array - { - $sitemapFiles = []; - - foreach ($categories as $category) { - $vcId = $category['vc_id']; - $vcNicheng = $category['vc_nicheng']; - $vcName = $category['vc_name']; - - $videoCount = VideoInfoModel::where('vc_id', $vcId)->count(); - $totalPages = ceil($videoCount / 30); - - $categoryUrls = []; - $categoryUrls[] = [ - 'url' => "/vodtype/{$vcNicheng}.html", - 'title' => "{$vcName} - 首页", - 'is_home' => true, - ]; - - for ($page = 2; $page <= $totalPages; $page++) { - $categoryUrls[] = [ - 'url' => "/vodshow/{$vcNicheng}--------{$page}---.html", - 'title' => "{$vcName} - 第{$page}页", - 'is_home' => false, - ]; - } - - $sitemapXml = $this->buildSitemap($categoryUrls, $strSiteDomain, true); - $filePath = public_path() . "sitemap_category_{$vcNicheng}.xml"; - file_put_contents($filePath, $sitemapXml); - $sitemapFiles[] = $filePath; - } - - return $sitemapFiles; - } - - private function buildSitemap(array $items, string $strSiteDomain, bool $isCategory = false): string - { - $xmlHeader = ''; - $urlsetStart = ''; - $urlsetEnd = ''; - - $urlEntries = ''; - foreach ($items as $item) { - $urlEntries .= ''; - $urlEntries .= 'https://' . $strSiteDomain . ($isCategory ? $item['url'] : '/voddetail/' . htmlspecialchars($item['v_pinyin'] . '.html', ENT_QUOTES)) . ''; - $urlEntries .= 'daily'; - $urlEntries .= '0.8'; - $urlEntries .= ''; - } - - return $xmlHeader . $urlsetStart . $urlEntries . $urlsetEnd; - } - - private function buildSitemapIndex(array $sitemapFiles, string $strSiteDomain): void - { - $xmlHeader = ''; - $sitemapIndexStart = ''; - $sitemapIndexEnd = ''; - - $sitemapEntries = ''; - foreach ($sitemapFiles as $file) { - $url = 'https://' . $strSiteDomain . '/' . basename($file); - $sitemapEntries .= ''; - $sitemapEntries .= '' . htmlspecialchars($url, ENT_QUOTES) . ''; - $sitemapEntries .= '' . date('Y-m-d') . ''; - $sitemapEntries .= ''; - } - - $sitemapIndexXml = $xmlHeader . $sitemapIndexStart . $sitemapEntries . $sitemapIndexEnd; - file_put_contents(public_path() . 'sitemap_index.xml', $sitemapIndexXml); - - var_dump(['message' => 'Sitemap Index 生成成功', 'file' => 'sitemap_index.xml']); - } -} diff --git a/code/app/task/collection/seo/GenerateSiteMapNovel.php b/code/app/task/collection/seo/GenerateSiteMapNovel.php deleted file mode 100644 index 73a524c..0000000 --- a/code/app/task/collection/seo/GenerateSiteMapNovel.php +++ /dev/null @@ -1,188 +0,0 @@ -generateNovelSitemaps($chunkSize, $totalNovelCount, $strSiteDomain); - - // 生成章节的 Sitemap - $novelZjSitemapFiles = $this->generateNovelZjSitemaps($chunkSize, $totalNovelZjCount, $strSiteDomain); - - // 生成分类的 Sitemap - $categorySitemapFiles = $this->generateCategorySitemaps($categories, $strSiteDomain); - - // 合并所有 Sitemap 文件路径 - $allSitemapFiles = array_merge($novelSitemapFiles, $categorySitemapFiles,$novelZjSitemapFiles); - - // 生成 Sitemap Index 文件 - $this->buildSitemapIndex($allSitemapFiles, $strSiteDomain); - - var_dump(['message' => '所有 Sitemap 生成成功', 'files' => $allSitemapFiles]); - } - - private function generateNovelZjSitemaps(int $chunkSize, int $totalCount, string $strSiteDomain): array - { - $totalFiles = ceil($totalCount / $chunkSize); - $sitemapFiles = []; - - for ($i = 0; $i < $totalFiles; $i++) { - $Novels = XiaoShuoZhangJieModel::field(['xsxq_id','xszj_id']) //intNovelId-:intNovelSourceId-:intNovelSourceSeoId - ->limit($i * $chunkSize, $chunkSize) - ->select() - ->toArray(); - - $sitemapXml = $this->buildZjSitemap($Novels, $strSiteDomain); - $filePath = public_path() . "sitemap_novel_zj_{$i}.xml"; - file_put_contents($filePath, $sitemapXml); - $sitemapFiles[] = $filePath; - } - - return $sitemapFiles; - } - - - private function generateNovelSitemaps(int $chunkSize, int $totalCount, string $strSiteDomain): array - { - $totalFiles = ceil($totalCount / $chunkSize); - $sitemapFiles = []; - - for ($i = 0; $i < $totalFiles; $i++) { - $Novels = XiaoShuoXiangQingModel::field(['xsxq_id','xsxq_source_id','xsxq_source_seo_id']) //intNovelId-:intNovelSourceId-:intNovelSourceSeoId - ->limit($i * $chunkSize, $chunkSize) - ->select() - ->toArray(); - - $sitemapXml = $this->buildSitemap($Novels, $strSiteDomain); - $filePath = public_path() . "sitemap_novel_{$i}.xml"; - file_put_contents($filePath, $sitemapXml); - $sitemapFiles[] = $filePath; - } - - return $sitemapFiles; - } - - private function generateCategorySitemaps(array $categories, string $strSiteDomain): array - { - $sitemapFiles = []; - - foreach ($categories as $category) { - $vcId = $category['xsfl_id']; - $vcName = $category['xsfl_name']; - - $intNovelCount = XiaoShuoFenLeiModel::where('xsfl_id', $vcId)->count(); - $totalPages = ceil($intNovelCount / 30); - - $categoryUrls = []; - for ($type = 0; $type <= 3; $type++) { - for ($page = 1; $page <= $totalPages; $page++) { - $categoryUrls[] = [ - // /fenlei/1-0-1.html - 'url' => "/fenlei/{$vcId}-{$type}-{$page}.html", - 'title' => "{$vcName} - 第{$page}页", - 'is_home' => false, - ]; - } - } - - $sitemapXml = $this->buildSitemap($categoryUrls, $strSiteDomain, true); - $filePath = public_path() . "sitemap_category_{$vcId}.xml"; - file_put_contents($filePath, $sitemapXml); - $sitemapFiles[] = $filePath; - } - - return $sitemapFiles; - } - - private function buildZjSitemap(array $items, string $strSiteDomain, bool $isCategory = false): string - { - $xmlHeader = ''; - $urlsetStart = ''; - $urlsetEnd = ''; - // zj/18648-206.html - $urlEntries = ''; - foreach ($items as $item) { - $urlEntries .= ''; - $urlEntries .= 'https://' . $strSiteDomain . ($isCategory ? $item['url'] : '/zj/' . htmlspecialchars($item['xsxq_id'].'-'.$item['xszj_id']. '.html', ENT_QUOTES)) . ''; - $urlEntries .= 'daily'; - $urlEntries .= '0.8'; - $urlEntries .= ''; - } - - return $xmlHeader . $urlsetStart . $urlEntries . $urlsetEnd; - } - - private function buildSitemap(array $items, string $strSiteDomain, bool $isCategory = false): string - { - $xmlHeader = ''; - $urlsetStart = ''; - $urlsetEnd = ''; - // /xq/20052-16107-161071.html - $urlEntries = ''; - foreach ($items as $item) { - $urlEntries .= ''; - $urlEntries .= 'https://' . $strSiteDomain . ($isCategory ? $item['url'] : '/xq/' . htmlspecialchars($item['xsxq_id'].'-'.$item['xsxq_source_id'].'-'.$item['xsxq_source_seo_id']. '.html', ENT_QUOTES)) . ''; - $urlEntries .= 'daily'; - $urlEntries .= '1.0'; - $urlEntries .= ''; - } - - return $xmlHeader . $urlsetStart . $urlEntries . $urlsetEnd; - } - - private function buildSitemapIndex(array $sitemapFiles, string $strSiteDomain): void - { - $xmlHeader = ''; - $sitemapIndexStart = ''; - $sitemapIndexEnd = ''; - - $sitemapEntries = ''; - foreach ($sitemapFiles as $file) { - $url = 'https://' . $strSiteDomain . '/' . basename($file); - $sitemapEntries .= ''; - $sitemapEntries .= '' . htmlspecialchars($url, ENT_QUOTES) . ''; - $sitemapEntries .= '' . date('Y-m-d') . ''; - $sitemapEntries .= ''; - } - - $sitemapIndexXml = $xmlHeader . $sitemapIndexStart . $sitemapEntries . $sitemapIndexEnd; - file_put_contents(public_path() . 'sitemap_index.xml', $sitemapIndexXml); - - var_dump(['message' => 'Sitemap Index 生成成功', 'file' => 'sitemap_index.xml']); - } -} diff --git a/code/app/task/collection/seo/GenerateSiteMapNovelSk.php b/code/app/task/collection/seo/GenerateSiteMapNovelSk.php deleted file mode 100644 index 1d1cbc0..0000000 --- a/code/app/task/collection/seo/GenerateSiteMapNovelSk.php +++ /dev/null @@ -1,242 +0,0 @@ -generateNovelSitemaps($chunkSize, $totalNovelCount, $strSiteDomain, $strXqPath); - - // 生成详情的 Sitemap 小说seo - $novelSeoSitemapFiles = $this->generateNovelSeoSitemaps($chunkSize, $totalNovelSeoCount, $strSiteDomain, $strSeoPath); - - // 生成章节的 Sitemap 小说章节 - $novelZjSitemapFiles = $this->generateNovelZjSitemaps($chunkSize, $totalNovelZjCount, $strSiteDomain, $strXqPath); - - // 生成分类的 Sitemap 小说分类 - $categorySitemapFiles = $this->generateCategorySitemaps($categories, $strSiteDomain, $strFenleiPath, $strXqPath); - - // 合并所有 Sitemap 文件路径 - $allSitemapFiles = array_merge($novelSitemapFiles, $novelSeoSitemapFiles, $categorySitemapFiles, $novelZjSitemapFiles); - - // 生成 Sitemap Index 文件 - $this->buildSitemapIndex($allSitemapFiles, $strSiteDomain); - - var_dump(['message' => '所有 Sitemap 生成成功', 'files' => $allSitemapFiles]); - } - - # 生成小说章节 xml文件 - private function generateNovelZjSitemaps(int $chunkSize, int $totalCount, string $strSiteDomain, string $strXqPath): array - { - $totalFiles = ceil($totalCount / $chunkSize); - $sitemapFiles = []; - - for ($i = 0; $i < $totalFiles; $i++) { - $Novels = XiaoShuoZhangJieModel::field(['xsxq_id', 'xszj_id']) //intNovelId-:intNovelSourceId-:intNovelSourceSeoId - ->limit($i * $chunkSize, $chunkSize) - ->select() - ->toArray(); - - $sitemapXml = $this->buildZjSitemap($Novels, $strSiteDomain, $strXqPath); - $filePath = public_path() . "sitemap_novel_zj_{$i}.xml"; - file_put_contents($filePath, $sitemapXml); - $sitemapFiles[] = $filePath; - } - - return $sitemapFiles; - } - - # 生成小说详情 xml文件 - private function generateNovelSitemaps(int $chunkSize, int $totalCount, string $strSiteDomain, string $strXqPath): array - { - $totalFiles = ceil($totalCount / $chunkSize); - $sitemapFiles = []; - - for ($i = 0; $i < $totalFiles; $i++) { - $Novels = XiaoShuoXiangQingModel::field(['xsxq_id', 'xsxq_source_id', 'xsxq_source_seo_id']) //intNovelId-:intNovelSourceId-:intNovelSourceSeoId - ->limit($i * $chunkSize, $chunkSize) - ->select() - ->toArray(); - - $sitemapXml = $this->buildSitemap($Novels, $strSiteDomain, $strXqPath); - $filePath = public_path() . "sitemap_novel_{$i}.xml"; - file_put_contents($filePath, $sitemapXml); - $sitemapFiles[] = $filePath; - } - - return $sitemapFiles; - } - - # 生成小说详情 xml文件 - private function generateNovelSeoSitemaps(int $chunkSize, int $totalCount, string $strSiteDomain, string $strSeoPath): array - { - $totalFiles = ceil($totalCount / $chunkSize); - $sitemapFiles = []; - - for ($i = 0; $i < $totalFiles; $i++) { - $Novels = NovelInfoSeoModel::field(['xsxq_id', 'nis_id']) //show/nis_id - ->limit($i * $chunkSize, $chunkSize) - ->select() - ->toArray(); - - $sitemapXml = $this->buildXiaoShuoSeoSitemap($Novels, $strSiteDomain, $strSeoPath); - $filePath = public_path() . "sitemap_novel_seo_{$i}.xml"; - file_put_contents($filePath, $sitemapXml); - $sitemapFiles[] = $filePath; - } - - return $sitemapFiles; - } - - // 生成分类链接 - private function generateCategorySitemaps(array $categories, string $strSiteDomain, string $strFenleiPath, string $strXqPath): array - { - $sitemapFiles = []; - - foreach ($categories as $category) { - $vcId = $category['xsfl_id']; - $vcName = $category['xsfl_name']; - - $intNovelCount = XiaoShuoFenLeiModel::where('xsfl_id', $vcId)->count(); - $totalPages = ceil($intNovelCount / 30); - - $categoryUrls = []; - for ($status = 0; $status <= 2; $status++) { - for ($sort = 0; $sort <= 4; $sort++) { - for ($page = 1; $page <= $totalPages; $page++) { - $categoryUrls[] = [ - // /fl/:intCId-:intSort-[:intPage] - 'url' => "/{$strFenleiPath}/{$vcId}-{$sort}-{$page}.html", - 'title' => "{$vcName} - 第{$page}页", - 'is_home' => false, - ]; - } - } - } - - $sitemapXml = $this->buildSitemap($categoryUrls, $strSiteDomain, $strXqPath, true); - $filePath = public_path() . "sitemap_category_{$vcId}.xml"; - file_put_contents($filePath, $sitemapXml); - $sitemapFiles[] = $filePath; - } - - return $sitemapFiles; - } - - //生成章节链接 - private function buildZjSitemap(array $items, string $strSiteDomain, string $strXqPath, bool $isCategory = false): string - { - $xmlHeader = ''; - $urlsetStart = ''; - $urlsetEnd = ''; - // /xq/:intNovelId/:intZhangJiePaiXu-[:strSort] - $urlEntries = ''; - foreach ($items as $item) { - $urlEntries .= ''; - $urlEntries .= 'https://' . $strSiteDomain . ($isCategory ? $item['url'] : '/' . $strXqPath . '/' . htmlspecialchars($item['xsxq_id'] . '-' . $item['xszj_pai_xu']. '-' . $item['xszj_sort'] . '.html', ENT_QUOTES)) . ''; - $urlEntries .= 'daily'; - $urlEntries .= '0.8'; - $urlEntries .= ''; - } - - return $xmlHeader . $urlsetStart . $urlEntries . $urlsetEnd; - } - - // 生成 详情链接 - private function buildSitemap(array $items, string $strSiteDomain, string $strXqPath, bool $isCategory = false): string - { - $xmlHeader = ''; - $urlsetStart = ''; - $urlsetEnd = ''; - // /xq/:intNovelId - $urlEntries = ''; - foreach ($items as $item) { - $urlEntries .= ''; - $urlEntries .= 'https://' . $strSiteDomain . ($isCategory ? $item['url'] : '/' . $strXqPath . '/' . htmlspecialchars($item['xsxq_id'] . '.html', ENT_QUOTES)) . ''; - $urlEntries .= 'daily'; - $urlEntries .= '1.0'; - $urlEntries .= ''; - } - - return $xmlHeader . $urlsetStart . $urlEntries . $urlsetEnd; - } - - // 生成详情 seo 链接 - private function buildXiaoShuoSeoSitemap(array $items, string $strSiteDomain, string $strSeoPath, bool $isCategory = false): string - { - $xmlHeader = ''; - $urlsetStart = ''; - $urlsetEnd = ''; - // /show/:intNovelId - $urlEntries = ''; - foreach ($items as $item) { - $urlEntries .= ''; - $urlEntries .= 'https://' . $strSiteDomain . ($isCategory ? $item['url'] : '/' . $strSeoPath . '/' . htmlspecialchars($item['nis_id'] . '.html', ENT_QUOTES)) . ''; - $urlEntries .= 'daily'; - $urlEntries .= '1.0'; - $urlEntries .= ''; - } - - return $xmlHeader . $urlsetStart . $urlEntries . $urlsetEnd; - } - - // 生成 xml 文件 - private function buildSitemapIndex(array $sitemapFiles, string $strSiteDomain): void - { - $xmlHeader = ''; - $sitemapIndexStart = ''; - $sitemapIndexEnd = ''; - - $sitemapEntries = ''; - foreach ($sitemapFiles as $file) { - $url = 'https://' . $strSiteDomain . '/' . basename($file); - $sitemapEntries .= ''; - $sitemapEntries .= '' . htmlspecialchars($url, ENT_QUOTES) . ''; - $sitemapEntries .= '' . date('Y-m-d') . ''; - $sitemapEntries .= ''; - } - - $sitemapIndexXml = $xmlHeader . $sitemapIndexStart . $sitemapEntries . $sitemapIndexEnd; - file_put_contents(public_path() . 'sitemap_index.xml', $sitemapIndexXml); - - var_dump(['message' => 'Sitemap Index 生成成功', 'file' => 'sitemap_index.xml']); - } -} diff --git a/code/app/task/collection/seo/GenerateSiteMapNovelZw.php b/code/app/task/collection/seo/GenerateSiteMapNovelZw.php deleted file mode 100644 index dc28062..0000000 --- a/code/app/task/collection/seo/GenerateSiteMapNovelZw.php +++ /dev/null @@ -1,242 +0,0 @@ -generateNovelSitemaps($chunkSize, $totalNovelCount, $strSiteDomain, $strXqPath); - - // 生成详情的 Sitemap 小说seo - $novelSeoSitemapFiles = $this->generateNovelSeoSitemaps($chunkSize, $totalNovelSeoCount, $strSiteDomain, $strSeoPath); - - // 生成章节的 Sitemap 小说章节 - $novelZjSitemapFiles = $this->generateNovelZjSitemaps($chunkSize, $totalNovelZjCount, $strSiteDomain, $strXqPath); - - // 生成分类的 Sitemap 小说分类 - $categorySitemapFiles = $this->generateCategorySitemaps($categories, $strSiteDomain, $strFenleiPath, $strXqPath); - - // 合并所有 Sitemap 文件路径 - $allSitemapFiles = array_merge($novelSitemapFiles, $novelSeoSitemapFiles, $categorySitemapFiles, $novelZjSitemapFiles); - - // 生成 Sitemap Index 文件 - $this->buildSitemapIndex($allSitemapFiles, $strSiteDomain); - - var_dump(['message' => '所有 Sitemap 生成成功', 'files' => $allSitemapFiles]); - } - - # 生成小说章节 xml文件 - private function generateNovelZjSitemaps(int $chunkSize, int $totalCount, string $strSiteDomain, string $strXqPath): array - { - $totalFiles = ceil($totalCount / $chunkSize); - $sitemapFiles = []; - - for ($i = 0; $i < $totalFiles; $i++) { - $Novels = XiaoShuoZhangJieModel::field(['xsxq_id', 'xszj_id']) //intNovelId-:intNovelSourceId-:intNovelSourceSeoId - ->limit($i * $chunkSize, $chunkSize) - ->select() - ->toArray(); - - $sitemapXml = $this->buildZjSitemap($Novels, $strSiteDomain, $strXqPath); - $filePath = public_path() . "sitemap_novel_zj_{$i}.xml"; - file_put_contents($filePath, $sitemapXml); - $sitemapFiles[] = $filePath; - } - - return $sitemapFiles; - } - - # 生成小说详情 xml文件 - private function generateNovelSitemaps(int $chunkSize, int $totalCount, string $strSiteDomain, string $strXqPath): array - { - $totalFiles = ceil($totalCount / $chunkSize); - $sitemapFiles = []; - - for ($i = 0; $i < $totalFiles; $i++) { - $Novels = XiaoShuoXiangQingModel::field(['xsxq_id', 'xsxq_source_id', 'xsxq_source_seo_id']) //intNovelId-:intNovelSourceId-:intNovelSourceSeoId - ->limit($i * $chunkSize, $chunkSize) - ->select() - ->toArray(); - - $sitemapXml = $this->buildSitemap($Novels, $strSiteDomain, $strXqPath); - $filePath = public_path() . "sitemap_novel_{$i}.xml"; - file_put_contents($filePath, $sitemapXml); - $sitemapFiles[] = $filePath; - } - - return $sitemapFiles; - } - - # 生成小说详情 xml文件 - private function generateNovelSeoSitemaps(int $chunkSize, int $totalCount, string $strSiteDomain, string $strSeoPath): array - { - $totalFiles = ceil($totalCount / $chunkSize); - $sitemapFiles = []; - - for ($i = 0; $i < $totalFiles; $i++) { - $Novels = NovelInfoSeoModel::field(['xsxq_id', 'nis_id']) //show/nis_id - ->limit($i * $chunkSize, $chunkSize) - ->select() - ->toArray(); - - $sitemapXml = $this->buildXiaoShuoSeoSitemap($Novels, $strSiteDomain, $strSeoPath); - $filePath = public_path() . "sitemap_novel_seo_{$i}.xml"; - file_put_contents($filePath, $sitemapXml); - $sitemapFiles[] = $filePath; - } - - return $sitemapFiles; - } - - // 生成分类链接 - private function generateCategorySitemaps(array $categories, string $strSiteDomain, string $strFenleiPath, string $strXqPath): array - { - $sitemapFiles = []; - - foreach ($categories as $category) { - $vcId = $category['xsfl_id']; - $vcName = $category['xsfl_name']; - - $intNovelCount = XiaoShuoFenLeiModel::where('xsfl_id', $vcId)->count(); - $totalPages = ceil($intNovelCount / 30); - - $categoryUrls = []; - for ($status = 0; $status <= 2; $status++) { - for ($sort = 0; $sort <= 4; $sort++) { - for ($page = 1; $page <= $totalPages; $page++) { - $categoryUrls[] = [ - // /fl/:intCId-:intSort-[:intPage] - 'url' => "/{$strFenleiPath}/{$vcId}-{$sort}-{$page}.html", - 'title' => "{$vcName} - 第{$page}页", - 'is_home' => false, - ]; - } - } - } - - $sitemapXml = $this->buildSitemap($categoryUrls, $strSiteDomain, $strXqPath, true); - $filePath = public_path() . "sitemap_category_{$vcId}.xml"; - file_put_contents($filePath, $sitemapXml); - $sitemapFiles[] = $filePath; - } - - return $sitemapFiles; - } - - //生成章节链接 - private function buildZjSitemap(array $items, string $strSiteDomain, string $strXqPath, bool $isCategory = false): string - { - $xmlHeader = ''; - $urlsetStart = ''; - $urlsetEnd = ''; - // /xq/:intNovelId/:intZhangJiePaiXu-[:strSort] - $urlEntries = ''; - foreach ($items as $item) { - $urlEntries .= ''; - $urlEntries .= 'https://' . $strSiteDomain . ($isCategory ? $item['url'] : '/' . $strXqPath . '/' . htmlspecialchars($item['xsxq_id'] . '-' . $item['xszj_pai_xu']. '-' . $item['xszj_sort'] . '.html', ENT_QUOTES)) . ''; - $urlEntries .= 'daily'; - $urlEntries .= '0.8'; - $urlEntries .= ''; - } - - return $xmlHeader . $urlsetStart . $urlEntries . $urlsetEnd; - } - - // 生成 详情链接 - private function buildSitemap(array $items, string $strSiteDomain, string $strXqPath, bool $isCategory = false): string - { - $xmlHeader = ''; - $urlsetStart = ''; - $urlsetEnd = ''; - // /xq/:intNovelId - $urlEntries = ''; - foreach ($items as $item) { - $urlEntries .= ''; - $urlEntries .= 'https://' . $strSiteDomain . ($isCategory ? $item['url'] : '/' . $strXqPath . '/' . htmlspecialchars($item['xsxq_id'] . '.html', ENT_QUOTES)) . ''; - $urlEntries .= 'daily'; - $urlEntries .= '1.0'; - $urlEntries .= ''; - } - - return $xmlHeader . $urlsetStart . $urlEntries . $urlsetEnd; - } - - // 生成详情 seo 链接 - private function buildXiaoShuoSeoSitemap(array $items, string $strSiteDomain, string $strSeoPath, bool $isCategory = false): string - { - $xmlHeader = ''; - $urlsetStart = ''; - $urlsetEnd = ''; - // /show/:intNovelId - $urlEntries = ''; - foreach ($items as $item) { - $urlEntries .= ''; - $urlEntries .= 'https://' . $strSiteDomain . ($isCategory ? $item['url'] : '/' . $strSeoPath . '/' . htmlspecialchars($item['nis_id'] . '.html', ENT_QUOTES)) . ''; - $urlEntries .= 'daily'; - $urlEntries .= '1.0'; - $urlEntries .= ''; - } - - return $xmlHeader . $urlsetStart . $urlEntries . $urlsetEnd; - } - - // 生成 xml 文件 - private function buildSitemapIndex(array $sitemapFiles, string $strSiteDomain): void - { - $xmlHeader = ''; - $sitemapIndexStart = ''; - $sitemapIndexEnd = ''; - - $sitemapEntries = ''; - foreach ($sitemapFiles as $file) { - $url = 'https://' . $strSiteDomain . '/' . basename($file); - $sitemapEntries .= ''; - $sitemapEntries .= '' . htmlspecialchars($url, ENT_QUOTES) . ''; - $sitemapEntries .= '' . date('Y-m-d') . ''; - $sitemapEntries .= ''; - } - - $sitemapIndexXml = $xmlHeader . $sitemapIndexStart . $sitemapEntries . $sitemapIndexEnd; - file_put_contents(public_path() . 'sitemap_index.xml', $sitemapIndexXml); - - var_dump(['message' => 'Sitemap Index 生成成功', 'file' => 'sitemap_index.xml']); - } -} diff --git a/code/app/task/collection/tianlai/TianLaiXiaoShuoCol.php b/code/app/task/collection/tianlai/TianLaiXiaoShuoCol.php deleted file mode 100644 index d49c299..0000000 --- a/code/app/task/collection/tianlai/TianLaiXiaoShuoCol.php +++ /dev/null @@ -1,538 +0,0 @@ -TaskCore; - $strUrl = $this->getTianLaiDomain(); - var_dump($strUrl); - $Client = $this->Client; - - try { - $arrCookie = $this->getAilisiCookie(); - $strHost = getDomainFromUrl($strUrl); - - $CookieJar = CookieJar::fromArray($arrCookie, $strHost); - $proxy = getRandomProxies(); - $userAgent = getRandomUserAgent(); - - - // $proxy = '127.0.0.1:8080'; // 代理地址 - // $response = fetchContentWithCurl($strUrl, [], 10, $proxy); - - $strResult = $Client->request('GET', $strUrl, [ - 'headers' => [ - 'User-Agent' => $userAgent, - 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', - 'Accept-Language' => 'en-US,en;q=0.5', - ], - 'cookies' => $CookieJar, - 'proxy' => $proxy, - 'timeout' => 30, - ]); - $strResult = (string)$strResult->getBody(); - var_dump($strResult); - // exit; - // $strResult = $response['data']; - $QueryList = QueryList::html($strResult); - - - $QueryList->find('.nav a')->map(function ($FenLeiQueryList) use ($strUrl, $TaskCore) { - $strName = trim($FenLeiQueryList->text()); - switch ($strName) { - - case '都市言情': - $strName = "都市言情"; - break; - case '历史军事': - $strName = "历史军事"; - break; - case '玄幻奇幻': - $strName = "玄幻奇幻"; - break; - case '武侠修真': - $strName = "武侠修真"; - break; - case '女生纯爱': - $strName = "女频言情"; - break; - case '科幻网游': - $strName = "科幻网游"; - break; - case '悬疑灵异': - $strName = "悬疑灵异"; - break; - default: - $strName = "其它"; - break; - } - $strFenLeiUrl = $strUrl . $FenLeiQueryList->attr('href'); - // $intFenLeiId = extractFilename($strFenLeiUrl."index.html"); - $intFenLeiId = extractNumberFromUrl($strFenLeiUrl); - if($intFenLeiId !== null){ - - $arrClass = [ - 'xsfl_source_id' => $intFenLeiId, - 'xsfl_source_code' => 'TIANLAI_FEN_LEI_ID_' . $intFenLeiId, - 'xsfl_name' => $strName, - ]; - var_dump($arrClass); - - $XiaoShuoFenLeiModel = XiaoShuoFenLeiModel::addXiaoShuoFenLei($arrClass); - - $arrTask = [ - 'callback' => [self::class, 'getNovelPage'], - 'data' => [ - 'url' => $strFenLeiUrl, - 'xsfl_source_id' => $intFenLeiId, - 'xsfl_name' => $strName, - 'xsfl_id' => $XiaoShuoFenLeiModel->xsfl_id, - 'xsfl_page' => 1, - 'xsfl_max_page' => 3000, - ] - ]; - - $TaskCore->set($arrTask); - } - - }); - } catch (\Throwable $T) { - //var_dump($T); - throw $T; - } - } - - public function getNovelPage($arrData) - { - - - var_dump('getNovelPage'); - var_dump($arrData); - - $strBasePage = $arrData['url']; - $Client = $this->Client; - - $TaskCore = $this->TaskCore; - - try { - $arrCookie = $this->getAilisiCookie(); - $strHost = getDomainFromUrl($strBasePage); - $CookieJar = CookieJar::fromArray($arrCookie, $strHost); - - $strSiteUrl = $this->getTianLaiDomain(); - - $intPage = $arrData['xsfl_page']; - $intMaxPage = $arrData['xsfl_max_page']; - - $strUrl = $strBasePage; - - var_dump($strUrl); - - - $proxy = getRandomProxies(); - $userAgent = getRandomUserAgent(); - $strResult = $Client->request('GET', $strUrl, [ - 'headers' => [ - 'User-Agent' => $userAgent - ], - 'cookies' => $CookieJar, - #'proxy' => $proxy, - 'timeout' => 10, - ]); - $strResult = (string)$strResult->getBody(); - - $QueryList = QueryList::html($strResult); - - - $intMaxPage = $QueryList->find('#newscontent h2 span')->eq(1)->text(); - $intMaxPage = ceil($intMaxPage / 100); - var_dump($intMaxPage); - $intMaxPage = 2; - - - var_dump('$strUrl'); - // 如果是首页 则根据最大页面吧所有分页 追加到任务 - if($intPage == 1){ - for ($intCurrentPage = 2; $intCurrentPage <= $intMaxPage; $intCurrentPage++) { - $strNewsFenleiUrl = replaceLastPageNumber($strBasePage, $intCurrentPage); - var_dump($strNewsFenleiUrl); - $arrTask = [ - 'callback' => [self::class, 'getNovelPage'], - 'data' => [ - 'url' => $strNewsFenleiUrl, - 'xsfl_source_id' => $arrData['xsfl_source_id'], - 'xsfl_name' => $arrData['xsfl_name'], - 'xsfl_id' => $arrData['xsfl_id'], - 'xsfl_page' => $intCurrentPage, - 'xsfl_max_page' => $intMaxPage, - ] - ]; - $TaskCore->set($arrTask); - } - } - var_dump('提取小说'); - $XiaoShuoListQueryList = $QueryList->find('#newscontent li'); - - if (!$XiaoShuoListQueryList || $XiaoShuoListQueryList->count() <= 0) { - var_dump('这个分类已经市最后一页'); - return true; - } - var_dump($XiaoShuoListQueryList->count()); - var_dump('提取小说2'); - // var_dump($XiaoShuoListQueryList); - $XiaoShuoListQueryList->map(function ($XiaoShuoQueryList) use ($strSiteUrl, $arrData, $TaskCore, $strUrl) { - // 提取小说链接 - // var_dump($XiaoShuoQueryList->html());exit; - var_dump('提取小说链接'); - $A = $XiaoShuoQueryList->find('a')->eq(0); - - $strXiaoShuoUrl = $A->attr('href'); - $strXiaoShuoUrl = strpos($strXiaoShuoUrl, 'https:') === 0 ? $strXiaoShuoUrl : 'https:' . $strXiaoShuoUrl; - var_dump('$A'); - var_dump($A); - var_dump($strXiaoShuoUrl); - exit; - // var_dump($A); - // 提取作者信息 - $strZuoZhe = trim($XiaoShuoQueryList->find('.s4')->eq(0)->text()); - - // 提取小说 ID - $arrXiaoShuoId = extractNumbersFromUrl($strXiaoShuoUrl); - if (!isset($arrXiaoShuoId['first'], $arrXiaoShuoId['second'])) { - return; // 如果 ID 提取失败，直接跳过 - } - - // 准备任务数据 - $arrXiaoShuo = array_merge($arrData, [ - 'xiaoshuo_url' => $strXiaoShuoUrl, - 'xiaoshuo_id' => $arrXiaoShuoId['first'], - 'xiaoshuo_seo_id' => $arrXiaoShuoId['second'], - 'xiaoshuo_zuozhe' => $strZuoZhe, - 'url' => $strUrl - ]); - var_dump($arrXiaoShuo); - $arrTask = [ - 'callback' => [self::class, 'getNovelInfo'], - 'data' => $arrXiaoShuo, - ]; - - // 将任务加入队列 - $TaskCore->set($arrTask); - var_dump("将任务加入队列getNovelInfo-{$strXiaoShuoUrl}"); - }); - var_dump('提取小说end'); - - } catch (\Throwable $T) { - throw $T; - } - } - - - - public function getNovelInfo($arrData) - { - var_dump('getNovelInfo'); - var_dump($arrData); - $Client = $this->Client; - $TaskCore = $this->TaskCore; - - // 获取 Redis 实例 - $redis = Cache::store('redis')->handler(); - $intXiaoShuoSourceId = $arrData['xiaoshuo_id']; - $intXiaoShuoSourceSeoId = $arrData['xiaoshuo_seo_id']; - $redisKey = "task:novel_source_id:" . $intXiaoShuoSourceId; - - try { - $arrCookie = $this->getAilisiCookie(); - $strHost = getDomainFromUrl($arrData['xiaoshuo_url']); - $CookieJar = CookieJar::fromArray($arrCookie, $strHost); - - $strSiteUrl = $this->getTianLaiDomain(); - - $proxy = getRandomProxies(); - $userAgent = getRandomUserAgent(); - $Response = $Client->request('GET', $arrData['xiaoshuo_url'], [ - 'headers' => [ - 'User-Agent' => $userAgent - ], - 'cookies' => $CookieJar, - #'proxy' => $proxy, - 'timeout' => 10, - ]); - $strResult = (string)$Response->getBody(); - unset($Response); - - $QueryList = QueryList::html($strResult); - - $strMingZi = $QueryList->find('#info h1')->eq(0)->text(); - $strZhuangTai = "状态：连载中"; - $strZiShu = 0; - $strZhuangTai = getStringAfterWord($strZhuangTai, '状态：'); - - // 提取作者信息 - $strZuoZhe = trim($QueryList->find('p a')->eq(0)->text()); - - // $strLastDate = $QueryList->find('meta[property="og:novel:update_time"]')->attr('content'); - $strLastDate = $QueryList->find('#info p')->eq(2)->text(); - $strLastDate = getStringAfterWord($strLastDate, '最后更新：'); - $strLastDate = trim((string)$strLastDate); - if (!isValidDateTime($strLastDate)) { - $strLastDate = date('Y-m-d H:i:s'); - } - - $arrTag = []; - // $QueryList->find('.tg span')->map(function ($TagQuery) use (&$arrTag) { - // $arrTag[] = trim($TagQuery->text()); - // }); - - $strJieShao = $QueryList->find('#intro')->eq(0)->text(); - $strJieShao = removeScriptTags('/