Merge branch 'dev' of ssh://git.bgbg.live:18150/root/SEONexus into dev

code/.example.env

@@ -42,18 +42,16 @@ QUEUE_EXEC_NUM_1 = 10
 QUEUE_NAME_2 = QueueKey02
 QUEUE_EXEC_NUM_2 = 10
 
-# MINIO
-MINIO_END_POINT = http://127.0.0.1:9200
-MINIO_KEY = FeJI6i2j15TK5lNNRuN2
-MINIO_SECRET = oTr9cVCYjql4SLehde0cXcFRCaA3QLnw4Yhnjw6N
-MINIO_REGION = hk-01
-MINIO_BUCKET = test001
-
 # LocalStorage
 STORAGE_LOCAL_DIR = /mnt/gluster
 
-
 # VIEW
 VIEW_DISPLAY_CACHE = false
 VIEW_TPL_CACHE = false
 VIEW_DATA_CACHE = false
+
+# PROXY
+PROXY_STATUS = false
+PROXY_CODE = XIONGMAO
+PROXY_ORDER = DT20250413204710pFcgjMR9
+PROXY_SECRET = 583b141edcf65ce94f83591c5cb592ed

code/app/task/crawler/zw630/Scheduler.php

@@ -8,12 +8,16 @@ use app\model\ChapterModel;
 use app\task\crawler\zw630\page\NovelPage;
 use app\task\crawler\zw630\page\Site;
 use microserver\QueueManage;
+use storage\StorageCore;
 
 /**
  * @mixin think\Model
  */
 class Scheduler
 {
+
+    static $intSlice = 3;
+
     /**
      * Undocumented variable
      *
@@ -33,7 +37,7 @@ class Scheduler
      */
     public function crawlFull()
     {
-        $intSliceSize = 3;
+        $intSliceSize = self::$intSlice;
 
         $arrFilter = [];
         for ($intNId = 1; $intNId <= 531208; $intNId++) {
@@ -61,7 +65,7 @@ class Scheduler
      */
     public function crawLatest()
     {
-        $intSliceSize = 3;
+        $intSliceSize =  self::$intSlice;
         $arrUri = [];
         $strUri = '/list/lastupdate_%s_0_0_%s.html';
 
@@ -108,7 +112,7 @@ class Scheduler
         $arrNSourceId = array_unique($arrNSourceId);
         $arrNSourceId  = array_values($arrNSourceId);
 
-        $intSliceSize = 3;
+        $intSliceSize =  self::$intSlice;
         $arrFilter = [];
 
         foreach ($arrNSourceId as $intKey => $intNSourceId) {
@@ -122,6 +126,7 @@ class Scheduler
                 $arrFilter = [];
             }
         }
+
         if (!empty($arrFilter)) {
             $this->fetchNovel($arrFilter);
         }
@@ -152,7 +157,7 @@ class Scheduler
      */
     public function fetchChapter(int $intNId, int $intPage, array $arrChapterList = [])
     {
-        $intSliceSize = 3;
+        $intSliceSize =  self::$intSlice;
         $intPageSize = 100;
 
         $arrTask = [
@@ -210,10 +215,16 @@ class Scheduler
                 ]);
             }
 
-            $arrCover[] = $NovelPage->getDownCoverInfo();
+            $arrCoverItem = $NovelPage->getDownCoverInfo();
+
+            if (!StorageCore::getInstance()->has($arrCoverItem['local_path'])) {
+                $arrCover[] = $arrCoverItem;
+            }
+
             $arrChapterList = $NovelPage->getChapters();
             $this->fetchChapter($arrNovel['n_id'], $NovelPage->getData()['page'], $arrChapterList);
         }
+
         if ($arrCover) {
             $Site->downRemoteImgToLocal($arrCover);
         }
@@ -242,7 +253,6 @@ class Scheduler
 
         $arrData['filter'] = array_values($arrData['filter']);
 
-
         $arrChapterPageList = $Site->getChapterPageList($arrData['filter']);
 
         $arrNextPageFilterList = [];

code/app/task/crawler/zw630/page/NovelPage.php

@@ -11,7 +11,6 @@ use app\model\NovelModel;
 
 class NovelPage extends BasePage
 {
-
     /**
      * Undocumented variable
      *
@@ -101,11 +100,14 @@ class NovelPage extends BasePage
     public function getNovelInfo($boolSave = true): array
     {
         if (empty($this->arrNovelInfo)) {
+
+            # 获取head字段
             $this->getQueryList()->find('meta[property]')->each(function ($Item) {
                 $strKey = str_replace(':', '_', $Item->attr('property'));
                 $this->arrNovelInfo[$strKey] = $Item->attr('content');
             });
 
+            # 获取 kan 和 shu 的关联数据
             $this->getQueryList()->find('.first_txt')->eq(0)->find("a")->map(function ($A) {
                 $strName =  $A->text();
                 $strUri =  $A->attr('href');
@@ -125,26 +127,38 @@ class NovelPage extends BasePage
                     ];
                 }
             });
+
             $this->arrNovelInfo['n_related_novel'] = $this->arrRelatedNovel;
             $this->arrNovelInfo['n_forge_novel'] = $this->arrForgeNovel;
-        }
 
+            $this->arrNovelInfo['og_novel_pin_yin'] = zhToPinYin($this->arrNovelInfo['og_novel_book_name'] ?? '');
+
+            if (!empty($this->arrNovelInfo['og_novel_category'])) {
+                $this->arrNovelInfo['category_pinyin'] = CategoryModel::getPinYinByZh($this->arrNovelInfo['og_novel_category']);
+                $this->arrNovelInfo['category_sex'] = CategoryModel::checkSex($this->arrNovelInfo['og_novel_category']);
+                $this->arrNovelInfo['category_sex_zh'] = CategoryModel::checkSexZh($this->arrNovelInfo['og_novel_category']);
+            }
+
+            $this->arrNovelInfo['n_site_code'] =  $this->getSite()->getSiteCode();
+            $this->arrNovelInfo['n_source_id'] = $this->arrData['n_source_id'];
+        }
 
         if ($boolSave && !empty($this->arrNovelInfo)) {
             $this->saveNovelInfo();
         }
+
         return  $this->arrNovelInfo;
     }
 
     /**
      * Undocumented function
      *
-     * @param [type] $strName
+     * @param string $strName
      * @return string
      */
     public function generateCoverPath($strName): string
     {
-        return  '/img/' . date('Y/m/d', time() - 24 * 3600 * 14) . '/' . $strName . '.jpg';
+        return sprintf('/img/%s/%s.jpg', date('Y/m/d', time() - 24 * 3600 * 14), $strName);
     }
 
     public function filterOtherFields(&$arrNovel)
@@ -197,17 +211,18 @@ class NovelPage extends BasePage
                 $arrExisting = (array) $arrExisting;
                 $this->arrNovelInfo = array_merge($arrExisting,   $this->arrNovelInfo);
 
-                if (!isset($this->arrNovelInfo['og_novel_pin_yin']) || empty($this->arrNovelInfo['og_novel_pin_yin'])) {
+                if (
-                    $this->arrNovelInfo['og_novel_pin_yin'] = zhToPinYin($this->arrNovelInfo['og_novel_book_name'] ?? '');
+                    !isset($this->arrNovelInfo['n_cover_path']) ||
-                }
+                    empty($this->arrNovelInfo['n_cover_path']) ||
+                    !file_exists($this->arrNovelInfo['n_cover_path'])  ||
 
-                if (!isset($this->arrNovelInfo['n_cover_path']) || empty($this->arrNovelInfo['n_cover_path'])) {
+                    (basename($this->arrNovelInfo['n_cover_path']) == '.jpg')
+                ) {
                     $this->arrNovelInfo['n_cover_path'] = $this->generateCoverPath($this->arrNovelInfo['og_novel_pin_yin']);
                 }
 
                 $this->arrNovelInfo['updated_at'] = new \MongoDB\BSON\UTCDateTime();
 
-
                 $arrUpdate =  $this->arrNovelInfo;
                 $this->filterOtherFields($arrUpdate);
 
@@ -216,23 +231,12 @@ class NovelPage extends BasePage
                     ['$set' => $arrUpdate],
                 );
             } else {
-                $this->arrNovelInfo['og_novel_pin_yin'] = zhToPinYin($this->arrNovelInfo['og_novel_book_name'] ?? '');
-
-                $this->arrNovelInfo['n_site_code'] =  $this->getSite()->getSiteCode();
                 $this->arrNovelInfo['n_site_uuid'] = $strUUId;
-                $this->arrNovelInfo['n_source_id'] = $this->arrData['n_source_id'];
                 $this->arrNovelInfo['n_id'] =  CountersModel::getInstance()->getNextSequence('novel');
                 $this->arrNovelInfo['n_level'] = 1;
                 $this->arrNovelInfo['n_have_chapter'] = 0;
                 $this->arrNovelInfo['n_cover_path'] =  $this->generateCoverPath($this->arrNovelInfo['og_novel_pin_yin']);
 
-
-                if (!empty($this->arrNovelInfo['og_novel_category'])) {
-                    $this->arrNovelInfo['category_pinyin'] = CategoryModel::getPinYinByZh($this->arrNovelInfo['og_novel_category']);
-                    $this->arrNovelInfo['category_sex'] = CategoryModel::checkSex($this->arrNovelInfo['og_novel_category']);
-                    $this->arrNovelInfo['category_sex_zh'] = CategoryModel::checkSexZh($this->arrNovelInfo['og_novel_category']);
-                }
-
                 $arrInsert =  $this->arrNovelInfo;
                 $this->filterOtherFields($arrInsert);
 

code/app/task/crawler/zw630/page/Site.php

@@ -66,17 +66,16 @@ class Site
                 CURLOPT_TCP_KEEPIDLE => 120, // Keep-Alive 空闲时间
                 CURLOPT_TCP_KEEPINTVL => 60, // Keep-Alive 探测间隔
                 // CURLOPT_SSL_VERIFYPEER => true, // 验证 SSL，已有，保持
-                // CURLOPT_SSL_VERIFYHOST => 2, // 确保主机名匹配
             ],
             'headers' => [
-                'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36 Edg/134.0.0.0',
+                'User-Agent' => 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36',
                 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
                 'Accept-Language' => 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
                 'Referer' => $this->strDomain,
                 'Connection' => 'keep-alive',
-                'sec-ch-ua' => '"Chromium";v="134", "Not:A-Brand";v="24", "Microsoft Edge";v="134"',
+                'sec-ch-ua' => '"Not/A)Brand";v="8", "Chromium";v="130", "Google Chrome";v="130"',
-                'sec-ch-ua-mobile' => '?0',
+                'sec-ch-ua-mobile' => '?1',
-                'sec-ch-ua-platform' => '"Windows"',
+                'sec-ch-ua-platform' => '"Android"',
                 'sec-fetch-dest' => 'document',
                 'sec-fetch-mode' => 'navigate',
                 'sec-fetch-site' => 'same-origin',
@@ -84,19 +83,23 @@ class Site
                 'upgrade-insecure-requests' => '1',
             ],
             'pool_size' => 5,
-            'verify' => false, // 忽略 SSL 证书验证
+            'verify' => false, 
         ];
 
-        $time = time();
+        $arrProxy = config('proxy');
-        $orderno = "DT20250413204710pFcgjMR9";
+        if ($arrProxy['proxy_status']) {
-        $secret = "583b141edcf65ce94f83591c5cb592ed";
+            $intTime = time();
+            $strOrderNo = $arrProxy['proxy_order'];
+            $strSecret = $arrProxy['proxy_secret'];
 
-        $txt = "orderno=" . $orderno . ",secret=" . $secret . ",timestamp=" . $time;
+            $strTxt = "orderno=" . $strOrderNo . ",secret=" . $strSecret . ",timestamp=" . $intTime;
-        $sign = strtoupper(md5($txt));
+            $strSign = strtoupper(md5($strTxt));
-        $auth = 'sign=' . $sign . '&orderno=' . $orderno . '&timestamp=' . $time;
+            $strAuth = 'sign=' . $strSign . '&orderno=' . $strOrderNo . '&timestamp=' . $intTime;
+
+            $arrConfig['proxy'] = 'http://dtbf.xiongmaodaili.com:8089';
+            $arrConfig['headers']['Proxy-Authorization'] =  $strAuth;
+        }
 
-        $arrConfig['proxy'] = 'http://dtbf.xiongmaodaili.com:8089';
-        $arrConfig['headers']['Proxy-Authorization'] =  $auth;
 
         $this->Client = new Client($arrConfig);
     }

code/config/proxy.php

@@ -0,0 +1,11 @@
+<?php
+// +----------------------------------------------------------------------
+// | 代理设置
+// +----------------------------------------------------------------------
+
+return [
+    'proxy_status' =>  env('PROXY_STATUS', false),
+    'proxy_code' =>  env('PROXY_CODE', ''),
+    'proxy_order' =>  env('PROXY_ORDER', ''),
+    'proxy_secret' =>  env('PROXY_SECRET', ''),
+];