Merge branch 'dev' of ssh://git.bgbg.live:18150/root/SEONexus into dev

code/app/task/crawler/zw/Scheduler.php

@@ -14,7 +14,7 @@ use storage\StorageCore;
  */
 class Scheduler
 {
-    protected $intSlice = 8;
+    protected $intSlice = 2;
 
     protected $intStartNSourceId = 1;
 
@@ -43,7 +43,7 @@ class Scheduler
      */
     public function init()
     {
-        $this->intSlice = 8;
+        $this->intSlice = 2;
     }
 
     /**
@@ -276,18 +276,39 @@ class Scheduler
      */
     public function getChapterInfo(array $arrData)
     {
+
         $Site = $this->getSite();
 
         $ChapterModel = ChapterModel::getInstance();
 
+        #FIXED 这里增加临时代码，将章节任务拆分为一个章节用一个进程请求，防止代理同时请求多个异常问题。
+        $arrTask = [
+            'callback' => [static::class, 'getChapterInfo'],
+            'data' => [
+                'n_id' => $arrData['n_id'],
+                'filter' => [],
+            ]
+        ];
+
         foreach ($arrData['filter'] as $intKey => $arrChapterFilter) {
             $strUUId = sprintf('%s-chapter-%s-%s',  $Site->getSiteCode(),  $arrChapterFilter['c_source_id'],  $arrChapterFilter['c_page']);
             $arrChapter = $ChapterModel->findOne(['c_source_uuid' => $strUUId]);
 
             if (!empty($arrChapter) && !empty($arrChapter['content'])) {
                 unset($arrData['filter'][$intKey]);
+            } else {
+                #FIXED 这里增加临时代码，将章节任务拆分为一个章节用一个进程请求，防止代理同时请求多个异常问题。
+
+                $arrFilter = [$arrChapterFilter];
+                $arrTask['data']['filter'] =  $arrFilter;
+                $this->QueueManage->set($arrTask);
             }
         }
+        if (count($arrData['filter']) > 1 || count($arrData['filter']) <= 0) {
+            #FIXED 这里增加临时代码，将章节任务拆分为一个章节用一个进程请求，防止代理同时请求多个异常问题。
+            return false;
+        }
+
 
         $arrData['filter'] = array_values($arrData['filter']);
 

code/app/task/crawler/zw74/Scheduler.php

@@ -19,7 +19,7 @@ class Scheduler extends ZwScheduler
      */
     public function init()
     {
-        $this->intSlice = 8;
+        $this->intSlice = 1;
     }
 
     /**

code/app/task/logic/SiteMapLogic.php

@@ -19,7 +19,7 @@ class SiteMapLogic
      */
     public $NovelModel;
 
-    public $intPageSize = 3;
+    public $intPageSize = 40000;
 
     public $strSiteMapPath = '';