diff --git a/code/app/task/crawler/zw/Scheduler.php b/code/app/task/crawler/zw/Scheduler.php index 27395b7..be22724 100644 --- a/code/app/task/crawler/zw/Scheduler.php +++ b/code/app/task/crawler/zw/Scheduler.php @@ -14,7 +14,7 @@ use storage\StorageCore; */ class Scheduler { - protected $intSlice = 8; + protected $intSlice = 2; protected $intStartNSourceId = 1; @@ -43,7 +43,7 @@ class Scheduler */ public function init() { - $this->intSlice = 8; + $this->intSlice = 2; } /** @@ -276,18 +276,39 @@ class Scheduler */ public function getChapterInfo(array $arrData) { + $Site = $this->getSite(); $ChapterModel = ChapterModel::getInstance(); + #FIXED 这里增加临时代码,将章节任务拆分为一个章节用一个进程请求,防止代理同时请求多个异常问题。 + $arrTask = [ + 'callback' => [static::class, 'getChapterInfo'], + 'data' => [ + 'n_id' => $arrData['n_id'], + 'filter' => [], + ] + ]; + foreach ($arrData['filter'] as $intKey => $arrChapterFilter) { $strUUId = sprintf('%s-chapter-%s-%s', $Site->getSiteCode(), $arrChapterFilter['c_source_id'], $arrChapterFilter['c_page']); $arrChapter = $ChapterModel->findOne(['c_source_uuid' => $strUUId]); if (!empty($arrChapter) && !empty($arrChapter['content'])) { unset($arrData['filter'][$intKey]); + } else { + #FIXED 这里增加临时代码,将章节任务拆分为一个章节用一个进程请求,防止代理同时请求多个异常问题。 + + $arrFilter = [$arrChapterFilter]; + $arrTask['data']['filter'] = $arrFilter; + $this->QueueManage->set($arrTask); } } + if (count($arrData['filter']) > 1 || count($arrData['filter']) <= 0) { + #FIXED 这里增加临时代码,将章节任务拆分为一个章节用一个进程请求,防止代理同时请求多个异常问题。 + return false; + } + $arrData['filter'] = array_values($arrData['filter']); diff --git a/code/app/task/crawler/zw74/Scheduler.php b/code/app/task/crawler/zw74/Scheduler.php index 6780f01..4697a85 100644 --- a/code/app/task/crawler/zw74/Scheduler.php +++ b/code/app/task/crawler/zw74/Scheduler.php @@ -19,7 +19,7 @@ class Scheduler extends ZwScheduler */ public function init() { - $this->intSlice = 8; + $this->intSlice = 1; } /** diff --git a/code/app/task/logic/SiteMapLogic.php b/code/app/task/logic/SiteMapLogic.php index 2f5e264..77807f7 100644 --- a/code/app/task/logic/SiteMapLogic.php +++ b/code/app/task/logic/SiteMapLogic.php @@ -19,7 +19,7 @@ class SiteMapLogic */ public $NovelModel; - public $intPageSize = 3; + public $intPageSize = 40000; public $strSiteMapPath = '';