From 321f5eda01633efa8ec69e9f4d13d809b3e8724f Mon Sep 17 00:00:00 2001 From: make Date: Thu, 19 Jun 2025 18:20:37 +0800 Subject: [PATCH] debug-novel-task --- code/app/task/crawler/zw/Scheduler.php | 69 ++------------------------ code/app/task/crawler/zw/page/Site.php | 65 +++++++++++++++++------- 2 files changed, 50 insertions(+), 84 deletions(-) diff --git a/code/app/task/crawler/zw/Scheduler.php b/code/app/task/crawler/zw/Scheduler.php index 0f71746..bae5f14 100644 --- a/code/app/task/crawler/zw/Scheduler.php +++ b/code/app/task/crawler/zw/Scheduler.php @@ -43,7 +43,7 @@ class Scheduler */ public function init() { - $this->intSlice = 2; + $this->intSlice = 1; } /** @@ -323,69 +323,6 @@ class Scheduler ]); } } - // public function getChapterInfo(array $arrData) - // { - - // $Site = $this->getSite(); - - // $ChapterModel = ChapterModel::getInstance(); - - // #FIXED 这里增加临时代码，将章节任务拆分为一个章节用一个进程请求，防止代理同时请求多个异常问题。 - // $arrTask = [ - // 'callback' => [static::class, 'getChapterInfo'], - // 'data' => [ - // 'n_id' => $arrData['n_id'], - // 'filter' => [], - // ] - // ]; - - // foreach ($arrData['filter'] as $intKey => $arrChapterFilter) { - // $strUUId = sprintf('%s-chapter-%s-%s', $Site->getSiteCode(), $arrChapterFilter['c_source_id'], $arrChapterFilter['c_page']); - - // $arrChapter = $ChapterModel->findOne(['c_source_uuid' => $strUUId]); - - // // 如果数据存在，且 内容不为空，删除任务 - // if (!empty($arrChapter) && !empty($arrChapter['c_content'])) { - // unset($arrData['filter'][$intKey]); - // } else { - // #FIXED 这里增加临时代码，将章节任务拆分为一个章节用一个进程请求，防止代理同时请求多个异常问题。 - - // $arrFilter = [$arrChapterFilter]; - // $arrTask['data']['filter'] = $arrFilter; - // $this->QueueManage->set($arrTask); - // } - // } - // if (count($arrData['filter']) > 1 || count($arrData['filter']) <= 0) { - // #FIXED 这里增加临时代码，将章节任务拆分为一个章节用一个进程请求，防止代理同时请求多个异常问题。 - // return false; - // } - - - // $arrData['filter'] = array_values($arrData['filter']); - - // $arrChapterPageList = $Site->getChapterPageList($arrData['filter']); - - // $arrNextPageFilterList = []; - // foreach ($arrChapterPageList as $ChapterPage) { - - // $ChapterPage->saveChapter($arrData['n_id']); - - // $arrNextPageFilter = $ChapterPage->getNextPage(); - - // if (!empty($arrNextPageFilter)) { - // $arrNextPageFilterList[] = $arrNextPageFilter; - // } - // } - - // if (!empty($arrNextPageFilterList)) { - - // $arrTask = $arrData; - // $arrTask['filter'] = $arrNextPageFilterList; - // $arrTask = [ - // 'callback' => [static::class, 'getChapterInfo'], - // 'data' => $arrTask - // ]; - // $this->QueueManage->set($arrTask); - // } - // } + + } diff --git a/code/app/task/crawler/zw/page/Site.php b/code/app/task/crawler/zw/page/Site.php index 0706b12..87d6df7 100644 --- a/code/app/task/crawler/zw/page/Site.php +++ b/code/app/task/crawler/zw/page/Site.php @@ -8,6 +8,8 @@ use GuzzleHttp\Client; use GuzzleHttp\Promise; use storage\StorageCore; use think\facade\Cache; +use app\task\crawler\zw74\Scheduler as Zw74Scheduler; +use microserver\QueueManage; /** * @mixin think\Model @@ -227,6 +229,14 @@ class Site continue; } + // 判断是否包含 $strContent ，不包含跳过 + if (strpos($strContent, '') === false) { + // 不包含该字符串，跳过 + var_dump(mb_substr($strContent, 0, 300)); + continue; + } + + $NovelPage = $this->getNovelPage($this, $arrFilter[$intKey]['uri']); $NovelPage->setContent($strContent); $NovelPage->setData($arrFilter[$intKey]); @@ -235,6 +245,15 @@ class Site $this->redisDailyInc(config('task.app_name').'_success_getNovelPageInfo:' . date('Y-m-d')); } elseif ($arrResult['state'] == 'rejected') { + + // 重新推送任务 + $arrTask = [ + 'callback' => [Zw74Scheduler::class, 'getNovelInfo'], + 'data' => $arrFilter, + ]; + $QueueManage = QueueManage::getInstance(); + $QueueManage->set($arrTask); + $incKey = config('task.app_name').'_errer_getNovelPageInfo:' . date('Y-m-d'); $this->redisDailyInc($incKey); @@ -247,22 +266,6 @@ class Site } } return $arrNovelPage; - // foreach ($arrSuccessfulResults as $intKey => $arrResult) { - // $Response = $arrResult['value']; - - // $strContent = $Response->getBody()->getContents(); - // if ($strContent == '索引文件不存在！') { - // continue; - // } - - // $NovelPage = $this->getNovelPage($this, $arrFilter[$intKey]['uri']); - - // $NovelPage->setContent($strContent); - // $NovelPage->setData($arrFilter[$intKey]); - - // $arrNovelPage[] = $NovelPage; - // } - } @@ -270,11 +273,12 @@ class Site * retrieve multiple chapter page model * 获取章节详情页面 * - * @param string $strUri + * @param array $arrData * @return ChapterPage[] */ - public function getChapterPageList(array $arrFilter): array + public function getChapterPageList(array $arrData): array { + $arrFilter = $arrData['filter']; /** @var ChapterPage[] */ $arrChapterPage = []; @@ -313,6 +317,15 @@ class Site $this->redisDailyInc(config('task.app_name') . '_success_getChapterPageList:' . date('Y-m-d')); } elseif ($arrResult['state'] === 'rejected') { + + // 重新推送任务 + $arrTask = [ + 'callback' => [Zw74Scheduler::class, 'getChapterInfo'], + 'data' => $arrData, + ]; + $QueueManage = QueueManage::getInstance(); + $QueueManage->set($arrTask); + $this->redisDailyInc(config('task.app_name') . '_errer_getChapterPageList:' . date('Y-m-d')); if (config('log.task_debug_log_open_status')) { @@ -369,6 +382,14 @@ class Site } elseif ($arrResult['state'] == 'rejected') { + // 重新推送任务 + $arrTask = [ + 'callback' => [Zw74Scheduler::class, 'getLatestChapter'], + 'data' => $arrFilter, + ]; + $QueueManage = QueueManage::getInstance(); + $QueueManage->set($arrTask); + $incKey = config('task.app_name').'_errer_getLatestListPageList:' . date('Y-m-d'); $this->redisDailyInc($incKey); @@ -427,6 +448,14 @@ class Site } elseif ($arrResult['state'] == 'rejected') { + // 重新推送任务 + $arrTask = [ + 'callback' => [$this::class, 'downRemoteImgToLocal'], + 'data' => $arrFilter, + ]; + $QueueManage = QueueManage::getInstance(); + $QueueManage->set($arrTask); + $incKey = config('task.app_name').'_errer_downRemoteImgToLocal:' . date('Y-m-d'); $this->redisDailyInc($incKey);