From d6e9171ba7c1b0c23d5b2c58a85462f3db7132a9 Mon Sep 17 00:00:00 2001 From: make Date: Wed, 18 Jun 2025 01:23:08 +0800 Subject: [PATCH] debug --- code/app/task/crawler/zw/page/BasePage.php | 17 +++++++ code/app/task/crawler/zw/page/ChapterPage.php | 13 ++++++ code/app/task/crawler/zw/page/Site.php | 44 ++++++++++++++----- 3 files changed, 63 insertions(+), 11 deletions(-) diff --git a/code/app/task/crawler/zw/page/BasePage.php b/code/app/task/crawler/zw/page/BasePage.php index e0c8c99..9c2eebc 100644 --- a/code/app/task/crawler/zw/page/BasePage.php +++ b/code/app/task/crawler/zw/page/BasePage.php @@ -10,6 +10,8 @@ use GuzzleHttp\Client; use microserver\QueueManage; use QL\QueryList; use Psr\Http\Message\ResponseInterface; +use think\facade\Cache; + /** * @mixin think\Model @@ -248,4 +250,19 @@ abstract class BasePage { return $this->Site; } + + /** + * Undocumented function + * + * @param string $key + * @param integer $expire + * @return void + */ + public function redisDailyInc(string $key, int $expire = 604800) { + $cache = Cache::store('redis'); + if (!$cache->has($key)) { + $cache->set($key, 0, $expire); + } + $cache->inc($key); + } } diff --git a/code/app/task/crawler/zw/page/ChapterPage.php b/code/app/task/crawler/zw/page/ChapterPage.php index fd8b799..456a4e1 100644 --- a/code/app/task/crawler/zw/page/ChapterPage.php +++ b/code/app/task/crawler/zw/page/ChapterPage.php @@ -115,6 +115,8 @@ class ChapterPage extends BasePage empty($arrPageChapter['c_page_data']) || count($arrPageChapter['c_page_data']) < 5 ) { + $incKey = config('task.app_name').'_empty_saveChapter:' . date('Y-m-d'); + $this->redisDailyInc($incKey); return false; } @@ -145,6 +147,9 @@ class ChapterPage extends BasePage ['$set' => $arrUpdate], ); $this->strCContentPath = $arrExistsChapter['c_content_path']; + + $incKey = config('task.app_name').'_update_saveChapter:' . date('Y-m-d'); + $this->redisDailyInc($incKey); } else { $arrInsertNovel = $arrChapter; @@ -154,10 +159,18 @@ class ChapterPage extends BasePage try { $this->getChapterModel()->insert($arrInsertNovel); + + $incKey = config('task.app_name').'_success_saveChapter:' . date('Y-m-d'); + $this->redisDailyInc($incKey); + $this->updateNovelHaveChapter($intNId); $this->strCContentPath = $arrInsertNovel['c_content_path']; } catch (\Exception $e) { if ($e->getCode() === 11000) { + + $incKey = config('task.app_name').'_11000_saveChapter:' . date('Y-m-d'); + $this->redisDailyInc($incKey); + return $this->saveChapter($intNId); } else { throw $e; diff --git a/code/app/task/crawler/zw/page/Site.php b/code/app/task/crawler/zw/page/Site.php index ace4504..7d37728 100644 --- a/code/app/task/crawler/zw/page/Site.php +++ b/code/app/task/crawler/zw/page/Site.php @@ -211,11 +211,12 @@ class Site $arrSuccessfulResults[$intKey] = $arrResult; $incKey = config('task.app_name').'_success_getNovelPageInfo:' . date('Y-m-d'); // 例如：success_counter:2025-06-15 - Cache::store('redis')->inc($incKey); - + $this->redisDailyInc($incKey); + // Cache::store('redis')->inc($incKey); } elseif ($arrResult['state'] == 'rejected') { $incKey = config('task.app_name').'_errer_getNovelPageInfo:' . date('Y-m-d'); // 例如：success_counter:2025-06-15 - Cache::store('redis')->inc($incKey); + $this->redisDailyInc($incKey); + // Cache::store('redis')->inc($incKey); if(config('log.task_debug_log_open_status')){ // 记录失败原因 @@ -276,12 +277,14 @@ class Site $arrSuccessfulResults[$intKey] = $arrResult; - $incKey = config('task.app_name').'_success_getChapterPageList:' . date('Y-m-d'); // 例如：success_counter:2025-06-15 - Cache::store('redis')->inc($incKey); + $incKey = config('task.app_name').'_success_getChapterPageList:' . date('Y-m-d'); + $this->redisDailyInc($incKey); + // Cache::store('redis')->inc($incKey); } elseif ($arrResult['state'] == 'rejected') { - $incKey = config('task.app_name').'_errer_getChapterPageList:' . date('Y-m-d'); // 例如：success_counter:2025-06-15 - Cache::store('redis')->inc($incKey); + $incKey = config('task.app_name').'_errer_getChapterPageList:' . date('Y-m-d'); + $this->redisDailyInc($incKey); + // Cache::store('redis')->inc($incKey); if(config('log.task_debug_log_open_status')){ // 记录失败原因 $reason = $arrResult['reason'] instanceof \Exception ? $arrResult['reason']->getMessage() : 'Unknown error'; @@ -340,12 +343,14 @@ class Site $arrSuccessfulResults[$intKey] = $arrResult; $incKey = config('task.app_name').'_success_getLatestListPageList:' . date('Y-m-d'); - Cache::store('redis')->inc($incKey); + $this->redisDailyInc($incKey); + // Cache::store('redis')->inc($incKey); } elseif ($arrResult['state'] == 'rejected') { $incKey = config('task.app_name').'_errer_getLatestListPageList:' . date('Y-m-d'); - Cache::store('redis')->inc($incKey); + $this->redisDailyInc($incKey); + // Cache::store('redis')->inc($incKey); if(config('log.task_debug_log_open_status')){ // 记录失败原因 $reason = $arrResult['reason'] instanceof \Exception ? $arrResult['reason']->getMessage() : 'Unknown error'; @@ -402,11 +407,13 @@ class Site $arrSuccessfulResults[$intKey] = $arrResult; $incKey = config('task.app_name').'_success_downRemoteImgToLocal:' . date('Y-m-d'); - Cache::store('redis')->inc($incKey); + $this->redisDailyInc($incKey); + // Cache::store('redis')->inc($incKey); } elseif ($arrResult['state'] == 'rejected') { $incKey = config('task.app_name').'_errer_downRemoteImgToLocal:' . date('Y-m-d'); - Cache::store('redis')->inc($incKey); + $this->redisDailyInc($incKey); + // Cache::store('redis')->inc($incKey); if(config('log.task_debug_log_open_status')){ // 记录失败原因 @@ -459,4 +466,19 @@ class Site { return new LatestListPage($Site, $strUri); } + + /** + * Undocumented function + * + * @param string $key + * @param integer $expire + * @return void + */ + public function redisDailyInc(string $key, int $expire = 604800) { + $cache = Cache::store('redis'); + if (!$cache->has($key)) { + $cache->set($key, 0, $expire); + } + $cache->inc($key); + } }