From 53b5ec4e5474dfdddee3ca3bc1a33a5273f6b3c6 Mon Sep 17 00:00:00 2001 From: make Date: Tue, 17 Jun 2025 13:57:41 +0800 Subject: [PATCH] debug --- code/.example.env | 2 + code/app/task/crawler/zw/Scheduler.php | 24 ++--- code/app/task/crawler/zw/page/Site.php | 120 +++++++++++++++++++++---- code/config/app.php | 4 + code/config/task.php | 2 + 5 files changed, 117 insertions(+), 35 deletions(-) diff --git a/code/.example.env b/code/.example.env index 579bf9e..5b9aa16 100644 --- a/code/.example.env +++ b/code/.example.env @@ -3,6 +3,7 @@ APP_DEBUG = true SHOW_ERROR_MSG = true DEFAULT_LANG = zh-cn DEFAULT_APP_TYPE = video +APP_NAME = novel # 数据库 DB_TYPE = mysql @@ -36,6 +37,7 @@ MONGO_DB = novel # TASK TASK_PROCESS_NUM_01 = 4 TASK_PROCESS_NUM_02 = 2 +TASK_DEBUG_LOG_OPEN_STATUS = false # QUEUEU QUEUE_NAME_1 = QueueKey01 diff --git a/code/app/task/crawler/zw/Scheduler.php b/code/app/task/crawler/zw/Scheduler.php index 45d24cb..a94854c 100644 --- a/code/app/task/crawler/zw/Scheduler.php +++ b/code/app/task/crawler/zw/Scheduler.php @@ -100,17 +100,11 @@ class Scheduler $Site = $this->getSite(); $arrNovelPageList = $Site->getNovelPageList($arrFilter); - // var_dump($arrNovelPageList); - // var_dump('$arrNovelPageList------------------------------------------------------------'); + $arrCover = []; foreach ($arrNovelPageList as $NovelPage) { - // $arrNovel = $NovelPage->getNovelInfo(); - // print_r($arrNovel); - // return; - // var_dump('$NovelPage------------------------------------------------------------'); + $boolResult = $NovelPage->saveNovelInfo(); - // var_dump($boolResult); - // return; if ($boolResult == false) { continue; @@ -144,6 +138,7 @@ class Scheduler /** * create a task to craw the latest chapter + * 创建 每日更新任务 * * @return void */ @@ -181,7 +176,7 @@ class Scheduler } /** - * Undocumented function + * 每日更新列表 * * @param array $arrData * @return void @@ -195,30 +190,21 @@ class Scheduler foreach ($arrLatestListPageList as $LatestListPage) { $arrResult = $LatestListPage->getNovelFilter(); - // var_dump($arrResult); $arrNSourceId = array_merge($arrNSourceId, $arrResult); } - // var_dump($arrNSourceId); $arrNSourceId = array_unique($arrNSourceId); $arrNSourceId = array_values($arrNSourceId); $intSliceSize = $this->intSlice; $arrFilter = []; - // var_dump($arrNSourceId); - // return ; - foreach ($arrNSourceId as $intKey => $intNSourceId) { $arrFilter[] = [ 'n_source_id' => $intNSourceId, 'page' => 0, ]; - // var_dump($intKey); - // var_dump($intSliceSize); - // var_dump($intKey % $intSliceSize); - // var_dump('---------------------------------------------'); + if ($intKey % $intSliceSize == 0) { - // var_dump('$intKey % $intSliceSize == 0'); $this->fetchNovel($arrFilter); $arrFilter = []; } diff --git a/code/app/task/crawler/zw/page/Site.php b/code/app/task/crawler/zw/page/Site.php index 50a1565..ef41547 100644 --- a/code/app/task/crawler/zw/page/Site.php +++ b/code/app/task/crawler/zw/page/Site.php @@ -7,6 +7,7 @@ namespace app\task\crawler\zw\page; use GuzzleHttp\Client; use GuzzleHttp\Promise; use storage\StorageCore; +use think\facade\Cache; /** * @mixin think\Model @@ -178,7 +179,7 @@ class Site } /** - * retrieve multiple novel page model + * 获取小说详情页面 * * @param string $strUri * @return NovelPage[] @@ -189,6 +190,8 @@ class Site /** @var NovelPage[] */ $arrNovelPage = []; + $arrSuccessfulResults = []; + foreach ($arrFilter as $intKey => $arrItem) { $arrFilter[$intKey]['uri'] = $this->getNovelPageUri($arrItem['n_source_id'], $arrItem['page']); @@ -199,9 +202,30 @@ class Site $arrResults = Promise\Utils::settle($arrPromises)->wait(); - $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) { - return $arrResult['state'] === 'fulfilled'; - }); + // $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) { + // return $arrResult['state'] === 'fulfilled'; + // }); + foreach ($arrResults as $intKey => $arrResult) { + if ($arrResult['state'] == 'fulfilled') { + + $arrSuccessfulResults[$intKey] = $arrResult; + + $incKey = config('task.app_name').'_success_getNovelPageInfo:' . date('Y-m-d'); // 例如：success_counter:2025-06-15 + Cache::store('redis')->inc($incKey); + + } elseif ($arrResult['state'] == 'rejected') { + $incKey = config('task.app_name').'_errer_getNovelPageInfo:' . date('Y-m-d'); // 例如：success_counter:2025-06-15 + Cache::store('redis')->inc($incKey); + + if(config('task.server.task_debug_log_open_status')){ + // 记录失败原因 + $reason = $arrResult['reason'] instanceof \Exception ? $arrResult['reason']->getMessage() : 'Unknown error'; + $strUri = $arrFilter[$intKey]['uri'] ?? ''; + \think\facade\Log::error("Request failed for getNovelPageInfo , URI: {$strUri }, Reason: {$reason}"); + var_dump($strUri .' => '.$arrResult['state']); + } + } + } foreach ($arrSuccessfulResults as $intKey => $arrResult) { $Response = $arrResult['value']; @@ -224,6 +248,7 @@ class Site /** * retrieve multiple chapter page model + * 获取章节详情页面 * * @param string $strUri * @return ChapterPage[] @@ -235,15 +260,37 @@ class Site $arrChapterPage = []; $arrPromises = []; + $arrSuccessfulResults = []; + foreach ($arrFilter as $intKey => $arrItem) { $arrPromises[$intKey] = $this->getClient()->getAsync($arrItem['uri']); } $arrResults = Promise\Utils::settle($arrPromises)->wait(); - $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) { - return $arrResult['state'] === 'fulfilled'; - }); + // $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) { + // return $arrResult['state'] === 'fulfilled'; + // }); + foreach ($arrResults as $intKey => $arrResult) { + if ($arrResult['state'] == 'fulfilled') { + + $arrSuccessfulResults[$intKey] = $arrResult; + + $incKey = config('task.app_name').'_success_getChapterPageList:' . date('Y-m-d'); // 例如：success_counter:2025-06-15 + Cache::store('redis')->inc($incKey); + + } elseif ($arrResult['state'] == 'rejected') { + $incKey = config('task.app_name').'_errer_getChapterPageList:' . date('Y-m-d'); // 例如：success_counter:2025-06-15 + Cache::store('redis')->inc($incKey); + if(config('task.server.task_debug_log_open_status')){ + // 记录失败原因 + $reason = $arrResult['reason'] instanceof \Exception ? $arrResult['reason']->getMessage() : 'Unknown error'; + $strUri = $arrFilter[$intKey]['uri'] ?? ''; + \think\facade\Log::error("Request failed for getNovelChapterInfo , URI: {$strUri }, Reason: {$reason}"); + var_dump($strUri .' => '.$arrResult['state']); + } + } + } foreach ($arrSuccessfulResults as $intKey => $arrResult) { $Response = $arrResult['value']; @@ -264,6 +311,7 @@ class Site /** * retrieve multiple chapter page model + * 每日更新列表 * * @param string $strUri * @return LatestListPage[] @@ -275,19 +323,38 @@ class Site $arrLatestListPage = []; $arrPromises = []; + $arrSuccessfulResults = []; foreach ($arrFilter as $intKey => $strUri) { $arrPromises[$intKey] = $this->getClient()->getAsync($strUri); } $arrResults = Promise\Utils::settle($arrPromises)->wait(); - // var_dump($strUri); - // var_dump($arrResults); - $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) { - // var_dump($arrResult); - return $arrResult['state'] === 'fulfilled'; - }); + // $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) { + // return $arrResult['state'] === 'fulfilled'; + // }); + foreach ($arrResults as $intKey => $arrResult) { + if ($arrResult['state'] == 'fulfilled') { + + $arrSuccessfulResults[$intKey] = $arrResult; + + $incKey = config('task.app_name').'_success_getLatestListPageList:' . date('Y-m-d'); + Cache::store('redis')->inc($incKey); + + } elseif ($arrResult['state'] == 'rejected') { + + $incKey = config('task.app_name').'_errer_getLatestListPageList:' . date('Y-m-d'); + Cache::store('redis')->inc($incKey); + if(config('task.server.task_debug_log_open_status')){ + // 记录失败原因 + $reason = $arrResult['reason'] instanceof \Exception ? $arrResult['reason']->getMessage() : 'Unknown error'; + $strUri = $arrFilter[$intKey] ?? ''; + \think\facade\Log::error("Request failed for getLatestListPageList , URI: {$strUri }, Reason: {$reason}"); + var_dump($strUri .' => '.$arrResult['state']); + } + } + } foreach ($arrSuccessfulResults as $intKey => $arrResult) { $Response = $arrResult['value']; @@ -316,6 +383,7 @@ class Site $StorageCore = StorageCore::getInstance(); $arrPromises = []; + $arrSuccessfulResults = []; foreach ($arrFilter as $intKey => $arrItem) { if (!$StorageCore->has($arrItem['local_path'])) { @@ -325,9 +393,29 @@ class Site $arrResults = Promise\Utils::settle($arrPromises)->wait(); - $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) { - return $arrResult['state'] === 'fulfilled'; - }); + // $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) { + // return $arrResult['state'] === 'fulfilled'; + // }); + foreach ($arrResults as $intKey => $arrResult) { + if ($arrResult['state'] == 'fulfilled') { + + $arrSuccessfulResults[$intKey] = $arrResult; + + $incKey = config('task.app_name').'_success_downRemoteImgToLocal:' . date('Y-m-d'); + Cache::store('redis')->inc($incKey); + + } elseif ($arrResult['state'] == 'rejected') { + $incKey = config('task.app_name').'_errer_downRemoteImgToLocal:' . date('Y-m-d'); + Cache::store('redis')->inc($incKey); + + if(config('task.server.task_debug_log_open_status')){ + // 记录失败原因 + $reason = $arrResult['reason'] instanceof \Exception ? $arrResult['reason']->getMessage() : 'Unknown error'; + \think\facade\Log::error("Request failed for downRemoteImgToLocal , Reason: {$reason}"); + var_dump(' => '.$arrResult['state']); + } + } + } foreach ($arrSuccessfulResults as $intKey => $arrResult) { $Response = $arrResult['value']; diff --git a/code/config/app.php b/code/config/app.php index e77ea36..9c29597 100644 --- a/code/config/app.php +++ b/code/config/app.php @@ -13,6 +13,10 @@ return [ // 默认应用 'default_app' => 'home', + // 应用名称 + 'app_name' => env('APP_NAME', 'novel'), + + // 默认应用类型 video/novel 'default_app_type' => env('DEFAULT_APP_TYPE', 'video'), diff --git a/code/config/task.php b/code/config/task.php index 7e45e0b..528dc35 100644 --- a/code/config/task.php +++ b/code/config/task.php @@ -35,6 +35,8 @@ return [ 'log_file' => runtime_path() . 'task.log', 'pid_file' => runtime_path() . 'task.pid', 'task_worker_num' => 0, + + 'task_debug_log_open_status' => env('TASK_DEBUG_LOG_OPEN_STATUS', false), ], # Process Pool, NUM 为设置的任务组进程数 , 根据自身服务器情况增减