From f7afa16a406777c185652a2d322217ed1ac9275a Mon Sep 17 00:00:00 2001 From: make Date: Thu, 19 Jun 2025 12:10:14 +0800 Subject: [PATCH] debug-novel-task --- code/app/task/crawler/zw/page/ChapterPage.php | 19 +- code/app/task/crawler/zw/page/Site.php | 167 ++++++------------ 2 files changed, 72 insertions(+), 114 deletions(-) diff --git a/code/app/task/crawler/zw/page/ChapterPage.php b/code/app/task/crawler/zw/page/ChapterPage.php index faed278..3c26414 100644 --- a/code/app/task/crawler/zw/page/ChapterPage.php +++ b/code/app/task/crawler/zw/page/ChapterPage.php @@ -71,6 +71,15 @@ class ChapterPage extends BasePage $this->arrChapterInfo['c_page_keywords'] = $this->getQueryList()->find('meta[name="keywords"]')->attr('content'); $this->arrChapterInfo['c_page_description'] = $this->getQueryList()->find('meta[name="description"]')->attr('content'); + // 记录失败原因 + if(empty($this->arrChapterInfo['c_page_title'])){ + $incKey = config('task.app_name').'_getChapterInfo_title_null:' . date('Y-m-d'); + $this->redisDailyInc($incKey); + if(config('log.task_debug_log_open_status')){ + \think\facade\Log::error("Request getChapterInfo_title == NULL , c_page_title == NULL, "); + } + } + // var lastread=new LastRead();lastread.set('529620', '184472933', '身为妖族的我，却在人族成家立业', '第321章 一人一兽联手对敌', '0','眯着眼','0'); $strPattern = "/lastread\.set\((.*?)\);/"; @@ -86,6 +95,15 @@ class ChapterPage extends BasePage $this->arrChapterInfo['c_page_data'] = $arrData; } else { $this->arrChapterInfo['c_page_data'] = NULL; + + if(config('log.task_debug_log_open_status')){ + + $incKey = config('task.app_name').'_getChapterInfo_lastread_null:' . date('Y-m-d'); + $this->redisDailyInc($incKey); + + // 记录失败原因 + \think\facade\Log::error("Request getChapterInfo_lastread == NULL , c_page_data == NULL, "); + } } } @@ -124,7 +142,6 @@ class ChapterPage extends BasePage $strTitle = $arrPageChapter['c_page_title'] ?? ''; $strPageData = $arrPageChapter['c_page_data'] ?? ''; \think\facade\Log::error("Request empty_saveChapter , strTitle: {$strTitle }, strPageData: {$strPageData}"); - var_dump($strTitle .' --empty_saveChapter-- '.$strPageData); } return false; } diff --git a/code/app/task/crawler/zw/page/Site.php b/code/app/task/crawler/zw/page/Site.php index 7d6e9f0..167ce93 100644 --- a/code/app/task/crawler/zw/page/Site.php +++ b/code/app/task/crawler/zw/page/Site.php @@ -219,44 +219,51 @@ class Site // }); foreach ($arrResults as $intKey => $arrResult) { if ($arrResult['state'] == 'fulfilled') { + $Response = $arrResult['value']; + $strContent = $Response->getBody()->getContents(); - $arrSuccessfulResults[$intKey] = $arrResult; + // 跳过空内容 + if (empty($strContent) || $strContent == '索引文件不存在！') { + continue; + } + + $NovelPage = $this->getNovelPage($this, $arrFilter[$intKey]['uri']); + $NovelPage->setContent($strContent); + $NovelPage->setData($arrFilter[$intKey]); + $arrNovelPage[] = $NovelPage; + + $this->redisDailyInc(config('task.app_name').'_success_getNovelPageInfo:' . date('Y-m-d')); - $incKey = config('task.app_name').'_success_getNovelPageInfo:' . date('Y-m-d'); // 例如：success_counter:2025-06-15 - $this->redisDailyInc($incKey); - // Cache::store('redis')->inc($incKey); } elseif ($arrResult['state'] == 'rejected') { - $incKey = config('task.app_name').'_errer_getNovelPageInfo:' . date('Y-m-d'); // 例如：success_counter:2025-06-15 + $incKey = config('task.app_name').'_errer_getNovelPageInfo:' . date('Y-m-d'); $this->redisDailyInc($incKey); - // Cache::store('redis')->inc($incKey); if(config('log.task_debug_log_open_status')){ // 记录失败原因 $reason = $arrResult['reason'] instanceof \Exception ? $arrResult['reason']->getMessage() : 'Unknown error'; $strUri = $arrFilter[$intKey]['uri'] ?? ''; \think\facade\Log::error("Request failed for getNovelPageInfo , URI: {$strUri }, Reason: {$reason}"); - var_dump($strUri .' => '.$arrResult['state']); } } } - - foreach ($arrSuccessfulResults as $intKey => $arrResult) { - $Response = $arrResult['value']; - - $strContent = $Response->getBody()->getContents(); - if ($strContent == '索引文件不存在！') { - continue; - } - - $NovelPage = $this->getNovelPage($this, $arrFilter[$intKey]['uri']); - - $NovelPage->setContent($strContent); - $NovelPage->setData($arrFilter[$intKey]); - - $arrNovelPage[] = $NovelPage; - } - return $arrNovelPage; + // foreach ($arrSuccessfulResults as $intKey => $arrResult) { + // $Response = $arrResult['value']; + + // $strContent = $Response->getBody()->getContents(); + // if ($strContent == '索引文件不存在！') { + // continue; + // } + + // $NovelPage = $this->getNovelPage($this, $arrFilter[$intKey]['uri']); + + // $NovelPage->setContent($strContent); + // $NovelPage->setData($arrFilter[$intKey]); + + // $arrNovelPage[] = $NovelPage; + // } + + } /** @@ -285,7 +292,7 @@ class Site $strContent = $Response->getBody()->getContents(); // 跳过空内容 - if (empty($strContent)) { + if (empty($strContent) || $strContent == '索引文件不存在！') { continue; } @@ -307,74 +314,12 @@ class Site : 'Unknown error'; $strUri = $arrFilter[$intKey]['uri'] ?? ''; \think\facade\Log::error("Request failed for getNovelChapterInfo, URI: {$strUri}, Reason: {$reason}"); - var_dump($strUri . ' => ' . $arrResult['state']); } } } return $arrChapterPage; - - // /** @var ChapterPage[] */ - // $arrChapterPage = []; - - // $arrPromises = []; - // $arrSuccessfulResults = []; - - // foreach ($arrFilter as $intKey => $arrItem) { - // $arrPromises[$intKey] = $this->getClient()->getAsync($arrItem['uri']); - // } - - // $arrResults = Promise\Utils::settle($arrPromises)->wait(); - - // // $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) { - // // return $arrResult['state'] === 'fulfilled'; - // // }); - // foreach ($arrResults as $intKey => $arrResult) { - // if ($arrResult['state'] == 'fulfilled') { - - // $arrSuccessfulResults[$intKey] = $arrResult; - - // $incKey = config('task.app_name').'_success_getChapterPageList:' . date('Y-m-d'); - // $this->redisDailyInc($incKey); - // // Cache::store('redis')->inc($incKey); - - // } elseif ($arrResult['state'] == 'rejected') { - // $incKey = config('task.app_name').'_errer_getChapterPageList:' . date('Y-m-d'); - // $this->redisDailyInc($incKey); - // // Cache::store('redis')->inc($incKey); - // if(config('log.task_debug_log_open_status')){ - // // 记录失败原因 - // $reason = $arrResult['reason'] instanceof \Exception ? $arrResult['reason']->getMessage() : 'Unknown error'; - // $strUri = $arrFilter[$intKey]['uri'] ?? ''; - // \think\facade\Log::error("Request failed for getNovelChapterInfo , URI: {$strUri }, Reason: {$reason}"); - // var_dump($strUri .' => '.$arrResult['state']); - // } - // } - // } - - // foreach ($arrSuccessfulResults as $intKey => $arrResult) { - // $Response = $arrResult['value']; - - // $strContent = $Response->getBody()->getContents(); - - // // ✳️ 如果响应内容为空或无效，跳过（你可以根据规则自行扩展） - // if (empty($strContent)) { - // continue; - // } - - // $ChapterPage = $this->getChapterPage($this, $arrFilter[$intKey]['uri']); - - // $ChapterPage->setContent($strContent); - // $ChapterPage->setData($arrFilter[$intKey]); - - // $arrChapterPage[] = $ChapterPage; - - // $incKey = config('task.app_name').'_success_quit_getChapterPageList:' . date('Y-m-d'); - // $this->redisDailyInc($incKey); - // } - - // return $arrChapterPage; } @@ -392,7 +337,7 @@ class Site $arrLatestListPage = []; $arrPromises = []; - $arrSuccessfulResults = []; + // $arrSuccessfulResults = []; foreach ($arrFilter as $intKey => $strUri) { $arrPromises[$intKey] = $this->getClient()->getAsync($strUri); @@ -406,40 +351,37 @@ class Site foreach ($arrResults as $intKey => $arrResult) { if ($arrResult['state'] == 'fulfilled') { - $arrSuccessfulResults[$intKey] = $arrResult; + $Response = $arrResult['value']; + $strContent = $Response->getBody()->getContents(); + $LatestListPage = $this->getLatestListPage($this, $arrFilter[$intKey]); + $LatestListPage->setContent($strContent); + $arrLatestListPage[] = $LatestListPage; $incKey = config('task.app_name').'_success_getLatestListPageList:' . date('Y-m-d'); $this->redisDailyInc($incKey); - // Cache::store('redis')->inc($incKey); } elseif ($arrResult['state'] == 'rejected') { $incKey = config('task.app_name').'_errer_getLatestListPageList:' . date('Y-m-d'); $this->redisDailyInc($incKey); - // Cache::store('redis')->inc($incKey); + if(config('log.task_debug_log_open_status')){ // 记录失败原因 $reason = $arrResult['reason'] instanceof \Exception ? $arrResult['reason']->getMessage() : 'Unknown error'; $strUri = $arrFilter[$intKey] ?? ''; \think\facade\Log::error("Request failed for getLatestListPageList , URI: {$strUri }, Reason: {$reason}"); - var_dump($strUri .' => '.$arrResult['state']); } } } - foreach ($arrSuccessfulResults as $intKey => $arrResult) { - $Response = $arrResult['value']; + // foreach ($arrSuccessfulResults as $intKey => $arrResult) { + // $Response = $arrResult['value']; + // $strContent = $Response->getBody()->getContents();; + // $LatestListPage = $this->getLatestListPage($this, $arrFilter[$intKey]); + // $LatestListPage->setContent($strContent); + // $arrLatestListPage[] = $LatestListPage; + // } - $strContent = $Response->getBody()->getContents();; - - $LatestListPage = $this->getLatestListPage($this, $arrFilter[$intKey]); - - $LatestListPage->setContent($strContent); - - $arrLatestListPage[] = $LatestListPage; - } - - // var_dump($arrLatestListPage); return $arrLatestListPage; } @@ -454,7 +396,7 @@ class Site $StorageCore = StorageCore::getInstance(); $arrPromises = []; - $arrSuccessfulResults = []; + //$arrSuccessfulResults = []; foreach ($arrFilter as $intKey => $arrItem) { if (!$StorageCore->has($arrItem['local_path'])) { @@ -470,31 +412,30 @@ class Site foreach ($arrResults as $intKey => $arrResult) { if ($arrResult['state'] == 'fulfilled') { - $arrSuccessfulResults[$intKey] = $arrResult; + $Response = $arrResult['value']; + $StorageCore->writeStream($arrFilter[$intKey]['local_path'], $Response->getBody()->detach()); $incKey = config('task.app_name').'_success_downRemoteImgToLocal:' . date('Y-m-d'); $this->redisDailyInc($incKey); - // Cache::store('redis')->inc($incKey); } elseif ($arrResult['state'] == 'rejected') { + $incKey = config('task.app_name').'_errer_downRemoteImgToLocal:' . date('Y-m-d'); $this->redisDailyInc($incKey); - // Cache::store('redis')->inc($incKey); if(config('log.task_debug_log_open_status')){ // 记录失败原因 $reason = $arrResult['reason'] instanceof \Exception ? $arrResult['reason']->getMessage() : 'Unknown error'; \think\facade\Log::error("Request failed for downRemoteImgToLocal , Reason: {$reason}"); - var_dump(' => '.$arrResult['state']); } } } - foreach ($arrSuccessfulResults as $intKey => $arrResult) { - $Response = $arrResult['value']; + // foreach ($arrSuccessfulResults as $intKey => $arrResult) { + // $Response = $arrResult['value']; - $StorageCore->writeStream($arrFilter[$intKey]['local_path'], $Response->getBody()->detach());; - } + // $StorageCore->writeStream($arrFilter[$intKey]['local_path'], $Response->getBody()->detach()); + // } } /**