admin/SEONexus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

debug-novel-task

Browse Source
This commit is contained in:
make
2025-06-19 12:10:14 +08:00
parent 250ef8bd1c
commit f7afa16a40
2 changed files with 72 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
code/app/task/crawler/zw/page/ChapterPage.php
View File

@@ -71,6 +71,15 @@ class ChapterPage extends BasePage
$this->arrChapterInfo['c_page_keywords'] = $this->getQueryList()->find('meta[name="keywords"]')->attr('content'); $this->arrChapterInfo['c_page_keywords'] = $this->getQueryList()->find('meta[name="keywords"]')->attr('content');
$this->arrChapterInfo['c_page_description'] = $this->getQueryList()->find('meta[name="description"]')->attr('content'); $this->arrChapterInfo['c_page_description'] = $this->getQueryList()->find('meta[name="description"]')->attr('content');
// 记录失败原因
if(empty($this->arrChapterInfo['c_page_title'])){
$incKey = config('task.app_name').'_getChapterInfo_title_null:' . date('Y-m-d');
$this->redisDailyInc($incKey);
if(config('log.task_debug_log_open_status')){
\think\facade\Log::error("Request getChapterInfo_title == NULL , c_page_title == NULL, ");
}
}
// var lastread=new LastRead();lastread.set('529620', '184472933', '身为妖族的我，却在人族成家立业', '第321章 一人一兽联手对敌', '0','眯着眼','0'); // var lastread=new LastRead();lastread.set('529620', '184472933', '身为妖族的我，却在人族成家立业', '第321章 一人一兽联手对敌', '0','眯着眼','0');
$strPattern = "/lastread\.set\((.*?)\);/"; $strPattern = "/lastread\.set\((.*?)\);/";
@@ -86,6 +95,15 @@ class ChapterPage extends BasePage
$this->arrChapterInfo['c_page_data'] = $arrData; $this->arrChapterInfo['c_page_data'] = $arrData;
} else { } else {
$this->arrChapterInfo['c_page_data'] = NULL; $this->arrChapterInfo['c_page_data'] = NULL;
if(config('log.task_debug_log_open_status')){
$incKey = config('task.app_name').'_getChapterInfo_lastread_null:' . date('Y-m-d');
$this->redisDailyInc($incKey);
// 记录失败原因
\think\facade\Log::error("Request getChapterInfo_lastread == NULL , c_page_data == NULL, ");
}
} }
} }
@@ -124,7 +142,6 @@ class ChapterPage extends BasePage
$strTitle = $arrPageChapter['c_page_title'] ?? ''; $strTitle = $arrPageChapter['c_page_title'] ?? '';
$strPageData = $arrPageChapter['c_page_data'] ?? ''; $strPageData = $arrPageChapter['c_page_data'] ?? '';
\think\facade\Log::error("Request empty_saveChapter , strTitle: {$strTitle }, strPageData: {$strPageData}"); \think\facade\Log::error("Request empty_saveChapter , strTitle: {$strTitle }, strPageData: {$strPageData}");
var_dump($strTitle .' --empty_saveChapter-- '.$strPageData);
} }
return false; return false;
} }

167
code/app/task/crawler/zw/page/Site.php
View File

@@ -219,44 +219,51 @@ class Site
// }); // });
foreach ($arrResults as $intKey => $arrResult) { foreach ($arrResults as $intKey => $arrResult) {
if ($arrResult['state'] == 'fulfilled') { if ($arrResult['state'] == 'fulfilled') {
$Response = $arrResult['value'];
$strContent = $Response->getBody()->getContents();
$arrSuccessfulResults[$intKey] = $arrResult; // 跳过空内容
if (empty($strContent) || $strContent == '索引文件不存在！') {
continue;
}
$NovelPage = $this->getNovelPage($this, $arrFilter[$intKey]['uri']);
$NovelPage->setContent($strContent);
$NovelPage->setData($arrFilter[$intKey]);
$arrNovelPage[] = $NovelPage;
$this->redisDailyInc(config('task.app_name').'_success_getNovelPageInfo:' . date('Y-m-d'));
$incKey = config('task.app_name').'_success_getNovelPageInfo:' . date('Y-m-d'); // 例如：success_counter:2025-06-15
$this->redisDailyInc($incKey);
// Cache::store('redis')->inc($incKey);
} elseif ($arrResult['state'] == 'rejected') { } elseif ($arrResult['state'] == 'rejected') {
$incKey = config('task.app_name').'_errer_getNovelPageInfo:' . date('Y-m-d'); // 例如：success_counter:2025-06-15 $incKey = config('task.app_name').'_errer_getNovelPageInfo:' . date('Y-m-d');
$this->redisDailyInc($incKey); $this->redisDailyInc($incKey);
// Cache::store('redis')->inc($incKey);
if(config('log.task_debug_log_open_status')){ if(config('log.task_debug_log_open_status')){
// 记录失败原因 // 记录失败原因
$reason = $arrResult['reason'] instanceof \Exception ? $arrResult['reason']->getMessage() : 'Unknown error'; $reason = $arrResult['reason'] instanceof \Exception ? $arrResult['reason']->getMessage() : 'Unknown error';
$strUri = $arrFilter[$intKey]['uri'] ?? ''; $strUri = $arrFilter[$intKey]['uri'] ?? '';
\think\facade\Log::error("Request failed for getNovelPageInfo , URI: {$strUri }, Reason: {$reason}"); \think\facade\Log::error("Request failed for getNovelPageInfo , URI: {$strUri }, Reason: {$reason}");
var_dump($strUri .' => '.$arrResult['state']);
} }
} }
} }
foreach ($arrSuccessfulResults as $intKey => $arrResult) {
$Response = $arrResult['value'];
$strContent = $Response->getBody()->getContents();
if ($strContent == '索引文件不存在！') {
continue;
}
$NovelPage = $this->getNovelPage($this, $arrFilter[$intKey]['uri']);
$NovelPage->setContent($strContent);
$NovelPage->setData($arrFilter[$intKey]);
$arrNovelPage[] = $NovelPage;
}
return $arrNovelPage; return $arrNovelPage;
// foreach ($arrSuccessfulResults as $intKey => $arrResult) {
// $Response = $arrResult['value'];
// $strContent = $Response->getBody()->getContents();
// if ($strContent == '索引文件不存在！') {
// continue;
// }
// $NovelPage = $this->getNovelPage($this, $arrFilter[$intKey]['uri']);
// $NovelPage->setContent($strContent);
// $NovelPage->setData($arrFilter[$intKey]);
// $arrNovelPage[] = $NovelPage;
// }
} }
/** /**
@@ -285,7 +292,7 @@ class Site
$strContent = $Response->getBody()->getContents(); $strContent = $Response->getBody()->getContents();
// 跳过空内容 // 跳过空内容
if (empty($strContent)) { if (empty($strContent) || $strContent == '索引文件不存在！') {
continue; continue;
} }
@@ -307,74 +314,12 @@ class Site
: 'Unknown error'; : 'Unknown error';
$strUri = $arrFilter[$intKey]['uri'] ?? ''; $strUri = $arrFilter[$intKey]['uri'] ?? '';
\think\facade\Log::error("Request failed for getNovelChapterInfo, URI: {$strUri}, Reason: {$reason}"); \think\facade\Log::error("Request failed for getNovelChapterInfo, URI: {$strUri}, Reason: {$reason}");
var_dump($strUri . ' => ' . $arrResult['state']);
} }
} }
} }
return $arrChapterPage; return $arrChapterPage;
// /** @var ChapterPage[] */
// $arrChapterPage = [];
// $arrPromises = [];
// $arrSuccessfulResults = [];
// foreach ($arrFilter as $intKey => $arrItem) {
// $arrPromises[$intKey] = $this->getClient()->getAsync($arrItem['uri']);
// }
// $arrResults = Promise\Utils::settle($arrPromises)->wait();
// // $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
// // return $arrResult['state'] === 'fulfilled';
// // });
// foreach ($arrResults as $intKey => $arrResult) {
// if ($arrResult['state'] == 'fulfilled') {
// $arrSuccessfulResults[$intKey] = $arrResult;
// $incKey = config('task.app_name').'_success_getChapterPageList:' . date('Y-m-d');
// $this->redisDailyInc($incKey);
// // Cache::store('redis')->inc($incKey);
// } elseif ($arrResult['state'] == 'rejected') {
// $incKey = config('task.app_name').'_errer_getChapterPageList:' . date('Y-m-d');
// $this->redisDailyInc($incKey);
// // Cache::store('redis')->inc($incKey);
// if(config('log.task_debug_log_open_status')){
// // 记录失败原因
// $reason = $arrResult['reason'] instanceof \Exception ? $arrResult['reason']->getMessage() : 'Unknown error';
// $strUri = $arrFilter[$intKey]['uri'] ?? '';
// \think\facade\Log::error("Request failed for getNovelChapterInfo , URI: {$strUri }, Reason: {$reason}");
// var_dump($strUri .' => '.$arrResult['state']);
// }
// }
// }
// foreach ($arrSuccessfulResults as $intKey => $arrResult) {
// $Response = $arrResult['value'];
// $strContent = $Response->getBody()->getContents();
// // ✳️ 如果响应内容为空或无效，跳过（你可以根据规则自行扩展）
// if (empty($strContent)) {
// continue;
// }
// $ChapterPage = $this->getChapterPage($this, $arrFilter[$intKey]['uri']);
// $ChapterPage->setContent($strContent);
// $ChapterPage->setData($arrFilter[$intKey]);
// $arrChapterPage[] = $ChapterPage;
// $incKey = config('task.app_name').'_success_quit_getChapterPageList:' . date('Y-m-d');
// $this->redisDailyInc($incKey);
// }
// return $arrChapterPage;
} }
@@ -392,7 +337,7 @@ class Site
$arrLatestListPage = []; $arrLatestListPage = [];
$arrPromises = []; $arrPromises = [];
$arrSuccessfulResults = []; // $arrSuccessfulResults = [];
foreach ($arrFilter as $intKey => $strUri) { foreach ($arrFilter as $intKey => $strUri) {
$arrPromises[$intKey] = $this->getClient()->getAsync($strUri); $arrPromises[$intKey] = $this->getClient()->getAsync($strUri);
@@ -406,40 +351,37 @@ class Site
foreach ($arrResults as $intKey => $arrResult) { foreach ($arrResults as $intKey => $arrResult) {
if ($arrResult['state'] == 'fulfilled') { if ($arrResult['state'] == 'fulfilled') {
$arrSuccessfulResults[$intKey] = $arrResult; $Response = $arrResult['value'];
$strContent = $Response->getBody()->getContents();
$LatestListPage = $this->getLatestListPage($this, $arrFilter[$intKey]);
$LatestListPage->setContent($strContent);
$arrLatestListPage[] = $LatestListPage;
$incKey = config('task.app_name').'_success_getLatestListPageList:' . date('Y-m-d'); $incKey = config('task.app_name').'_success_getLatestListPageList:' . date('Y-m-d');
$this->redisDailyInc($incKey); $this->redisDailyInc($incKey);
// Cache::store('redis')->inc($incKey);
} elseif ($arrResult['state'] == 'rejected') { } elseif ($arrResult['state'] == 'rejected') {
$incKey = config('task.app_name').'_errer_getLatestListPageList:' . date('Y-m-d'); $incKey = config('task.app_name').'_errer_getLatestListPageList:' . date('Y-m-d');
$this->redisDailyInc($incKey); $this->redisDailyInc($incKey);
// Cache::store('redis')->inc($incKey);
if(config('log.task_debug_log_open_status')){ if(config('log.task_debug_log_open_status')){
// 记录失败原因 // 记录失败原因
$reason = $arrResult['reason'] instanceof \Exception ? $arrResult['reason']->getMessage() : 'Unknown error'; $reason = $arrResult['reason'] instanceof \Exception ? $arrResult['reason']->getMessage() : 'Unknown error';
$strUri = $arrFilter[$intKey] ?? ''; $strUri = $arrFilter[$intKey] ?? '';
\think\facade\Log::error("Request failed for getLatestListPageList , URI: {$strUri }, Reason: {$reason}"); \think\facade\Log::error("Request failed for getLatestListPageList , URI: {$strUri }, Reason: {$reason}");
var_dump($strUri .' => '.$arrResult['state']);
} }
} }
} }
foreach ($arrSuccessfulResults as $intKey => $arrResult) { // foreach ($arrSuccessfulResults as $intKey => $arrResult) {
$Response = $arrResult['value']; // $Response = $arrResult['value'];
// $strContent = $Response->getBody()->getContents();;
// $LatestListPage = $this->getLatestListPage($this, $arrFilter[$intKey]);
// $LatestListPage->setContent($strContent);
// $arrLatestListPage[] = $LatestListPage;
// }
$strContent = $Response->getBody()->getContents();;
$LatestListPage = $this->getLatestListPage($this, $arrFilter[$intKey]);
$LatestListPage->setContent($strContent);
$arrLatestListPage[] = $LatestListPage;
}
// var_dump($arrLatestListPage);
return $arrLatestListPage; return $arrLatestListPage;
} }
@@ -454,7 +396,7 @@ class Site
$StorageCore = StorageCore::getInstance(); $StorageCore = StorageCore::getInstance();
$arrPromises = []; $arrPromises = [];
$arrSuccessfulResults = []; //$arrSuccessfulResults = [];
foreach ($arrFilter as $intKey => $arrItem) { foreach ($arrFilter as $intKey => $arrItem) {
if (!$StorageCore->has($arrItem['local_path'])) { if (!$StorageCore->has($arrItem['local_path'])) {
@@ -470,31 +412,30 @@ class Site
foreach ($arrResults as $intKey => $arrResult) { foreach ($arrResults as $intKey => $arrResult) {
if ($arrResult['state'] == 'fulfilled') { if ($arrResult['state'] == 'fulfilled') {
$arrSuccessfulResults[$intKey] = $arrResult; $Response = $arrResult['value'];
$StorageCore->writeStream($arrFilter[$intKey]['local_path'], $Response->getBody()->detach());
$incKey = config('task.app_name').'_success_downRemoteImgToLocal:' . date('Y-m-d'); $incKey = config('task.app_name').'_success_downRemoteImgToLocal:' . date('Y-m-d');
$this->redisDailyInc($incKey); $this->redisDailyInc($incKey);
// Cache::store('redis')->inc($incKey);
} elseif ($arrResult['state'] == 'rejected') { } elseif ($arrResult['state'] == 'rejected') {
$incKey = config('task.app_name').'_errer_downRemoteImgToLocal:' . date('Y-m-d'); $incKey = config('task.app_name').'_errer_downRemoteImgToLocal:' . date('Y-m-d');
$this->redisDailyInc($incKey); $this->redisDailyInc($incKey);
// Cache::store('redis')->inc($incKey);
if(config('log.task_debug_log_open_status')){ if(config('log.task_debug_log_open_status')){
// 记录失败原因 // 记录失败原因
$reason = $arrResult['reason'] instanceof \Exception ? $arrResult['reason']->getMessage() : 'Unknown error'; $reason = $arrResult['reason'] instanceof \Exception ? $arrResult['reason']->getMessage() : 'Unknown error';
\think\facade\Log::error("Request failed for downRemoteImgToLocal , Reason: {$reason}"); \think\facade\Log::error("Request failed for downRemoteImgToLocal , Reason: {$reason}");
var_dump(' => '.$arrResult['state']);
} }
} }
} }
foreach ($arrSuccessfulResults as $intKey => $arrResult) { // foreach ($arrSuccessfulResults as $intKey => $arrResult) {
$Response = $arrResult['value']; // $Response = $arrResult['value'];
$StorageCore->writeStream($arrFilter[$intKey]['local_path'], $Response->getBody()->detach());; // $StorageCore->writeStream($arrFilter[$intKey]['local_path'], $Response->getBody()->detach());
} // }
} }
/** /**