admin/SEONexus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

debug

Browse Source
This commit is contained in:
make
2025-06-17 13:57:41 +08:00
parent 499882f1a4
commit 53b5ec4e54
5 changed files with 117 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

120
code/app/task/crawler/zw/page/Site.php
View File

@@ -7,6 +7,7 @@ namespace app\task\crawler\zw\page;
use GuzzleHttp\Client;
use GuzzleHttp\Promise;
use storage\StorageCore;
use think\facade\Cache;
/**
* @mixin think\Model
@@ -178,7 +179,7 @@ class Site
}
/**
* retrieve multiple novel page model
* 获取小说详情页面
*
* @param string $strUri
* @return NovelPage[]
@@ -189,6 +190,8 @@ class Site
/** @var NovelPage[] */
$arrNovelPage = [];
$arrSuccessfulResults = [];
foreach ($arrFilter as $intKey => $arrItem) {
$arrFilter[$intKey]['uri'] = $this->getNovelPageUri($arrItem['n_source_id'], $arrItem['page']);
@@ -199,9 +202,30 @@ class Site
$arrResults = Promise\Utils::settle($arrPromises)->wait();
$arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
return $arrResult['state'] === 'fulfilled';
});
// $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
// return $arrResult['state'] === 'fulfilled';
// });
foreach ($arrResults as $intKey => $arrResult) {
if ($arrResult['state'] == 'fulfilled') {
$arrSuccessfulResults[$intKey] = $arrResult;
$incKey = config('task.app_name').'_success_getNovelPageInfo:' . date('Y-m-d'); // 例如：success_counter:2025-06-15
Cache::store('redis')->inc($incKey);
} elseif ($arrResult['state'] == 'rejected') {
$incKey = config('task.app_name').'_errer_getNovelPageInfo:' . date('Y-m-d'); // 例如：success_counter:2025-06-15
Cache::store('redis')->inc($incKey);
if(config('task.server.task_debug_log_open_status')){
// 记录失败原因
$reason = $arrResult['reason'] instanceof \Exception ? $arrResult['reason']->getMessage() : 'Unknown error';
$strUri = $arrFilter[$intKey]['uri'] ?? '';
\think\facade\Log::error("Request failed for getNovelPageInfo , URI: {$strUri }, Reason: {$reason}");
var_dump($strUri .' => '.$arrResult['state']);
}
}
}
foreach ($arrSuccessfulResults as $intKey => $arrResult) {
$Response = $arrResult['value'];
@@ -224,6 +248,7 @@ class Site
/**
* retrieve multiple chapter page model
* 获取章节详情页面
*
* @param string $strUri
* @return ChapterPage[]
@@ -235,15 +260,37 @@ class Site
$arrChapterPage = [];
$arrPromises = [];
$arrSuccessfulResults = [];
foreach ($arrFilter as $intKey => $arrItem) {
$arrPromises[$intKey] = $this->getClient()->getAsync($arrItem['uri']);
}
$arrResults = Promise\Utils::settle($arrPromises)->wait();
$arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
return $arrResult['state'] === 'fulfilled';
});
// $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
// return $arrResult['state'] === 'fulfilled';
// });
foreach ($arrResults as $intKey => $arrResult) {
if ($arrResult['state'] == 'fulfilled') {
$arrSuccessfulResults[$intKey] = $arrResult;
$incKey = config('task.app_name').'_success_getChapterPageList:' . date('Y-m-d'); // 例如：success_counter:2025-06-15
Cache::store('redis')->inc($incKey);
} elseif ($arrResult['state'] == 'rejected') {
$incKey = config('task.app_name').'_errer_getChapterPageList:' . date('Y-m-d'); // 例如：success_counter:2025-06-15
Cache::store('redis')->inc($incKey);
if(config('task.server.task_debug_log_open_status')){
// 记录失败原因
$reason = $arrResult['reason'] instanceof \Exception ? $arrResult['reason']->getMessage() : 'Unknown error';
$strUri = $arrFilter[$intKey]['uri'] ?? '';
\think\facade\Log::error("Request failed for getNovelChapterInfo , URI: {$strUri }, Reason: {$reason}");
var_dump($strUri .' => '.$arrResult['state']);
}
}
}
foreach ($arrSuccessfulResults as $intKey => $arrResult) {
$Response = $arrResult['value'];
@@ -264,6 +311,7 @@ class Site
/**
* retrieve multiple chapter page model
* 每日更新列表
*
* @param string $strUri
* @return LatestListPage[]
@@ -275,19 +323,38 @@ class Site
$arrLatestListPage = [];
$arrPromises = [];
$arrSuccessfulResults = [];
foreach ($arrFilter as $intKey => $strUri) {
$arrPromises[$intKey] = $this->getClient()->getAsync($strUri);
}
$arrResults = Promise\Utils::settle($arrPromises)->wait();
// var_dump($strUri);
// var_dump($arrResults);
$arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
// var_dump($arrResult);
return $arrResult['state'] === 'fulfilled';
});
// $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
// return $arrResult['state'] === 'fulfilled';
// });
foreach ($arrResults as $intKey => $arrResult) {
if ($arrResult['state'] == 'fulfilled') {
$arrSuccessfulResults[$intKey] = $arrResult;
$incKey = config('task.app_name').'_success_getLatestListPageList:' . date('Y-m-d');
Cache::store('redis')->inc($incKey);
} elseif ($arrResult['state'] == 'rejected') {
$incKey = config('task.app_name').'_errer_getLatestListPageList:' . date('Y-m-d');
Cache::store('redis')->inc($incKey);
if(config('task.server.task_debug_log_open_status')){
// 记录失败原因
$reason = $arrResult['reason'] instanceof \Exception ? $arrResult['reason']->getMessage() : 'Unknown error';
$strUri = $arrFilter[$intKey] ?? '';
\think\facade\Log::error("Request failed for getLatestListPageList , URI: {$strUri }, Reason: {$reason}");
var_dump($strUri .' => '.$arrResult['state']);
}
}
}
foreach ($arrSuccessfulResults as $intKey => $arrResult) {
$Response = $arrResult['value'];
@@ -316,6 +383,7 @@ class Site
$StorageCore = StorageCore::getInstance();
$arrPromises = [];
$arrSuccessfulResults = [];
foreach ($arrFilter as $intKey => $arrItem) {
if (!$StorageCore->has($arrItem['local_path'])) {
@@ -325,9 +393,29 @@ class Site
$arrResults = Promise\Utils::settle($arrPromises)->wait();
$arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
return $arrResult['state'] === 'fulfilled';
});
// $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
// return $arrResult['state'] === 'fulfilled';
// });
foreach ($arrResults as $intKey => $arrResult) {
if ($arrResult['state'] == 'fulfilled') {
$arrSuccessfulResults[$intKey] = $arrResult;
$incKey = config('task.app_name').'_success_downRemoteImgToLocal:' . date('Y-m-d');
Cache::store('redis')->inc($incKey);
} elseif ($arrResult['state'] == 'rejected') {
$incKey = config('task.app_name').'_errer_downRemoteImgToLocal:' . date('Y-m-d');
Cache::store('redis')->inc($incKey);
if(config('task.server.task_debug_log_open_status')){
// 记录失败原因
$reason = $arrResult['reason'] instanceof \Exception ? $arrResult['reason']->getMessage() : 'Unknown error';
\think\facade\Log::error("Request failed for downRemoteImgToLocal , Reason: {$reason}");
var_dump(' => '.$arrResult['state']);
}
}
}
foreach ($arrSuccessfulResults as $intKey => $arrResult) {
$Response = $arrResult['value'];