admin/SEONexus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

debug

Browse Source
This commit is contained in:
make
2025-06-17 13:57:41 +08:00
parent 499882f1a4
commit 53b5ec4e54
5 changed files with 117 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
code/app/task/crawler/zw/Scheduler.php
View File

@@ -100,17 +100,11 @@ class Scheduler
$Site = $this->getSite();
$arrNovelPageList = $Site->getNovelPageList($arrFilter);
// var_dump($arrNovelPageList);
// var_dump('$arrNovelPageList------------------------------------------------------------');
$arrCover = [];
foreach ($arrNovelPageList as $NovelPage) {
// $arrNovel = $NovelPage->getNovelInfo();
// print_r($arrNovel);
// return;
// var_dump('$NovelPage------------------------------------------------------------');
$boolResult = $NovelPage->saveNovelInfo();
// var_dump($boolResult);
// return;
if ($boolResult == false) {
continue;
@@ -144,6 +138,7 @@ class Scheduler
/**
* create a task to craw the latest chapter
* 创建 每日更新任务
*
* @return void
*/
@@ -181,7 +176,7 @@ class Scheduler
}
/**
* Undocumented function
* 每日更新列表
*
* @param array $arrData
* @return void
@@ -195,30 +190,21 @@ class Scheduler
foreach ($arrLatestListPageList as $LatestListPage) {
$arrResult = $LatestListPage->getNovelFilter();
// var_dump($arrResult);
$arrNSourceId = array_merge($arrNSourceId, $arrResult);
}
// var_dump($arrNSourceId);
$arrNSourceId = array_unique($arrNSourceId);
$arrNSourceId = array_values($arrNSourceId);
$intSliceSize = $this->intSlice;
$arrFilter = [];
// var_dump($arrNSourceId);
// return ;
foreach ($arrNSourceId as $intKey => $intNSourceId) {
$arrFilter[] = [
'n_source_id' => $intNSourceId,
'page' => 0,
];
// var_dump($intKey);
// var_dump($intSliceSize);
// var_dump($intKey % $intSliceSize);
// var_dump('---------------------------------------------');
if ($intKey % $intSliceSize == 0) {
// var_dump('$intKey % $intSliceSize == 0');
$this->fetchNovel($arrFilter);
$arrFilter = [];
}

120
code/app/task/crawler/zw/page/Site.php
View File

@@ -7,6 +7,7 @@ namespace app\task\crawler\zw\page;
use GuzzleHttp\Client;
use GuzzleHttp\Promise;
use storage\StorageCore;
use think\facade\Cache;
/**
* @mixin think\Model
@@ -178,7 +179,7 @@ class Site
}
/**
* retrieve multiple novel page model
* 获取小说详情页面
*
* @param string $strUri
* @return NovelPage[]
@@ -189,6 +190,8 @@ class Site
/** @var NovelPage[] */
$arrNovelPage = [];
$arrSuccessfulResults = [];
foreach ($arrFilter as $intKey => $arrItem) {
$arrFilter[$intKey]['uri'] = $this->getNovelPageUri($arrItem['n_source_id'], $arrItem['page']);
@@ -199,9 +202,30 @@ class Site
$arrResults = Promise\Utils::settle($arrPromises)->wait();
$arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
return $arrResult['state'] === 'fulfilled';
});
// $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
// return $arrResult['state'] === 'fulfilled';
// });
foreach ($arrResults as $intKey => $arrResult) {
if ($arrResult['state'] == 'fulfilled') {
$arrSuccessfulResults[$intKey] = $arrResult;
$incKey = config('task.app_name').'_success_getNovelPageInfo:' . date('Y-m-d'); // 例如：success_counter:2025-06-15
Cache::store('redis')->inc($incKey);
} elseif ($arrResult['state'] == 'rejected') {
$incKey = config('task.app_name').'_errer_getNovelPageInfo:' . date('Y-m-d'); // 例如：success_counter:2025-06-15
Cache::store('redis')->inc($incKey);
if(config('task.server.task_debug_log_open_status')){
// 记录失败原因
$reason = $arrResult['reason'] instanceof \Exception ? $arrResult['reason']->getMessage() : 'Unknown error';
$strUri = $arrFilter[$intKey]['uri'] ?? '';
\think\facade\Log::error("Request failed for getNovelPageInfo , URI: {$strUri }, Reason: {$reason}");
var_dump($strUri .' => '.$arrResult['state']);
}
}
}
foreach ($arrSuccessfulResults as $intKey => $arrResult) {
$Response = $arrResult['value'];
@@ -224,6 +248,7 @@ class Site
/**
* retrieve multiple chapter page model
* 获取章节详情页面
*
* @param string $strUri
* @return ChapterPage[]
@@ -235,15 +260,37 @@ class Site
$arrChapterPage = [];
$arrPromises = [];
$arrSuccessfulResults = [];
foreach ($arrFilter as $intKey => $arrItem) {
$arrPromises[$intKey] = $this->getClient()->getAsync($arrItem['uri']);
}
$arrResults = Promise\Utils::settle($arrPromises)->wait();
$arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
return $arrResult['state'] === 'fulfilled';
});
// $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
// return $arrResult['state'] === 'fulfilled';
// });
foreach ($arrResults as $intKey => $arrResult) {
if ($arrResult['state'] == 'fulfilled') {
$arrSuccessfulResults[$intKey] = $arrResult;
$incKey = config('task.app_name').'_success_getChapterPageList:' . date('Y-m-d'); // 例如：success_counter:2025-06-15
Cache::store('redis')->inc($incKey);
} elseif ($arrResult['state'] == 'rejected') {
$incKey = config('task.app_name').'_errer_getChapterPageList:' . date('Y-m-d'); // 例如：success_counter:2025-06-15
Cache::store('redis')->inc($incKey);
if(config('task.server.task_debug_log_open_status')){
// 记录失败原因
$reason = $arrResult['reason'] instanceof \Exception ? $arrResult['reason']->getMessage() : 'Unknown error';
$strUri = $arrFilter[$intKey]['uri'] ?? '';
\think\facade\Log::error("Request failed for getNovelChapterInfo , URI: {$strUri }, Reason: {$reason}");
var_dump($strUri .' => '.$arrResult['state']);
}
}
}
foreach ($arrSuccessfulResults as $intKey => $arrResult) {
$Response = $arrResult['value'];
@@ -264,6 +311,7 @@ class Site
/**
* retrieve multiple chapter page model
* 每日更新列表
*
* @param string $strUri
* @return LatestListPage[]
@@ -275,19 +323,38 @@ class Site
$arrLatestListPage = [];
$arrPromises = [];
$arrSuccessfulResults = [];
foreach ($arrFilter as $intKey => $strUri) {
$arrPromises[$intKey] = $this->getClient()->getAsync($strUri);
}
$arrResults = Promise\Utils::settle($arrPromises)->wait();
// var_dump($strUri);
// var_dump($arrResults);
$arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
// var_dump($arrResult);
return $arrResult['state'] === 'fulfilled';
});
// $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
// return $arrResult['state'] === 'fulfilled';
// });
foreach ($arrResults as $intKey => $arrResult) {
if ($arrResult['state'] == 'fulfilled') {
$arrSuccessfulResults[$intKey] = $arrResult;
$incKey = config('task.app_name').'_success_getLatestListPageList:' . date('Y-m-d');
Cache::store('redis')->inc($incKey);
} elseif ($arrResult['state'] == 'rejected') {
$incKey = config('task.app_name').'_errer_getLatestListPageList:' . date('Y-m-d');
Cache::store('redis')->inc($incKey);
if(config('task.server.task_debug_log_open_status')){
// 记录失败原因
$reason = $arrResult['reason'] instanceof \Exception ? $arrResult['reason']->getMessage() : 'Unknown error';
$strUri = $arrFilter[$intKey] ?? '';
\think\facade\Log::error("Request failed for getLatestListPageList , URI: {$strUri }, Reason: {$reason}");
var_dump($strUri .' => '.$arrResult['state']);
}
}
}
foreach ($arrSuccessfulResults as $intKey => $arrResult) {
$Response = $arrResult['value'];
@@ -316,6 +383,7 @@ class Site
$StorageCore = StorageCore::getInstance();
$arrPromises = [];
$arrSuccessfulResults = [];
foreach ($arrFilter as $intKey => $arrItem) {
if (!$StorageCore->has($arrItem['local_path'])) {
@@ -325,9 +393,29 @@ class Site
$arrResults = Promise\Utils::settle($arrPromises)->wait();
$arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
return $arrResult['state'] === 'fulfilled';
});
// $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
// return $arrResult['state'] === 'fulfilled';
// });
foreach ($arrResults as $intKey => $arrResult) {
if ($arrResult['state'] == 'fulfilled') {
$arrSuccessfulResults[$intKey] = $arrResult;
$incKey = config('task.app_name').'_success_downRemoteImgToLocal:' . date('Y-m-d');
Cache::store('redis')->inc($incKey);
} elseif ($arrResult['state'] == 'rejected') {
$incKey = config('task.app_name').'_errer_downRemoteImgToLocal:' . date('Y-m-d');
Cache::store('redis')->inc($incKey);
if(config('task.server.task_debug_log_open_status')){
// 记录失败原因
$reason = $arrResult['reason'] instanceof \Exception ? $arrResult['reason']->getMessage() : 'Unknown error';
\think\facade\Log::error("Request failed for downRemoteImgToLocal , Reason: {$reason}");
var_dump(' => '.$arrResult['state']);
}
}
}
foreach ($arrSuccessfulResults as $intKey => $arrResult) {
$Response = $arrResult['value'];