admin/SEONexus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

debug

Browse Source
This commit is contained in:
make
2025-06-17 13:57:41 +08:00
parent 499882f1a4
commit 53b5ec4e54
5 changed files with 117 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
code/.example.env
View File

@@ -3,6 +3,7 @@ APP_DEBUG = true
SHOW_ERROR_MSG = true SHOW_ERROR_MSG = true
DEFAULT_LANG = zh-cn DEFAULT_LANG = zh-cn
DEFAULT_APP_TYPE = video DEFAULT_APP_TYPE = video
APP_NAME = novel
# 数据库 # 数据库
DB_TYPE = mysql DB_TYPE = mysql
@@ -36,6 +37,7 @@ MONGO_DB = novel
# TASK # TASK
TASK_PROCESS_NUM_01 = 4 TASK_PROCESS_NUM_01 = 4
TASK_PROCESS_NUM_02 = 2 TASK_PROCESS_NUM_02 = 2
TASK_DEBUG_LOG_OPEN_STATUS = false
# QUEUEU # QUEUEU
QUEUE_NAME_1 = QueueKey01 QUEUE_NAME_1 = QueueKey01

24
code/app/task/crawler/zw/Scheduler.php
View File

@@ -100,17 +100,11 @@ class Scheduler
$Site = $this->getSite(); $Site = $this->getSite();
$arrNovelPageList = $Site->getNovelPageList($arrFilter); $arrNovelPageList = $Site->getNovelPageList($arrFilter);
// var_dump($arrNovelPageList);
// var_dump('$arrNovelPageList------------------------------------------------------------');
$arrCover = []; $arrCover = [];
foreach ($arrNovelPageList as $NovelPage) { foreach ($arrNovelPageList as $NovelPage) {
// $arrNovel = $NovelPage->getNovelInfo();
// print_r($arrNovel);
// return;
// var_dump('$NovelPage------------------------------------------------------------');
$boolResult = $NovelPage->saveNovelInfo(); $boolResult = $NovelPage->saveNovelInfo();
// var_dump($boolResult);
// return;
if ($boolResult == false) { if ($boolResult == false) {
continue; continue;
@@ -144,6 +138,7 @@ class Scheduler
/** /**
* create a task to craw the latest chapter * create a task to craw the latest chapter
* 创建 每日更新任务
* *
* @return void * @return void
*/ */
@@ -181,7 +176,7 @@ class Scheduler
} }
/** /**
* Undocumented function * 每日更新列表
* *
* @param array $arrData * @param array $arrData
* @return void * @return void
@@ -195,30 +190,21 @@ class Scheduler
foreach ($arrLatestListPageList as $LatestListPage) { foreach ($arrLatestListPageList as $LatestListPage) {
$arrResult = $LatestListPage->getNovelFilter(); $arrResult = $LatestListPage->getNovelFilter();
// var_dump($arrResult);
$arrNSourceId = array_merge($arrNSourceId, $arrResult); $arrNSourceId = array_merge($arrNSourceId, $arrResult);
} }
// var_dump($arrNSourceId);
$arrNSourceId = array_unique($arrNSourceId); $arrNSourceId = array_unique($arrNSourceId);
$arrNSourceId = array_values($arrNSourceId); $arrNSourceId = array_values($arrNSourceId);
$intSliceSize = $this->intSlice; $intSliceSize = $this->intSlice;
$arrFilter = []; $arrFilter = [];
// var_dump($arrNSourceId);
// return ;
foreach ($arrNSourceId as $intKey => $intNSourceId) { foreach ($arrNSourceId as $intKey => $intNSourceId) {
$arrFilter[] = [ $arrFilter[] = [
'n_source_id' => $intNSourceId, 'n_source_id' => $intNSourceId,
'page' => 0, 'page' => 0,
]; ];
// var_dump($intKey);
// var_dump($intSliceSize);
// var_dump($intKey % $intSliceSize);
// var_dump('---------------------------------------------');
if ($intKey % $intSliceSize == 0) { if ($intKey % $intSliceSize == 0) {
// var_dump('$intKey % $intSliceSize == 0');
$this->fetchNovel($arrFilter); $this->fetchNovel($arrFilter);
$arrFilter = []; $arrFilter = [];
} }

120
code/app/task/crawler/zw/page/Site.php
View File

@@ -7,6 +7,7 @@ namespace app\task\crawler\zw\page;
use GuzzleHttp\Client; use GuzzleHttp\Client;
use GuzzleHttp\Promise; use GuzzleHttp\Promise;
use storage\StorageCore; use storage\StorageCore;
use think\facade\Cache;
/** /**
* @mixin think\Model * @mixin think\Model
@@ -178,7 +179,7 @@ class Site
} }
/** /**
* retrieve multiple novel page model * 获取小说详情页面
* *
* @param string $strUri * @param string $strUri
* @return NovelPage[] * @return NovelPage[]
@@ -189,6 +190,8 @@ class Site
/** @var NovelPage[] */ /** @var NovelPage[] */
$arrNovelPage = []; $arrNovelPage = [];
$arrSuccessfulResults = [];
foreach ($arrFilter as $intKey => $arrItem) { foreach ($arrFilter as $intKey => $arrItem) {
$arrFilter[$intKey]['uri'] = $this->getNovelPageUri($arrItem['n_source_id'], $arrItem['page']); $arrFilter[$intKey]['uri'] = $this->getNovelPageUri($arrItem['n_source_id'], $arrItem['page']);
@@ -199,9 +202,30 @@ class Site
$arrResults = Promise\Utils::settle($arrPromises)->wait(); $arrResults = Promise\Utils::settle($arrPromises)->wait();
$arrSuccessfulResults = array_filter($arrResults, function ($arrResult) { // $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
return $arrResult['state'] === 'fulfilled'; // return $arrResult['state'] === 'fulfilled';
}); // });
foreach ($arrResults as $intKey => $arrResult) {
if ($arrResult['state'] == 'fulfilled') {
$arrSuccessfulResults[$intKey] = $arrResult;
$incKey = config('task.app_name').'_success_getNovelPageInfo:' . date('Y-m-d'); // 例如：success_counter:2025-06-15
Cache::store('redis')->inc($incKey);
} elseif ($arrResult['state'] == 'rejected') {
$incKey = config('task.app_name').'_errer_getNovelPageInfo:' . date('Y-m-d'); // 例如：success_counter:2025-06-15
Cache::store('redis')->inc($incKey);
if(config('task.server.task_debug_log_open_status')){
// 记录失败原因
$reason = $arrResult['reason'] instanceof \Exception ? $arrResult['reason']->getMessage() : 'Unknown error';
$strUri = $arrFilter[$intKey]['uri'] ?? '';
\think\facade\Log::error("Request failed for getNovelPageInfo , URI: {$strUri }, Reason: {$reason}");
var_dump($strUri .' => '.$arrResult['state']);
}
}
}
foreach ($arrSuccessfulResults as $intKey => $arrResult) { foreach ($arrSuccessfulResults as $intKey => $arrResult) {
$Response = $arrResult['value']; $Response = $arrResult['value'];
@@ -224,6 +248,7 @@ class Site
/** /**
* retrieve multiple chapter page model * retrieve multiple chapter page model
* 获取章节详情页面
* *
* @param string $strUri * @param string $strUri
* @return ChapterPage[] * @return ChapterPage[]
@@ -235,15 +260,37 @@ class Site
$arrChapterPage = []; $arrChapterPage = [];
$arrPromises = []; $arrPromises = [];
$arrSuccessfulResults = [];
foreach ($arrFilter as $intKey => $arrItem) { foreach ($arrFilter as $intKey => $arrItem) {
$arrPromises[$intKey] = $this->getClient()->getAsync($arrItem['uri']); $arrPromises[$intKey] = $this->getClient()->getAsync($arrItem['uri']);
} }
$arrResults = Promise\Utils::settle($arrPromises)->wait(); $arrResults = Promise\Utils::settle($arrPromises)->wait();
$arrSuccessfulResults = array_filter($arrResults, function ($arrResult) { // $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
return $arrResult['state'] === 'fulfilled'; // return $arrResult['state'] === 'fulfilled';
}); // });
foreach ($arrResults as $intKey => $arrResult) {
if ($arrResult['state'] == 'fulfilled') {
$arrSuccessfulResults[$intKey] = $arrResult;
$incKey = config('task.app_name').'_success_getChapterPageList:' . date('Y-m-d'); // 例如：success_counter:2025-06-15
Cache::store('redis')->inc($incKey);
} elseif ($arrResult['state'] == 'rejected') {
$incKey = config('task.app_name').'_errer_getChapterPageList:' . date('Y-m-d'); // 例如：success_counter:2025-06-15
Cache::store('redis')->inc($incKey);
if(config('task.server.task_debug_log_open_status')){
// 记录失败原因
$reason = $arrResult['reason'] instanceof \Exception ? $arrResult['reason']->getMessage() : 'Unknown error';
$strUri = $arrFilter[$intKey]['uri'] ?? '';
\think\facade\Log::error("Request failed for getNovelChapterInfo , URI: {$strUri }, Reason: {$reason}");
var_dump($strUri .' => '.$arrResult['state']);
}
}
}
foreach ($arrSuccessfulResults as $intKey => $arrResult) { foreach ($arrSuccessfulResults as $intKey => $arrResult) {
$Response = $arrResult['value']; $Response = $arrResult['value'];
@@ -264,6 +311,7 @@ class Site
/** /**
* retrieve multiple chapter page model * retrieve multiple chapter page model
* 每日更新列表
* *
* @param string $strUri * @param string $strUri
* @return LatestListPage[] * @return LatestListPage[]
@@ -275,19 +323,38 @@ class Site
$arrLatestListPage = []; $arrLatestListPage = [];
$arrPromises = []; $arrPromises = [];
$arrSuccessfulResults = [];
foreach ($arrFilter as $intKey => $strUri) { foreach ($arrFilter as $intKey => $strUri) {
$arrPromises[$intKey] = $this->getClient()->getAsync($strUri); $arrPromises[$intKey] = $this->getClient()->getAsync($strUri);
} }
$arrResults = Promise\Utils::settle($arrPromises)->wait(); $arrResults = Promise\Utils::settle($arrPromises)->wait();
// var_dump($strUri);
// var_dump($arrResults);
$arrSuccessfulResults = array_filter($arrResults, function ($arrResult) { // $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
// var_dump($arrResult); // return $arrResult['state'] === 'fulfilled';
return $arrResult['state'] === 'fulfilled'; // });
}); foreach ($arrResults as $intKey => $arrResult) {
if ($arrResult['state'] == 'fulfilled') {
$arrSuccessfulResults[$intKey] = $arrResult;
$incKey = config('task.app_name').'_success_getLatestListPageList:' . date('Y-m-d');
Cache::store('redis')->inc($incKey);
} elseif ($arrResult['state'] == 'rejected') {
$incKey = config('task.app_name').'_errer_getLatestListPageList:' . date('Y-m-d');
Cache::store('redis')->inc($incKey);
if(config('task.server.task_debug_log_open_status')){
// 记录失败原因
$reason = $arrResult['reason'] instanceof \Exception ? $arrResult['reason']->getMessage() : 'Unknown error';
$strUri = $arrFilter[$intKey] ?? '';
\think\facade\Log::error("Request failed for getLatestListPageList , URI: {$strUri }, Reason: {$reason}");
var_dump($strUri .' => '.$arrResult['state']);
}
}
}
foreach ($arrSuccessfulResults as $intKey => $arrResult) { foreach ($arrSuccessfulResults as $intKey => $arrResult) {
$Response = $arrResult['value']; $Response = $arrResult['value'];
@@ -316,6 +383,7 @@ class Site
$StorageCore = StorageCore::getInstance(); $StorageCore = StorageCore::getInstance();
$arrPromises = []; $arrPromises = [];
$arrSuccessfulResults = [];
foreach ($arrFilter as $intKey => $arrItem) { foreach ($arrFilter as $intKey => $arrItem) {
if (!$StorageCore->has($arrItem['local_path'])) { if (!$StorageCore->has($arrItem['local_path'])) {
@@ -325,9 +393,29 @@ class Site
$arrResults = Promise\Utils::settle($arrPromises)->wait(); $arrResults = Promise\Utils::settle($arrPromises)->wait();
$arrSuccessfulResults = array_filter($arrResults, function ($arrResult) { // $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
return $arrResult['state'] === 'fulfilled'; // return $arrResult['state'] === 'fulfilled';
}); // });
foreach ($arrResults as $intKey => $arrResult) {
if ($arrResult['state'] == 'fulfilled') {
$arrSuccessfulResults[$intKey] = $arrResult;
$incKey = config('task.app_name').'_success_downRemoteImgToLocal:' . date('Y-m-d');
Cache::store('redis')->inc($incKey);
} elseif ($arrResult['state'] == 'rejected') {
$incKey = config('task.app_name').'_errer_downRemoteImgToLocal:' . date('Y-m-d');
Cache::store('redis')->inc($incKey);
if(config('task.server.task_debug_log_open_status')){
// 记录失败原因
$reason = $arrResult['reason'] instanceof \Exception ? $arrResult['reason']->getMessage() : 'Unknown error';
\think\facade\Log::error("Request failed for downRemoteImgToLocal , Reason: {$reason}");
var_dump(' => '.$arrResult['state']);
}
}
}
foreach ($arrSuccessfulResults as $intKey => $arrResult) { foreach ($arrSuccessfulResults as $intKey => $arrResult) {
$Response = $arrResult['value']; $Response = $arrResult['value'];

4
code/config/app.php
View File

@@ -13,6 +13,10 @@ return [
// 默认应用 // 默认应用
'default_app' => 'home', 'default_app' => 'home',
// 应用名称
'app_name' => env('APP_NAME', 'novel'),
// 默认应用类型 video/novel // 默认应用类型 video/novel
'default_app_type' => env('DEFAULT_APP_TYPE', 'video'), 'default_app_type' => env('DEFAULT_APP_TYPE', 'video'),

2
code/config/task.php
View File

@@ -35,6 +35,8 @@ return [
'log_file' => runtime_path() . 'task.log', 'log_file' => runtime_path() . 'task.log',
'pid_file' => runtime_path() . 'task.pid', 'pid_file' => runtime_path() . 'task.pid',
'task_worker_num' => 0, 'task_worker_num' => 0,
'task_debug_log_open_status' => env('TASK_DEBUG_LOG_OPEN_STATUS', false),
], ],
# Process Pool, NUM 为设置的任务组进程数 , 根据自身服务器情况增减 # Process Pool, NUM 为设置的任务组进程数 , 根据自身服务器情况增减