admin/SEONexus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

debug

Browse Source
This commit is contained in:
make
2025-06-18 13:54:03 +08:00
parent ae48449a35
commit d87ae4c27a
2 changed files with 97 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

131
code/app/task/crawler/zw/Scheduler.php
View File

@@ -218,9 +218,9 @@ class Scheduler
/**
* Undocumented function
*
* @param integer $intNId
* @param integer $intPage
* @param array $arrChapterList
* @param integer $intNId 小说id
* @param integer $intPage 分页
* @param array $arrChapterList 章节列表
* @return void
*/
public function fetchChapter(int $intNId, int $intPage, array $arrChapterList = [])
@@ -276,6 +276,7 @@ class Scheduler
/**
* Undocumented function
* 获取每个章节详情
*
* @param array $arrData
* @return void
@@ -284,63 +285,107 @@ class Scheduler
{
$Site = $this->getSite();
$ChapterModel = ChapterModel::getInstance();
#FIXED 这里增加临时代码，将章节任务拆分为一个章节用一个进程请求，防止代理同时请求多个异常问题。
$arrTask = [
'callback' => [static::class, 'getChapterInfo'],
'data' => [
'n_id' => $arrData['n_id'],
'filter' => [],
]
];
// 过滤已有章节内容
$arrData['filter'] = array_filter($arrData['filter'], function ($filter) use ($Site, $ChapterModel) {
$strUUId = sprintf('%s-chapter-%s-%s', $Site->getSiteCode(), $filter['c_source_id'], $filter['c_page']);
$chapter = $ChapterModel->findOne(['c_source_uuid' => $strUUId]);
return empty($chapter) || empty($chapter['c_content']);
});
foreach ($arrData['filter'] as $intKey => $arrChapterFilter) {
$strUUId = sprintf('%s-chapter-%s-%s', $Site->getSiteCode(), $arrChapterFilter['c_source_id'], $arrChapterFilter['c_page']);
$arrChapter = $ChapterModel->findOne(['c_source_uuid' => $strUUId]);
$arrData['filter'] = array_values($arrData['filter']); // 重新索引
if (!empty($arrChapter) && !empty($arrChapter['content'])) {
unset($arrData['filter'][$intKey]);
} else {
#FIXED 这里增加临时代码，将章节任务拆分为一个章节用一个进程请求，防止代理同时请求多个异常问题。
$arrFilter = [$arrChapterFilter];
$arrTask['data']['filter'] = $arrFilter;
$this->QueueManage->set($arrTask);
}
// 如果全部都已存在，直接返回
if (empty($arrData['filter'])) {
return;
}
if (count($arrData['filter']) > 1 || count($arrData['filter']) <= 0) {
#FIXED 这里增加临时代码，将章节任务拆分为一个章节用一个进程请求，防止代理同时请求多个异常问题。
return false;
}
$arrData['filter'] = array_values($arrData['filter']);
$arrChapterPageList = $Site->getChapterPageList($arrData['filter']);
$arrNextPageFilterList = [];
foreach ($arrChapterPageList as $ChapterPage) {
$ChapterPage->saveChapter($arrData['n_id']);
$arrNextPageFilter = $ChapterPage->getNextPage();
if (!empty($arrNextPageFilter)) {
$arrNextPageFilterList[] = $arrNextPageFilter;
$next = $ChapterPage->getNextPage();
if (!empty($next)) {
$arrNextPageFilterList[] = $next;
}
}
if (!empty($arrNextPageFilterList)) {
$arrTask = $arrData;
$arrTask['filter'] = $arrNextPageFilterList;
$arrTask = [
$this->QueueManage->set([
'callback' => [static::class, 'getChapterInfo'],
'data' => $arrTask
];
$this->QueueManage->set($arrTask);
'data' => [
'n_id' => $arrData['n_id'],
'filter' => $arrNextPageFilterList
]
]);
}
}
// public function getChapterInfo(array $arrData)
// {
// $Site = $this->getSite();
// $ChapterModel = ChapterModel::getInstance();
// #FIXED 这里增加临时代码，将章节任务拆分为一个章节用一个进程请求，防止代理同时请求多个异常问题。
// $arrTask = [
// 'callback' => [static::class, 'getChapterInfo'],
// 'data' => [
// 'n_id' => $arrData['n_id'],
// 'filter' => [],
// ]
// ];
// foreach ($arrData['filter'] as $intKey => $arrChapterFilter) {
// $strUUId = sprintf('%s-chapter-%s-%s', $Site->getSiteCode(), $arrChapterFilter['c_source_id'], $arrChapterFilter['c_page']);
// $arrChapter = $ChapterModel->findOne(['c_source_uuid' => $strUUId]);
// // 如果数据存在，且 内容不为空，删除任务
// if (!empty($arrChapter) && !empty($arrChapter['c_content'])) {
// unset($arrData['filter'][$intKey]);
// } else {
// #FIXED 这里增加临时代码，将章节任务拆分为一个章节用一个进程请求，防止代理同时请求多个异常问题。
// $arrFilter = [$arrChapterFilter];
// $arrTask['data']['filter'] = $arrFilter;
// $this->QueueManage->set($arrTask);
// }
// }
// if (count($arrData['filter']) > 1 || count($arrData['filter']) <= 0) {
// #FIXED 这里增加临时代码，将章节任务拆分为一个章节用一个进程请求，防止代理同时请求多个异常问题。
// return false;
// }
// $arrData['filter'] = array_values($arrData['filter']);
// $arrChapterPageList = $Site->getChapterPageList($arrData['filter']);
// $arrNextPageFilterList = [];
// foreach ($arrChapterPageList as $ChapterPage) {
// $ChapterPage->saveChapter($arrData['n_id']);
// $arrNextPageFilter = $ChapterPage->getNextPage();
// if (!empty($arrNextPageFilter)) {
// $arrNextPageFilterList[] = $arrNextPageFilter;
// }
// }
// if (!empty($arrNextPageFilterList)) {
// $arrTask = $arrData;
// $arrTask['filter'] = $arrNextPageFilterList;
// $arrTask = [
// 'callback' => [static::class, 'getChapterInfo'],
// 'data' => $arrTask
// ];
// $this->QueueManage->set($arrTask);
// }
// }
}

9
code/app/task/crawler/zw/page/ChapterPage.php
View File

@@ -71,6 +71,7 @@ class ChapterPage extends BasePage
$this->arrChapterInfo['c_page_keywords'] = $this->getQueryList()->find('meta[name="keywords"]')->attr('content');
$this->arrChapterInfo['c_page_description'] = $this->getQueryList()->find('meta[name="description"]')->attr('content');
// var lastread=new LastRead();lastread.set('529620', '184472933', '身为妖族的我，却在人族成家立业', '第321章 一人一兽联手对敌', '0','眯着眼','0');
$strPattern = "/lastread\.set\((.*?)\);/";
if (preg_match($strPattern, $this->getContent(), $arrMatches)) {
@@ -117,6 +118,14 @@ class ChapterPage extends BasePage
) {
$incKey = config('task.app_name').'_empty_saveChapter:' . date('Y-m-d');
$this->redisDailyInc($incKey);
if(config('log.task_debug_log_open_status')){
// 记录失败原因
$strTitle = $arrPageChapter['c_page_title'] ?? '';
$strPageData = $arrPageChapter['c_page_data'] ?? '';
\think\facade\Log::error("Request empty_saveChapter , strTitle: {$strTitle }, strPageData: {$strPageData}");
var_dump($strTitle .' ---- '.$strPageData);
}
return false;
}