admin/SEONexus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

debug-novel-task

Browse Source
This commit is contained in:
make
2025-06-19 18:20:37 +08:00
parent 1664db7a3a
commit 321f5eda01
2 changed files with 50 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
code/app/task/crawler/zw/Scheduler.php
View File

@@ -43,7 +43,7 @@ class Scheduler
*/ */
public function init() public function init()
{ {
$this->intSlice = 2; $this->intSlice = 1;
} }
/** /**
@@ -323,69 +323,6 @@ class Scheduler
]); ]);
} }
} }
// public function getChapterInfo(array $arrData)
// {
// $Site = $this->getSite();
// $ChapterModel = ChapterModel::getInstance();
// #FIXED 这里增加临时代码，将章节任务拆分为一个章节用一个进程请求，防止代理同时请求多个异常问题。
// $arrTask = [
// 'callback' => [static::class, 'getChapterInfo'],
// 'data' => [
// 'n_id' => $arrData['n_id'],
// 'filter' => [],
// ]
// ];
// foreach ($arrData['filter'] as $intKey => $arrChapterFilter) {
// $strUUId = sprintf('%s-chapter-%s-%s', $Site->getSiteCode(), $arrChapterFilter['c_source_id'], $arrChapterFilter['c_page']);
// $arrChapter = $ChapterModel->findOne(['c_source_uuid' => $strUUId]);
// // 如果数据存在，且 内容不为空，删除任务
// if (!empty($arrChapter) && !empty($arrChapter['c_content'])) {
// unset($arrData['filter'][$intKey]);
// } else {
// #FIXED 这里增加临时代码，将章节任务拆分为一个章节用一个进程请求，防止代理同时请求多个异常问题。
// $arrFilter = [$arrChapterFilter];
// $arrTask['data']['filter'] = $arrFilter;
// $this->QueueManage->set($arrTask);
// }
// }
// if (count($arrData['filter']) > 1 || count($arrData['filter']) <= 0) {
// #FIXED 这里增加临时代码，将章节任务拆分为一个章节用一个进程请求，防止代理同时请求多个异常问题。
// return false;
// }
// $arrData['filter'] = array_values($arrData['filter']);
// $arrChapterPageList = $Site->getChapterPageList($arrData['filter']);
// $arrNextPageFilterList = [];
// foreach ($arrChapterPageList as $ChapterPage) {
// $ChapterPage->saveChapter($arrData['n_id']);
// $arrNextPageFilter = $ChapterPage->getNextPage();
// if (!empty($arrNextPageFilter)) {
// $arrNextPageFilterList[] = $arrNextPageFilter;
// }
// }
// if (!empty($arrNextPageFilterList)) {
// $arrTask = $arrData;
// $arrTask['filter'] = $arrNextPageFilterList;
// $arrTask = [
// 'callback' => [static::class, 'getChapterInfo'],
// 'data' => $arrTask
// ];
// $this->QueueManage->set($arrTask);
// }
// }
} }

65
code/app/task/crawler/zw/page/Site.php
View File

@@ -8,6 +8,8 @@ use GuzzleHttp\Client;
use GuzzleHttp\Promise; use GuzzleHttp\Promise;
use storage\StorageCore; use storage\StorageCore;
use think\facade\Cache; use think\facade\Cache;
use app\task\crawler\zw74\Scheduler as Zw74Scheduler;
use microserver\QueueManage;
/** /**
* @mixin think\Model * @mixin think\Model
@@ -227,6 +229,14 @@ class Site
continue; continue;
} }
// 判断是否包含 $strContent <body class="novel_info">，不包含跳过
if (strpos($strContent, '<body class="novel_info">') === false) {
// 不包含该字符串，跳过
var_dump(mb_substr($strContent, 0, 300));
continue;
}
$NovelPage = $this->getNovelPage($this, $arrFilter[$intKey]['uri']); $NovelPage = $this->getNovelPage($this, $arrFilter[$intKey]['uri']);
$NovelPage->setContent($strContent); $NovelPage->setContent($strContent);
$NovelPage->setData($arrFilter[$intKey]); $NovelPage->setData($arrFilter[$intKey]);
@@ -235,6 +245,15 @@ class Site
$this->redisDailyInc(config('task.app_name').'_success_getNovelPageInfo:' . date('Y-m-d')); $this->redisDailyInc(config('task.app_name').'_success_getNovelPageInfo:' . date('Y-m-d'));
} elseif ($arrResult['state'] == 'rejected') { } elseif ($arrResult['state'] == 'rejected') {
// 重新推送任务
$arrTask = [
'callback' => [Zw74Scheduler::class, 'getNovelInfo'],
'data' => $arrFilter,
];
$QueueManage = QueueManage::getInstance();
$QueueManage->set($arrTask);
$incKey = config('task.app_name').'_errer_getNovelPageInfo:' . date('Y-m-d'); $incKey = config('task.app_name').'_errer_getNovelPageInfo:' . date('Y-m-d');
$this->redisDailyInc($incKey); $this->redisDailyInc($incKey);
@@ -247,22 +266,6 @@ class Site
} }
} }
return $arrNovelPage; return $arrNovelPage;
// foreach ($arrSuccessfulResults as $intKey => $arrResult) {
// $Response = $arrResult['value'];
// $strContent = $Response->getBody()->getContents();
// if ($strContent == '索引文件不存在！') {
// continue;
// }
// $NovelPage = $this->getNovelPage($this, $arrFilter[$intKey]['uri']);
// $NovelPage->setContent($strContent);
// $NovelPage->setData($arrFilter[$intKey]);
// $arrNovelPage[] = $NovelPage;
// }
} }
@@ -270,11 +273,12 @@ class Site
* retrieve multiple chapter page model * retrieve multiple chapter page model
* 获取章节详情页面 * 获取章节详情页面
* *
* @param string $strUri * @param array $arrData
* @return ChapterPage[] * @return ChapterPage[]
*/ */
public function getChapterPageList(array $arrFilter): array public function getChapterPageList(array $arrData): array
{ {
$arrFilter = $arrData['filter'];
/** @var ChapterPage[] */ /** @var ChapterPage[] */
$arrChapterPage = []; $arrChapterPage = [];
@@ -313,6 +317,15 @@ class Site
$this->redisDailyInc(config('task.app_name') . '_success_getChapterPageList:' . date('Y-m-d')); $this->redisDailyInc(config('task.app_name') . '_success_getChapterPageList:' . date('Y-m-d'));
} elseif ($arrResult['state'] === 'rejected') { } elseif ($arrResult['state'] === 'rejected') {
// 重新推送任务
$arrTask = [
'callback' => [Zw74Scheduler::class, 'getChapterInfo'],
'data' => $arrData,
];
$QueueManage = QueueManage::getInstance();
$QueueManage->set($arrTask);
$this->redisDailyInc(config('task.app_name') . '_errer_getChapterPageList:' . date('Y-m-d')); $this->redisDailyInc(config('task.app_name') . '_errer_getChapterPageList:' . date('Y-m-d'));
if (config('log.task_debug_log_open_status')) { if (config('log.task_debug_log_open_status')) {
@@ -369,6 +382,14 @@ class Site
} elseif ($arrResult['state'] == 'rejected') { } elseif ($arrResult['state'] == 'rejected') {
// 重新推送任务
$arrTask = [
'callback' => [Zw74Scheduler::class, 'getLatestChapter'],
'data' => $arrFilter,
];
$QueueManage = QueueManage::getInstance();
$QueueManage->set($arrTask);
$incKey = config('task.app_name').'_errer_getLatestListPageList:' . date('Y-m-d'); $incKey = config('task.app_name').'_errer_getLatestListPageList:' . date('Y-m-d');
$this->redisDailyInc($incKey); $this->redisDailyInc($incKey);
@@ -427,6 +448,14 @@ class Site
} elseif ($arrResult['state'] == 'rejected') { } elseif ($arrResult['state'] == 'rejected') {
// 重新推送任务
$arrTask = [
'callback' => [$this::class, 'downRemoteImgToLocal'],
'data' => $arrFilter,
];
$QueueManage = QueueManage::getInstance();
$QueueManage->set($arrTask);
$incKey = config('task.app_name').'_errer_downRemoteImgToLocal:' . date('Y-m-d'); $incKey = config('task.app_name').'_errer_downRemoteImgToLocal:' . date('Y-m-d');
$this->redisDailyInc($incKey); $this->redisDailyInc($incKey);