admin/SEONexus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

debug-novel-task

Browse Source
This commit is contained in:
make
2025-06-19 18:20:37 +08:00
parent 1664db7a3a
commit 321f5eda01
2 changed files with 50 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
code/app/task/crawler/zw/page/Site.php
View File

@@ -8,6 +8,8 @@ use GuzzleHttp\Client;
use GuzzleHttp\Promise;
use storage\StorageCore;
use think\facade\Cache;
use app\task\crawler\zw74\Scheduler as Zw74Scheduler;
use microserver\QueueManage;
/**
* @mixin think\Model
@@ -227,6 +229,14 @@ class Site
continue;
}
// 判断是否包含 $strContent <body class="novel_info">，不包含跳过
if (strpos($strContent, '<body class="novel_info">') === false) {
// 不包含该字符串，跳过
var_dump(mb_substr($strContent, 0, 300));
continue;
}
$NovelPage = $this->getNovelPage($this, $arrFilter[$intKey]['uri']);
$NovelPage->setContent($strContent);
$NovelPage->setData($arrFilter[$intKey]);
@@ -235,6 +245,15 @@ class Site
$this->redisDailyInc(config('task.app_name').'_success_getNovelPageInfo:' . date('Y-m-d'));
} elseif ($arrResult['state'] == 'rejected') {
// 重新推送任务
$arrTask = [
'callback' => [Zw74Scheduler::class, 'getNovelInfo'],
'data' => $arrFilter,
];
$QueueManage = QueueManage::getInstance();
$QueueManage->set($arrTask);
$incKey = config('task.app_name').'_errer_getNovelPageInfo:' . date('Y-m-d');
$this->redisDailyInc($incKey);
@@ -247,22 +266,6 @@ class Site
}
}
return $arrNovelPage;
// foreach ($arrSuccessfulResults as $intKey => $arrResult) {
// $Response = $arrResult['value'];
// $strContent = $Response->getBody()->getContents();
// if ($strContent == '索引文件不存在！') {
// continue;
// }
// $NovelPage = $this->getNovelPage($this, $arrFilter[$intKey]['uri']);
// $NovelPage->setContent($strContent);
// $NovelPage->setData($arrFilter[$intKey]);
// $arrNovelPage[] = $NovelPage;
// }
}
@@ -270,11 +273,12 @@ class Site
* retrieve multiple chapter page model
* 获取章节详情页面
*
* @param string $strUri
* @param array $arrData
* @return ChapterPage[]
*/
public function getChapterPageList(array $arrFilter): array
public function getChapterPageList(array $arrData): array
{
$arrFilter = $arrData['filter'];
/** @var ChapterPage[] */
$arrChapterPage = [];
@@ -313,6 +317,15 @@ class Site
$this->redisDailyInc(config('task.app_name') . '_success_getChapterPageList:' . date('Y-m-d'));
} elseif ($arrResult['state'] === 'rejected') {
// 重新推送任务
$arrTask = [
'callback' => [Zw74Scheduler::class, 'getChapterInfo'],
'data' => $arrData,
];
$QueueManage = QueueManage::getInstance();
$QueueManage->set($arrTask);
$this->redisDailyInc(config('task.app_name') . '_errer_getChapterPageList:' . date('Y-m-d'));
if (config('log.task_debug_log_open_status')) {
@@ -369,6 +382,14 @@ class Site
} elseif ($arrResult['state'] == 'rejected') {
// 重新推送任务
$arrTask = [
'callback' => [Zw74Scheduler::class, 'getLatestChapter'],
'data' => $arrFilter,
];
$QueueManage = QueueManage::getInstance();
$QueueManage->set($arrTask);
$incKey = config('task.app_name').'_errer_getLatestListPageList:' . date('Y-m-d');
$this->redisDailyInc($incKey);
@@ -427,6 +448,14 @@ class Site
} elseif ($arrResult['state'] == 'rejected') {
// 重新推送任务
$arrTask = [
'callback' => [$this::class, 'downRemoteImgToLocal'],
'data' => $arrFilter,
];
$QueueManage = QueueManage::getInstance();
$QueueManage->set($arrTask);
$incKey = config('task.app_name').'_errer_downRemoteImgToLocal:' . date('Y-m-d');
$this->redisDailyInc($incKey);