admin/SEONexus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

debug

Browse Source
This commit is contained in:
Default
2025-04-14 15:08:34 +08:00
parent ea2c6119f1
commit f0188c94cc
6 changed files with 130 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
code/app/task/crawler/zw630/Scheduler.php
View File

@@ -8,12 +8,16 @@ use app\model\ChapterModel;
use app\task\crawler\zw630\page\NovelPage;
use app\task\crawler\zw630\page\Site;
use microserver\QueueManage;
use storage\StorageCore;
/**
* @mixin think\Model
*/
class Scheduler
{
static $intSlice = 3;
/**
* Undocumented variable
*
@@ -33,7 +37,7 @@ class Scheduler
*/
public function crawlFull()
{
$intSliceSize = 3;
$intSliceSize = self::$intSlice;
$arrFilter = [];
for ($intNId = 1; $intNId <= 531208; $intNId++) {
@@ -61,7 +65,7 @@ class Scheduler
*/
public function crawLatest()
{
$intSliceSize = 3;
$intSliceSize = self::$intSlice;
$arrUri = [];
$strUri = '/list/lastupdate_%s_0_0_%s.html';
@@ -108,7 +112,7 @@ class Scheduler
$arrNSourceId = array_unique($arrNSourceId);
$arrNSourceId = array_values($arrNSourceId);
$intSliceSize = 3;
$intSliceSize = self::$intSlice;
$arrFilter = [];
foreach ($arrNSourceId as $intKey => $intNSourceId) {
@@ -122,6 +126,7 @@ class Scheduler
$arrFilter = [];
}
}
if (!empty($arrFilter)) {
$this->fetchNovel($arrFilter);
}
@@ -152,7 +157,7 @@ class Scheduler
*/
public function fetchChapter(int $intNId, int $intPage, array $arrChapterList = [])
{
$intSliceSize = 3;
$intSliceSize = self::$intSlice;
$intPageSize = 100;
$arrTask = [
@@ -210,10 +215,16 @@ class Scheduler
]);
}
$arrCover[] = $NovelPage->getDownCoverInfo();
$arrCoverItem = $NovelPage->getDownCoverInfo();
if (!StorageCore::getInstance()->has($arrCoverItem['local_path'])) {
$arrCover[] = $arrCoverItem;
}
$arrChapterList = $NovelPage->getChapters();
$this->fetchChapter($arrNovel['n_id'], $NovelPage->getData()['page'], $arrChapterList);
}
if ($arrCover) {
$Site->downRemoteImgToLocal($arrCover);
}
@@ -242,7 +253,6 @@ class Scheduler
$arrData['filter'] = array_values($arrData['filter']);
$arrChapterPageList = $Site->getChapterPageList($arrData['filter']);
$arrNextPageFilterList = [];

44
code/app/task/crawler/zw630/page/NovelPage.php
View File

@@ -11,7 +11,6 @@ use app\model\NovelModel;
class NovelPage extends BasePage
{
/**
* Undocumented variable
*
@@ -101,11 +100,14 @@ class NovelPage extends BasePage
public function getNovelInfo($boolSave = true): array
{
if (empty($this->arrNovelInfo)) {
# 获取head字段
$this->getQueryList()->find('meta[property]')->each(function ($Item) {
$strKey = str_replace(':', '_', $Item->attr('property'));
$this->arrNovelInfo[$strKey] = $Item->attr('content');
});
# 获取 kan 和 shu 的关联数据
$this->getQueryList()->find('.first_txt')->eq(0)->find("a")->map(function ($A) {
$strName = $A->text();
$strUri = $A->attr('href');
@@ -125,26 +127,38 @@ class NovelPage extends BasePage
];
}
});
$this->arrNovelInfo['n_related_novel'] = $this->arrRelatedNovel;
$this->arrNovelInfo['n_forge_novel'] = $this->arrForgeNovel;
}
$this->arrNovelInfo['og_novel_pin_yin'] = zhToPinYin($this->arrNovelInfo['og_novel_book_name'] ?? '');
if (!empty($this->arrNovelInfo['og_novel_category'])) {
$this->arrNovelInfo['category_pinyin'] = CategoryModel::getPinYinByZh($this->arrNovelInfo['og_novel_category']);
$this->arrNovelInfo['category_sex'] = CategoryModel::checkSex($this->arrNovelInfo['og_novel_category']);
$this->arrNovelInfo['category_sex_zh'] = CategoryModel::checkSexZh($this->arrNovelInfo['og_novel_category']);
}
$this->arrNovelInfo['n_site_code'] = $this->getSite()->getSiteCode();
$this->arrNovelInfo['n_source_id'] = $this->arrData['n_source_id'];
}
if ($boolSave && !empty($this->arrNovelInfo)) {
$this->saveNovelInfo();
}
return $this->arrNovelInfo;
}
/**
* Undocumented function
*
* @param [type] $strName
* @param string $strName
* @return string
*/
public function generateCoverPath($strName): string
{
return '/img/' . date('Y/m/d', time() - 24 * 3600 * 14) . '/' . $strName . '.jpg';
return sprintf('/img/%s/%s.jpg', date('Y/m/d', time() - 24 * 3600 * 14), $strName);
}
public function filterOtherFields(&$arrNovel)
@@ -197,17 +211,18 @@ class NovelPage extends BasePage
$arrExisting = (array) $arrExisting;
$this->arrNovelInfo = array_merge($arrExisting, $this->arrNovelInfo);
if (!isset($this->arrNovelInfo['og_novel_pin_yin']) || empty($this->arrNovelInfo['og_novel_pin_yin'])) {
$this->arrNovelInfo['og_novel_pin_yin'] = zhToPinYin($this->arrNovelInfo['og_novel_book_name'] ?? '');
}
if (
!isset($this->arrNovelInfo['n_cover_path']) ||
empty($this->arrNovelInfo['n_cover_path']) ||
!file_exists($this->arrNovelInfo['n_cover_path']) ||
if (!isset($this->arrNovelInfo['n_cover_path']) || empty($this->arrNovelInfo['n_cover_path'])) {
(basename($this->arrNovelInfo['n_cover_path']) == '.jpg')
) {
$this->arrNovelInfo['n_cover_path'] = $this->generateCoverPath($this->arrNovelInfo['og_novel_pin_yin']);
}
$this->arrNovelInfo['updated_at'] = new \MongoDB\BSON\UTCDateTime();
$arrUpdate = $this->arrNovelInfo;
$this->filterOtherFields($arrUpdate);
@@ -216,23 +231,12 @@ class NovelPage extends BasePage
['$set' => $arrUpdate],
);
} else {
$this->arrNovelInfo['og_novel_pin_yin'] = zhToPinYin($this->arrNovelInfo['og_novel_book_name'] ?? '');
$this->arrNovelInfo['n_site_code'] = $this->getSite()->getSiteCode();
$this->arrNovelInfo['n_site_uuid'] = $strUUId;
$this->arrNovelInfo['n_source_id'] = $this->arrData['n_source_id'];
$this->arrNovelInfo['n_id'] = CountersModel::getInstance()->getNextSequence('novel');
$this->arrNovelInfo['n_level'] = 1;
$this->arrNovelInfo['n_have_chapter'] = 0;
$this->arrNovelInfo['n_cover_path'] = $this->generateCoverPath($this->arrNovelInfo['og_novel_pin_yin']);
if (!empty($this->arrNovelInfo['og_novel_category'])) {
$this->arrNovelInfo['category_pinyin'] = CategoryModel::getPinYinByZh($this->arrNovelInfo['og_novel_category']);
$this->arrNovelInfo['category_sex'] = CategoryModel::checkSex($this->arrNovelInfo['og_novel_category']);
$this->arrNovelInfo['category_sex_zh'] = CategoryModel::checkSexZh($this->arrNovelInfo['og_novel_category']);
}
$arrInsert = $this->arrNovelInfo;
$this->filterOtherFields($arrInsert);

33
code/app/task/crawler/zw630/page/Site.php
View File

@@ -66,17 +66,16 @@ class Site
CURLOPT_TCP_KEEPIDLE => 120, // Keep-Alive 空闲时间
CURLOPT_TCP_KEEPINTVL => 60, // Keep-Alive 探测间隔
// CURLOPT_SSL_VERIFYPEER => true, // 验证 SSL，已有，保持
// CURLOPT_SSL_VERIFYHOST => 2, // 确保主机名匹配
],
'headers' => [
'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36 Edg/134.0.0.0',
'User-Agent' => 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36',
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'Accept-Language' => 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
'Referer' => $this->strDomain,
'Connection' => 'keep-alive',
'sec-ch-ua' => '"Chromium";v="134", "Not:A-Brand";v="24", "Microsoft Edge";v="134"',
'sec-ch-ua-mobile' => '?0',
'sec-ch-ua-platform' => '"Windows"',
'sec-ch-ua' => '"Not/A)Brand";v="8", "Chromium";v="130", "Google Chrome";v="130"',
'sec-ch-ua-mobile' => '?1',
'sec-ch-ua-platform' => '"Android"',
'sec-fetch-dest' => 'document',
'sec-fetch-mode' => 'navigate',
'sec-fetch-site' => 'same-origin',
@@ -84,19 +83,23 @@ class Site
'upgrade-insecure-requests' => '1',
],
'pool_size' => 5,
'verify' => false, // 忽略 SSL 证书验证
'verify' => false,
];
$time = time();
$orderno = "DT20250413204710pFcgjMR9";
$secret = "583b141edcf65ce94f83591c5cb592ed";
$arrProxy = config('proxy');
if ($arrProxy['proxy_status']) {
$intTime = time();
$strOrderNo = $arrProxy['proxy_order'];
$strSecret = $arrProxy['proxy_secret'];
$txt = "orderno=" . $orderno . ",secret=" . $secret . ",timestamp=" . $time;
$sign = strtoupper(md5($txt));
$auth = 'sign=' . $sign . '&orderno=' . $orderno . '&timestamp=' . $time;
$strTxt = "orderno=" . $strOrderNo . ",secret=" . $strSecret . ",timestamp=" . $intTime;
$strSign = strtoupper(md5($strTxt));
$strAuth = 'sign=' . $strSign . '&orderno=' . $strOrderNo . '&timestamp=' . $intTime;
$arrConfig['proxy'] = 'http://dtbf.xiongmaodaili.com:8089';
$arrConfig['headers']['Proxy-Authorization'] = $strAuth;
}
$arrConfig['proxy'] = 'http://dtbf.xiongmaodaili.com:8089';
$arrConfig['headers']['Proxy-Authorization'] = $auth;
$this->Client = new Client($arrConfig);
}
@@ -148,7 +151,7 @@ class Site
}
$arrResults = Promise\Utils::settle($arrPromises)->wait();
$arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
return $arrResult['state'] === 'fulfilled';
});