admin/SEONexus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

debug

Browse Source
This commit is contained in:
Default
2025-04-14 15:08:34 +08:00
parent ea2c6119f1
commit f0188c94cc
6 changed files with 130 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
code/.example.env
View File

@@ -42,18 +42,16 @@ QUEUE_EXEC_NUM_1 = 10
QUEUE_NAME_2 = QueueKey02
QUEUE_EXEC_NUM_2 = 10
# MINIO
MINIO_END_POINT = http://127.0.0.1:9200
MINIO_KEY = FeJI6i2j15TK5lNNRuN2
MINIO_SECRET = oTr9cVCYjql4SLehde0cXcFRCaA3QLnw4Yhnjw6N
MINIO_REGION = hk-01
MINIO_BUCKET = test001
# LocalStorage
STORAGE_LOCAL_DIR = /mnt/gluster
# VIEW
VIEW_DISPLAY_CACHE = false
VIEW_TPL_CACHE = false
VIEW_DATA_CACHE = false
# PROXY
PROXY_STATUS = false
PROXY_CODE = XIONGMAO
PROXY_ORDER = DT20250413204710pFcgjMR9
PROXY_SECRET = 583b141edcf65ce94f83591c5cb592ed

22
code/app/task/crawler/zw630/Scheduler.php
View File

@@ -8,12 +8,16 @@ use app\model\ChapterModel;
use app\task\crawler\zw630\page\NovelPage;
use app\task\crawler\zw630\page\Site;
use microserver\QueueManage;
use storage\StorageCore;
/**
* @mixin think\Model
*/
class Scheduler
{
static $intSlice = 3;
/**
* Undocumented variable
*
@@ -33,7 +37,7 @@ class Scheduler
*/
public function crawlFull()
{
$intSliceSize = 3;
$intSliceSize = self::$intSlice;
$arrFilter = [];
for ($intNId = 1; $intNId <= 531208; $intNId++) {
@@ -61,7 +65,7 @@ class Scheduler
*/
public function crawLatest()
{
$intSliceSize = 3;
$intSliceSize = self::$intSlice;
$arrUri = [];
$strUri = '/list/lastupdate_%s_0_0_%s.html';
@@ -108,7 +112,7 @@ class Scheduler
$arrNSourceId = array_unique($arrNSourceId);
$arrNSourceId = array_values($arrNSourceId);
$intSliceSize = 3;
$intSliceSize = self::$intSlice;
$arrFilter = [];
foreach ($arrNSourceId as $intKey => $intNSourceId) {
@@ -122,6 +126,7 @@ class Scheduler
$arrFilter = [];
}
}
if (!empty($arrFilter)) {
$this->fetchNovel($arrFilter);
}
@@ -152,7 +157,7 @@ class Scheduler
*/
public function fetchChapter(int $intNId, int $intPage, array $arrChapterList = [])
{
$intSliceSize = 3;
$intSliceSize = self::$intSlice;
$intPageSize = 100;
$arrTask = [
@@ -210,10 +215,16 @@ class Scheduler
]);
}
$arrCover[] = $NovelPage->getDownCoverInfo();
$arrCoverItem = $NovelPage->getDownCoverInfo();
if (!StorageCore::getInstance()->has($arrCoverItem['local_path'])) {
$arrCover[] = $arrCoverItem;
}
$arrChapterList = $NovelPage->getChapters();
$this->fetchChapter($arrNovel['n_id'], $NovelPage->getData()['page'], $arrChapterList);
}
if ($arrCover) {
$Site->downRemoteImgToLocal($arrCover);
}
@@ -242,7 +253,6 @@ class Scheduler
$arrData['filter'] = array_values($arrData['filter']);
$arrChapterPageList = $Site->getChapterPageList($arrData['filter']);
$arrNextPageFilterList = [];

42
code/app/task/crawler/zw630/page/NovelPage.php
View File

@@ -11,7 +11,6 @@ use app\model\NovelModel;
class NovelPage extends BasePage
{
/**
* Undocumented variable
*
@@ -101,11 +100,14 @@ class NovelPage extends BasePage
public function getNovelInfo($boolSave = true): array
{
if (empty($this->arrNovelInfo)) {
# 获取head字段
$this->getQueryList()->find('meta[property]')->each(function ($Item) {
$strKey = str_replace(':', '_', $Item->attr('property'));
$this->arrNovelInfo[$strKey] = $Item->attr('content');
});
# 获取 kan 和 shu 的关联数据
$this->getQueryList()->find('.first_txt')->eq(0)->find("a")->map(function ($A) {
$strName = $A->text();
$strUri = $A->attr('href');
@@ -125,26 +127,38 @@ class NovelPage extends BasePage
];
}
});
$this->arrNovelInfo['n_related_novel'] = $this->arrRelatedNovel;
$this->arrNovelInfo['n_forge_novel'] = $this->arrForgeNovel;
$this->arrNovelInfo['og_novel_pin_yin'] = zhToPinYin($this->arrNovelInfo['og_novel_book_name'] ?? '');
if (!empty($this->arrNovelInfo['og_novel_category'])) {
$this->arrNovelInfo['category_pinyin'] = CategoryModel::getPinYinByZh($this->arrNovelInfo['og_novel_category']);
$this->arrNovelInfo['category_sex'] = CategoryModel::checkSex($this->arrNovelInfo['og_novel_category']);
$this->arrNovelInfo['category_sex_zh'] = CategoryModel::checkSexZh($this->arrNovelInfo['og_novel_category']);
}
$this->arrNovelInfo['n_site_code'] = $this->getSite()->getSiteCode();
$this->arrNovelInfo['n_source_id'] = $this->arrData['n_source_id'];
}
if ($boolSave && !empty($this->arrNovelInfo)) {
$this->saveNovelInfo();
}
return $this->arrNovelInfo;
}
/**
* Undocumented function
*
* @param [type] $strName
* @param string $strName
* @return string
*/
public function generateCoverPath($strName): string
{
return '/img/' . date('Y/m/d', time() - 24 * 3600 * 14) . '/' . $strName . '.jpg';
return sprintf('/img/%s/%s.jpg', date('Y/m/d', time() - 24 * 3600 * 14), $strName);
}
public function filterOtherFields(&$arrNovel)
@@ -197,17 +211,18 @@ class NovelPage extends BasePage
$arrExisting = (array) $arrExisting;
$this->arrNovelInfo = array_merge($arrExisting, $this->arrNovelInfo);
if (!isset($this->arrNovelInfo['og_novel_pin_yin']) || empty($this->arrNovelInfo['og_novel_pin_yin'])) {
$this->arrNovelInfo['og_novel_pin_yin'] = zhToPinYin($this->arrNovelInfo['og_novel_book_name'] ?? '');
}
if (
!isset($this->arrNovelInfo['n_cover_path']) ||
empty($this->arrNovelInfo['n_cover_path']) ||
!file_exists($this->arrNovelInfo['n_cover_path']) ||
if (!isset($this->arrNovelInfo['n_cover_path']) || empty($this->arrNovelInfo['n_cover_path'])) {
(basename($this->arrNovelInfo['n_cover_path']) == '.jpg')
) {
$this->arrNovelInfo['n_cover_path'] = $this->generateCoverPath($this->arrNovelInfo['og_novel_pin_yin']);
}
$this->arrNovelInfo['updated_at'] = new \MongoDB\BSON\UTCDateTime();
$arrUpdate = $this->arrNovelInfo;
$this->filterOtherFields($arrUpdate);
@@ -216,23 +231,12 @@ class NovelPage extends BasePage
['$set' => $arrUpdate],
);
} else {
$this->arrNovelInfo['og_novel_pin_yin'] = zhToPinYin($this->arrNovelInfo['og_novel_book_name'] ?? '');
$this->arrNovelInfo['n_site_code'] = $this->getSite()->getSiteCode();
$this->arrNovelInfo['n_site_uuid'] = $strUUId;
$this->arrNovelInfo['n_source_id'] = $this->arrData['n_source_id'];
$this->arrNovelInfo['n_id'] = CountersModel::getInstance()->getNextSequence('novel');
$this->arrNovelInfo['n_level'] = 1;
$this->arrNovelInfo['n_have_chapter'] = 0;
$this->arrNovelInfo['n_cover_path'] = $this->generateCoverPath($this->arrNovelInfo['og_novel_pin_yin']);
if (!empty($this->arrNovelInfo['og_novel_category'])) {
$this->arrNovelInfo['category_pinyin'] = CategoryModel::getPinYinByZh($this->arrNovelInfo['og_novel_category']);
$this->arrNovelInfo['category_sex'] = CategoryModel::checkSex($this->arrNovelInfo['og_novel_category']);
$this->arrNovelInfo['category_sex_zh'] = CategoryModel::checkSexZh($this->arrNovelInfo['og_novel_category']);
}
$arrInsert = $this->arrNovelInfo;
$this->filterOtherFields($arrInsert);

29
code/app/task/crawler/zw630/page/Site.php
View File

@@ -66,17 +66,16 @@ class Site
CURLOPT_TCP_KEEPIDLE => 120, // Keep-Alive 空闲时间
CURLOPT_TCP_KEEPINTVL => 60, // Keep-Alive 探测间隔
// CURLOPT_SSL_VERIFYPEER => true, // 验证 SSL，已有，保持
// CURLOPT_SSL_VERIFYHOST => 2, // 确保主机名匹配
],
'headers' => [
'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36 Edg/134.0.0.0',
'User-Agent' => 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36',
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'Accept-Language' => 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
'Referer' => $this->strDomain,
'Connection' => 'keep-alive',
'sec-ch-ua' => '"Chromium";v="134", "Not:A-Brand";v="24", "Microsoft Edge";v="134"',
'sec-ch-ua-mobile' => '?0',
'sec-ch-ua-platform' => '"Windows"',
'sec-ch-ua' => '"Not/A)Brand";v="8", "Chromium";v="130", "Google Chrome";v="130"',
'sec-ch-ua-mobile' => '?1',
'sec-ch-ua-platform' => '"Android"',
'sec-fetch-dest' => 'document',
'sec-fetch-mode' => 'navigate',
'sec-fetch-site' => 'same-origin',
@@ -84,19 +83,23 @@ class Site
'upgrade-insecure-requests' => '1',
],
'pool_size' => 5,
'verify' => false, // 忽略 SSL 证书验证
'verify' => false,
];
$time = time();
$orderno = "DT20250413204710pFcgjMR9";
$secret = "583b141edcf65ce94f83591c5cb592ed";
$arrProxy = config('proxy');
if ($arrProxy['proxy_status']) {
$intTime = time();
$strOrderNo = $arrProxy['proxy_order'];
$strSecret = $arrProxy['proxy_secret'];
$txt = "orderno=" . $orderno . ",secret=" . $secret . ",timestamp=" . $time;
$sign = strtoupper(md5($txt));
$auth = 'sign=' . $sign . '&orderno=' . $orderno . '&timestamp=' . $time;
$strTxt = "orderno=" . $strOrderNo . ",secret=" . $strSecret . ",timestamp=" . $intTime;
$strSign = strtoupper(md5($strTxt));
$strAuth = 'sign=' . $strSign . '&orderno=' . $strOrderNo . '&timestamp=' . $intTime;
$arrConfig['proxy'] = 'http://dtbf.xiongmaodaili.com:8089';
$arrConfig['headers']['Proxy-Authorization'] = $auth;
$arrConfig['headers']['Proxy-Authorization'] = $strAuth;
}
$this->Client = new Client($arrConfig);
}

108
code/composer.lock generated
View File

@@ -302,16 +302,16 @@
},
{
"name": "guzzlehttp/guzzle",
"version": "7.9.3",
"version": "7.9.2",
"source": {
"type": "git",
"url": "https://github.com/guzzle/guzzle.git",
"reference": "7b2f29fe81dc4da0ca0ea7d42107a0845946ea77"
"reference": "d281ed313b989f213357e3be1a179f02196ac99b"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/guzzle/guzzle/zipball/7b2f29fe81dc4da0ca0ea7d42107a0845946ea77",
"reference": "7b2f29fe81dc4da0ca0ea7d42107a0845946ea77",
"url": "https://api.github.com/repos/guzzle/guzzle/zipball/d281ed313b989f213357e3be1a179f02196ac99b",
"reference": "d281ed313b989f213357e3be1a179f02196ac99b",
"shasum": ""
},
"require": {
@@ -408,7 +408,7 @@
],
"support": {
"issues": "https://github.com/guzzle/guzzle/issues",
"source": "https://github.com/guzzle/guzzle/tree/7.9.3"
"source": "https://github.com/guzzle/guzzle/tree/7.9.2"
},
"funding": [
{
@@ -424,20 +424,20 @@
"type": "tidelift"
}
],
"time": "2025-03-27T13:37:11+00:00"
"time": "2024-07-24T11:22:20+00:00"
},
{
"name": "guzzlehttp/promises",
"version": "2.2.0",
"version": "2.0.4",
"source": {
"type": "git",
"url": "https://github.com/guzzle/promises.git",
"reference": "7c69f28996b0a6920945dd20b3857e499d9ca96c"
"reference": "f9c436286ab2892c7db7be8c8da4ef61ccf7b455"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/guzzle/promises/zipball/7c69f28996b0a6920945dd20b3857e499d9ca96c",
"reference": "7c69f28996b0a6920945dd20b3857e499d9ca96c",
"url": "https://api.github.com/repos/guzzle/promises/zipball/f9c436286ab2892c7db7be8c8da4ef61ccf7b455",
"reference": "f9c436286ab2892c7db7be8c8da4ef61ccf7b455",
"shasum": ""
},
"require": {
@@ -491,7 +491,7 @@
],
"support": {
"issues": "https://github.com/guzzle/promises/issues",
"source": "https://github.com/guzzle/promises/tree/2.2.0"
"source": "https://github.com/guzzle/promises/tree/2.0.4"
},
"funding": [
{
@@ -507,20 +507,20 @@
"type": "tidelift"
}
],
"time": "2025-03-27T13:27:01+00:00"
"time": "2024-10-17T10:06:22+00:00"
},
{
"name": "guzzlehttp/psr7",
"version": "2.7.1",
"version": "2.7.0",
"source": {
"type": "git",
"url": "https://github.com/guzzle/psr7.git",
"reference": "c2270caaabe631b3b44c85f99e5a04bbb8060d16"
"reference": "a70f5c95fb43bc83f07c9c948baa0dc1829bf201"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/guzzle/psr7/zipball/c2270caaabe631b3b44c85f99e5a04bbb8060d16",
"reference": "c2270caaabe631b3b44c85f99e5a04bbb8060d16",
"url": "https://api.github.com/repos/guzzle/psr7/zipball/a70f5c95fb43bc83f07c9c948baa0dc1829bf201",
"reference": "a70f5c95fb43bc83f07c9c948baa0dc1829bf201",
"shasum": ""
},
"require": {
@@ -607,7 +607,7 @@
],
"support": {
"issues": "https://github.com/guzzle/psr7/issues",
"source": "https://github.com/guzzle/psr7/tree/2.7.1"
"source": "https://github.com/guzzle/psr7/tree/2.7.0"
},
"funding": [
{
@@ -623,7 +623,7 @@
"type": "tidelift"
}
],
"time": "2025-03-27T12:30:47+00:00"
"time": "2024-07-18T11:15:46+00:00"
},
{
"name": "jaeger/g-http",
@@ -1887,16 +1887,16 @@
},
{
"name": "symfony/cache",
"version": "v7.2.5",
"version": "v7.2.4",
"source": {
"type": "git",
"url": "https://github.com/symfony/cache.git",
"reference": "9131e3018872d2ebb6fe8a9a4d6631273513d42c"
"reference": "d33cd9e14326e14a4145c21e600602eaf17cc9e7"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/cache/zipball/9131e3018872d2ebb6fe8a9a4d6631273513d42c",
"reference": "9131e3018872d2ebb6fe8a9a4d6631273513d42c",
"url": "https://api.github.com/repos/symfony/cache/zipball/d33cd9e14326e14a4145c21e600602eaf17cc9e7",
"reference": "d33cd9e14326e14a4145c21e600602eaf17cc9e7",
"shasum": ""
},
"require": {
@@ -1965,7 +1965,7 @@
"psr6"
],
"support": {
"source": "https://github.com/symfony/cache/tree/v7.2.5"
"source": "https://github.com/symfony/cache/tree/v7.2.4"
},
"funding": [
{
@@ -1981,7 +1981,7 @@
"type": "tidelift"
}
],
"time": "2025-03-25T15:54:33+00:00"
"time": "2025-02-26T09:57:54+00:00"
},
{
"name": "symfony/cache-contracts",
@@ -2374,16 +2374,16 @@
},
{
"name": "symfony/var-exporter",
"version": "v7.2.5",
"version": "v7.2.4",
"source": {
"type": "git",
"url": "https://github.com/symfony/var-exporter.git",
"reference": "c37b301818bd7288715d40de634f05781b686ace"
"reference": "4ede73aa7a73d81506002d2caadbbdad1ef5b69a"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/symfony/var-exporter/zipball/c37b301818bd7288715d40de634f05781b686ace",
"reference": "c37b301818bd7288715d40de634f05781b686ace",
"url": "https://api.github.com/repos/symfony/var-exporter/zipball/4ede73aa7a73d81506002d2caadbbdad1ef5b69a",
"reference": "4ede73aa7a73d81506002d2caadbbdad1ef5b69a",
"shasum": ""
},
"require": {
@@ -2430,7 +2430,7 @@
"serialize"
],
"support": {
"source": "https://github.com/symfony/var-exporter/tree/v7.2.5"
"source": "https://github.com/symfony/var-exporter/tree/v7.2.4"
},
"funding": [
{
@@ -2446,7 +2446,7 @@
"type": "tidelift"
}
],
"time": "2025-03-13T12:21:46+00:00"
"time": "2025-02-13T10:27:23+00:00"
},
{
"name": "topthink/framework",
@@ -2611,16 +2611,16 @@
},
{
"name": "topthink/think-helper",
"version": "v3.1.11",
"version": "v3.1.10",
"source": {
"type": "git",
"url": "https://github.com/top-think/think-helper.git",
"reference": "1d6ada9b9f3130046bf6922fe1bd159c8d88a33c"
"reference": "ac66cc0859a12cd5d73258f50f338aadc95e9b46"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/top-think/think-helper/zipball/1d6ada9b9f3130046bf6922fe1bd159c8d88a33c",
"reference": "1d6ada9b9f3130046bf6922fe1bd159c8d88a33c",
"url": "https://api.github.com/repos/top-think/think-helper/zipball/ac66cc0859a12cd5d73258f50f338aadc95e9b46",
"reference": "ac66cc0859a12cd5d73258f50f338aadc95e9b46",
"shasum": ""
},
"require": {
@@ -2651,9 +2651,9 @@
"description": "The ThinkPHP6 Helper Package",
"support": {
"issues": "https://github.com/top-think/think-helper/issues",
"source": "https://github.com/top-think/think-helper/tree/v3.1.11"
"source": "https://github.com/top-think/think-helper/tree/v3.1.10"
},
"time": "2025-04-07T06:55:59+00:00"
"time": "2024-11-21T01:47:51+00:00"
},
{
"name": "topthink/think-migration",
@@ -2762,16 +2762,16 @@
},
{
"name": "topthink/think-orm",
"version": "v4.0.16",
"version": "v4.0.3",
"source": {
"type": "git",
"url": "https://github.com/top-think/think-orm.git",
"reference": "1692d2c85eaca0b66935ca04d6cefa77b2fce43e"
"reference": "0c951d1eb08b48211d31079a616164e195028e44"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/top-think/think-orm/zipball/1692d2c85eaca0b66935ca04d6cefa77b2fce43e",
"reference": "1692d2c85eaca0b66935ca04d6cefa77b2fce43e",
"url": "https://api.github.com/repos/top-think/think-orm/zipball/0c951d1eb08b48211d31079a616164e195028e44",
"reference": "0c951d1eb08b48211d31079a616164e195028e44",
"shasum": ""
},
"require": {
@@ -2815,9 +2815,9 @@
],
"support": {
"issues": "https://github.com/top-think/think-orm/issues",
"source": "https://github.com/top-think/think-orm/tree/v4.0.16"
"source": "https://github.com/top-think/think-orm/tree/v4.0.3"
},
"time": "2025-04-10T23:40:03+00:00"
"time": "2025-02-13T06:09:45+00:00"
},
{
"name": "topthink/think-template",
@@ -2959,16 +2959,16 @@
},
{
"name": "topthink/think-view",
"version": "v2.0.5",
"version": "v2.0.4",
"source": {
"type": "git",
"url": "https://github.com/top-think/think-view.git",
"reference": "b42009b98199b5a3833d3d6fd18c8a55aa511fad"
"reference": "e6a68e4179baf8d795700d62b710da86d7c3f783"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/top-think/think-view/zipball/b42009b98199b5a3833d3d6fd18c8a55aa511fad",
"reference": "b42009b98199b5a3833d3d6fd18c8a55aa511fad",
"url": "https://api.github.com/repos/top-think/think-view/zipball/e6a68e4179baf8d795700d62b710da86d7c3f783",
"reference": "e6a68e4179baf8d795700d62b710da86d7c3f783",
"shasum": ""
},
"require": {
@@ -2994,24 +2994,24 @@
"description": "thinkphp template driver",
"support": {
"issues": "https://github.com/top-think/think-view/issues",
"source": "https://github.com/top-think/think-view/tree/v2.0.5"
"source": "https://github.com/top-think/think-view/tree/v2.0.4"
},
"time": "2025-03-19T07:04:19+00:00"
"time": "2025-01-10T08:49:57+00:00"
}
],
"packages-dev": [
{
"name": "topthink/think-dumper",
"version": "v1.0.5",
"version": "v1.0.4",
"source": {
"type": "git",
"url": "https://github.com/top-think/think-dumper.git",
"reference": "eba662a1843d5db68059050c530f7d43287289fc"
"reference": "49e6975cec4ffe7997c05fc902f98f1ab1f54150"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/top-think/think-dumper/zipball/eba662a1843d5db68059050c530f7d43287289fc",
"reference": "eba662a1843d5db68059050c530f7d43287289fc",
"url": "https://api.github.com/repos/top-think/think-dumper/zipball/49e6975cec4ffe7997c05fc902f98f1ab1f54150",
"reference": "49e6975cec4ffe7997c05fc902f98f1ab1f54150",
"shasum": ""
},
"require": {
@@ -3044,9 +3044,9 @@
"description": "Dumper extend for thinkphp",
"support": {
"issues": "https://github.com/top-think/think-dumper/issues",
"source": "https://github.com/top-think/think-dumper/tree/v1.0.5"
"source": "https://github.com/top-think/think-dumper/tree/v1.0.4"
},
"time": "2025-03-21T07:15:45+00:00"
"time": "2025-01-17T09:41:46+00:00"
},
{
"name": "topthink/think-trace",

11
code/config/proxy.php Normal file
View File

@@ -0,0 +1,11 @@
<?php
// +----------------------------------------------------------------------
// | 代理设置
// +----------------------------------------------------------------------
return [
'proxy_status' => env('PROXY_STATUS', false),
'proxy_code' => env('PROXY_CODE', ''),
'proxy_order' => env('PROXY_ORDER', ''),
'proxy_secret' => env('PROXY_SECRET', ''),
];