admin/SEONexus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

debug

Browse Source
This commit is contained in:
Default
2025-04-15 21:16:14 +08:00
parent 694d5fd14c
commit d19ea97481
47 changed files with 907 additions and 769 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
code/app/task/crawler/zw/Scheduler.php
View File

@@ -16,9 +16,9 @@ class Scheduler
{
protected $intSlice = 8;
protected $intStartNSourceId = 7377;
protected $intStartNSourceId = 1;
protected $intEndNSourceId = 7377;
protected $intEndNSourceId = 100;
/**
* Undocumented variable
@@ -102,11 +102,9 @@ class Scheduler
$arrCover = [];
foreach ($arrNovelPageList as $NovelPage) {
$arrNovel = $NovelPage->getNovelInfo();
// $boolResult = $NovelPage->saveNovelInfo();
return;
// $arrNovel = $NovelPage->getNovelInfo();
$boolResult = $NovelPage->saveNovelInfo();
// return;
if ($boolResult == false) {
continue;
@@ -129,7 +127,7 @@ class Scheduler
}
$arrChapterList = $NovelPage->getChapters();
$this->fetchChapter($arrNovel['n_id'], $NovelPage->getData()['page'], $arrChapterList);
$this->fetchChapter($NovelPage->intNId, $NovelPage->getData()['page'], $arrChapterList);
}
if ($arrCover) {
@@ -284,7 +282,7 @@ class Scheduler
foreach ($arrData['filter'] as $intKey => $arrChapterFilter) {
$strUUId = sprintf('%s-chapter-%s-%s', $Site->getSiteCode(), $arrChapterFilter['c_source_id'], $arrChapterFilter['c_page']);
$arrChapter = $ChapterModel->findOne(['c_site_uuid' => $strUUId]);
$arrChapter = $ChapterModel->findOne(['c_source_uuid' => $strUUId]);
if (!empty($arrChapter) && !empty($arrChapter['content'])) {
unset($arrData['filter'][$intKey]);

135
code/app/task/crawler/zw/page/ChapterPage.php
View File

@@ -70,6 +70,22 @@ class ChapterPage extends BasePage
$this->arrChapterInfo['c_page_title'] = $this->getQueryList()->find('title')->text();
$this->arrChapterInfo['c_page_keywords'] = $this->getQueryList()->find('meta[name="keywords"]')->attr('content');
$this->arrChapterInfo['c_page_description'] = $this->getQueryList()->find('meta[name="description"]')->attr('content');
$strPattern = "/lastread\.set\((.*?)\);/";
if (preg_match($strPattern, $this->getContent(), $arrMatches)) {
$strMatch = $arrMatches[1]; // 提取括号内的内容
$arrData = array_map('trim', explode(",", $strMatch));
$arrData = array_map(function ($item) {
return trim($item, "'");
}, $arrData);
$this->arrChapterInfo['c_page_data'] = $arrData;
} else {
$this->arrChapterInfo['c_page_data'] = NULL;
}
}
return $this->arrChapterInfo;
@@ -83,85 +99,78 @@ class ChapterPage extends BasePage
protected $arrChapterInfo;
public function filterOtherFields(&$arrChapter)
/**
* Undocumented function
*
* @param integer $intNId
* @return boolean
*/
public function saveChapter(int $intNId): bool
{
$arrFields = [
'n_id',
'name',
'c_sort_num',
'c_page',
'c_source_id',
'c_site_code',
'c_site_uuid',
'c_content_path',
'created_at',
'updated_at',
$arrPageChapter = $this->getChapterInfo();
if (
empty($arrPageChapter) ||
empty($arrPageChapter['c_page_title']) ||
empty($arrPageChapter['c_page_data']) ||
count($arrPageChapter['c_page_data']) < 5
) {
return false;
}
$strUUId = $this->getSite()->getSiteUUId('chapter-' . $this->arrData['c_source_id'] . '-' . $this->arrData['c_page']);
$arrChapter = [
'n_id' => $intNId,
'c_name' => $arrPageChapter['c_page_data'][3],
'c_sort_num' => $this->arrData['c_sort_num'],
'c_page' => $this->arrData['c_page'],
'c_source_id' => $this->arrData['c_source_id'],
'c_site_code' => $this->getSite()->getSiteCode(),
'c_source_uuid' => $strUUId,
];
foreach ($arrChapter as $strKey => $strVal) {
if (!in_array($strKey, $arrFields)) {
unset($arrChapter[$strKey]);
}
}
}
$arrExistsChapter = $this->getChapterModel()->findOne(['c_source_uuid' => $strUUId], $this->arrOptions);
public function saveChapter(int $intNId): array
{
if ($arrExistsChapter) {
try {
$this->getChapterInfo();
$arrExistsChapter = (array) $arrExistsChapter;
$strUUId = $this->getSite()->getSiteUUId('chapter-' . $this->arrData['c_source_id'] . '-' . $this->arrData['c_page']);
$arrExisting = $this->getChapterModel()->findOne(['c_site_uuid' => $strUUId], $this->arrOptions);
$arrUpdate = [
'updated_at' => new \MongoDB\BSON\UTCDateTime(),
];
if ($arrExisting) {
$this->getChapterModel()->updateOne(
['c_source_uuid' => $strUUId],
['$set' => $arrUpdate],
);
$this->strCContentPath = $arrExistsChapter['c_content_path'];
} else {
$arrExisting = (array) $arrExisting;
$this->arrChapterInfo = array_merge($arrExisting, $this->getChapterInfo());
$arrInsertNovel = $arrChapter;
$this->arrChapterInfo['updated_at'] = new \MongoDB\BSON\UTCDateTime();
$arrUpdate = $this->arrChapterInfo;
$this->filterOtherFields($arrUpdate);
$this->getChapterModel()->updateOne(
['c_site_uuid' => $strUUId],
['$set' => $arrUpdate],
);
} else {
$this->arrChapterInfo['n_id'] = $intNId;
$this->arrChapterInfo['name'] = $this->arrData['name'];
$this->arrChapterInfo['c_sort_num'] = $this->arrData['c_sort_num'];
$this->arrChapterInfo['c_page'] = $this->arrData['c_page'];
$this->arrChapterInfo['c_source_id'] = $this->arrData['c_source_id'];
$this->arrChapterInfo['c_site_code'] = $this->getSite()->getSiteCode();
$this->arrChapterInfo['c_site_uuid'] = $strUUId;
$this->arrChapterInfo['c_content_path'] = '/novel/' . ($intNId % 100) . '/' . $intNId . '/' . $this->arrChapterInfo['c_sort_num'] . '_' . $this->arrChapterInfo['c_page'] . '.txt';
$this->arrChapterInfo['created_at'] = new \MongoDB\BSON\UTCDateTime();
$arrInsert = $this->arrChapterInfo;
$this->filterOtherFields($arrInsert);
$this->getChapterModel()->insert($arrInsert);
$arrInsertNovel['c_content_path'] = '/novel/' . ($intNId % 100) . '/' . $intNId . '/' . $arrChapter['c_sort_num'] . '_' . $arrChapter['c_page'] . '.txt';
$arrInsertNovel['created_at'] = new \MongoDB\BSON\UTCDateTime();
try {
$this->getChapterModel()->insert($arrInsertNovel);
$this->updateNovelHaveChapter($intNId);
}
if (mb_strlen($this->strChapterContent) > 20) {
if (!StorageCore::getInstance()->has($this->arrChapterInfo['c_content_path'])) {
StorageCore::getInstance()->set($this->arrChapterInfo['c_content_path'], $this->strChapterContent);
$this->strCContentPath = $arrInsertNovel['c_content_path'];
} catch (\Exception $e) {
if ($e->getCode() === 11000) {
return $this->saveChapter($intNId);
} else {
throw $e;
}
}
return $this->arrChapterInfo;
} catch (\Exception $e) {
throw new \Exception("Upsert failed: " . $e->getMessage());
}
StorageCore::getInstance()->put($this->strCContentPath, $this->strChapterContent);
return true;
}
public $strCContentPath;
/**
* Undocumented function
*
@@ -179,7 +188,7 @@ class ChapterPage extends BasePage
if (!empty($arrLasteChapter)) {
$arrUpdate['$set']['n_have_chapter'] = 1;
$arrUpdate['$set']['og_novel_latest_chapter_name'] = $arrLasteChapter['name'];
$arrUpdate['$set']['n_latest_chapter_name'] = $arrLasteChapter['c_name'];
$this->getMongoBase()->updateOne('novel', $arrFilter, $arrUpdate);
}
}

246
code/app/task/crawler/zw/page/NovelPage.php
View File

@@ -7,7 +7,7 @@ namespace app\task\crawler\zw\page;
use app\model\CategoryModel;
use app\model\CountersModel;
use app\model\NovelModel;
use storage\StorageCore;
class NovelPage extends BasePage
{
@@ -71,22 +71,31 @@ class NovelPage extends BasePage
{
$strUUId = $this->getSite()->getSiteUUId($intNSourceId);
$arrExisting = $this->getNovelModel()->findOne(['n_site_uuid' => $strUUId], $this->arrOptions);
$arrExistsNovel = $this->getNovelModel()->findOne(['n_source_uuid' => $strUUId], $this->arrOptions);
if ($arrExisting) {
$arrNovelInfo = (array) $arrExisting;
if ($arrExistsNovel) {
$arrNovelInfo = (array) $arrExistsNovel;
} else {
$arrNovelInfo = [
'og_novel_book_name' => $strName,
'og_novel_pin_yin' => zhToPinYin($strName),
'n_id' => CountersModel::getInstance()->getNextSequence('novel'),
'n_name' => $strName,
'n_name_pinyin' => zhToPinYin($strName),
'n_site_code' => $this->getSite()->getSiteCode(),
'n_site_uuid' => $strUUId,
'n_source_uuid' => $strUUId,
'n_source_id' => $intNSourceId,
'n_level' => 1,
'n_have_chapter' => 0,
'n_id' => CountersModel::getInstance()->getNextSequence('novel'),
'created_at' => new \MongoDB\BSON\UTCDateTime(),
];
$this->getNovelModel()->insert($arrNovelInfo);
try {
$this->getNovelModel()->insert($arrNovelInfo);
} catch (\Exception $e) {
if ($e->getCode() === 11000) {
return $this->findOrCreateNovel($intNSourceId, $strName);
} else {
throw $e;
}
}
}
return $arrNovelInfo;
}
@@ -116,24 +125,8 @@ class NovelPage extends BasePage
});
$this->getReletedNovel();
$this->arrNovelInfo['n_related_novel'] = $this->arrRelatedNovel;
$this->arrNovelInfo['n_forge_novel'] = $this->arrForgeNovel;
// $this->arrNovelInfo['og_novel_pin_yin'] = zhToPinYin($this->arrNovelInfo['og_novel_book_name'] ?? '');
// if (!empty($this->arrNovelInfo['og_novel_category'])) {
// $this->arrNovelInfo['category_pinyin'] = CategoryModel::getPinYinByZh($this->arrNovelInfo['og_novel_category']);
// $this->arrNovelInfo['category_sex'] = CategoryModel::checkSex($this->arrNovelInfo['og_novel_category']);
// $this->arrNovelInfo['category_sex_zh'] = CategoryModel::checkSexZh($this->arrNovelInfo['og_novel_category']);
// }
// $this->arrNovelInfo['n_site_code'] = $this->getSite()->getSiteCode();
// $this->arrNovelInfo['n_source_id'] = $this->arrData['n_source_id'];
}
print_r($this->arrNovelInfo);
return $this->arrNovelInfo;
}
@@ -145,29 +138,31 @@ class NovelPage extends BasePage
*/
public function generateCoverPath($strName): string
{
return sprintf('/img/%s/%s.jpg', date('Y/m/d', time() - 24 * 3600 * 14), $strName);
$intTime = time() - mt_rand(1, 2 * 24 * 3600 * 360);
return sprintf('/img/%s/%s.jpg', date('Y/m/d/H', $intTime), $strName);
}
public function filterOtherFields(&$arrNovel)
{
$arrFields = [
'n_id',
'og_novel_book_name',
'og_novel_pin_yin',
'og_novel_status',
'og_novel_author',
'og_description',
'og_novel_category',
'category_pinyin',
'category_sex',
'category_sex_zh',
'n_name',
'n_name_pinyin',
'n_status',
'n_author',
'n_description',
'n_category',
'n_category_pinyin',
'n_category_sex_en',
'n_category_sex_zh',
'n_cover_path',
'og_novel_latest_chapter_name',
'og_novel_update_time',
'n_latest_chapter_name',
'n_update_time',
'n_forge_novel',
'n_related_novel',
'og_site_code',
'n_site_uuid',
'n_site_code',
'n_source_uuid',
'n_source_id',
'n_have_chapter',
'created_at',
@@ -184,65 +179,123 @@ class NovelPage extends BasePage
/**
* Undocumented function
*
* @return void
* @return bool
*/
public function saveNovelInfo()
public function saveNovelInfo(): bool
{
try {
$this->getNovelInfo();
if (!isset($this->arrNovelInfo['og_novel_book_name'])) {
return false;
}
$strUUId = $this->getSite()->getSiteUUId($this->arrData['n_source_id']);
$arrExisting = $this->getNovelModel()->findOne(['n_site_uuid' => $strUUId], $this->arrOptions);
if ($arrExisting) {
$arrExisting = (array) $arrExisting;
$this->arrNovelInfo = array_merge($arrExisting, $this->arrNovelInfo);
if (
!isset($this->arrNovelInfo['n_cover_path']) ||
empty($this->arrNovelInfo['n_cover_path']) ||
!file_exists($this->arrNovelInfo['n_cover_path']) ||
(basename($this->arrNovelInfo['n_cover_path']) == '.jpg')
) {
$this->arrNovelInfo['n_cover_path'] = $this->generateCoverPath($this->arrNovelInfo['og_novel_pin_yin']);
}
$this->arrNovelInfo['updated_at'] = new \MongoDB\BSON\UTCDateTime();
$arrUpdate = $this->arrNovelInfo;
$this->filterOtherFields($arrUpdate);
$this->getNovelModel()->updateOne(
['n_site_uuid' => $strUUId],
['$set' => $arrUpdate],
);
} else {
$this->arrNovelInfo['n_site_uuid'] = $strUUId;
$this->arrNovelInfo['n_id'] = CountersModel::getInstance()->getNextSequence('novel');
$this->arrNovelInfo['n_level'] = 1;
$this->arrNovelInfo['n_have_chapter'] = 0;
$this->arrNovelInfo['n_cover_path'] = $this->generateCoverPath($this->arrNovelInfo['og_novel_pin_yin']);
$arrInsert = $this->arrNovelInfo;
$this->filterOtherFields($arrInsert);
$this->getNovelModel()->insert($arrInsert);
}
return $this->arrNovelInfo;
} catch (\Exception $e) {
throw new \Exception("Upsert failed: " . $e->getMessage());
$arrPageInfo = $this->getNovelInfo();
if (
empty($arrPageInfo) ||
empty($arrPageInfo['og_title']) ||
empty($arrPageInfo['og_image']) ||
empty($arrPageInfo['og_novel_category'])
) {
return false;
}
$arrNovelInfo = [
// 'n_id'=> $arrPageInfo['xxxxx'],
'n_name' => $arrPageInfo['og_title'],
'n_name_pinyin' => zhToPinYin($arrPageInfo['og_title']),
'n_status' => $arrPageInfo['og_novel_status'],
'n_author' => $arrPageInfo['og_novel_author'],
'n_description' => $arrPageInfo['og_description'],
'n_category' => $arrPageInfo['og_novel_category'],
// 'n_category_pinyin' => $arrPageInfo['xxxxx'],
// 'n_category_sex_en' => $arrPageInfo['xxxxx'],
// 'n_category_sex_zh' => $arrPageInfo['xxxxx'],
// 'n_cover_path' => $arrPageInfo['xxxxx'],
'n_latest_chapter_name' => $arrPageInfo['og_novel_latest_chapter_name'],
'n_update_time' => $arrPageInfo['og_novel_update_time'],
'n_forge_novel' => $this->arrForgeNovel,
'n_related_novel' => $this->arrRelatedNovel,
'n_site_code' => $this->getSite()->getSiteCode(),
'n_source_id' => $this->arrData['n_source_id'],
'n_source_uuid' => $this->getSite()->getSiteUUId($this->arrData['n_source_id']),
// 'n_have_chapter' => $arrPageInfo['xxxxx'],
// 'created_at' => $arrPageInfo['xxxxx'],
// 'updated_at' => $arrPageInfo['xxxxx'],
];
$arrExistsNovel = $this->getNovelModel()->findOne(['n_source_uuid' => $arrNovelInfo['n_source_uuid']], $this->arrOptions);
if ($arrExistsNovel) {
$arrExistsNovel = (array) $arrExistsNovel;
$arrUpdate = [
'n_latest_chapter_name' => $arrNovelInfo['n_latest_chapter_name'],
'n_update_time' => $arrNovelInfo['n_update_time'],
'updated_at' => new \MongoDB\BSON\UTCDateTime(),
];
# 如果没有分类，说明这条数据是最初从关联shu那里采集的，需要补全全部信息
if (empty($arrExistsNovel['n_category'])) {
$arrUpdate = array_merge($arrUpdate, $arrNovelInfo);
$arrUpdate['n_related_novel'] = $this->arrRelatedNovel;
$arrUpdate['n_forge_novel'] = $this->arrForgeNovel;
$arrUpdate['n_category_pinyin'] = CategoryModel::getPinYin($this->arrNovelInfo['og_novel_category']);
$arrUpdate['n_category_sex_en'] = CategoryModel::getSexEn($this->arrNovelInfo['og_novel_category']);
$arrUpdate['n_category_sex_zh'] = CategoryModel::getSexZh($this->arrNovelInfo['og_novel_category']);
$arrUpdate['n_related_novel'] = $this->arrRelatedNovel;
$arrUpdate['n_forge_novel'] = $this->arrForgeNovel;
}
# 图片不存在的时候，需要重新生成
if (
!isset($arrExistsNovel['n_cover_path']) ||
empty($arrExistsNovel['n_cover_path']) ||
!StorageCore::getInstance()->has($arrExistsNovel['n_cover_path']) ||
(basename($arrExistsNovel['n_cover_path']) == '.jpg')
) {
$arrUpdate['n_cover_path'] = $this->generateCoverPath($arrNovelInfo['n_name_pinyin']);
$this->strCoverLocalPath = $arrUpdate['n_cover_path'];
} else {
$this->strCoverLocalPath = $arrExistsNovel['n_cover_path'];
}
$this->getNovelModel()->updateOne(
['n_source_uuid' => $arrNovelInfo['n_source_uuid']],
['$set' => $arrUpdate],
);
$this->intNId = $arrExistsNovel['n_id'];
} else {
$arrNovelInfo['n_category_pinyin'] = CategoryModel::getPinYin($this->arrNovelInfo['og_novel_category']);
$arrNovelInfo['n_category_sex_en'] = CategoryModel::getSexEn($this->arrNovelInfo['og_novel_category']);
$arrNovelInfo['n_category_sex_zh'] = CategoryModel::getSexZh($this->arrNovelInfo['og_novel_category']);
$arrNovelInfo['n_id'] = CountersModel::getInstance()->getNextSequence('novel');
$arrNovelInfo['n_level'] = 1;
$arrNovelInfo['n_have_chapter'] = 0;
$arrNovelInfo['n_cover_path'] = $this->generateCoverPath($arrNovelInfo['n_name_pinyin']);
$arrNovelInfo['n_related_novel'] = $this->arrRelatedNovel;
$arrNovelInfo['n_forge_novel'] = $this->arrForgeNovel;
$arrNovelInfo['created_at'] = new \MongoDB\BSON\UTCDateTime();
try {
$this->getNovelModel()->insert($arrNovelInfo);
$this->intNId = $arrNovelInfo['n_id'];
$this->strCoverLocalPath = $arrNovelInfo['n_cover_path'];
} catch (\Exception $e) {
if ($e->getCode() === 11000) {
return $this->saveNovelInfo();
} else {
throw $e;
}
}
}
return true;
}
public $strCoverLocalPath = '';
public $intNId;
/**
* Undocumented function
*
@@ -250,14 +303,9 @@ class NovelPage extends BasePage
*/
public function getDownCoverInfo(): array
{
if (!key_exists('og_image', $this->arrNovelInfo)) {
print_r($this->arrNovelInfo);
echo $this->getContent();
}
return [
'uri' => $this->arrNovelInfo['og_image'],
'local_path' => $this->arrNovelInfo['n_cover_path'],
'local_path' => $this->strCoverLocalPath,
];
}

66
code/app/task/crawler/zw/page/Site.php
View File

@@ -42,7 +42,71 @@ class Site
$this->initClient();
}
/**
* Undocumented function
*/
public function initClient()
{
$arrConfig = [
'http_version' => '1.1', // 强制用 HTTP/1.1，已有，保持
'base_uri' => $this->strDomain, // 基础域名，已有，保持
'http_errors' => false, // 不抛 HTTP 错误，已有，保持
'timeout' => 60, // 从 160 秒改为 60 秒
'connect_timeout' => 10, // 从 30 秒改为 10 秒
'curl' => [
CURLOPT_TCP_KEEPALIVE => 1, // 启用 TCP Keep-Alive，已有，保持
CURLOPT_TCP_KEEPIDLE => 120, // Keep-Alive 空闲时间
CURLOPT_TCP_KEEPINTVL => 60, // Keep-Alive 探测间隔
// CURLOPT_SSL_VERIFYPEER => true, // 验证 SSL，已有，保持
],
'headers' => [
'User-Agent' => 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36',
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'Accept-Language' => 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
'Referer' => $this->strDomain,
'Connection' => 'keep-alive',
'sec-ch-ua' => '"Not/A)Brand";v="8", "Chromium";v="130", "Google Chrome";v="130"',
'sec-ch-ua-mobile' => '?1',
'sec-ch-ua-platform' => '"Android"',
'sec-fetch-dest' => 'document',
'sec-fetch-mode' => 'navigate',
'sec-fetch-site' => 'same-origin',
'sec-fetch-user' => '?1',
'upgrade-insecure-requests' => '1',
],
'pool_size' => 10,
'verify' => false,
];
$arrProxy = config('proxy');
if ($arrProxy['proxy_status']) {
$strCode = $arrProxy['proxy_code'];
$strProxyHost = $arrProxy['proxy_host'];
$strKey = $arrProxy['proxy_key'];
$strSecret = $arrProxy['proxy_secret'];
switch ($strCode) {
case 'XIONGMAO':
$intTime = time();
$strTxt = "orderno=" . $strKey . ",secret=" . $strSecret . ",timestamp=" . $intTime;
$strSign = strtoupper(md5($strTxt));
$strAuth = 'sign=' . $strSign . '&orderno=' . $strKey . '&timestamp=' . $intTime;
$arrConfig['proxy'] = $strProxyHost;
$arrConfig['headers']['Proxy-Authorization'] = $strAuth;
break;
case 'KUAI':
$arrConfig['proxy'] = $strProxyHost;
$arrConfig['curl'][CURLOPT_PROXYAUTH] = CURLAUTH_BASIC;
$arrConfig['curl'][CURLOPT_PROXYUSERPWD] = "$strKey:$strSecret";
break;
}
}
$this->Client = new Client($arrConfig);
}
/**
* Undocumented function