admin/SEONexus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

debug

Browse Source
This commit is contained in:
Default
2025-04-10 02:33:28 +08:00
parent 096e79039e
commit fb2a5a30f8
10 changed files with 295 additions and 124 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
code/app/task/crawler/zw630/Scheduler.php
View File

@@ -37,7 +37,7 @@ class Scheduler
$arrFilter = [];
// for ($intNId = 1; $intNId <= 531208; $intNId++) {
// for ($intNId = 1; $intNId <= 10000; $intNId++) {
for ($intNId = 1; $intNId <= 6; $intNId++) {
for ($intNId = 1; $intNId <= 1; $intNId++) {
$arrFilter[] = [
'n_source_id' => $intNId,
'page' => 0,

15
code/app/task/crawler/zw630/page/BasePage.php
View File

@@ -239,20 +239,5 @@ abstract class BasePage
return $this->Site;
}
/**
* Undocumented variable
*
* @var boolean
*/
protected $boolIsFirstSave = false;
/**
* Undocumented function
*
* @return boolean
*/
public function getIsFirstSave(): bool
{
return $this->boolIsFirstSave;
}
}

88
code/app/task/crawler/zw630/page/ChapterPage.php
View File

@@ -16,6 +16,34 @@ class ChapterPage extends BasePage
*/
protected $strChapterContent = '';
/**
* Undocumented variable
*
* @var ChapterModel
*/
protected $ChapterModel;
/**
* Undocumented function
*
* @return ChapterModel
*/
public function getChapterModel(): ChapterModel
{
if ($this->ChapterModel == NULL) {
$this->ChapterModel = ChapterModel::getInstance();
}
return $this->ChapterModel;
}
static public $arrOptions = [
'typeMap' => [
'root' => 'array',
'document' => 'array',
'array' => 'array',
],
];
/**
* Undocumented function
*
@@ -37,7 +65,6 @@ class ChapterPage extends BasePage
$strContent .= base64_decode($str);
}
// $this->arrChapterInfo['c_content'] = $strContent;
$this->strChapterContent = $strContent;
$this->arrChapterInfo['c_page_title'] = $this->getQueryList()->find('title')->text();
@@ -55,16 +82,38 @@ class ChapterPage extends BasePage
*/
protected $arrChapterInfo;
public function filterOtherFields(&$arrChapter)
{
$arrFields = [
'n_id',
'name',
'c_sort_num',
'c_page',
'c_source_id',
'c_site_code',
'c_site_uuid',
'c_content_path',
'created_at',
'updated_at',
];
foreach ($arrChapter as $strKey => $strVal) {
if (!in_array($strKey, $arrFields)) {
unset($arrChapter[$strKey]);
}
}
}
public function saveChapter(int $intNId): array
{
try {
$this->getChapterInfo();
$strUUId = 'zw630-chapter-' . $this->arrData['c_source_id'] . '-' . $this->arrData['c_page'];
$Col = $this->getMongoBase()->getDb()->selectCollection('chapter');
$arrExisting = $Col->findOne(['c_site_uuid' => $strUUId]);
$strUUId = $this->getSite()->getSiteUUId('chapter-' . $this->arrData['c_source_id'] . '-' . $this->arrData['c_page']);
$arrExisting = $this->getChapterModel()->findOne(['c_site_uuid' => $strUUId], self::$arrOptions);
if ($arrExisting) {
@@ -72,39 +121,42 @@ class ChapterPage extends BasePage
$this->arrChapterInfo = array_merge($arrExisting, $this->getChapterInfo());
$this->arrChapterInfo['updated_at'] = new \MongoDB\BSON\UTCDateTime();
$Col->updateOne(
$arrUpdate = $this->arrChapterInfo;
$this->filterOtherFields($arrUpdate);
$this->getChapterModel()->updateOne(
['c_site_uuid' => $strUUId],
['$set' => $this->arrChapterInfo],
// ['upsert' => false]
['$set' => $arrUpdate],
);
} else {
$this->arrChapterInfo['n_id'] = $intNId;
$this->arrChapterInfo['name'] = $this->arrData['name'];
$this->arrChapterInfo['c_sort_num'] = $this->arrData['c_sort_num'];
$this->arrChapterInfo['c_page'] = $this->arrData['c_page'];
$this->arrChapterInfo['c_source_id'] = $this->arrData['c_source_id'];
$this->arrChapterInfo['c_source_uri'] = $this->arrData['uri'];
$this->arrChapterInfo['c_site_name'] = '恋上你中文';
$this->arrChapterInfo['c_site_code'] = 'zw630';
$this->arrChapterInfo['c_site_code'] = $this->getSite()->getSiteCode();
$this->arrChapterInfo['c_site_uuid'] = $strUUId;
$this->arrChapterInfo['c_content_path'] = '/novel/' . ($intNId % 100) . '/' . $intNId . '/' . $this->arrChapterInfo['c_sort_num'] . '_' . $this->arrChapterInfo['c_page'] . '.txt';
$this->getMongoBase()->insertOne('chapter', $this->arrChapterInfo);
$arrInsert = $this->arrChapterInfo;
$this->filterOtherFields($arrInsert);
$this->getChapterModel()->insert($arrInsert);
$this->updateNovelHaveChapter($intNId);
}
if (mb_strlen($this->strChapterContent) > 20) {
if (!StorageCore::getInstance()->has($this->arrChapterInfo['c_content_path'])) {
StorageCore::getInstance()->set($this->arrChapterInfo['c_content_path'], $this->strChapterContent);
}
$this->boolIsFirstSave = true;
}
return $this->arrChapterInfo;
} catch (\MongoDB\Exception\Exception $e) {
} catch (\Exception $e) {
throw new \Exception("Upsert failed: " . $e->getMessage());
}
}

182
code/app/task/crawler/zw630/page/NovelPage.php
View File

@@ -5,8 +5,8 @@ declare(strict_types=1);
namespace app\task\crawler\zw630\page;
use app\model\CategoryModel;
use ddl\DDLManage;
use QL\QueryList;
use app\model\CountersModel;
use app\model\NovelModel;
class NovelPage extends BasePage
@@ -33,60 +33,65 @@ class NovelPage extends BasePage
*/
protected $arrForgeNovel = [];
/**
* Undocumented variable
*
* @var NovelModel
*/
protected $NovelModel;
/**
* Undocumented function
*
* @return NovelModel
*/
public function getNovelModel(): NovelModel
{
if ($this->NovelModel == NULL) {
$this->NovelModel = NovelModel::getInstance();
}
return $this->NovelModel;
}
static public $arrOptions = [
'typeMap' => [
'root' => 'array',
'document' => 'array',
'array' => 'array',
],
];
/**
* 查询看的小说
*
* @param integer $intNSourceId
* @param string $strName
* @return array
*/
private function findOrCreateNovel(int $intNSourceId, string $strName): array
{
$strUUId = $this->getSite()->getSiteUUId($intNSourceId);
$strUUId = 'zw630-' . $intNSourceId;
$Col = $this->getMongoBase()->getDb()->selectCollection('novel');
$arrExisting = $Col->findOne(['n_site_uuid' => $strUUId]);
$arrExisting = $this->getNovelModel()->findOne(['n_site_uuid' => $strUUId], self::$arrOptions);
if ($arrExisting) {
$arrNovelInfo = (array) $arrExisting;
} else {
$arrNovelInfo['og_novel_book_name'] = $strName;
$arrNovelInfo['og_novel_pin_yin'] = zhToPinYin($strName);
$arrNovelInfo['n_site_name'] = '恋上你中文';
$arrNovelInfo['n_site_code'] = 'zw630';
$arrNovelInfo['n_site_uuid'] = $strUUId;
$arrNovelInfo['n_source_id'] = $intNSourceId;
$arrNovelInfo['n_level'] = 1;
$arrNovelInfo['n_have_chapter'] = 0;
$arrNovelInfo['n_id'] = $this->getMongoBase()->getNextSequence('novel');
$this->getMongoBase()->insertOne('novel', $arrNovelInfo);
$arrNovelInfo = [
'og_novel_book_name' => $strName,
'og_novel_pin_yin' => zhToPinYin($strName),
'n_site_code' => $this->getSite()->getSiteCode(),
'n_site_uuid' => $strUUId,
'n_source_id' => $intNSourceId,
'n_level' => 1,
'n_have_chapter' => 0,
'n_id' => CountersModel::getInstance()->getNextSequence('novel'),
];
$this->getNovelModel()->insert($arrNovelInfo);
}
return $arrNovelInfo;
}
/**
* Find or create a category by name, handling concurrency safely.
*
* @param string $strCategory
* @return array|null The category document (existing or newly created)
*/
public function findOrCreateCategory(string $strCategory)
{
if (empty($strCategory)) {
return false;
}
$Col = $this->getMongoBase()->getDb()->selectCollection('category');
$arrCategory = ['ca_name' => $strCategory];
try {
$Result = $Col->findOneAndUpdate(
$arrCategory,
['$set' => $arrCategory],
[
'upsert' => true,
'returnDocument' => \MongoDB\Operation\FindOneAndUpdate::RETURN_DOCUMENT_AFTER
]
);
return $Result;
} catch (\MongoDB\Driver\Exception\Exception $e) {
return null;
}
}
/**
* Undocumented function
@@ -101,10 +106,6 @@ class NovelPage extends BasePage
$this->arrNovelInfo[$strKey] = $Item->attr('content');
});
// $this->arrNovelInfo['n_page_title'] = $this->getQueryList()->find('title')->text();
// $this->arrNovelInfo['n_page_keywords'] = $this->getQueryList()->find('meta[name="keywords"]')->attr('content');
// $this->arrNovelInfo['n_page_description'] = $this->getQueryList()->find('meta[name="description"]')->attr('content');
$this->getQueryList()->find('.first_txt')->eq(0)->find("a")->map(function ($A) {
$strName = $A->text();
$strUri = $A->attr('href');
@@ -131,12 +132,54 @@ class NovelPage extends BasePage
if ($boolSave && !empty($this->arrNovelInfo)) {
$this->saveNovelInfo();
$this->findOrCreateCategory($this->arrNovelInfo['og_novel_category']);
}
return $this->arrNovelInfo;
}
/**
* Undocumented function
*
* @param [type] $strName
* @return string
*/
public function generateCoverPath($strName): string
{
return '/img/' . date('Y/m/d', time() - 24 * 3600 * 14) . '/' . $strName . '.jpg';
}
public function filterOtherFields(&$arrNovel)
{
$arrFields = [
'n_id',
'og_novel_book_name',
'og_novel_pin_yin',
'og_novel_status',
'og_novel_author',
'og_description',
'og_novel_category',
'category_pinyin',
'category_sex',
'category_sex_zh',
'n_cover_path',
'og_novel_latest_chapter_name',
'og_novel_update_time',
'n_forge_novel',
'n_related_novel',
'og_site_code',
'n_site_uuid',
'n_source_id',
'n_have_chapter',
'created_at',
'updated_at',
];
foreach ($arrNovel as $strKey => $strVal) {
if (!in_array($strKey, $arrFields)) {
unset($arrNovel[$strKey]);
}
}
}
/**
* Undocumented function
*
@@ -145,10 +188,9 @@ class NovelPage extends BasePage
public function saveNovelInfo()
{
try {
$strUUId = 'zw630-' . $this->arrData['n_source_id'];
$Col = $this->getMongoBase()->getDb()->selectCollection('novel');
$strUUId = $this->getSite()->getSiteUUId($this->arrData['n_source_id']);
$arrExisting = $Col->findOne(['n_site_uuid' => $strUUId]);
$arrExisting = $this->getNovelModel()->findOne(['n_site_uuid' => $strUUId], self::$arrOptions);
if ($arrExisting) {
@@ -160,40 +202,44 @@ class NovelPage extends BasePage
}
if (!isset($this->arrNovelInfo['n_cover_path']) || empty($this->arrNovelInfo['n_cover_path'])) {
$this->arrNovelInfo['n_cover_path'] = '/img/' . date('Y/m/d', time() - 24 * 3600 * 14) . '/' . $this->arrNovelInfo['n_id'] . '.jpg';
$this->arrNovelInfo['n_cover_path'] = $this->generateCoverPath($this->arrNovelInfo['og_novel_pin_yin']);
}
$this->arrNovelInfo['updated_at'] = new \MongoDB\BSON\UTCDateTime();
$Col->updateOne(
$arrUpdate = $this->arrNovelInfo;
$this->filterOtherFields($arrUpdate);
$this->getNovelModel()->updateOne(
['n_site_uuid' => $strUUId],
['$set' => $this->arrNovelInfo],
// ['upsert' => false]
['$set' => $arrUpdate],
);
} else {
$this->arrNovelInfo['og_novel_pin_yin'] = zhToPinYin($this->arrNovelInfo['og_novel_book_name'] ?? '');
$this->arrNovelInfo['n_site_name'] = '恋上你中文';
$this->arrNovelInfo['n_site_code'] = 'zw630';
$this->arrNovelInfo['n_site_code'] = $this->getSite()->getSiteCode();
$this->arrNovelInfo['n_site_uuid'] = $strUUId;
$this->arrNovelInfo['n_source_id'] = $this->arrData['n_source_id'];
$this->arrNovelInfo['n_id'] = $this->getMongoBase()->getNextSequence('novel');
$this->arrNovelInfo['n_id'] = CountersModel::getInstance()->getNextSequence('novel');
$this->arrNovelInfo['n_level'] = 1;
$this->arrNovelInfo['n_have_chapter'] = 0;
$this->arrNovelInfo['n_cover_path'] = '/img/' . date('Y/m/d', time() - 24 * 3600 * 14) . '/' . $this->arrNovelInfo['n_id'] . '.jpg';
$this->arrNovelInfo['n_cover_path'] = $this->generateCoverPath($this->arrNovelInfo['og_novel_pin_yin']);
if (!empty($this->arrNovelInfo['og_novel_category'])) {
$this->arrNovelInfo['category_pinyin'] = CategoryModel::getPinYinByZh($this->arrNovelInfo['og_novel_category']);
$this->arrNovelInfo['category_sex'] = CategoryModel::checkSex($this->arrNovelInfo['og_novel_category']);
$this->arrNovelInfo['category_sex_zh'] = CategoryModel::checkSexZh($this->arrNovelInfo['og_novel_category']);
}
$arrInsert = $this->arrNovelInfo;
$this->filterOtherFields($arrInsert);
$this->getMongoBase()->insertOne('novel', $this->arrNovelInfo);
$this->boolIsFirstSave = true;
$this->getNovelModel()->insert($arrInsert);
}
return $this->arrNovelInfo;
} catch (\MongoDB\Exception\Exception $e) {
return $this->arrNovelInfo;
} catch (\Exception $e) {
throw new \Exception("Upsert failed: " . $e->getMessage());
}
}

25
code/app/task/crawler/zw630/page/Site.php
View File

@@ -27,6 +27,29 @@ class Site
*/
protected $strDomain = 'https://www.630zw.com';
protected $strSiteCode = 'zw630';
/**
* Undocumented function
*
* @return string
*/
public function getSiteCode(): string
{
return $this->strSiteCode;
}
/**
* Undocumented function
*
* @param integer|string $strSign
* @return string
*/
public function getSiteUUId(int|string $strSign): string
{
return $this->getSiteCode() . '-' . $strSign;
}
/**
* Undocumented function
*/
@@ -93,7 +116,7 @@ class Site
// 'sec-fetch-mode' => 'navigate',
// 'sec-fetch-site' => 'same-origin',
// 'sec-fetch-user' => '?1',
// PC
'sec-ch-ua' => '"Chromium";v="134", "Not:A-Brand";v="24", "Microsoft Edge";v="134"',