admin/SEONexus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

debug

Browse Source
This commit is contained in:
Default
2025-04-10 02:33:28 +08:00
parent 096e79039e
commit fb2a5a30f8
10 changed files with 295 additions and 124 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
code/app/home/controller/Novel.php
View File

@@ -244,7 +244,7 @@ class Novel extends BaseController
// echo "<pre>";
// var_dump($arrNewsUpdateNovel);exit;
// TODO n_page_description 改成 og_description
// TODO og_description 改成 og_description
// 更新值
ConverterMovel::setVal([
'intPage' => 1,
@@ -428,7 +428,7 @@ class Novel extends BaseController
'strNovelCategoryName' => $arrNovel['og_novel_category'],
'strNovelSex' => $arrNovel['category_sex'] == 'man' ? '男生' : '女生',
'strNovelStatus' => $arrNovel['og_novel_status'],
'strNovelDescription' => $arrNovel['n_page_description'],
'strNovelDescription' => $arrNovel['og_description'],
]);
$strTitleTemplate = $Request->DomainModel->getFomartSubject('NOVEL@GETNOVELINFO@TITLE');
@@ -441,7 +441,7 @@ class Novel extends BaseController
// js使用到
$arrNovelJs = $arrNovel;
unset($arrNovelJs['n_related_novel'], $arrNovelJs['n_forge_novel'], $arrNovelJs['n_page_description']);
unset($arrNovelJs['n_related_novel'], $arrNovelJs['n_forge_novel'], $arrNovelJs['og_description']);
View::assign([
'arrRelateveNovel' => $arrRelateveNovel,
@@ -523,7 +523,7 @@ class Novel extends BaseController
'strNovelCategoryName' => $arrNovel['og_novel_category'],
'strNovelSex' => $arrNovel['category_sex'] == 'man' ? '男生' : '女生',
'strNovelStatus' => $arrNovel['og_novel_status'],
'strNovelDescription' => $arrNovel['n_page_description'],
'strNovelDescription' => $arrNovel['og_description'],
]);
$strTitleTemplate = $Request->DomainModel->getFomartSubject(getFomartSubjectKey(__METHOD__) . '@TITLE');
@@ -603,7 +603,7 @@ class Novel extends BaseController
'strNovelAuthor' => $arrNovel['og_novel_author'],
'strNovelCategoryName' => $arrNovel['og_novel_category'],
'strNovelStatus' => $arrNovel['og_novel_status'],
'strNovelDescription' => $arrNovel['n_page_description'],
'strNovelDescription' => $arrNovel['og_description'],
'strNovelChapterName' => $arrChapter['name'] ?? '',
'strNovelChapterSort' => $arrChapter['c_sort_num'] ?? '',
]);
@@ -683,7 +683,7 @@ class Novel extends BaseController
'strNovelAuthor' => $arrNovel['og_novel_author'],
'strNovelCategoryName' => $arrNovel['og_novel_category'],
'strNovelStatus' => $arrNovel['og_novel_status'],
'strNovelDescription' => $arrNovel['n_page_description'],
'strNovelDescription' => $arrNovel['og_description'],
'strNovelChapterName' => $arrChapter['name'],
'strNovelChapterSort' => $arrChapter['c_sort_num'],
]);
@@ -766,7 +766,7 @@ class Novel extends BaseController
'strNovelAuthor' => $arrNovel['og_novel_author'],
'strNovelCategoryName' => $arrNovel['og_novel_category'],
'strNovelStatus' => $arrNovel['og_novel_status'],
'strNovelDescription' => $arrNovel['n_page_description'],
'strNovelDescription' => $arrNovel['og_description'],
]);
$strTitleTemplate = $Request->DomainModel->getFomartSubject(getFomartSubjectKey(__METHOD__) . '@TITLE');

5
code/app/model/ChapterModel.php
View File

@@ -35,12 +35,7 @@ class ChapterModel extends MongoModel
],
'projection' => [
'_id' => 0,
'c_page_title' => 0,
'c_page_keywords' => 0,
'c_page_description' => 0,
'c_source_uri' => 0,
'c_source_id' => 0,
'c_site_name' => 0,
'c_site_code' => 0,
'c_site_uuid' => 0,

63
code/app/model/CountersModel.php Normal file
View File

@@ -0,0 +1,63 @@
<?php
declare(strict_types=1);
namespace app\model;
/**
* @mixin think\Model
*/
class CountersModel extends MongoModel
{
/**
* Undocumented variable
*
* @var self
*/
static public $obj;
static public function getInstance(): self
{
if (self::$obj == NULL) {
self::$obj = new self;
}
return self::$obj;
}
/**
* Undocumented variable
*
* @var string
*/
protected $strCollection = 'counters';
static public $arrOptions = [
'typeMap' => [
'root' => 'array',
'document' => 'array',
'array' => 'array',
],
'projection' => [
'_id' => 0,
],
];
/**
* 获取下一个自增序列值
* @param string $strSequenceName 序列名称
* @return int 自增 ID
*/
public function getNextSequence(string $strSequenceName): int
{
$Result = $this->getCol()->findOneAndUpdate(
['_id' => $strSequenceName],
['$inc' => ['sequence_value' => 1]],
[
'upsert' => true,
'returnDocument' => \MongoDB\Operation\FindOneAndUpdate::RETURN_DOCUMENT_AFTER
]
);
return $Result->sequence_value ?? 1;
}
}

15
code/app/model/MongoModel.php
View File

@@ -129,4 +129,19 @@ abstract class MongoModel
$Result = $this->getCol()->insertOne($arrData);
return $Result->getInsertedId();
}
/**
* 更新单条数据
* @param string $strColName 集合名称
* @param array $arrFilter 查询条件
* @param array $arrUpdate 更新内容
* @param array $arrOptions 更新选项
* @return int 受影响的文档数
*/
public function updateOne(array $arrFilter, array $arrUpdate, array $arrOptions = []): int
{
$Result = $this->getCol()->updateOne($arrFilter, $arrUpdate, $arrOptions);
return $Result->getModifiedCount();
}
}

10
code/app/model/NovelModel.php
View File

@@ -43,17 +43,9 @@ class NovelModel extends MongoModel
],
'projection' => [
'_id' => 0,
'og_title' => 0,
'n_site_name' => 0,
'n_site_code' => 0,
'n_site_uuid' => 0,
'n_source_id' => 0,
'n_page_title' => 0,
'og_image' => 0,
'og_novel_latest_chapter_url' => 0,
'og_novel_read_url' => 0,
'og_type' => 0,
'og_url' => 0,
],
];
@@ -171,7 +163,7 @@ class NovelModel extends MongoModel
$arrOptions = self::$arrOptions;
}
$arrNovel = $this->getOne($arrFilter, $arrOptions);
$arrNovel = $this->getOne($arrFilter, $arrOptions);
if (!empty($arrNovel)) {
applyToKeys($arrNovel, ['created_at', 'updated_at'], 'formatMongoDate');

2
code/app/task/crawler/zw630/Scheduler.php
View File

@@ -37,7 +37,7 @@ class Scheduler
$arrFilter = [];
// for ($intNId = 1; $intNId <= 531208; $intNId++) {
// for ($intNId = 1; $intNId <= 10000; $intNId++) {
for ($intNId = 1; $intNId <= 6; $intNId++) {
for ($intNId = 1; $intNId <= 1; $intNId++) {
$arrFilter[] = [
'n_source_id' => $intNId,
'page' => 0,

15
code/app/task/crawler/zw630/page/BasePage.php
View File

@@ -239,20 +239,5 @@ abstract class BasePage
return $this->Site;
}
/**
* Undocumented variable
*
* @var boolean
*/
protected $boolIsFirstSave = false;
/**
* Undocumented function
*
* @return boolean
*/
public function getIsFirstSave(): bool
{
return $this->boolIsFirstSave;
}
}

88
code/app/task/crawler/zw630/page/ChapterPage.php
View File

@@ -16,6 +16,34 @@ class ChapterPage extends BasePage
*/
protected $strChapterContent = '';
/**
* Undocumented variable
*
* @var ChapterModel
*/
protected $ChapterModel;
/**
* Undocumented function
*
* @return ChapterModel
*/
public function getChapterModel(): ChapterModel
{
if ($this->ChapterModel == NULL) {
$this->ChapterModel = ChapterModel::getInstance();
}
return $this->ChapterModel;
}
static public $arrOptions = [
'typeMap' => [
'root' => 'array',
'document' => 'array',
'array' => 'array',
],
];
/**
* Undocumented function
*
@@ -37,7 +65,6 @@ class ChapterPage extends BasePage
$strContent .= base64_decode($str);
}
// $this->arrChapterInfo['c_content'] = $strContent;
$this->strChapterContent = $strContent;
$this->arrChapterInfo['c_page_title'] = $this->getQueryList()->find('title')->text();
@@ -55,16 +82,38 @@ class ChapterPage extends BasePage
*/
protected $arrChapterInfo;
public function filterOtherFields(&$arrChapter)
{
$arrFields = [
'n_id',
'name',
'c_sort_num',
'c_page',
'c_source_id',
'c_site_code',
'c_site_uuid',
'c_content_path',
'created_at',
'updated_at',
];
foreach ($arrChapter as $strKey => $strVal) {
if (!in_array($strKey, $arrFields)) {
unset($arrChapter[$strKey]);
}
}
}
public function saveChapter(int $intNId): array
{
try {
$this->getChapterInfo();
$strUUId = 'zw630-chapter-' . $this->arrData['c_source_id'] . '-' . $this->arrData['c_page'];
$Col = $this->getMongoBase()->getDb()->selectCollection('chapter');
$arrExisting = $Col->findOne(['c_site_uuid' => $strUUId]);
$strUUId = $this->getSite()->getSiteUUId('chapter-' . $this->arrData['c_source_id'] . '-' . $this->arrData['c_page']);
$arrExisting = $this->getChapterModel()->findOne(['c_site_uuid' => $strUUId], self::$arrOptions);
if ($arrExisting) {
@@ -72,39 +121,42 @@ class ChapterPage extends BasePage
$this->arrChapterInfo = array_merge($arrExisting, $this->getChapterInfo());
$this->arrChapterInfo['updated_at'] = new \MongoDB\BSON\UTCDateTime();
$Col->updateOne(
$arrUpdate = $this->arrChapterInfo;
$this->filterOtherFields($arrUpdate);
$this->getChapterModel()->updateOne(
['c_site_uuid' => $strUUId],
['$set' => $this->arrChapterInfo],
// ['upsert' => false]
['$set' => $arrUpdate],
);
} else {
$this->arrChapterInfo['n_id'] = $intNId;
$this->arrChapterInfo['name'] = $this->arrData['name'];
$this->arrChapterInfo['c_sort_num'] = $this->arrData['c_sort_num'];
$this->arrChapterInfo['c_page'] = $this->arrData['c_page'];
$this->arrChapterInfo['c_source_id'] = $this->arrData['c_source_id'];
$this->arrChapterInfo['c_source_uri'] = $this->arrData['uri'];
$this->arrChapterInfo['c_site_name'] = '恋上你中文';
$this->arrChapterInfo['c_site_code'] = 'zw630';
$this->arrChapterInfo['c_site_code'] = $this->getSite()->getSiteCode();
$this->arrChapterInfo['c_site_uuid'] = $strUUId;
$this->arrChapterInfo['c_content_path'] = '/novel/' . ($intNId % 100) . '/' . $intNId . '/' . $this->arrChapterInfo['c_sort_num'] . '_' . $this->arrChapterInfo['c_page'] . '.txt';
$this->getMongoBase()->insertOne('chapter', $this->arrChapterInfo);
$arrInsert = $this->arrChapterInfo;
$this->filterOtherFields($arrInsert);
$this->getChapterModel()->insert($arrInsert);
$this->updateNovelHaveChapter($intNId);
}
if (mb_strlen($this->strChapterContent) > 20) {
if (!StorageCore::getInstance()->has($this->arrChapterInfo['c_content_path'])) {
StorageCore::getInstance()->set($this->arrChapterInfo['c_content_path'], $this->strChapterContent);
}
$this->boolIsFirstSave = true;
}
return $this->arrChapterInfo;
} catch (\MongoDB\Exception\Exception $e) {
} catch (\Exception $e) {
throw new \Exception("Upsert failed: " . $e->getMessage());
}
}

180
code/app/task/crawler/zw630/page/NovelPage.php
View File

@@ -5,8 +5,8 @@ declare(strict_types=1);
namespace app\task\crawler\zw630\page;
use app\model\CategoryModel;
use ddl\DDLManage;
use QL\QueryList;
use app\model\CountersModel;
use app\model\NovelModel;
class NovelPage extends BasePage
@@ -33,60 +33,65 @@ class NovelPage extends BasePage
*/
protected $arrForgeNovel = [];
/**
* Undocumented variable
*
* @var NovelModel
*/
protected $NovelModel;
/**
* Undocumented function
*
* @return NovelModel
*/
public function getNovelModel(): NovelModel
{
if ($this->NovelModel == NULL) {
$this->NovelModel = NovelModel::getInstance();
}
return $this->NovelModel;
}
static public $arrOptions = [
'typeMap' => [
'root' => 'array',
'document' => 'array',
'array' => 'array',
],
];
/**
* 查询看的小说
*
* @param integer $intNSourceId
* @param string $strName
* @return array
*/
private function findOrCreateNovel(int $intNSourceId, string $strName): array
{
$strUUId = $this->getSite()->getSiteUUId($intNSourceId);
$strUUId = 'zw630-' . $intNSourceId;
$Col = $this->getMongoBase()->getDb()->selectCollection('novel');
$arrExisting = $Col->findOne(['n_site_uuid' => $strUUId]);
$arrExisting = $this->getNovelModel()->findOne(['n_site_uuid' => $strUUId], self::$arrOptions);
if ($arrExisting) {
$arrNovelInfo = (array) $arrExisting;
} else {
$arrNovelInfo['og_novel_book_name'] = $strName;
$arrNovelInfo['og_novel_pin_yin'] = zhToPinYin($strName);
$arrNovelInfo['n_site_name'] = '恋上你中文';
$arrNovelInfo['n_site_code'] = 'zw630';
$arrNovelInfo['n_site_uuid'] = $strUUId;
$arrNovelInfo['n_source_id'] = $intNSourceId;
$arrNovelInfo['n_level'] = 1;
$arrNovelInfo['n_have_chapter'] = 0;
$arrNovelInfo['n_id'] = $this->getMongoBase()->getNextSequence('novel');
$this->getMongoBase()->insertOne('novel', $arrNovelInfo);
$arrNovelInfo = [
'og_novel_book_name' => $strName,
'og_novel_pin_yin' => zhToPinYin($strName),
'n_site_code' => $this->getSite()->getSiteCode(),
'n_site_uuid' => $strUUId,
'n_source_id' => $intNSourceId,
'n_level' => 1,
'n_have_chapter' => 0,
'n_id' => CountersModel::getInstance()->getNextSequence('novel'),
];
$this->getNovelModel()->insert($arrNovelInfo);
}
return $arrNovelInfo;
}
/**
* Find or create a category by name, handling concurrency safely.
*
* @param string $strCategory
* @return array|null The category document (existing or newly created)
*/
public function findOrCreateCategory(string $strCategory)
{
if (empty($strCategory)) {
return false;
}
$Col = $this->getMongoBase()->getDb()->selectCollection('category');
$arrCategory = ['ca_name' => $strCategory];
try {
$Result = $Col->findOneAndUpdate(
$arrCategory,
['$set' => $arrCategory],
[
'upsert' => true,
'returnDocument' => \MongoDB\Operation\FindOneAndUpdate::RETURN_DOCUMENT_AFTER
]
);
return $Result;
} catch (\MongoDB\Driver\Exception\Exception $e) {
return null;
}
}
/**
* Undocumented function
@@ -101,10 +106,6 @@ class NovelPage extends BasePage
$this->arrNovelInfo[$strKey] = $Item->attr('content');
});
// $this->arrNovelInfo['n_page_title'] = $this->getQueryList()->find('title')->text();
// $this->arrNovelInfo['n_page_keywords'] = $this->getQueryList()->find('meta[name="keywords"]')->attr('content');
// $this->arrNovelInfo['n_page_description'] = $this->getQueryList()->find('meta[name="description"]')->attr('content');
$this->getQueryList()->find('.first_txt')->eq(0)->find("a")->map(function ($A) {
$strName = $A->text();
$strUri = $A->attr('href');
@@ -131,12 +132,54 @@ class NovelPage extends BasePage
if ($boolSave && !empty($this->arrNovelInfo)) {
$this->saveNovelInfo();
$this->findOrCreateCategory($this->arrNovelInfo['og_novel_category']);
}
return $this->arrNovelInfo;
}
/**
* Undocumented function
*
* @param [type] $strName
* @return string
*/
public function generateCoverPath($strName): string
{
return '/img/' . date('Y/m/d', time() - 24 * 3600 * 14) . '/' . $strName . '.jpg';
}
public function filterOtherFields(&$arrNovel)
{
$arrFields = [
'n_id',
'og_novel_book_name',
'og_novel_pin_yin',
'og_novel_status',
'og_novel_author',
'og_description',
'og_novel_category',
'category_pinyin',
'category_sex',
'category_sex_zh',
'n_cover_path',
'og_novel_latest_chapter_name',
'og_novel_update_time',
'n_forge_novel',
'n_related_novel',
'og_site_code',
'n_site_uuid',
'n_source_id',
'n_have_chapter',
'created_at',
'updated_at',
];
foreach ($arrNovel as $strKey => $strVal) {
if (!in_array($strKey, $arrFields)) {
unset($arrNovel[$strKey]);
}
}
}
/**
* Undocumented function
*
@@ -145,10 +188,9 @@ class NovelPage extends BasePage
public function saveNovelInfo()
{
try {
$strUUId = 'zw630-' . $this->arrData['n_source_id'];
$Col = $this->getMongoBase()->getDb()->selectCollection('novel');
$strUUId = $this->getSite()->getSiteUUId($this->arrData['n_source_id']);
$arrExisting = $Col->findOne(['n_site_uuid' => $strUUId]);
$arrExisting = $this->getNovelModel()->findOne(['n_site_uuid' => $strUUId], self::$arrOptions);
if ($arrExisting) {
@@ -160,25 +202,29 @@ class NovelPage extends BasePage
}
if (!isset($this->arrNovelInfo['n_cover_path']) || empty($this->arrNovelInfo['n_cover_path'])) {
$this->arrNovelInfo['n_cover_path'] = '/img/' . date('Y/m/d', time() - 24 * 3600 * 14) . '/' . $this->arrNovelInfo['n_id'] . '.jpg';
$this->arrNovelInfo['n_cover_path'] = $this->generateCoverPath($this->arrNovelInfo['og_novel_pin_yin']);
}
$this->arrNovelInfo['updated_at'] = new \MongoDB\BSON\UTCDateTime();
$Col->updateOne(
$arrUpdate = $this->arrNovelInfo;
$this->filterOtherFields($arrUpdate);
$this->getNovelModel()->updateOne(
['n_site_uuid' => $strUUId],
['$set' => $this->arrNovelInfo],
// ['upsert' => false]
['$set' => $arrUpdate],
);
} else {
$this->arrNovelInfo['og_novel_pin_yin'] = zhToPinYin($this->arrNovelInfo['og_novel_book_name'] ?? '');
$this->arrNovelInfo['n_site_name'] = '恋上你中文';
$this->arrNovelInfo['n_site_code'] = 'zw630';
$this->arrNovelInfo['n_site_code'] = $this->getSite()->getSiteCode();
$this->arrNovelInfo['n_site_uuid'] = $strUUId;
$this->arrNovelInfo['n_source_id'] = $this->arrData['n_source_id'];
$this->arrNovelInfo['n_id'] = $this->getMongoBase()->getNextSequence('novel');
$this->arrNovelInfo['n_id'] = CountersModel::getInstance()->getNextSequence('novel');
$this->arrNovelInfo['n_level'] = 1;
$this->arrNovelInfo['n_have_chapter'] = 0;
$this->arrNovelInfo['n_cover_path'] = '/img/' . date('Y/m/d', time() - 24 * 3600 * 14) . '/' . $this->arrNovelInfo['n_id'] . '.jpg';
$this->arrNovelInfo['n_cover_path'] = $this->generateCoverPath($this->arrNovelInfo['og_novel_pin_yin']);
if (!empty($this->arrNovelInfo['og_novel_category'])) {
@@ -187,13 +233,13 @@ class NovelPage extends BasePage
$this->arrNovelInfo['category_sex_zh'] = CategoryModel::checkSexZh($this->arrNovelInfo['og_novel_category']);
}
$arrInsert = $this->arrNovelInfo;
$this->filterOtherFields($arrInsert);
$this->getMongoBase()->insertOne('novel', $this->arrNovelInfo);
$this->boolIsFirstSave = true;
$this->getNovelModel()->insert($arrInsert);
}
return $this->arrNovelInfo;
} catch (\MongoDB\Exception\Exception $e) {
return $this->arrNovelInfo;
} catch (\Exception $e) {
throw new \Exception("Upsert failed: " . $e->getMessage());
}
}

23
code/app/task/crawler/zw630/page/Site.php
View File

@@ -27,6 +27,29 @@ class Site
*/
protected $strDomain = 'https://www.630zw.com';
protected $strSiteCode = 'zw630';
/**
* Undocumented function
*
* @return string
*/
public function getSiteCode(): string
{
return $this->strSiteCode;
}
/**
* Undocumented function
*
* @param integer|string $strSign
* @return string
*/
public function getSiteUUId(int|string $strSign): string
{
return $this->getSiteCode() . '-' . $strSign;
}
/**
* Undocumented function
*/