admin/SEONexus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

DEBUG

Browse Source
This commit is contained in:
Default
2025-04-21 11:56:50 +08:00
parent 3216cffb5e
commit 65b8710bff
17 changed files with 2432 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
code/app/home/view/default/test/test01/index.html
View File

@@ -1,7 +1,26 @@
{novel:pagerexp page="1" limit="10" n_category="$Request.route.category" d_key="d_key" d_val="novel" p_val="page_data"
{site:forgelist count="3" d_key="d_key" d_val="novel" cache_life="1000"}
<li>
Index: {$d_key}
Title: {$novel.n_name}
Author: {$novel.n_category}
</li>
{/site:forgelist}
<!-- {novel:ranklist count="10" sort_type="total" n_status="wanjie" n_sex="man" d_key="d_key" d_val="novel" cache_life="1000"}
<li>
Index: {$d_key}
Title: {$novel.n_name}
Author: {$novel.n_category}
</li>
{/novel:ranklist} -->
<!-- {novel:pagerexp page="1" limit="10" n_category="$Request.route.category" d_key="d_key" d_val="novel" p_val="page_data"
cache_life="1000" func="generateSearchPager" export_name="fadfads" /}
{php}var_dump($fadfads['p_data']);{/php}
{php}var_dump($fadfads['p_data']);{/php} -->

18
code/app/model/NovelModel.php
View File

@@ -242,7 +242,7 @@ class NovelModel extends MongoModel
): array {
try {
$arrFilter['n_have_chapter'] = 1;
$arrResult = $this->checkCacheByKey($strKey);
if ($arrResult !== NULL) {
return $arrResult;
@@ -417,10 +417,16 @@ class NovelModel extends MongoModel
* * total 总
* @return array
*/
public function getRankNovelOnCache(string $strKey, int $intCount = 10, int $intType = 0, $strDateType = 'daily', $strStatus = NULL): array
{
public function getRankNovelOnCache(
string $strKey,
int $intCount = 10,
int $intType = 0,
$strDateType = 'daily',
$strStatus = NULL,
$intLifeTime = 24 * 3600,
): array {
try {
$intLifeTime = 24 * 3600;
$arrResult = $this->checkCacheByKey($strKey);
if ($arrResult !== NULL) {
@@ -477,8 +483,4 @@ class NovelModel extends MongoModel
return [];
}
}
}

124
code/app/model/VideoClicksModel.php Normal file
View File

@@ -0,0 +1,124 @@
<?php
declare(strict_types=1);
namespace app\model;
use DateTime;
use think\facade\Cache;
use think\Model;
/**
* @mixin think\Model
*/
class VideoClicksModel extends MongoModel
{
/**
* VideoClicksModel
*
* @var VideoClicksModel
*/
static public $VideoClicksModel;
static public function getInstance(): self
{
if (self::$VideoClicksModel == NULL) {
self::$VideoClicksModel = new self;
}
return self::$VideoClicksModel;
}
protected $strCollection = 'video_clicks';
/**
* addStatis
*
* @param integer $intVId
* @return void
*/
public function addStatis(int $intVId)
{
$NovelModel = NovelModel::getInstance();
$Novel = $NovelModel->getCol()->findOne([
"v_id" => $intVId,
]);
if (empty($Novel)) {
return false;
}
$Now = new DateTime();
$strDailyDate = $Now->format('Y-m-d');
$strWeeklyDate = $Now->modify('monday this week')->format('Y-m-d');
$MonthlyDate = $Now->format('Y-m-01');
$arrClickTypes = [
'daily' => $strDailyDate,
'weekly' => $strWeeklyDate,
'monthly' => $MonthlyDate,
'total' => null,
];
foreach ($arrClickTypes as $strType => $strDate) {
$this->getCol()->updateOne(
[
'v_id' => $intVId,
'vc_date' => $strDate,
'vc_type' => $strType,
],
[
'$inc' => ['vc_clicks' => 1],
'$set' => ['v_category_id' => $Novel->v_category_id]
],
[
'upsert' => true
]
);
}
return true;
}
/**
* GetStats
*
* @param integer $intVId
* @return array
*/
public function getStats(int $intVId): array
{
$Now = new DateTime();
$strDailyDate = $Now->format('Y-m-d');
$strWeeklyDate = $Now->modify('monday this week')->format('Y-m-d');
$strMonthlyDate = $Now->format('Y-m-01');
$query = [
'v_id' => $intVId,
'$or' => [
['vc_type' => 'daily', 'vc_date' => $strDailyDate],
['vc_type' => 'weekly', 'vc_date' => $strWeeklyDate],
['vc_type' => 'monthly', 'vc_date' => $strMonthlyDate],
['vc_type' => 'total', 'vc_date' => null]
]
];
$arrOptions = [
'projection' => ['vc_type' => 1, 'vc_clicks' => 1, '_id' => 0],
'typeMap' => ['root' => 'array', 'document' => 'array', 'array' => 'array']
];
$arrResults = $this->getCol()->find($query, $arrOptions)->toArray();
$arrStats = array_column($arrResults, 'vc_clicks', 'vc_type');
$defaultStats = [
'daily' => 0,
'weekly' => 0,
'monthly' => 0,
'total' => 0
];
$arrStats = array_merge($defaultStats, $arrStats);
return $arrStats;
}
}

484
code/app/model/VideoModel.php Normal file
View File

@@ -0,0 +1,484 @@
<?php
declare(strict_types=1);
namespace app\model;
use Exception;
/**
* @mixin think\Model
*/
class VideoModel extends MongoModel
{
/**
* Undocumented variable
*
* @var self
*/
static public $obj;
static public function getInstance(): self
{
if (self::$obj == NULL) {
self::$obj = new self;
}
return self::$obj;
}
/**
* Undocumented variable
*
* @var string
*/
protected $strCollection = 'video';
static public $arrOptions = [
'typeMap' => [
'root' => 'array',
'document' => 'array',
'array' => 'array',
],
'projection' => [
'_id' => 0,
'n_site_code' => 0,
'n_source_uuid' => 0,
'n_source_id' => 0,
],
];
/**
* 根据关键词数组更新小说推荐等级
* @param array $arrKeywords 关键词数组，例如 ["斗罗", "遮天", "凡人"]
* @param int $intLevel 推荐等级1~9，例如 3
* @return array 更新结果
*/
public function updateByKeywords(array $arrKeywords, int $intLevel)
{
if (empty($arrKeywords)) {
return false;
}
try {
$arrConditions = array_map(function ($strKeyword) {
return ['n_name' => ['$regex' => trim($strKeyword), '$options' => 'i']];
}, $arrKeywords);
$UpdateResult = $this->getCol()->updateMany(
['$or' => $arrConditions],
['$set' => ['n_level' => $intLevel]]
);
$UpdateResult->getModifiedCount();
} catch (Exception $e) {
return false;
}
return true;
}
/**
* 直接读库查询大量小说
*
* @param integer $intNId
* @return null|array
*/
public function findMany(array $arrFilter, int $intCount = 10, array $arrSort = [], array $arrOptions = []): null|array
{
if (empty($arrOptions)) {
$arrOptions = self::$arrOptions;
}
if ($intCount > 0) {
$arrOptions['limit'] = $intCount;
}
if (empty($arrSort)) {
$arrOptions['sort'] = $arrSort;
}
$arrFilter['n_have_chapter'] = 1;
$Cursor = $this->getCol()->find($arrFilter, $arrOptions);
$arrResult = iterator_to_array($Cursor);
foreach ($arrResult as &$arrItem) {
applyToKeys($arrItem, ['created_at', 'updated_at'], 'formatMongoDate');
}
return $arrResult;
}
/**
* 查询多条数据并缓存
*
* @param array $arrFilter
* @param integer $intCount
* @param array $arrSort
* @param array $arrOptions
* @param string $strKey
* @param integer $intLifeTime
* @return null|array
*/
public function findManyWithCache(
array $arrFilter,
int $intCount = 0,
array $arrSort = [],
array $arrOptions = [],
string $strKey = '',
int $intLifeTime = 24 * 3600,
): null|array {
try {
$arrResult = $this->checkCacheByKey($strKey);
if ($arrResult !== NULL) {
return $arrResult;
}
$arrResult = $this->findMany($arrFilter, $intCount, $arrSort, $arrOptions);
$this->setCacheByKey($strKey, $arrResult, $intLifeTime);
return $arrResult;
} catch (\Exception $e) {
return [];
}
}
/**
* find one Video
*
* @param array $arrFilter
* @param array $arrOptions
* @return null|array
*/
public function findOne(array $arrFilter, array $arrOptions = []): null|array
{
if (empty($arrOptions)) {
$arrOptions = self::$arrOptions;
}
$arrVideo = $this->getOne($arrFilter, $arrOptions);
if (!empty($arrVideo)) {
applyToKeys($arrVideo, ['created_at', 'updated_at'], 'formatMongoDate');
}
return $arrVideo;
}
/**
* 根据小说ID查询指定小说
*
* @param integer $intNId
* @return array
*/
public function getVideoByNId(int $intNId): null|array
{
$arrFilter = ['n_id' => $intNId, 'n_have_chapter' => 1];
return $this->findOne($arrFilter);
}
/**
* 查询符合条件的随机多条小说
*
* @param string $strKey
* @param array $arrFilter
* @param integer $intCount
* @param integer $intLifeTime
* @return array
*/
public function getRandVideoWithCache(array $arrFilter = [], int $intCount = 10, string $strKey = '', int $intLifeTime = 24 * 3600,): array
{
try {
$arrResult = $this->checkCacheByKey($strKey);
if ($arrResult !== NULL) {
return $arrResult;
}
$arrFilter['n_have_chapter'] = 1;
$arrPipeline = [
['$match' => $arrFilter],
['$sample' => ['size' => $intCount]]
];
$Video = $this->getCol()->aggregate($arrPipeline);
$arrResult = iterator_to_array($Video);
$this->setCacheByKey($strKey, $arrResult, $intLifeTime);
return $arrResult;
} catch (\Exception $e) {
return [];
}
}
/**
* 通用小说分页
*
* @param array $arrFilter
* @param integer $intPage
* @param integer $intLimit
* @param array $arrSort
* @param string $strKey
* @param int $intLifeTime
* @return array
*/
public function findPaginatedWithCache(
array $arrFilter,
int $intPage = 1,
int $intLimit = 20,
array $arrSort = ['n_level' => -1],
string $strKey = '',
int $intLifeTime = 24 * 3600,
): array {
try {
$arrFilter['n_have_chapter'] = 1;
$arrResult = $this->checkCacheByKey($strKey);
if ($arrResult !== NULL) {
return $arrResult;
}
$arrOptions = self::$arrOptions;
$arrOptions['sort'] = $arrSort;
$arrOptions['skip'] = ($intPage - 1) * $intLimit;
$arrOptions['limit'] = $intLimit;
unset($arrOptions['typeMap']);
$Cursor = $this->getCol()->find($arrFilter, $arrOptions);
$intTotal = $this->getCol()->countDocuments($arrFilter);
$arrResult = iterator_to_array($Cursor);
shuffle($arrResult);
$arrResult = [
'data' => $arrResult,
'total' => $intTotal,
'page' => $intPage,
'limit' => $intLimit,
'pages' => ceil($intTotal / $intLimit)
];
$this->setCacheByKey($strKey, $arrResult, $intLifeTime);
return $arrResult;
} catch (\Exception $e) {
return [];
}
}
/**
* 用于Home下的书库分页
*
* @param string $strKey
* @param integer $intPage
* @param integer $intLimit
* @param int $intLifeTime
* @param integer $intType
* @param string $strStatus
* @param array $arrSort
* @return array
*/
public function getCustomPage01(
string $strKey,
int $intPage = 1,
int $intLimit = 20,
int $intLifeTime = 24 * 3600,
int $intType = 0,
$strStatus = NULL,
array $arrSort = ['n_level' => -1]
): array {
$arrFilter = ['n_have_chapter' => 1,];
switch ($intType) {
case 1:
$arrCategory = array_values(CategoryModel::getManCategory());
$arrFilter['n_category'] = ['$in' => $arrCategory];
break;
case 2:
$arrCategory = array_values(CategoryModel::getWomenCategory());
$arrFilter['n_category'] = ['$in' => $arrCategory];
break;
}
if ($strStatus !== NULL) {
$arrFilter['n_status'] = $strStatus;
}
return $this->findPaginatedWithCache($arrFilter, $intPage, $intLimit, $arrSort, $strKey, $intLifeTime,);
}
/**
* Undocumented function
*
* @param array $arrFilter
* @param integer $intCount
* @param array $arrSort
* @param string $strKey
* @param integer $intLifeTime
* @return array
*/
public function findManyShuffledWithCache(
array $arrFilter,
int $intCount = 10,
array $arrSort = [],
string $strKey = '',
int $intLifeTime = 24 * 3600,
): array {
try {
$arrResult = $this->checkCacheByKey($strKey);
if ($arrResult !== NULL) {
return $arrResult;
}
$arrFilter['n_have_chapter'] = 1;
$arrOptions = self::$arrOptions;
if (!empty($arrSort)) {
$arrOptions['sort'] = $arrSort;
}
$arrOptions['limit'] = $intCount * 3;
unset($arrOptions['typeMap']);
$Cursor = $this->getCol()->find($arrFilter, $arrOptions);
$arrResult = iterator_to_array($Cursor);
shuffle($arrResult);
$arrResult = array_slice($arrResult, 0, $intCount);
$this->setCacheByKey($strKey, $arrResult, $intLifeTime);
return $arrResult;
} catch (\Exception $e) {
return [];
}
}
/**
* Undocumented function
*
* @param string $strKey
* @param int $intLifeTime
* @param integer $intCount
* @param integer $intType
* @param string $strStatus
* @param array $arrSort
* @return array
*/
public function getCommonVideoOnCache(
string $strKey,
int $intLifeTime = 24 * 3600,
int $intCount = 10,
int $intType = 0,
$strStatus = NULL,
array $arrSort = ['n_level' => -1]
): array {
try {
$arrFilter = [];
if (in_array($intType, [1, 2])) {
$arrFilter['n_category'] = ['$in' => CategoryModel::getCategoryByType($intType)];
}
if ($strStatus !== NULL) {
$arrFilter['n_status'] = $strStatus;
}
return $this->findManyShuffledWithCache($arrFilter, $intCount, $arrSort, $strKey, $intLifeTime);
} catch (\Exception $e) {
return [];
}
}
/**
* Undocumented function
*
* @param string $strSign
* @param integer $intCount
* @param integer $intType
* @param string strDateType
* * daily 天
* * weekly 周
* * monthly 月
* * total 总
* @return array
*/
public function getRankVideoOnCache(string $strKey, int $intCount = 10, int $intType = 0, $strDateType = 'daily', $strStatus = NULL): array
{
try {
$intLifeTime = 24 * 3600;
$arrResult = $this->checkCacheByKey($strKey);
if ($arrResult !== NULL) {
return $arrResult;
}
$arrFilter = [
'nc_type' => $strDateType,
];
if ($strStatus !== NULL) {
$arrFilter['n_status'] = $strStatus;
}
if (in_array($intType, [1, 2])) {
$arrFilter['n_category'] = ['$in' => CategoryModel::getCategoryByType($intType)];
}
$arrOptions = [
'sort' => ['nc_clicks' => -1],
'limit' => $intCount * 3,
'projection' => [
'n_id' => 1,
],
];
$VideoClicksModel = VideoClicksModel::getInstance();
$Cursor = $VideoClicksModel->getCol()->find($arrFilter, $arrOptions);
$arrNId = [];
foreach ($Cursor as $Doc) {
$arrNId[] = $Doc['n_id'];
}
$arrFilter = ['n_id' => ['$in' => $arrNId]];
$arrOptions = self::$arrOptions;
unset($arrOptions['typeMap']);
$Cursor = $this->getCol()->find(
$arrFilter,
$arrOptions
);
$arrResult = iterator_to_array($Cursor);
shuffle($arrResult);
$arrResult = array_slice($arrResult, 0, $intCount);
$this->setCacheByKey($strKey, $arrResult, $intLifeTime);
return $arrResult;
} catch (\Exception $e) {
return [];
}
}
}

114
code/app/services/NovelService.php
View File

@@ -216,6 +216,13 @@ class NovelService
'update' => '最新更新',
];
public $arrRankSortType = [
'daily' => '天',
'weekly' => '周',
'monthly' => '月',
'total' => '总',
];
public $arrStatus = [
'all' => '所有',
'lianzai' => '连载中',
@@ -234,18 +241,108 @@ class NovelService
*
* @return array
*/
public function getCategoryFilter(string $n_sex=''): array
public function getCategoryFilter(string $strNSex = ''): array
{
if($n_sex == 'man'){
if ($strNSex == 'man') {
return CategoryModel::$arrCategoryAll['man'];
}else if($n_sex == 'women'){
} else if ($strNSex == 'women') {
return CategoryModel::$arrCategoryAll['women'];
}else{
} else {
return array_combine(CategoryModel::$arrCategoryPinYin, CategoryModel::$arrCategory);
}
}
/**
* 获取排行榜小说
*
* @param array $arrParams
* @return array
*/
public function getRankList(array $arrParams): array
{
$intCount = (int)$arrParams['count'] ?? 10;
$strSortType = (string)$arrParams['sort_type'] ?? '';
$strNStatus = (string) $arrParams['n_status'] ?? '';
$strNSex = (string) $arrParams['n_sex'] ?? '';
$strDiffKey = (string) $arrParams['diff_key'] ?? '';
$intCacheLifeTime = (int)$arrParams['cache_life'] ?? 0;
$arrFilter = [];
# n_status filter
if (!empty($strNStatus)) {
switch ($strNStatus) {
case 'lianzai':
$arrFilter['n_status'] = $this->arrStatus[$strNStatus];
break;
case 'wanjie':
$arrFilter['n_status'] = $this->arrStatus[$strNStatus];
break;
}
}
# n_category_sex_zh filter
if (!empty($strNSex)) {
switch ($strNSex) {
case 'man':
$arrFilter['n_category_sex_zh'] = 1;
break;
case 'women':
$arrFilter['n_category_sex_zh'] = 2;
break;
}
}
$arrSort = [];
# sort filter
if (!empty($strSortType)) {
switch ($strSortType) {
case 'daily':
$arrSort['n_sort_type'] = 'daily';
break;
case 'weekly':
$arrSort['n_sort_type'] = 'weekly';
break;
case 'monthly':
$arrSort['n_sort_type'] = 'monthly';
break;
case 'total':
$arrSort['n_sort_type'] = 'total';
break;
}
}
$strCacheKey = '';
$intLifeTime = 0;
# cache filter
if ($intCacheLifeTime > 0) {
$strCacheKey = sprintf(
'NovelService:getRankList:%s:%s:%s:%s:%s',
$this->SiteContext->DomainModel->d_domain,
md5(serialize($arrFilter)),
md5(serialize($arrSort)),
$intCount,
$strDiffKey,
);
$intLifeTime = $intLifeTime;
}
$arrNovel = $this->NovelModel->getRankNovelOnCache(
$strCacheKey,
$intCount,
$arrFilter['n_category_sex_zh'] ?? 0,
$arrSort['n_sort_type'] ?? 'daily',
$arrFilter['n_status'] ?? NULL,
$intLifeTime,
);
return $arrNovel;
}
/**
* 获取小说数据
*
@@ -319,7 +416,7 @@ class NovelService
# cache filter
if ($intCacheLifeTime > 0) {
$strCacheKey = sprintf(
'Service:getNovelList:%s:%s:%s:%s:%s',
'NovelService:getNovelList:%s:%s:%s:%s:%s',
$this->SiteContext->DomainModel->d_domain,
md5(serialize($arrFilter)),
md5(serialize($arrSort)),
@@ -464,7 +561,7 @@ class NovelService
# cache filter
if ($intCacheLifeTime > 0) {
$strCacheKey = sprintf(
'Service:getNovelList:%s:%s:%s:%s:%s:%s',
'NovelService:getNovelList:%s:%s:%s:%s:%s:%s',
$this->SiteContext->DomainModel->d_domain,
md5(serialize($arrFilter)),
md5(serialize($arrSort)),
@@ -619,4 +716,7 @@ class NovelService
return $arrNovel;
}
}

134
code/app/services/SiteContext.php
View File

@@ -7,6 +7,9 @@ use app\model\DomainModel;
use app\model\GanRaoMaModel;
use app\model\TemplatesModel;
use app\model\CategoryModel;
use app\model\ChapterModel;
use app\model\NovelClicksModel;
use app\model\NovelModel;
use think\facade\Request;
use think\facade\Config;
use think\exception\HttpException;
@@ -35,9 +38,32 @@ class SiteContext
*/
public $TemplatesModel;
/**
* NovelModel
*
* @var NovelModel
*/
public $NovelModel;
/**
* ChapterModel
*
* @var ChapterModel
*/
public $ChapterModel;
/**
* NovelClicksModel
*
* @var NovelClicksModel
*/
public $NovelClicksModel;
public function __construct()
{
$this->Request = request();
$this->NovelModel = NovelModel::getInstance();
$this->ChapterModel = ChapterModel::getInstance();
$this->NovelClicksModel = NovelClicksModel::getInstance();
}
/**
@@ -81,32 +107,32 @@ class SiteContext
*/
public function initCfg()
{
// 全局使用的URL模板
$strPcUrlTemp = $this->DomainModel->getFomartSubject('PC_URL_PATH','',false);
$strNanUrlTemp = $this->DomainModel->getFomartSubject('NAN_SHENG_URL','',false);
$strNvUrlTemp = $this->DomainModel->getFomartSubject('NV_SHENG_URL','',false);
$strCategoryIndexUrlTemp = $this->DomainModel->getFomartSubject('CATEGORY_INDEX_URL','',false);
$strRankUrlTemp = $this->DomainModel->getFomartSubject('RANK_INFO_URL','',false);
$strSearchIndexUrlTemp = $this->DomainModel->getFomartSubject('SEARCH_INDEX_URL','',false);
$strHistoryUrlTemp = $this->DomainModel->getFomartSubject('HISTORY_INFO_URL','',false);
$strBookShelfUrlTemp = $this->DomainModel->getFomartSubject('SHUJIA_INFO_URL','',false);
$strUserUrlTemp = $this->DomainModel->getFomartSubject('USER_INFO_URL','',false);
// 全局使用的URL模板
$strPcUrlTemp = $this->DomainModel->getFomartSubject('PC_URL_PATH', '', false);
$strNanUrlTemp = $this->DomainModel->getFomartSubject('NAN_SHENG_URL', '', false);
$strNvUrlTemp = $this->DomainModel->getFomartSubject('NV_SHENG_URL', '', false);
$strCategoryIndexUrlTemp = $this->DomainModel->getFomartSubject('CATEGORY_INDEX_URL', '', false);
$strRankUrlTemp = $this->DomainModel->getFomartSubject('RANK_INFO_URL', '', false);
$strSearchIndexUrlTemp = $this->DomainModel->getFomartSubject('SEARCH_INDEX_URL', '', false);
$strHistoryUrlTemp = $this->DomainModel->getFomartSubject('HISTORY_INFO_URL', '', false);
$strBookShelfUrlTemp = $this->DomainModel->getFomartSubject('SHUJIA_INFO_URL', '', false);
$strUserUrlTemp = $this->DomainModel->getFomartSubject('USER_INFO_URL', '', false);
$strNanUrl = str_replace("/", "", $strNanUrlTemp);
$strNvUrl = str_replace("/", "", $strNvUrlTemp);
View::assign("strPcPath", $strPcUrlTemp);
View::assign("strNanUrlTemp", $strNanUrlTemp);
View::assign("strNvUrlTemp", $strNvUrlTemp);
View::assign("strNanUrl", $strNanUrl);
View::assign("strNvUrl", $strNvUrl);
View::assign("strCategoryIndexUrlTemp", $strCategoryIndexUrlTemp);
View::assign("strRankUrlTemp", $strRankUrlTemp);
View::assign("strSearchIndexUrlTemp", $strSearchIndexUrlTemp);
View::assign("strHistoryUrlTemp", $strHistoryUrlTemp);
View::assign("strBookShelfUrlTemp", $strBookShelfUrlTemp);
View::assign("strUserUrlTemp", $strUserUrlTemp);
View::assign("strPcPath", $strPcUrlTemp);
View::assign("strNanUrlTemp", $strNanUrlTemp);
View::assign("strNvUrlTemp", $strNvUrlTemp);
View::assign("strNanUrl", $strNanUrl);
View::assign("strNvUrl", $strNvUrl);
View::assign("strCategoryIndexUrlTemp", $strCategoryIndexUrlTemp);
View::assign("strRankUrlTemp", $strRankUrlTemp);
View::assign("strSearchIndexUrlTemp", $strSearchIndexUrlTemp);
View::assign("strHistoryUrlTemp", $strHistoryUrlTemp);
View::assign("strBookShelfUrlTemp", $strBookShelfUrlTemp);
View::assign("strUserUrlTemp", $strUserUrlTemp);
View::assign('Request', $this->Request);
View::assign('DomainModel', $this->DomainModel);
View::assign('TemplatesModel', $this->TemplatesModel);
@@ -166,14 +192,72 @@ class SiteContext
} else {
$strView = 'index/index.html';
}
}else{
} else {
if ($this->DomainModel->n_id > 0) {
$strView = 'pc/getNovelInfo.html';
} else {
$strView = 'pc/index.html';
}
}
return $strView;
}
/**
* 获取伪造小说列表
*
* @param array $arrParams
* @return array
*/
public function getForgeList(array $arrParams): array
{
$intCount = (int)$arrParams['count'] ?? 10;
$strDiffKey = (string) $arrParams['diff_key'] ?? '';
$intCacheLifeTime = (int)$arrParams['cache_life'] ?? 0;
$arrFilter = [
'n_forge_novel' => [
'$ne' => null, // 不为 null
'$ne' => '', // 不为空字符串
'$ne' => [], // 不为空数组
'$exists' => true // 确保字段存在
],
];
$strCacheKey = '';
$intLifeTime = 0;
# cache filter
if ($intCacheLifeTime > 0) {
$strCacheKey = sprintf(
'SiteContext:getForgeList:%s:%s:%s:%s',
$this->DomainModel->d_domain,
md5(serialize($arrFilter)),
$intCount,
$strDiffKey,
);
$intLifeTime = $intLifeTime;
}
$arrAllNovel = $this->NovelModel->getRandNovelWithCache(
$arrFilter,
$intCount,
$strCacheKey,
$intLifeTime,
);
$arrResult = [];
foreach ($arrAllNovel as $arrNovel) {
$arrResult[] = [
'n_id' => $arrNovel['n_id'],
'n_name_pinyin' => $arrNovel['n_name_pinyin'],
'n_forge_id' => $arrNovel['n_forge_novel'][0]->n_forge_id,
'n_forge_title' => $arrNovel['n_forge_novel'][0]->n_forge_title,
];
}
return $arrResult;
}
}

122
code/app/task/crawler/youzhi/Scheduler.php Normal file
View File

@@ -0,0 +1,122 @@
<?php
declare(strict_types=1);
namespace app\task\crawler\youzhi;
use app\model\ChapterModel;
use app\task\crawler\youzhi\page\Site;
use microserver\QueueManage;
use storage\StorageCore;
/**
* @mixin think\Model
*/
class Scheduler
{
protected $intSlice = 8;
protected $intStartNSourceId = 1;
protected $intEndNSourceId = 500000;
/**
* Undocumented variable
*
* @var \microserver\QueueManage
*/
protected $QueueManage = NULL;
/**
* Undocumented variable
*
* @var Site
*/
public $Site;
/**
* Undocumented function
*/
public function __construct()
{
$this->QueueManage = new QueueManage();
$this->init();
}
/**
* Undocumented function
*
* @return void
*/
public function init()
{
$this->intSlice = 8;
}
/**
* get site
*
* @return Site
*/
public function getSite()
{
if ($this->Site == NULL) {
$this->Site = new Site;
}
return $this->Site;
}
/**
* create a site-wide craw task
*
* @return void
*/
public function crawlFull()
{
$intLimit = 10;
$Site = $this->getSite();
$arrAllCategory = $Site->getCategory();
foreach ($arrAllCategory as $arrCategory) {
$intTotalPage = ceil($arrCategory['total'] / $intLimit);
for ($intStartPage = 1; $intStartPage <= $intTotalPage; $intStartPage++) {
$arrFilter = [
[
'v_category_id' => $arrCategory['v_category_id'],
'page' => $intStartPage,
],
];
$arrVideoPageList = $Site->getVideoPageList($arrFilter);
foreach ($arrVideoPageList as $VideoPage) {
$arrAllVideoPager = $VideoPage->getVideoPager();
$arrVSourceId = [];
foreach ($arrAllVideoPager['list'] as $arrVideoPager) {
$arrVSourceId[] = ['v_source_id' => (int)$arrVideoPager['vod_id']];
}
$arrAllVideoInfoPage = $Site->getVideoInfoPageList($arrVSourceId);
foreach ($arrAllVideoInfoPage as $VideoInfoPage) {
$arrVideo = $VideoInfoPage->getVideo();
print_r($arrVideo);
}
}
}
}
$arrCover = [];
if ($arrCover) {
// $Site->downRemoteImgToLocal($arrCover);
}
return;
}
}

251
code/app/task/crawler/youzhi/page/BasePage.php Normal file
View File

@@ -0,0 +1,251 @@
<?php
declare(strict_types=1);
namespace app\task\crawler\youzhi\page;
use db\mongo\MongoBase;
use ddl\DDLManage;
use GuzzleHttp\Client;
use microserver\QueueManage;
use QL\QueryList;
use Psr\Http\Message\ResponseInterface;
/**
* @mixin think\Model
*/
abstract class BasePage
{
/**
* Undocumented variable
*
* @var string
*/
// protected $strDomain;
/**
* Undocumented variable
*
* @var string
*/
protected $strUri;
/**
* Undocumented variable
*
* @var MongoBase
*/
protected $MongoBase;
/**
* Undocumented variable
*
* @var \GuzzleHttp\Client
*/
protected $Client = NULL;
/**
* Undocumented variable
*
* @var \microserver\QueueManage
*/
protected $QueueManage = NULL;
/**
* Undocumented variable
*
* @var array
*/
protected $arrCaiJiPeiZhi = [];
/**
* Undocumented variable
*
* @var string
*/
protected $strContent;
/**
* Undocumented variable
*
* @var QueryList
*/
protected $QueryList;
/**
* Undocumented variable
*
* @var array
*/
protected $arrData = [];
/**
* Undocumented variable
*
* @var bool
*/
protected $boolIsMobile;
/**
* Undocumented variable
*
* @var ResponseInterface
*/
private $Response;
/**
* Undocumented function
*
* @param string $strUri
*/
public function __construct($Site, string $strUri)
{
$this->setSite($Site);
$this->strUri = $strUri;
}
/**
* Undocumented function
*
* @return string
*/
public function getUri(): string
{
return $this->strUri;
}
/**
* Undocumented function
*
* @param array $arrData
* @return void
*/
public function setData(array $arrData)
{
$this->arrData = $arrData;
}
/**
* Undocumented function
*
* @return array
*/
public function getData(): array
{
return $this->arrData;
}
/**
* Undocumented function
*
* @return QueueManage
*/
public function getQueueManage(): QueueManage
{
if ($this->QueueManage == NULL) {
$this->QueueManage = QueueManage::getInstance();
}
return $this->QueueManage;
}
/**
* Undocumented function
*
* @return MongoBase
*/
public function getMongoBase(): MongoBase
{
if ($this->MongoBase == NULL) {
DDLManage::load('MongoBase');
$this->MongoBase = MongoBase::getInstance();
}
return $this->MongoBase;
}
public function getQueryList(): QueryList
{
if ($this->QueryList == NULL) {
$this->QueryList = QueryList::html($this->getContent());
}
return $this->QueryList;
}
/**
* Undocumented function
*
* @return string
*/
public function getContent(): string
{
if ($this->strContent == NULL) {
$this->Response = $this->getClient()->get($this->strUri);
$this->strContent = $this->Response->getBody()->getContents();
}
return $this->strContent;
}
/**
* force set page content
*
* @param string $strContent
* @return void
*/
public function setContent($strContent)
{
$this->strContent = $strContent;
}
/**
* Undocumented function
*
* @return boolean
*/
public function isMobile($strMark = 'content="mobile"'): bool
{
if ($this->boolIsMobile === NULL) {
$this->boolIsMobile = is_numeric(strpos($this->getContent(), $strMark));
}
return $this->boolIsMobile;
}
/**
* Undocumented function
*
* @return Client
*/
protected function getClient(): Client
{
if ($this->Client == NULL) {
$this->Client = $this->getSite()->getClient();
}
return $this->Client;
}
/**
* Undocumented variable
*
* @var Site
*/
protected $Site;
public function setSite(Site $Site)
{
$this->Site = $Site;
}
public function getSite()
{
return $this->Site;
}
}

369
code/app/task/crawler/youzhi/page/Site.php Normal file
View File

@@ -0,0 +1,369 @@
<?php
declare(strict_types=1);
namespace app\task\crawler\youzhi\page;
use GuzzleHttp\Client;
use GuzzleHttp\Promise;
use storage\StorageCore;
/**
* @mixin think\Model
*/
class Site
{
/**
* Undocumented variable
*
* @var Client
*/
protected $Client;
/**
* Undocumented variable
*
* @var string
*/
protected $strDomain = 'https://api.yzzy-api.com';
/**
* Undocumented variable
*
* @var string
*/
protected $strSiteCode = 'youzhi';
/**
* use proxy
*
* @var boolean
*/
protected $boolProxy = false;
/**
* Undocumented function
*/
public function __construct()
{
$this->initClient();
}
/**
* Undocumented function
*/
public function initClient()
{
$arrConfig = [
'http_version' => '1.1', // 强制用 HTTP/1.1，已有，保持
'base_uri' => $this->strDomain, // 基础域名，已有，保持
'http_errors' => false, // 不抛 HTTP 错误，已有，保持
'timeout' => 60, // 从 160 秒改为 60 秒
'connect_timeout' => 10, // 从 30 秒改为 10 秒
'curl' => [
CURLOPT_TCP_KEEPALIVE => 1, // 启用 TCP Keep-Alive，已有，保持
CURLOPT_TCP_KEEPIDLE => 120, // Keep-Alive 空闲时间
CURLOPT_TCP_KEEPINTVL => 60, // Keep-Alive 探测间隔
// CURLOPT_SSL_VERIFYPEER => true, // 验证 SSL，已有，保持
],
'headers' => [
'User-Agent' => 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36',
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'Accept-Language' => 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
'Referer' => $this->strDomain,
'Connection' => 'keep-alive',
'sec-ch-ua' => '"Not/A)Brand";v="8", "Chromium";v="130", "Google Chrome";v="130"',
'sec-ch-ua-mobile' => '?1',
'sec-ch-ua-platform' => '"Android"',
'sec-fetch-dest' => 'document',
'sec-fetch-mode' => 'navigate',
'sec-fetch-site' => 'same-origin',
'sec-fetch-user' => '?1',
'upgrade-insecure-requests' => '1',
],
'pool_size' => 20,
'verify' => false,
];
$arrProxy = config('proxy');
if ($this->boolProxy && $arrProxy['proxy_status']) {
$strCode = $arrProxy['proxy_code'];
$strProxyHost = $arrProxy['proxy_host'];
$strKey = $arrProxy['proxy_key'];
$strSecret = $arrProxy['proxy_secret'];
switch ($strCode) {
case 'XIONGMAO':
$intTime = time();
$strTxt = "orderno=" . $strKey . ",secret=" . $strSecret . ",timestamp=" . $intTime;
$strSign = strtoupper(md5($strTxt));
$strAuth = 'sign=' . $strSign . '&orderno=' . $strKey . '&timestamp=' . $intTime;
$arrConfig['proxy'] = $strProxyHost;
$arrConfig['headers']['Proxy-Authorization'] = $strAuth;
break;
case 'KUAI':
$arrConfig['proxy'] = $strProxyHost;
$arrConfig['curl'][CURLOPT_PROXYAUTH] = CURLAUTH_BASIC;
$arrConfig['curl'][CURLOPT_PROXYUSERPWD] = "$strKey:$strSecret";
break;
}
}
$this->Client = new Client($arrConfig);
}
/**
* Undocumented function
*
* @return string
*/
public function getSiteCode(): string
{
return $this->strSiteCode;
}
/**
* Undocumented function
*
* @param integer|string $strSign
* @return string
*/
public function getSiteUUId(int|string $strSign): string
{
return $this->getSiteCode() . '-' . $strSign;
}
/**
* Undocumented function
*
* @return Client
*/
public function getClient(): Client
{
return $this->Client;
}
/**
* retrieve multiple novel page model
*
* @param string $strUri
* @return VideoPage[]
*/
public function getCategory(): array
{
$strUri = '/inc/api_mac10.php?ac=list&t=1&page=1';
$Response = $this->getClient()->get($strUri);
$strContent = $Response->getBody()->getContents();
$arrContent = json_decode($strContent, true);
$arrFilter = $arrContent['class'];
$arrCategory = [];
foreach ($arrFilter as $intKey => $arrItem) {
$arrFilter[$intKey]['uri'] = $this->getVideoPageUri((int)$arrItem['type_id'], 1);
$arrPromises[$intKey] = $this->getClient()->getAsync($arrFilter[$intKey]['uri']);
}
$arrResults = Promise\Utils::settle($arrPromises)->wait();
$arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
return $arrResult['state'] === 'fulfilled';
});
foreach ($arrSuccessfulResults as $intKey => $arrResult) {
$Response = $arrResult['value'];
$strContent = $Response->getBody()->getContents();
$arrTemp = json_decode($strContent, true);
$arrCategory[] = [
'v_category_id' => (int)$arrFilter[$intKey]['type_id'],
'v_category_name' => $arrFilter[$intKey]['type_name'],
'v_category_name_en' => zhToPinYin($arrFilter[$intKey]['type_name']),
'total' => $arrTemp['total'],
];
}
return $arrCategory;
}
/**
* generate video list uri
*
* @param integer $intVCategoryId
* @param integer $intPage
* @return string
*/
public function getVideoPageUri(int $intVCategoryId, int $intPage): string
{
return sprintf('/inc/api_mac10.php?ac=list&t=%s&page=%s', $intVCategoryId, $intPage);
}
/**
* retrieve multiple novel page model
*
* @param string $strUri
* @return VideoPage[]
*/
public function getVideoPageList(array $arrFilter): array
{
/** @var VideoPage[] */
$arrVideoPage = [];
foreach ($arrFilter as $intKey => $arrItem) {
$arrFilter[$intKey]['uri'] = $this->getVideoPageUri($arrItem['v_category_id'], $arrItem['page']);
$arrPromises[$intKey] = $this->getClient()->getAsync($arrFilter[$intKey]['uri']);
}
$arrResults = Promise\Utils::settle($arrPromises)->wait();
$arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
return $arrResult['state'] === 'fulfilled';
});
foreach ($arrSuccessfulResults as $intKey => $arrResult) {
$Response = $arrResult['value'];
$strContent = $Response->getBody()->getContents();
$VideoPage = $this->getVideoPage($this, $arrFilter[$intKey]['uri']);
$VideoPage->setContent($strContent);
$VideoPage->setData($arrFilter[$intKey]);
$arrVideoPage[] = $VideoPage;
}
return $arrVideoPage;
}
/**
* generate video list uri
*
* @param integer $intVCategoryId
* @param integer $intPage
* @return string
*/
public function getVideoUri(int $intVSourceId): string
{
return sprintf('/inc/api_mac10.php?ac=detail&tds=%s', $intVSourceId);
}
/**
* retrieve multiple chapter page model
*
* @param string $strUri
* @return VideoInfoPage[]
*/
public function getVideoInfoPageList(array $arrFilter): array
{
/** @var VideoInfoPage[] */
$arrVideoInfoPage = [];
$arrPromises = [];
foreach ($arrFilter as $intKey => $arrItem) {
$arrFilter[$intKey]['uri'] = $this->getVideoUri($arrItem['v_source_id']);
$arrPromises[$intKey] = $arrFilter[$intKey]['uri'];
}
print_r($arrFilter);
$arrResults = Promise\Utils::settle($arrPromises)->wait();
var_dump($arrResults);
return [];
$arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
return $arrResult['state'] === 'fulfilled';
});
foreach ($arrSuccessfulResults as $intKey => $arrResult) {
$Response = $arrResult['value'];
var_dump($arrResult);
$strContent = $Response->getBody()->getContents();;
$VideoInfoPage = $this->getVideoInfoPage($this, $arrFilter[$intKey]['uri']);
$VideoInfoPage->setContent($strContent);
$VideoInfoPage->setData($arrFilter[$intKey]);
$arrVideoInfoPage[] = $VideoInfoPage;
}
return $arrVideoInfoPage;
}
/**
* download the image to the local system
*
* @param array $arrFilter
* @return void
*/
public function downRemoteImgToLocal(array $arrFilter)
{
$StorageCore = StorageCore::getInstance();
$arrPromises = [];
foreach ($arrFilter as $intKey => $arrItem) {
if (!$StorageCore->has($arrItem['local_path'])) {
$arrPromises[$intKey] = $this->getClient()->getAsync($arrItem['uri']);
}
}
$arrResults = Promise\Utils::settle($arrPromises)->wait();
$arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
return $arrResult['state'] === 'fulfilled';
});
foreach ($arrSuccessfulResults as $intKey => $arrResult) {
$Response = $arrResult['value'];
$StorageCore->writeStream($arrFilter[$intKey]['local_path'], $Response->getBody()->detach());;
}
}
/**
* Undocumented function
*
* @param object $Site
* @param string $strUri
* @return VideoPage
*/
public function getVideoPage($Site, $strUri)
{
return new VideoPage($Site, $strUri);
}
/**
* Undocumented function
*
* @param object $Site
* @param string $strUri
* @return VideoInfoPage
*/
public function getVideoInfoPage($Site, $strUri)
{
return new VideoInfoPage($Site, $strUri);
}
}

194
code/app/task/crawler/youzhi/page/VideoInfoPage.php Normal file
View File

@@ -0,0 +1,194 @@
<?php
declare(strict_types=1);
namespace app\task\crawler\youzhi\page;
use app\model\VideoModel;
use storage\StorageCore;
class VideoInfoPage extends BasePage
{
/**
* Undocumented variable
*
* @var string
*/
protected $strChapterContent = '';
/**
* Undocumented variable
*
* @var VideoModel
*/
protected $VideoModel;
/**
* Undocumented function
*
* @return VideoModel
*/
public function getVideoModel(): VideoModel
{
if ($this->VideoModel == NULL) {
$this->VideoModel = VideoModel::getInstance();
}
return $this->VideoModel;
}
public $arrOptions = [
'typeMap' => [
'root' => 'array',
'document' => 'array',
'array' => 'array',
],
];
/**
* Undocumented function
*
* @return array
*/
public function getVideo(): array
{
if (empty($this->arrVideo)) {
$this->arrVideo = json_decode($this->getContent(), true);
}
return $this->arrVideo;
}
/**
* Undocumented variable
*
* @var array
*/
protected $arrVideo;
/**
* Undocumented function
*
* @param integer $intNId
* @return boolean
*/
public function saveChapter(int $intNId): bool
{
$arrPageChapter = $this->getVideo();
if (
empty($arrPageChapter) ||
empty($arrPageChapter['c_page_title']) ||
empty($arrPageChapter['c_page_data']) ||
count($arrPageChapter['c_page_data']) < 5
) {
return false;
}
$strUUId = $this->getSite()->getSiteUUId('chapter-' . $this->arrData['c_source_id'] . '-' . $this->arrData['c_page']);
$arrChapter = [
'n_id' => $intNId,
'c_name' => $arrPageChapter['c_page_data'][3],
'c_sort_num' => $this->arrData['c_sort_num'],
'c_page' => $this->arrData['c_page'],
'c_source_id' => $this->arrData['c_source_id'],
'c_site_code' => $this->getSite()->getSiteCode(),
'c_source_uuid' => $strUUId,
];
$arrExistsChapter = $this->getVideoModel()->findOne(['c_source_uuid' => $strUUId], $this->arrOptions);
if ($arrExistsChapter) {
$arrExistsChapter = (array) $arrExistsChapter;
$arrUpdate = [
'updated_at' => new \MongoDB\BSON\UTCDateTime(),
];
$this->getVideoModel()->updateOne(
['c_source_uuid' => $strUUId],
['$set' => $arrUpdate],
);
$this->strCContentPath = $arrExistsChapter['c_content_path'];
} else {
$arrInsertNovel = $arrChapter;
$arrInsertNovel['c_content_path'] = '/novel/' . ($intNId % 100) . '/' . $intNId . '/' . $arrChapter['c_sort_num'] . '_' . $arrChapter['c_page'] . '.txt';
$arrInsertNovel['created_at'] = new \MongoDB\BSON\UTCDateTime();
try {
$this->getVideoModel()->insert($arrInsertNovel);
$this->updateNovelHaveChapter($intNId);
$this->strCContentPath = $arrInsertNovel['c_content_path'];
} catch (\Exception $e) {
if ($e->getCode() === 11000) {
return $this->saveChapter($intNId);
} else {
throw $e;
}
}
}
StorageCore::getInstance()->put($this->strCContentPath, $this->strChapterContent);
return true;
}
public $strCContentPath;
/**
* Undocumented function
*
* @param integer $intNId
* @return void
*/
protected function updateNovelHaveChapter(int $intNId)
{
$arrFilter = [
'n_id' => $intNId,
];
$VideoModel = VideoModel::getInstance();
$arrLasteChapter = $VideoModel->getLatestChapterByNId($intNId, false);
if (!empty($arrLasteChapter)) {
$arrUpdate['$set']['n_have_chapter'] = 1;
$arrUpdate['$set']['n_latest_chapter_name'] = $arrLasteChapter['c_name'];
$this->getMongoBase()->updateOne('novel', $arrFilter, $arrUpdate);
}
}
protected $arrNextPage;
/**
* Undocumented function
*
* @return array
*/
public function getNextPage(): array
{
if ($this->arrNextPage === NULL) {
$this->arrNextPage = [];
if ($this->isMobile()) {
$strPattern = "/var\s+hhekgsv\s*=\s*['\"](.*?)['\"]/";
preg_match_all($strPattern, $this->getContent(), $arrMatches);
$strNextPageUri = end($arrMatches[1]) ?: '';
} else {
$strHref = $this->getQueryList()->find('.read_btn a:contains("下一章")')->attr('href');
$strNextPageUri = $strHref ?: '';
}
if (is_numeric(strpos($strNextPageUri, '_'))) {
$this->arrNextPage = $this->arrData;
$this->arrNextPage['c_page'] += 1;
$this->arrNextPage['uri'] = $strNextPageUri;
}
}
return $this->arrNextPage;
}
}

417
code/app/task/crawler/youzhi/page/VideoPage.php Normal file
View File

@@ -0,0 +1,417 @@
<?php
declare(strict_types=1);
namespace app\task\crawler\youzhi\page;
use app\model\CategoryModel;
use app\model\CountersModel;
use app\model\NovelModel;
use storage\StorageCore;
class VideoPage extends BasePage
{
/**
* Undocumented variable
*
* @var array
*/
protected $arrVideo = [];
/**
* Undocumented variable
*
* @var array
*/
protected $arrRelatedNovel = [];
/**
* Undocumented variable
*
* @var array
*/
protected $arrForgeNovel = [];
/**
* Undocumented variable
*
* @var NovelModel
*/
protected $NovelModel;
/**
* Undocumented function
*
* @return NovelModel
*/
public function getNovelModel(): NovelModel
{
if ($this->NovelModel == NULL) {
$this->NovelModel = NovelModel::getInstance();
}
return $this->NovelModel;
}
public $arrOptions = [
'typeMap' => [
'root' => 'array',
'document' => 'array',
'array' => 'array',
],
];
/**
* 查询看的小说
*
* @param integer $intNSourceId
* @param string $strName
* @return array
*/
protected function findOrCreateNovel(int $intNSourceId, string $strName): array
{
$strUUId = $this->getSite()->getSiteUUId($intNSourceId);
$arrExistsNovel = $this->getNovelModel()->findOne(['n_source_uuid' => $strUUId], $this->arrOptions);
if ($arrExistsNovel) {
$arrVideo = (array) $arrExistsNovel;
} else {
$arrVideo = [
'n_id' => CountersModel::getInstance()->getNextSequence('novel'),
'n_name' => $strName,
'n_name_pinyin' => zhToPinYin($strName),
'n_site_code' => $this->getSite()->getSiteCode(),
'n_source_uuid' => $strUUId,
'n_source_id' => $intNSourceId,
'n_level' => 1,
'n_have_chapter' => 0,
'created_at' => new \MongoDB\BSON\UTCDateTime(),
];
try {
$this->getNovelModel()->insert($arrVideo);
} catch (\Exception $e) {
if ($e->getCode() === 11000) {
return $this->findOrCreateNovel($intNSourceId, $strName);
} else {
throw $e;
}
}
}
return $arrVideo;
}
/**
* 分析关联数据
*
* @return void
*/
public function getReletedNovel() {}
/**
* Undocumented function
*
* @return array
*/
public function getVideoPager(): array
{
if (empty($this->arrVideo)) {
$this->arrVideo = json_decode($this->getContent(), true);
$this->getReletedNovel();
}
return $this->arrVideo;
}
/**
* Undocumented function
*
* @param string $strName
* @return string
*/
public function generateCoverPath($strName): string
{
$intTime = time() - mt_rand(1, 2 * 24 * 3600 * 360);
return sprintf('/img/%s/%s.jpg', date('Y/m/d/H', $intTime), $strName);
}
public function filterOtherFields(&$arrNovel)
{
$arrFields = [
'n_id',
'n_name',
'n_name_pinyin',
'n_status',
'n_author',
'n_description',
'n_category',
'n_category_pinyin',
'n_category_sex_en',
'n_category_sex_zh',
'n_cover_path',
'n_latest_chapter_name',
'n_update_time',
'n_forge_novel',
'n_related_novel',
'n_site_code',
'n_source_uuid',
'n_source_id',
'n_have_chapter',
'created_at',
'updated_at',
];
foreach ($arrNovel as $strKey => $strVal) {
if (!in_array($strKey, $arrFields)) {
unset($arrNovel[$strKey]);
}
}
}
/**
* Undocumented function
*
* @return bool
*/
public function saveVideo(): bool
{
$arrPageInfo = $this->getVideo();
print_r($arrPageInfo);
return false;
if (
empty($arrPageInfo) ||
empty($arrPageInfo['og_title']) ||
empty($arrPageInfo['og_image']) ||
empty($arrPageInfo['og_novel_category'])
) {
return false;
}
$arrVideo = [
// 'n_id'=> $arrPageInfo['xxxxx'],
'n_name' => $arrPageInfo['og_title'],
'n_name_pinyin' => zhToPinYin($arrPageInfo['og_title']),
'n_status' => $arrPageInfo['og_novel_status'],
'n_author' => $arrPageInfo['og_novel_author'],
'n_description' => $arrPageInfo['og_description'],
'n_category' => $arrPageInfo['og_novel_category'],
// 'n_category_pinyin' => $arrPageInfo['xxxxx'],
// 'n_category_sex_en' => $arrPageInfo['xxxxx'],
// 'n_category_sex_zh' => $arrPageInfo['xxxxx'],
// 'n_cover_path' => $arrPageInfo['xxxxx'],
'n_latest_chapter_name' => $arrPageInfo['og_novel_latest_chapter_name'],
'n_update_time' => $arrPageInfo['og_novel_update_time'],
'n_forge_novel' => $this->arrForgeNovel,
'n_related_novel' => $this->arrRelatedNovel,
'n_site_code' => $this->getSite()->getSiteCode(),
'n_source_id' => $this->arrData['n_source_id'],
'n_source_uuid' => $this->getSite()->getSiteUUId($this->arrData['n_source_id']),
// 'n_have_chapter' => $arrPageInfo['xxxxx'],
// 'created_at' => $arrPageInfo['xxxxx'],
// 'updated_at' => $arrPageInfo['xxxxx'],
];
$arrExistsNovel = $this->getNovelModel()->findOne(['n_source_uuid' => $arrVideo['n_source_uuid']], $this->arrOptions);
if ($arrExistsNovel) {
$arrExistsNovel = (array) $arrExistsNovel;
$arrUpdate = [
'n_latest_chapter_name' => $arrVideo['n_latest_chapter_name'],
'n_update_time' => $arrVideo['n_update_time'],
'updated_at' => new \MongoDB\BSON\UTCDateTime(),
];
# 如果没有分类，说明这条数据是最初从关联shu那里采集的，需要补全全部信息
if (empty($arrExistsNovel['n_category'])) {
$arrUpdate = array_merge($arrUpdate, $arrVideo);
$arrUpdate['n_related_novel'] = $this->arrRelatedNovel;
$arrUpdate['n_forge_novel'] = $this->arrForgeNovel;
$arrUpdate['n_category_pinyin'] = CategoryModel::getPinYin($this->arrVideo['og_novel_category']);
$arrUpdate['n_category_sex_en'] = CategoryModel::getSexEn($this->arrVideo['og_novel_category']);
$arrUpdate['n_category_sex_zh'] = CategoryModel::getSexZh($this->arrVideo['og_novel_category']);
$arrUpdate['n_related_novel'] = $this->arrRelatedNovel;
$arrUpdate['n_forge_novel'] = $this->arrForgeNovel;
}
# 图片不存在的时候，需要重新生成
if (
!isset($arrExistsNovel['n_cover_path']) ||
empty($arrExistsNovel['n_cover_path']) ||
!StorageCore::getInstance()->has($arrExistsNovel['n_cover_path']) ||
(basename($arrExistsNovel['n_cover_path']) == '.jpg')
) {
$arrUpdate['n_cover_path'] = $this->generateCoverPath($arrVideo['n_name_pinyin']);
$this->strCoverLocalPath = $arrUpdate['n_cover_path'];
} else {
$this->strCoverLocalPath = $arrExistsNovel['n_cover_path'];
}
$this->getNovelModel()->updateOne(
['n_source_uuid' => $arrVideo['n_source_uuid']],
['$set' => $arrUpdate],
);
$this->intNId = $arrExistsNovel['n_id'];
} else {
$arrVideo['n_category_pinyin'] = CategoryModel::getPinYin($this->arrVideo['og_novel_category']);
$arrVideo['n_category_sex_en'] = CategoryModel::getSexEn($this->arrVideo['og_novel_category']);
$arrVideo['n_category_sex_zh'] = CategoryModel::getSexZh($this->arrVideo['og_novel_category']);
$arrVideo['n_id'] = CountersModel::getInstance()->getNextSequence('novel');
$arrVideo['n_level'] = 1;
$arrVideo['n_have_chapter'] = 0;
$arrVideo['n_cover_path'] = $this->generateCoverPath($arrVideo['n_name_pinyin']);
$arrVideo['n_related_novel'] = $this->arrRelatedNovel;
$arrVideo['n_forge_novel'] = $this->arrForgeNovel;
$arrVideo['created_at'] = new \MongoDB\BSON\UTCDateTime();
try {
$this->getNovelModel()->insert($arrVideo);
$this->intNId = $arrVideo['n_id'];
$this->strCoverLocalPath = $arrVideo['n_cover_path'];
} catch (\Exception $e) {
if ($e->getCode() === 11000) {
return $this->saveVideo();
} else {
throw $e;
}
}
}
return true;
}
public $strCoverLocalPath = '';
public $intNId;
/**
* Undocumented function
*
* @return array
*/
public function getDownCoverInfo(): array
{
return [
'uri' => $this->arrVideo['og_image'],
'local_path' => $this->strCoverLocalPath,
];
}
/**
* Undocumented variable
*
* @var array
*/
protected $arrLatestChapters = NULL;
/**
* Undocumented variable
*
* @var array
*/
protected $arrChapters = NULL;
public function getLatestChapterByNIdsList(): array
{
if ($this->arrLatestChapters === NULL) {
$this->arrLatestChapters = [];
if ($this->isMobile()) {
$strClass = '.chapter-list';
} else {
$strClass = '.section-list';
}
if ($this->arrData['page'] == 0) {
$this->getQueryList()->find($strClass)->eq(0)->find("a")->map(function ($A) {
$this->arrLatestChapters[] = [
'name' => $A->text(),
'uri' => $A->attr('href'),
];
});
}
}
return $this->arrLatestChapters;
}
/**
* Undocumented function
*
* @return array
*/
public function getChapters(): array
{
if ($this->arrChapters === NULL) {
$this->arrChapters = [];
if ($this->isMobile()) {
$strClass = '.chapter-list';
} else {
$strClass = '.section-list';
}
$intDivIndex = 0;
if ($this->arrData['page'] == 0) {
$intDivIndex = 1;
}
$this->getQueryList()->find($strClass)->eq($intDivIndex)->find("a")->map(function ($A) {
$this->arrChapters[] = [
'name' => $A->text(),
'uri' => $A->attr('href'),
];
});
}
return $this->arrChapters;
}
protected $arrNextPageArgs = NULL;
/**
* Undocumented function
*
* @return array
*/
public function getNextPageArgs(): array
{
$this->arrNextPageArgs = [];
if ($this->arrData['page'] == 0) {
$this->getLatestChapterByNIdsList();
$this->getChapters();
$arrLatestChapterFirstBlock = count($this->arrLatestChapters) > 0 ? $this->arrLatestChapters[0] : null;
$arrLatestChapterSecondBlock = count($this->arrChapters) > 0 ? $this->arrChapters[count($this->arrChapters) - 1] : null;
if (
!empty($arrLatestChapterFirstBlock) && !empty($arrLatestChapterSecondBlock)
&& ($arrLatestChapterFirstBlock['uri'] != $arrLatestChapterSecondBlock['uri'])
) {
$this->arrNextPageArgs = [
'n_source_id' => $this->arrData['n_source_id'],
'page' => $this->arrData['page'] + 1,
];
}
} else {
$strLastUri = $this->getQueryList()->find('select:first option:last')->attr('value');
if ($strLastUri != '' && $strLastUri != $this->strUri) {
$this->arrNextPageArgs = [
'n_source_id' => $this->arrData['n_source_id'],
'page' => $this->arrData['page'] + 1,
];
}
}
return $this->arrNextPageArgs;
}
}

25
code/app/task/module/PlanTask.php
View File

@@ -7,6 +7,8 @@ namespace app\task\module;
use app\model\PlanTaskModel;
use app\task\crawler\zw630\Scheduler as Zw630Scheduler;
use app\task\crawler\zw74\Scheduler as Zw74Scheduler;
use app\task\crawler\youzhi\Scheduler as YouZhiScheduler;
use microserver\QueueManage;
use microserver\ProcessSanitizer;
@@ -42,11 +44,30 @@ class PlanTask
case 'PULL_74_ZW_NOVEL_LATEST':
self::pull74ZwNovelLatest();
break;
default:
case 'PULL_YOU_ZHI_SHI_PIN':
self::pullYouZhiShiPin();
break;
}
}
}
/**
* Undocumented function
*
* @return void
*/
public static function pullYouZhiShiPin()
{
$arrTask = [
'callback' => [YouZhiScheduler::class, 'crawlFull'],
'data' => []
];
$QueueManage = new QueueManage();
$QueueManage->set($arrTask);
}
/**
* Undocumented function
@@ -81,7 +102,7 @@ class PlanTask
}
/**
/**
* Undocumented function
*
* @return void

8
code/database/seeders/PlanTaskSeeder.php
View File

@@ -44,6 +44,14 @@ class PlanTaskSeeder
'pt_enable' => 0,
'pt_limit' => 1 * 24 * 3600,
],
[
'pt_name' => '优质资源库全量采集',
'pt_code' => 'PULL_YOU_ZHI_SHI_PIN',
'pt_enable' => 0,
'pt_limit' => 1 * 24 * 3600,
],
];
foreach ($arrDataNovel as $arrPlanTask) {

2
code/extend/template/taglib/Chapter.php
View File

@@ -7,7 +7,7 @@ use think\template\TagLib;
class Chapter extends TagLib
{
protected $tags = [
'list' => ['attr' => 'limit,n_category,sort_type,n_status,n_sex,key,diff_key,cache_life,d_key,d_val', 'close' => 1],
'list' => ['attr' => 'count,n_category,sort_type,n_status,n_sex,key,diff_key,cache_life,d_key,d_val', 'close' => 1],
'pager' => ['attr' => 'page,limit,n_category,sort_type,n_status,n_sex,key,diff_key,cache_life,d_key,d_val,p_val,export_name', 'close' => 1],
'pagerexp' => ['attr' => 'page,limit,n_category,sort_type,n_status,n_sex,key,diff_key,cache_life,d_key,d_val,p_val,func,export_name', 'close' => 0],
'info' => ['attr' => 'n_id,n_key', 'close' => 0],

77
code/extend/template/taglib/Novel.php
View File

@@ -7,7 +7,7 @@ use think\template\TagLib;
class Novel extends TagLib
{
protected $tags = [
'list' => ['attr' => 'limit,n_category,sort_type,n_status,n_sex,key,diff_key,cache_life,d_key,d_val', 'close' => 1],
'list' => ['attr' => 'count,n_category,sort_type,n_status,n_sex,diff_key,cache_life,d_key,d_val', 'close' => 1],
'pager' => ['attr' => 'page,limit,n_category,sort_type,n_status,n_sex,key,diff_key,cache_life,d_key,d_val,p_val,func,export_name', 'close' => 1],
'pagerexp' => ['attr' => 'page,limit,n_category,sort_type,n_status,n_sex,key,diff_key,cache_life,d_key,d_val,p_val,func,export_name', 'close' => 0],
'info' => ['attr' => 'n_id,n_key', 'close' => 0],
@@ -15,6 +15,8 @@ class Novel extends TagLib
'status' => ['attr' => 'd_key,d_val'],
'sex' => ['attr' => 'd_key,d_val'],
'category' => ['attr' => 'd_key,d_val,n_sex'],
'ranklist' => ['attr' => 'count,sort_type,n_status,n_sex,diff_key,cache_life,d_key,d_val', 'close' => 1],
];
private function customBuildVar($mixedArgs)
@@ -27,8 +29,67 @@ class Novel extends TagLib
}
/**
* novel 自定义标签
* tagRankList
*
* @param array $tag
* @param string $content
* @return void
*/
public function tagRankList($tag, $content)
{
$count = $tag['count'] ?? '10';
$sort_type = $tag['sort_type'] ?? "";
$n_status = $tag['n_status'] ?? "all";
$n_sex = $tag['n_sex'] ?? "all";
$diff_key = $tag['diff_key'] ?? "";
$cache_life = $tag['cache_life'] ?? '0';
$d_key = $tag['d_key'] ?? 'd_key';
$d_val = $tag['d_val'] ?? 'd_val';
$sort_type = $this->customBuildVar($sort_type);
$n_status = $this->customBuildVar($n_status);
$n_sex = $this->customBuildVar($n_sex);
$diff_key = $this->customBuildVar($diff_key);
$cache_life = $this->customBuildVar($cache_life);
$arrDataName = randomString(5);
$strParse = <<<EOT
<?php
\${$arrDataName} = app(\\app\\services\\NovelService::class)->getRankList([
'count' => {$count},
'sort_type' => {$sort_type},
'n_status' => {$n_status},
'n_sex' => {$n_sex},
'diff_key' => {$diff_key},
'cache_life' => {$cache_life}
]);
if (!empty(\${$arrDataName}) && is_array(\${$arrDataName})):
?>
EOT;
$strParse .= <<<EOT
<?php foreach (\${$arrDataName} as \${$d_key} => \${$d_val}): ?>
EOT;
$strParse .= $content;
$strParse .= <<<EOT
<?php endforeach; ?>
<?php endif; ?>
EOT;
return $strParse;
}
/**
* tagPagerExp
*
* @param array $tag
* @param string $content
* @return void
*/
public function tagPagerExp($tag, $content)
{
@@ -36,7 +97,11 @@ class Novel extends TagLib
}
/**
* novel 自定义标签
* tagPager
*
* @param array $tag
* @param string $content
* @return void
*/
public function tagPager($tag, $content)
{
@@ -115,7 +180,11 @@ EOT;
}
/**
* novel 自定义标签
* tagList
*
* @param array $tag
* @param string $content
* @return void
*/
public function tagList($tag, $content)
{

46
code/extend/template/taglib/Site.php
View File

@@ -18,6 +18,8 @@ class Site extends TagLib
'nflurl' => ['attr' => 'gender,category,status,order,page', 'close' => 0],
'grm' => ['attr' => 'domain,code,line,diff,start,end,index,type', 'close' => 0],
'replace' => ['attr' => 'code', 'close' => 0],
'forgelist' => ['attr' => 'count,diff_key,cache_life,d_key,d_val', 'close' => 1],
];
private function customBuildVar($mixedArgs)
@@ -29,6 +31,50 @@ class Site extends TagLib
}
}
/**
* tagRankList
*
* @param array $tag
* @param string $content
* @return void
*/
public function tagForgeList($tag, $content)
{
$count = $tag['count'] ?? '10';
$diff_key = $tag['diff_key'] ?? "";
$cache_life = $tag['cache_life'] ?? '0';
$d_key = $tag['d_key'] ?? 'd_key';
$d_val = $tag['d_val'] ?? 'd_val';
$diff_key = $this->customBuildVar($diff_key);
$cache_life = $this->customBuildVar($cache_life);
$arrDataName = randomString(5);
$strParse = <<<EOT
<?php
\${$arrDataName} = app(\\app\\services\\SiteContext::class)->getForgeList([
'count' => {$count},
'diff_key' => {$diff_key},
'cache_life' => {$cache_life}
]);
if (!empty(\${$arrDataName}) && is_array(\${$arrDataName})):
?>
EOT;
$strParse .= <<<EOT
<?php foreach (\${$arrDataName} as \${$d_key} => \${$d_val}): ?>
EOT;
$strParse .= $content;
$strParse .= <<<EOT
<?php endforeach; ?>
<?php endif; ?>
EOT;
return $strParse;
}
/**
* cfg 标签处理函数-系统配置
*