admin/SEONexus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'dev' of ssh://git.bgbg.live:18150/root/SEONexus into dev

Browse Source
This commit is contained in:
make
2025-04-23 10:49:09 +08:00
parent 0f24fae826 7f89da9839
commit eb0c21dff9
12 changed files with 228 additions and 507 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
code/app/common.php
View File

@@ -815,15 +815,51 @@ namespace {
* @param integer $length
* @return string
*/
function randomString(int $length): string {
function randomString(int $length): string
{
$chars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ';
$max = 51; // 字符集长度减1 (0-51)
$result = '';
for ($i = 0; $i < $length; $i++) {
$result .= $chars[random_int(0, $max)];
}
return $result;
}
/**
* 图像转换函数
*
* @param string $strInputFile
* @param string $strOutputFile
* @param integer $intQuality
* @return boolean
*/
function convertToWebP(string $strInputFile, string $strOutputFile, int $intQuality = 80): bool
{
$strCwebpPath = root_path() . '/extend/tools/cwebp';
$strCMD = escapeshellcmd("$strCwebpPath -q $intQuality " . escapeshellarg($strInputFile) . " -o " . escapeshellarg($strOutputFile));
exec($strCMD . ' 2>&1', $strOutput, $intReturnCode);
if ($intReturnCode === 0) {
return true;
} else {
echo "Conversion failed: " . implode("\n", $strOutput);
return false;
}
}
function ensureUtf8(string $input): string
{
// 检查是否为有效的 UTF-8
if (mb_check_encoding($input, 'UTF-8')) {
return $input; // 已经是有效的 UTF-8
}
// 尝试将非 UTF-8 转换为 UTF-8，忽略无效字符
return iconv('UTF-8', 'UTF-8//IGNORE', $input);
}
}

2
code/app/model/ConverterMovel.php
View File

@@ -88,7 +88,7 @@ class ConverterMovel
* Undocumented function
*
* @param string $strKey
* @return void
* @return string
*/
static public function getVal(string $strKey)
{

1
code/app/model/VideoClicksModel.php
View File

@@ -70,7 +70,6 @@ class VideoClicksModel extends MongoModel
],
[
'$inc' => ['vc_clicks' => 1],
'$set' => ['v_category_id' => $Novel->v_category_id]
],
[
'upsert' => true

45
code/app/model/VideoModel.php
View File

@@ -43,9 +43,6 @@ class VideoModel extends MongoModel
],
'projection' => [
'_id' => 0,
'n_site_code' => 0,
'n_source_uuid' => 0,
'n_source_id' => 0,
],
];
@@ -63,12 +60,12 @@ class VideoModel extends MongoModel
try {
$arrConditions = array_map(function ($strKeyword) {
return ['n_name' => ['$regex' => trim($strKeyword), '$options' => 'i']];
return ['v_name' => ['$regex' => trim($strKeyword), '$options' => 'i']];
}, $arrKeywords);
$UpdateResult = $this->getCol()->updateMany(
['$or' => $arrConditions],
['$set' => ['n_level' => $intLevel]]
['$set' => ['v_level' => $intLevel]]
);
$UpdateResult->getModifiedCount();
@@ -99,7 +96,6 @@ class VideoModel extends MongoModel
$arrOptions['sort'] = $arrSort;
}
$arrFilter['n_have_chapter'] = 1;
$Cursor = $this->getCol()->find($arrFilter, $arrOptions);
@@ -179,7 +175,7 @@ class VideoModel extends MongoModel
*/
public function getVideoByNId(int $intNId): null|array
{
$arrFilter = ['n_id' => $intNId, 'n_have_chapter' => 1];
$arrFilter = ['v_id' => $intNId];
return $this->findOne($arrFilter);
}
@@ -202,7 +198,6 @@ class VideoModel extends MongoModel
return $arrResult;
}
$arrFilter['n_have_chapter'] = 1;
$arrPipeline = [
['$match' => $arrFilter],
@@ -236,12 +231,11 @@ class VideoModel extends MongoModel
array $arrFilter,
int $intPage = 1,
int $intLimit = 20,
array $arrSort = ['n_level' => -1],
array $arrSort = ['v_level' => -1],
string $strKey = '',
int $intLifeTime = 24 * 3600,
): array {
try {
$arrFilter['n_have_chapter'] = 1;
$arrResult = $this->checkCacheByKey($strKey);
if ($arrResult !== NULL) {
@@ -295,23 +289,23 @@ class VideoModel extends MongoModel
int $intLifeTime = 24 * 3600,
int $intType = 0,
$strStatus = NULL,
array $arrSort = ['n_level' => -1]
array $arrSort = ['v_level' => -1]
): array {
$arrFilter = ['n_have_chapter' => 1,];
$arrFilter = [];
switch ($intType) {
case 1:
$arrCategory = array_values(CategoryModel::getManCategory());
$arrFilter['n_category'] = ['$in' => $arrCategory];
$arrFilter['v_category'] = ['$in' => $arrCategory];
break;
case 2:
$arrCategory = array_values(CategoryModel::getWomenCategory());
$arrFilter['n_category'] = ['$in' => $arrCategory];
$arrFilter['v_category'] = ['$in' => $arrCategory];
break;
}
if ($strStatus !== NULL) {
$arrFilter['n_status'] = $strStatus;
$arrFilter['v_status'] = $strStatus;
}
return $this->findPaginatedWithCache($arrFilter, $intPage, $intLimit, $arrSort, $strKey, $intLifeTime,);
}
@@ -340,7 +334,6 @@ class VideoModel extends MongoModel
return $arrResult;
}
$arrFilter['n_have_chapter'] = 1;
$arrOptions = self::$arrOptions;
@@ -385,17 +378,17 @@ class VideoModel extends MongoModel
int $intCount = 10,
int $intType = 0,
$strStatus = NULL,
array $arrSort = ['n_level' => -1]
array $arrSort = ['v_level' => -1]
): array {
try {
$arrFilter = [];
if (in_array($intType, [1, 2])) {
$arrFilter['n_category'] = ['$in' => CategoryModel::getCategoryByType($intType)];
$arrFilter['v_category'] = ['$in' => CategoryModel::getCategoryByType($intType)];
}
if ($strStatus !== NULL) {
$arrFilter['n_status'] = $strStatus;
$arrFilter['v_status'] = $strStatus;
}
return $this->findManyShuffledWithCache($arrFilter, $intCount, $arrSort, $strKey, $intLifeTime);
@@ -409,7 +402,6 @@ class VideoModel extends MongoModel
*
* @param string $strSign
* @param integer $intCount
* @param integer $intType
* @param string strDateType
* * daily 天
* * weekly 周
@@ -417,7 +409,7 @@ class VideoModel extends MongoModel
* * total 总
* @return array
*/
public function getRankVideoOnCache(string $strKey, int $intCount = 10, int $intType = 0, $strDateType = 'daily', $strStatus = NULL): array
public function getRankVideoOnCache(string $strKey, int $intCount = 10, $strDateType = 'daily', $strStatus = NULL): array
{
try {
$intLifeTime = 24 * 3600;
@@ -431,18 +423,15 @@ class VideoModel extends MongoModel
];
if ($strStatus !== NULL) {
$arrFilter['n_status'] = $strStatus;
$arrFilter['v_status'] = $strStatus;
}
if (in_array($intType, [1, 2])) {
$arrFilter['n_category'] = ['$in' => CategoryModel::getCategoryByType($intType)];
}
$arrOptions = [
'sort' => ['nc_clicks' => -1],
'limit' => $intCount * 3,
'projection' => [
'n_id' => 1,
'v_id' => 1,
],
];
@@ -452,10 +441,10 @@ class VideoModel extends MongoModel
$arrNId = [];
foreach ($Cursor as $Doc) {
$arrNId[] = $Doc['n_id'];
$arrNId[] = $Doc['v_id'];
}
$arrFilter = ['n_id' => ['$in' => $arrNId]];
$arrFilter = ['v_id' => ['$in' => $arrNId]];
$arrOptions = self::$arrOptions;
unset($arrOptions['typeMap']);

34
code/app/task/crawler/youzhi/Scheduler.php
View File

@@ -74,15 +74,12 @@ class Scheduler
*/
public function crawlFull()
{
// $Site = $this->getSite();
// $arrVSourceId = [];
// $arrVSourceId[] = ['v_source_id' => 14256];
// $arrAllVideoInfoPage = $Site->getVideoInfoPageList($arrVSourceId);
// foreach ($arrAllVideoInfoPage as $VideoInfoPage) {
// $arrVideo = $VideoInfoPage->getVideo();
// $VideoInfoPage->saveVideo();
// print_r($arrVideo);
// }
// $arrVSourceId = [['v_source_id' => 14256]];
// $arrTask = [
// 'callback' => [static::class, 'fetchVideo'],
// 'data' => $arrVSourceId,
// ];
// $this->QueueManage->set($arrTask);
// return;
$Site = $this->getSite();
@@ -141,13 +138,12 @@ class Scheduler
'data' => [],
];
# 访问分页，获取 v_source_id 集合
$arrAllVideoPageList = $Site->getVideoPageList($arrFilter);
$arrAllVideoPagerPage = $Site->getVideoPagerPageList($arrFilter);
$arrVSourceId = [];
foreach ($arrAllVideoPageList as $VideoPageList) {
$arrVideoPager = $VideoPageList->getVideoPager();
foreach ($arrAllVideoPagerPage as $VideoPagerPage) {
$arrVideoPager = $VideoPagerPage->getVideoPager();
foreach ($arrVideoPager['list'] as $arrVideoPager) {
$arrVSourceId[] = ['v_source_id' => (int)$arrVideoPager['vod_id']];
@@ -177,8 +173,20 @@ class Scheduler
$Site = $this->getSite();
$arrAllVideoInfoPage = $Site->getVideoInfoPageList($arrVSourceId);
$arrPic = [];
foreach ($arrAllVideoInfoPage as $VideoInfoPage) {
$VideoInfoPage->saveVideo();
$arrPicItem = $VideoInfoPage->getDownPicInfo();
if (!StorageCore::getInstance()->has($arrPicItem['local_path'])) {
$arrPic[] = $arrPicItem;
}
}
if ($arrPic) {
$Site->downRemoteImgToLocal($arrPic);
}
}
}

62
code/app/task/crawler/youzhi/page/Site.php
View File

@@ -142,7 +142,7 @@ class Site
* retrieve multiple novel page model
*
* @param string $strUri
* @return VideoPage[]
* @return array
*/
public function getCategory(): array
{
@@ -204,13 +204,13 @@ class Site
* retrieve multiple novel page model
*
* @param string $strUri
* @return VideoPage[]
* @return VideoPagerPage[]
*/
public function getVideoPageList(array $arrFilter): array
public function getVideoPagerPageList(array $arrFilter): array
{
/** @var VideoPage[] */
$arrVideoPage = [];
/** @var VideoPagerPage[] */
$arrVideoPagerPage = [];
foreach ($arrFilter as $intKey => $arrItem) {
@@ -232,15 +232,15 @@ class Site
$strContent = $Response->getBody()->getContents();
$VideoPage = $this->getVideoPage($this, $arrFilter[$intKey]['uri']);
$VideoPagerPage = $this->getVideoPagerPage($this, $arrFilter[$intKey]['uri']);
$VideoPage->setContent($strContent);
$VideoPage->setData($arrFilter[$intKey]);
$VideoPagerPage->setContent($strContent);
$VideoPagerPage->setData($arrFilter[$intKey]);
$arrVideoPage[] = $VideoPage;
$arrVideoPagerPage[] = $VideoPagerPage;
}
return $arrVideoPage;
return $arrVideoPagerPage;
}
/**
@@ -306,25 +306,31 @@ class Site
{
$StorageCore = StorageCore::getInstance();
$arrPromises = [];
$Client = new Client();
foreach ($arrFilter as $intKey => $arrItem) {
foreach ($arrFilter as $arrItem) {
if (!$StorageCore->has($arrItem['local_path'])) {
$arrPromises[$intKey] = $this->getClient()->getAsync($arrItem['uri']);
try {
$Response = $Client->get($arrItem['uri']);
if ($Response->getStatusCode() === 200) {
if (str_ends_with(strtolower($arrItem['local_path']), '.webp')) {
$strTmpFile = str_replace('.webp', '.jpg', $arrItem['local_path']);
$StorageCore->writeStream($strTmpFile, $Response->getBody()->detach());
convertToWebP(
$StorageCore->getRealPath($strTmpFile),
$StorageCore->getRealPath($arrItem['local_path']),
40
);
@unlink($StorageCore->getRealPath($strTmpFile));
} else {
$StorageCore->writeStream($arrItem['local_path'], $Response->getBody()->detach());
}
}
} catch (\Exception $e) {
// echo $e->getMessage();
}
}
}
$arrResults = Promise\Utils::settle($arrPromises)->wait();
$arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
return $arrResult['state'] === 'fulfilled';
});
foreach ($arrSuccessfulResults as $intKey => $arrResult) {
$Response = $arrResult['value'];
$StorageCore->writeStream($arrFilter[$intKey]['local_path'], $Response->getBody()->detach());;
}
}
/**
@@ -332,11 +338,11 @@ class Site
*
* @param object $Site
* @param string $strUri
* @return VideoPage
* @return VideoPagerPage
*/
public function getVideoPage($Site, $strUri)
public function getVideoPagerPage($Site, $strUri)
{
return new VideoPage($Site, $strUri);
return new VideoPagerPage($Site, $strUri);
}
/**

74
code/app/task/crawler/youzhi/page/VideoInfoPage.php
View File

@@ -24,6 +24,26 @@ class VideoInfoPage extends BasePage
*/
protected $VideoModel;
/**
* Undocumented variable
*
* @var array
*/
protected $arrVideo;
/**
* Undocumented variable
*
* @var array
*/
public $arrOptions = [
'typeMap' => [
'root' => 'array',
'document' => 'array',
'array' => 'array',
],
];
/**
* Undocumented function
*
@@ -37,13 +57,6 @@ class VideoInfoPage extends BasePage
return $this->VideoModel;
}
public $arrOptions = [
'typeMap' => [
'root' => 'array',
'document' => 'array',
'array' => 'array',
],
];
/**
* Undocumented function
@@ -67,7 +80,7 @@ class VideoInfoPage extends BasePage
foreach ($arrPlayUrl as $intKey => $arrVal) {
$arrPlayUrl[$intKey] = [
'name' => $arrVal[0],
'url' => $arrVal[1]?? $arrVal[0],
'url' => $arrVal[1] ?? $arrVal[0],
];
}
$arrVideo['vod_play_url'][$this->getsite()->getSiteCode()] = $arrPlayUrl;
@@ -75,7 +88,9 @@ class VideoInfoPage extends BasePage
$arrVideo['vod_director'] = explode(',', $arrVideo['vod_director']);
$arrVideo['vod_class'] = explode(',', $arrVideo['vod_class']);
$arrVideo['vod_content'] = trim($arrVideo['vod_content']);
$arrVideo['vod_content'] = trim($arrVideo['vod_content'], '\　');
$arrVideo['vod_content'] = ensureUtf8($arrVideo['vod_content']);
foreach ($arrVideo as $strKey => $mixedVal) {
$strKey = str_replace('vod', 'v', $strKey);
@@ -102,12 +117,6 @@ class VideoInfoPage extends BasePage
return $this->arrVideo;
}
/**
* Undocumented variable
*
* @var array
*/
protected $arrVideo;
/**
* Undocumented function
@@ -143,6 +152,7 @@ class VideoInfoPage extends BasePage
['v_name' => $arrPageVideo['v_name']],
['$set' => $arrUpdate],
);
$this->strPicLocalPath = $arrExistsVideo['pic'];
} else {
$arrInsertVideo = [
@@ -155,7 +165,7 @@ class VideoInfoPage extends BasePage
'v_director' => $arrPageVideo['v_director'],
'v_class' => $arrPageVideo['v_class'],
'v_letter' => $arrPageVideo['v_letter'],
'v_pic' => $arrPageVideo['v_pic'],
// 'v_pic' => $arrPageVideo['v_pic'],
'v_lang' => $arrPageVideo['v_lang'],
'v_area' => $arrPageVideo['v_area'],
'v_year' => $arrPageVideo['v_year'],
@@ -167,6 +177,9 @@ class VideoInfoPage extends BasePage
'v_publish_date' => $arrPageVideo['v_publish_date'],
'created_at' => new \MongoDB\BSON\UTCDateTime(),
];
$arrInsertVideo['v_pic'] = $this->generateVideoPath($arrInsertVideo['v_name_en']);
$this->strPicLocalPath = $arrInsertVideo['v_pic'];
try {
$this->getVideoModel()->insert($arrInsertVideo);
@@ -181,4 +194,33 @@ class VideoInfoPage extends BasePage
return true;
}
private $strPicLocalPath = '';
/**
* Undocumented function
*
* @param string $strName
* @return string
*/
public function generateVideoPath($strName): string
{
$intTime = time() - mt_rand(1, 2 * 24 * 3600 * 360);
return sprintf('/img/video/%s/%s.webp', date('Y/m/d/H', $intTime), $strName);
}
/**
* Undocumented function
*
* @return array
*/
public function getDownPicInfo(): array
{
return [
'uri' => $this->getVideo()['v_pic'],
'local_path' => $this->strPicLocalPath,
];
}
}

417
code/app/task/crawler/youzhi/page/VideoPage.php
View File

@@ -1,417 +0,0 @@
<?php
declare(strict_types=1);
namespace app\task\crawler\youzhi\page;
use app\model\CategoryModel;
use app\model\CountersModel;
use app\model\NovelModel;
use storage\StorageCore;
class VideoPage extends BasePage
{
/**
* Undocumented variable
*
* @var array
*/
protected $arrVideo = [];
/**
* Undocumented variable
*
* @var array
*/
protected $arrRelatedNovel = [];
/**
* Undocumented variable
*
* @var array
*/
protected $arrForgeNovel = [];
/**
* Undocumented variable
*
* @var NovelModel
*/
protected $NovelModel;
/**
* Undocumented function
*
* @return NovelModel
*/
public function getNovelModel(): NovelModel
{
if ($this->NovelModel == NULL) {
$this->NovelModel = NovelModel::getInstance();
}
return $this->NovelModel;
}
public $arrOptions = [
'typeMap' => [
'root' => 'array',
'document' => 'array',
'array' => 'array',
],
];
/**
* 查询看的小说
*
* @param integer $intNSourceId
* @param string $strName
* @return array
*/
protected function findOrCreateNovel(int $intNSourceId, string $strName): array
{
$strUUId = $this->getSite()->getSiteUUId($intNSourceId);
$arrExistsNovel = $this->getNovelModel()->findOne(['n_source_uuid' => $strUUId], $this->arrOptions);
if ($arrExistsNovel) {
$arrVideo = (array) $arrExistsNovel;
} else {
$arrVideo = [
'n_id' => CountersModel::getInstance()->getNextSequence('novel'),
'n_name' => $strName,
'n_name_pinyin' => zhToPinYin($strName),
'n_site_code' => $this->getSite()->getSiteCode(),
'n_source_uuid' => $strUUId,
'n_source_id' => $intNSourceId,
'n_level' => 1,
'n_have_chapter' => 0,
'created_at' => new \MongoDB\BSON\UTCDateTime(),
];
try {
$this->getNovelModel()->insert($arrVideo);
} catch (\Exception $e) {
if ($e->getCode() === 11000) {
return $this->findOrCreateNovel($intNSourceId, $strName);
} else {
throw $e;
}
}
}
return $arrVideo;
}
/**
* 分析关联数据
*
* @return void
*/
public function getReletedNovel() {}
/**
* Undocumented function
*
* @return array
*/
public function getVideoPager(): array
{
if (empty($this->arrVideo)) {
$this->arrVideo = json_decode($this->getContent(), true);
$this->getReletedNovel();
}
return $this->arrVideo;
}
/**
* Undocumented function
*
* @param string $strName
* @return string
*/
public function generateCoverPath($strName): string
{
$intTime = time() - mt_rand(1, 2 * 24 * 3600 * 360);
return sprintf('/img/%s/%s.jpg', date('Y/m/d/H', $intTime), $strName);
}
public function filterOtherFields(&$arrNovel)
{
$arrFields = [
'n_id',
'n_name',
'n_name_pinyin',
'n_status',
'n_author',
'n_description',
'n_category',
'n_category_pinyin',
'n_category_sex_en',
'n_category_sex_zh',
'n_cover_path',
'n_latest_chapter_name',
'n_update_time',
'n_forge_novel',
'n_related_novel',
'n_site_code',
'n_source_uuid',
'n_source_id',
'n_have_chapter',
'created_at',
'updated_at',
];
foreach ($arrNovel as $strKey => $strVal) {
if (!in_array($strKey, $arrFields)) {
unset($arrNovel[$strKey]);
}
}
}
/**
* Undocumented function
*
* @return bool
*/
public function saveVideo(): bool
{
$arrPageInfo = $this->getVideo();
print_r($arrPageInfo);
return false;
if (
empty($arrPageInfo) ||
empty($arrPageInfo['og_title']) ||
empty($arrPageInfo['og_image']) ||
empty($arrPageInfo['og_novel_category'])
) {
return false;
}
$arrVideo = [
// 'n_id'=> $arrPageInfo['xxxxx'],
'n_name' => $arrPageInfo['og_title'],
'n_name_pinyin' => zhToPinYin($arrPageInfo['og_title']),
'n_status' => $arrPageInfo['og_novel_status'],
'n_author' => $arrPageInfo['og_novel_author'],
'n_description' => $arrPageInfo['og_description'],
'n_category' => $arrPageInfo['og_novel_category'],
// 'n_category_pinyin' => $arrPageInfo['xxxxx'],
// 'n_category_sex_en' => $arrPageInfo['xxxxx'],
// 'n_category_sex_zh' => $arrPageInfo['xxxxx'],
// 'n_cover_path' => $arrPageInfo['xxxxx'],
'n_latest_chapter_name' => $arrPageInfo['og_novel_latest_chapter_name'],
'n_update_time' => $arrPageInfo['og_novel_update_time'],
'n_forge_novel' => $this->arrForgeNovel,
'n_related_novel' => $this->arrRelatedNovel,
'n_site_code' => $this->getSite()->getSiteCode(),
'n_source_id' => $this->arrData['n_source_id'],
'n_source_uuid' => $this->getSite()->getSiteUUId($this->arrData['n_source_id']),
// 'n_have_chapter' => $arrPageInfo['xxxxx'],
// 'created_at' => $arrPageInfo['xxxxx'],
// 'updated_at' => $arrPageInfo['xxxxx'],
];
$arrExistsNovel = $this->getNovelModel()->findOne(['n_source_uuid' => $arrVideo['n_source_uuid']], $this->arrOptions);
if ($arrExistsNovel) {
$arrExistsNovel = (array) $arrExistsNovel;
$arrUpdate = [
'n_latest_chapter_name' => $arrVideo['n_latest_chapter_name'],
'n_update_time' => $arrVideo['n_update_time'],
'updated_at' => new \MongoDB\BSON\UTCDateTime(),
];
# 如果没有分类，说明这条数据是最初从关联shu那里采集的，需要补全全部信息
if (empty($arrExistsNovel['n_category'])) {
$arrUpdate = array_merge($arrUpdate, $arrVideo);
$arrUpdate['n_related_novel'] = $this->arrRelatedNovel;
$arrUpdate['n_forge_novel'] = $this->arrForgeNovel;
$arrUpdate['n_category_pinyin'] = CategoryModel::getPinYin($this->arrVideo['og_novel_category']);
$arrUpdate['n_category_sex_en'] = CategoryModel::getSexEn($this->arrVideo['og_novel_category']);
$arrUpdate['n_category_sex_zh'] = CategoryModel::getSexZh($this->arrVideo['og_novel_category']);
$arrUpdate['n_related_novel'] = $this->arrRelatedNovel;
$arrUpdate['n_forge_novel'] = $this->arrForgeNovel;
}
# 图片不存在的时候，需要重新生成
if (
!isset($arrExistsNovel['n_cover_path']) ||
empty($arrExistsNovel['n_cover_path']) ||
!StorageCore::getInstance()->has($arrExistsNovel['n_cover_path']) ||
(basename($arrExistsNovel['n_cover_path']) == '.jpg')
) {
$arrUpdate['n_cover_path'] = $this->generateCoverPath($arrVideo['n_name_pinyin']);
$this->strCoverLocalPath = $arrUpdate['n_cover_path'];
} else {
$this->strCoverLocalPath = $arrExistsNovel['n_cover_path'];
}
$this->getNovelModel()->updateOne(
['n_source_uuid' => $arrVideo['n_source_uuid']],
['$set' => $arrUpdate],
);
$this->intNId = $arrExistsNovel['n_id'];
} else {
$arrVideo['n_category_pinyin'] = CategoryModel::getPinYin($this->arrVideo['og_novel_category']);
$arrVideo['n_category_sex_en'] = CategoryModel::getSexEn($this->arrVideo['og_novel_category']);
$arrVideo['n_category_sex_zh'] = CategoryModel::getSexZh($this->arrVideo['og_novel_category']);
$arrVideo['n_id'] = CountersModel::getInstance()->getNextSequence('novel');
$arrVideo['n_level'] = 1;
$arrVideo['n_have_chapter'] = 0;
$arrVideo['n_cover_path'] = $this->generateCoverPath($arrVideo['n_name_pinyin']);
$arrVideo['n_related_novel'] = $this->arrRelatedNovel;
$arrVideo['n_forge_novel'] = $this->arrForgeNovel;
$arrVideo['created_at'] = new \MongoDB\BSON\UTCDateTime();
try {
$this->getNovelModel()->insert($arrVideo);
$this->intNId = $arrVideo['n_id'];
$this->strCoverLocalPath = $arrVideo['n_cover_path'];
} catch (\Exception $e) {
if ($e->getCode() === 11000) {
return $this->saveVideo();
} else {
throw $e;
}
}
}
return true;
}
public $strCoverLocalPath = '';
public $intNId;
/**
* Undocumented function
*
* @return array
*/
public function getDownCoverInfo(): array
{
return [
'uri' => $this->arrVideo['og_image'],
'local_path' => $this->strCoverLocalPath,
];
}
/**
* Undocumented variable
*
* @var array
*/
protected $arrLatestChapters = NULL;
/**
* Undocumented variable
*
* @var array
*/
protected $arrChapters = NULL;
public function getLatestChapterByNIdsList(): array
{
if ($this->arrLatestChapters === NULL) {
$this->arrLatestChapters = [];
if ($this->isMobile()) {
$strClass = '.chapter-list';
} else {
$strClass = '.section-list';
}
if ($this->arrData['page'] == 0) {
$this->getQueryList()->find($strClass)->eq(0)->find("a")->map(function ($A) {
$this->arrLatestChapters[] = [
'name' => $A->text(),
'uri' => $A->attr('href'),
];
});
}
}
return $this->arrLatestChapters;
}
/**
* Undocumented function
*
* @return array
*/
public function getChapters(): array
{
if ($this->arrChapters === NULL) {
$this->arrChapters = [];
if ($this->isMobile()) {
$strClass = '.chapter-list';
} else {
$strClass = '.section-list';
}
$intDivIndex = 0;
if ($this->arrData['page'] == 0) {
$intDivIndex = 1;
}
$this->getQueryList()->find($strClass)->eq($intDivIndex)->find("a")->map(function ($A) {
$this->arrChapters[] = [
'name' => $A->text(),
'uri' => $A->attr('href'),
];
});
}
return $this->arrChapters;
}
protected $arrNextPageArgs = NULL;
/**
* Undocumented function
*
* @return array
*/
public function getNextPageArgs(): array
{
$this->arrNextPageArgs = [];
if ($this->arrData['page'] == 0) {
$this->getLatestChapterByNIdsList();
$this->getChapters();
$arrLatestChapterFirstBlock = count($this->arrLatestChapters) > 0 ? $this->arrLatestChapters[0] : null;
$arrLatestChapterSecondBlock = count($this->arrChapters) > 0 ? $this->arrChapters[count($this->arrChapters) - 1] : null;
if (
!empty($arrLatestChapterFirstBlock) && !empty($arrLatestChapterSecondBlock)
&& ($arrLatestChapterFirstBlock['uri'] != $arrLatestChapterSecondBlock['uri'])
) {
$this->arrNextPageArgs = [
'n_source_id' => $this->arrData['n_source_id'],
'page' => $this->arrData['page'] + 1,
];
}
} else {
$strLastUri = $this->getQueryList()->find('select:first option:last')->attr('value');
if ($strLastUri != '' && $strLastUri != $this->strUri) {
$this->arrNextPageArgs = [
'n_source_id' => $this->arrData['n_source_id'],
'page' => $this->arrData['page'] + 1,
];
}
}
return $this->arrNextPageArgs;
}
}

43
code/app/task/crawler/youzhi/page/VideoPagerPage.php Normal file
View File

@@ -0,0 +1,43 @@
<?php
declare(strict_types=1);
namespace app\task\crawler\youzhi\page;
use app\model\CategoryModel;
use app\model\CountersModel;
use app\model\NovelModel;
use storage\StorageCore;
class VideoPagerPage extends BasePage
{
/**
* Undocumented variable
*
* @var array
*/
protected $arrVideo = [];
public $arrOptions = [
'typeMap' => [
'root' => 'array',
'document' => 'array',
'array' => 'array',
],
];
/**
* Undocumented function
*
* @return array
*/
public function getVideoPager(): array
{
if (empty($this->arrVideo)) {
$this->arrVideo = json_decode($this->getContent(), true);
}
return $this->arrVideo;
}
}

10
code/database/mongo/migrations/video_clicks.php Normal file
View File

@@ -0,0 +1,10 @@
<?php
return [
'collection' => 'video_clicks',
'indexes' => [
[
'key' => ['v_id' => 1, 'vc_date' => 1, 'vc_type' => 1],
'options' => ['unique' => true]
],
]
];

5
code/extend/storage/StorageCore.php
View File

@@ -95,4 +95,9 @@ class StorageCore
{
return $this->Filesystem->has($strKey);
}
public function getRealPath($strKey): string
{
return $this->Adapter->applyPathPrefix($strKey);
}
}

BIN
code/extend/tools/cwebp Normal file
View File

Binary file not shown.