admin/SEONexus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

debug

Browse Source
This commit is contained in:
Default
2025-04-22 22:10:45 +08:00
parent 06ba3c8411
commit 1c1734bd60
12 changed files with 228 additions and 507 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
code/app/task/crawler/youzhi/Scheduler.php
View File

@@ -74,15 +74,12 @@ class Scheduler
*/
public function crawlFull()
{
// $Site = $this->getSite();
// $arrVSourceId = [];
// $arrVSourceId[] = ['v_source_id' => 14256];
// $arrAllVideoInfoPage = $Site->getVideoInfoPageList($arrVSourceId);
// foreach ($arrAllVideoInfoPage as $VideoInfoPage) {
// $arrVideo = $VideoInfoPage->getVideo();
// $VideoInfoPage->saveVideo();
// print_r($arrVideo);
// }
// $arrVSourceId = [['v_source_id' => 14256]];
// $arrTask = [
// 'callback' => [static::class, 'fetchVideo'],
// 'data' => $arrVSourceId,
// ];
// $this->QueueManage->set($arrTask);
// return;
$Site = $this->getSite();
@@ -141,13 +138,12 @@ class Scheduler
'data' => [],
];
# 访问分页，获取 v_source_id 集合
$arrAllVideoPageList = $Site->getVideoPageList($arrFilter);
$arrAllVideoPagerPage = $Site->getVideoPagerPageList($arrFilter);
$arrVSourceId = [];
foreach ($arrAllVideoPageList as $VideoPageList) {
$arrVideoPager = $VideoPageList->getVideoPager();
foreach ($arrAllVideoPagerPage as $VideoPagerPage) {
$arrVideoPager = $VideoPagerPage->getVideoPager();
foreach ($arrVideoPager['list'] as $arrVideoPager) {
$arrVSourceId[] = ['v_source_id' => (int)$arrVideoPager['vod_id']];
@@ -177,8 +173,20 @@ class Scheduler
$Site = $this->getSite();
$arrAllVideoInfoPage = $Site->getVideoInfoPageList($arrVSourceId);
$arrPic = [];
foreach ($arrAllVideoInfoPage as $VideoInfoPage) {
$VideoInfoPage->saveVideo();
$arrPicItem = $VideoInfoPage->getDownPicInfo();
if (!StorageCore::getInstance()->has($arrPicItem['local_path'])) {
$arrPic[] = $arrPicItem;
}
}
if ($arrPic) {
$Site->downRemoteImgToLocal($arrPic);
}
}
}

62
code/app/task/crawler/youzhi/page/Site.php
View File

@@ -142,7 +142,7 @@ class Site
* retrieve multiple novel page model
*
* @param string $strUri
* @return VideoPage[]
* @return array
*/
public function getCategory(): array
{
@@ -204,13 +204,13 @@ class Site
* retrieve multiple novel page model
*
* @param string $strUri
* @return VideoPage[]
* @return VideoPagerPage[]
*/
public function getVideoPageList(array $arrFilter): array
public function getVideoPagerPageList(array $arrFilter): array
{
/** @var VideoPage[] */
$arrVideoPage = [];
/** @var VideoPagerPage[] */
$arrVideoPagerPage = [];
foreach ($arrFilter as $intKey => $arrItem) {
@@ -232,15 +232,15 @@ class Site
$strContent = $Response->getBody()->getContents();
$VideoPage = $this->getVideoPage($this, $arrFilter[$intKey]['uri']);
$VideoPagerPage = $this->getVideoPagerPage($this, $arrFilter[$intKey]['uri']);
$VideoPage->setContent($strContent);
$VideoPage->setData($arrFilter[$intKey]);
$VideoPagerPage->setContent($strContent);
$VideoPagerPage->setData($arrFilter[$intKey]);
$arrVideoPage[] = $VideoPage;
$arrVideoPagerPage[] = $VideoPagerPage;
}
return $arrVideoPage;
return $arrVideoPagerPage;
}
/**
@@ -306,25 +306,31 @@ class Site
{
$StorageCore = StorageCore::getInstance();
$arrPromises = [];
$Client = new Client();
foreach ($arrFilter as $intKey => $arrItem) {
foreach ($arrFilter as $arrItem) {
if (!$StorageCore->has($arrItem['local_path'])) {
$arrPromises[$intKey] = $this->getClient()->getAsync($arrItem['uri']);
try {
$Response = $Client->get($arrItem['uri']);
if ($Response->getStatusCode() === 200) {
if (str_ends_with(strtolower($arrItem['local_path']), '.webp')) {
$strTmpFile = str_replace('.webp', '.jpg', $arrItem['local_path']);
$StorageCore->writeStream($strTmpFile, $Response->getBody()->detach());
convertToWebP(
$StorageCore->getRealPath($strTmpFile),
$StorageCore->getRealPath($arrItem['local_path']),
40
);
@unlink($StorageCore->getRealPath($strTmpFile));
} else {
$StorageCore->writeStream($arrItem['local_path'], $Response->getBody()->detach());
}
}
} catch (\Exception $e) {
// echo $e->getMessage();
}
}
}
$arrResults = Promise\Utils::settle($arrPromises)->wait();
$arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
return $arrResult['state'] === 'fulfilled';
});
foreach ($arrSuccessfulResults as $intKey => $arrResult) {
$Response = $arrResult['value'];
$StorageCore->writeStream($arrFilter[$intKey]['local_path'], $Response->getBody()->detach());;
}
}
/**
@@ -332,11 +338,11 @@ class Site
*
* @param object $Site
* @param string $strUri
* @return VideoPage
* @return VideoPagerPage
*/
public function getVideoPage($Site, $strUri)
public function getVideoPagerPage($Site, $strUri)
{
return new VideoPage($Site, $strUri);
return new VideoPagerPage($Site, $strUri);
}
/**

74
code/app/task/crawler/youzhi/page/VideoInfoPage.php
View File

@@ -24,6 +24,26 @@ class VideoInfoPage extends BasePage
*/
protected $VideoModel;
/**
* Undocumented variable
*
* @var array
*/
protected $arrVideo;
/**
* Undocumented variable
*
* @var array
*/
public $arrOptions = [
'typeMap' => [
'root' => 'array',
'document' => 'array',
'array' => 'array',
],
];
/**
* Undocumented function
*
@@ -37,13 +57,6 @@ class VideoInfoPage extends BasePage
return $this->VideoModel;
}
public $arrOptions = [
'typeMap' => [
'root' => 'array',
'document' => 'array',
'array' => 'array',
],
];
/**
* Undocumented function
@@ -67,7 +80,7 @@ class VideoInfoPage extends BasePage
foreach ($arrPlayUrl as $intKey => $arrVal) {
$arrPlayUrl[$intKey] = [
'name' => $arrVal[0],
'url' => $arrVal[1]?? $arrVal[0],
'url' => $arrVal[1] ?? $arrVal[0],
];
}
$arrVideo['vod_play_url'][$this->getsite()->getSiteCode()] = $arrPlayUrl;
@@ -75,7 +88,9 @@ class VideoInfoPage extends BasePage
$arrVideo['vod_director'] = explode(',', $arrVideo['vod_director']);
$arrVideo['vod_class'] = explode(',', $arrVideo['vod_class']);
$arrVideo['vod_content'] = trim($arrVideo['vod_content']);
$arrVideo['vod_content'] = trim($arrVideo['vod_content'], '\　');
$arrVideo['vod_content'] = ensureUtf8($arrVideo['vod_content']);
foreach ($arrVideo as $strKey => $mixedVal) {
$strKey = str_replace('vod', 'v', $strKey);
@@ -102,12 +117,6 @@ class VideoInfoPage extends BasePage
return $this->arrVideo;
}
/**
* Undocumented variable
*
* @var array
*/
protected $arrVideo;
/**
* Undocumented function
@@ -143,6 +152,7 @@ class VideoInfoPage extends BasePage
['v_name' => $arrPageVideo['v_name']],
['$set' => $arrUpdate],
);
$this->strPicLocalPath = $arrExistsVideo['pic'];
} else {
$arrInsertVideo = [
@@ -155,7 +165,7 @@ class VideoInfoPage extends BasePage
'v_director' => $arrPageVideo['v_director'],
'v_class' => $arrPageVideo['v_class'],
'v_letter' => $arrPageVideo['v_letter'],
'v_pic' => $arrPageVideo['v_pic'],
// 'v_pic' => $arrPageVideo['v_pic'],
'v_lang' => $arrPageVideo['v_lang'],
'v_area' => $arrPageVideo['v_area'],
'v_year' => $arrPageVideo['v_year'],
@@ -167,6 +177,9 @@ class VideoInfoPage extends BasePage
'v_publish_date' => $arrPageVideo['v_publish_date'],
'created_at' => new \MongoDB\BSON\UTCDateTime(),
];
$arrInsertVideo['v_pic'] = $this->generateVideoPath($arrInsertVideo['v_name_en']);
$this->strPicLocalPath = $arrInsertVideo['v_pic'];
try {
$this->getVideoModel()->insert($arrInsertVideo);
@@ -181,4 +194,33 @@ class VideoInfoPage extends BasePage
return true;
}
private $strPicLocalPath = '';
/**
* Undocumented function
*
* @param string $strName
* @return string
*/
public function generateVideoPath($strName): string
{
$intTime = time() - mt_rand(1, 2 * 24 * 3600 * 360);
return sprintf('/img/video/%s/%s.webp', date('Y/m/d/H', $intTime), $strName);
}
/**
* Undocumented function
*
* @return array
*/
public function getDownPicInfo(): array
{
return [
'uri' => $this->getVideo()['v_pic'],
'local_path' => $this->strPicLocalPath,
];
}
}

417
code/app/task/crawler/youzhi/page/VideoPage.php
View File

@@ -1,417 +0,0 @@
<?php
declare(strict_types=1);
namespace app\task\crawler\youzhi\page;
use app\model\CategoryModel;
use app\model\CountersModel;
use app\model\NovelModel;
use storage\StorageCore;
class VideoPage extends BasePage
{
/**
* Undocumented variable
*
* @var array
*/
protected $arrVideo = [];
/**
* Undocumented variable
*
* @var array
*/
protected $arrRelatedNovel = [];
/**
* Undocumented variable
*
* @var array
*/
protected $arrForgeNovel = [];
/**
* Undocumented variable
*
* @var NovelModel
*/
protected $NovelModel;
/**
* Undocumented function
*
* @return NovelModel
*/
public function getNovelModel(): NovelModel
{
if ($this->NovelModel == NULL) {
$this->NovelModel = NovelModel::getInstance();
}
return $this->NovelModel;
}
public $arrOptions = [
'typeMap' => [
'root' => 'array',
'document' => 'array',
'array' => 'array',
],
];
/**
* 查询看的小说
*
* @param integer $intNSourceId
* @param string $strName
* @return array
*/
protected function findOrCreateNovel(int $intNSourceId, string $strName): array
{
$strUUId = $this->getSite()->getSiteUUId($intNSourceId);
$arrExistsNovel = $this->getNovelModel()->findOne(['n_source_uuid' => $strUUId], $this->arrOptions);
if ($arrExistsNovel) {
$arrVideo = (array) $arrExistsNovel;
} else {
$arrVideo = [
'n_id' => CountersModel::getInstance()->getNextSequence('novel'),
'n_name' => $strName,
'n_name_pinyin' => zhToPinYin($strName),
'n_site_code' => $this->getSite()->getSiteCode(),
'n_source_uuid' => $strUUId,
'n_source_id' => $intNSourceId,
'n_level' => 1,
'n_have_chapter' => 0,
'created_at' => new \MongoDB\BSON\UTCDateTime(),
];
try {
$this->getNovelModel()->insert($arrVideo);
} catch (\Exception $e) {
if ($e->getCode() === 11000) {
return $this->findOrCreateNovel($intNSourceId, $strName);
} else {
throw $e;
}
}
}
return $arrVideo;
}
/**
* 分析关联数据
*
* @return void
*/
public function getReletedNovel() {}
/**
* Undocumented function
*
* @return array
*/
public function getVideoPager(): array
{
if (empty($this->arrVideo)) {
$this->arrVideo = json_decode($this->getContent(), true);
$this->getReletedNovel();
}
return $this->arrVideo;
}
/**
* Undocumented function
*
* @param string $strName
* @return string
*/
public function generateCoverPath($strName): string
{
$intTime = time() - mt_rand(1, 2 * 24 * 3600 * 360);
return sprintf('/img/%s/%s.jpg', date('Y/m/d/H', $intTime), $strName);
}
public function filterOtherFields(&$arrNovel)
{
$arrFields = [
'n_id',
'n_name',
'n_name_pinyin',
'n_status',
'n_author',
'n_description',
'n_category',
'n_category_pinyin',
'n_category_sex_en',
'n_category_sex_zh',
'n_cover_path',
'n_latest_chapter_name',
'n_update_time',
'n_forge_novel',
'n_related_novel',
'n_site_code',
'n_source_uuid',
'n_source_id',
'n_have_chapter',
'created_at',
'updated_at',
];
foreach ($arrNovel as $strKey => $strVal) {
if (!in_array($strKey, $arrFields)) {
unset($arrNovel[$strKey]);
}
}
}
/**
* Undocumented function
*
* @return bool
*/
public function saveVideo(): bool
{
$arrPageInfo = $this->getVideo();
print_r($arrPageInfo);
return false;
if (
empty($arrPageInfo) ||
empty($arrPageInfo['og_title']) ||
empty($arrPageInfo['og_image']) ||
empty($arrPageInfo['og_novel_category'])
) {
return false;
}
$arrVideo = [
// 'n_id'=> $arrPageInfo['xxxxx'],
'n_name' => $arrPageInfo['og_title'],
'n_name_pinyin' => zhToPinYin($arrPageInfo['og_title']),
'n_status' => $arrPageInfo['og_novel_status'],
'n_author' => $arrPageInfo['og_novel_author'],
'n_description' => $arrPageInfo['og_description'],
'n_category' => $arrPageInfo['og_novel_category'],
// 'n_category_pinyin' => $arrPageInfo['xxxxx'],
// 'n_category_sex_en' => $arrPageInfo['xxxxx'],
// 'n_category_sex_zh' => $arrPageInfo['xxxxx'],
// 'n_cover_path' => $arrPageInfo['xxxxx'],
'n_latest_chapter_name' => $arrPageInfo['og_novel_latest_chapter_name'],
'n_update_time' => $arrPageInfo['og_novel_update_time'],
'n_forge_novel' => $this->arrForgeNovel,
'n_related_novel' => $this->arrRelatedNovel,
'n_site_code' => $this->getSite()->getSiteCode(),
'n_source_id' => $this->arrData['n_source_id'],
'n_source_uuid' => $this->getSite()->getSiteUUId($this->arrData['n_source_id']),
// 'n_have_chapter' => $arrPageInfo['xxxxx'],
// 'created_at' => $arrPageInfo['xxxxx'],
// 'updated_at' => $arrPageInfo['xxxxx'],
];
$arrExistsNovel = $this->getNovelModel()->findOne(['n_source_uuid' => $arrVideo['n_source_uuid']], $this->arrOptions);
if ($arrExistsNovel) {
$arrExistsNovel = (array) $arrExistsNovel;
$arrUpdate = [
'n_latest_chapter_name' => $arrVideo['n_latest_chapter_name'],
'n_update_time' => $arrVideo['n_update_time'],
'updated_at' => new \MongoDB\BSON\UTCDateTime(),
];
# 如果没有分类，说明这条数据是最初从关联shu那里采集的，需要补全全部信息
if (empty($arrExistsNovel['n_category'])) {
$arrUpdate = array_merge($arrUpdate, $arrVideo);
$arrUpdate['n_related_novel'] = $this->arrRelatedNovel;
$arrUpdate['n_forge_novel'] = $this->arrForgeNovel;
$arrUpdate['n_category_pinyin'] = CategoryModel::getPinYin($this->arrVideo['og_novel_category']);
$arrUpdate['n_category_sex_en'] = CategoryModel::getSexEn($this->arrVideo['og_novel_category']);
$arrUpdate['n_category_sex_zh'] = CategoryModel::getSexZh($this->arrVideo['og_novel_category']);
$arrUpdate['n_related_novel'] = $this->arrRelatedNovel;
$arrUpdate['n_forge_novel'] = $this->arrForgeNovel;
}
# 图片不存在的时候，需要重新生成
if (
!isset($arrExistsNovel['n_cover_path']) ||
empty($arrExistsNovel['n_cover_path']) ||
!StorageCore::getInstance()->has($arrExistsNovel['n_cover_path']) ||
(basename($arrExistsNovel['n_cover_path']) == '.jpg')
) {
$arrUpdate['n_cover_path'] = $this->generateCoverPath($arrVideo['n_name_pinyin']);
$this->strCoverLocalPath = $arrUpdate['n_cover_path'];
} else {
$this->strCoverLocalPath = $arrExistsNovel['n_cover_path'];
}
$this->getNovelModel()->updateOne(
['n_source_uuid' => $arrVideo['n_source_uuid']],
['$set' => $arrUpdate],
);
$this->intNId = $arrExistsNovel['n_id'];
} else {
$arrVideo['n_category_pinyin'] = CategoryModel::getPinYin($this->arrVideo['og_novel_category']);
$arrVideo['n_category_sex_en'] = CategoryModel::getSexEn($this->arrVideo['og_novel_category']);
$arrVideo['n_category_sex_zh'] = CategoryModel::getSexZh($this->arrVideo['og_novel_category']);
$arrVideo['n_id'] = CountersModel::getInstance()->getNextSequence('novel');
$arrVideo['n_level'] = 1;
$arrVideo['n_have_chapter'] = 0;
$arrVideo['n_cover_path'] = $this->generateCoverPath($arrVideo['n_name_pinyin']);
$arrVideo['n_related_novel'] = $this->arrRelatedNovel;
$arrVideo['n_forge_novel'] = $this->arrForgeNovel;
$arrVideo['created_at'] = new \MongoDB\BSON\UTCDateTime();
try {
$this->getNovelModel()->insert($arrVideo);
$this->intNId = $arrVideo['n_id'];
$this->strCoverLocalPath = $arrVideo['n_cover_path'];
} catch (\Exception $e) {
if ($e->getCode() === 11000) {
return $this->saveVideo();
} else {
throw $e;
}
}
}
return true;
}
public $strCoverLocalPath = '';
public $intNId;
/**
* Undocumented function
*
* @return array
*/
public function getDownCoverInfo(): array
{
return [
'uri' => $this->arrVideo['og_image'],
'local_path' => $this->strCoverLocalPath,
];
}
/**
* Undocumented variable
*
* @var array
*/
protected $arrLatestChapters = NULL;
/**
* Undocumented variable
*
* @var array
*/
protected $arrChapters = NULL;
public function getLatestChapterByNIdsList(): array
{
if ($this->arrLatestChapters === NULL) {
$this->arrLatestChapters = [];
if ($this->isMobile()) {
$strClass = '.chapter-list';
} else {
$strClass = '.section-list';
}
if ($this->arrData['page'] == 0) {
$this->getQueryList()->find($strClass)->eq(0)->find("a")->map(function ($A) {
$this->arrLatestChapters[] = [
'name' => $A->text(),
'uri' => $A->attr('href'),
];
});
}
}
return $this->arrLatestChapters;
}
/**
* Undocumented function
*
* @return array
*/
public function getChapters(): array
{
if ($this->arrChapters === NULL) {
$this->arrChapters = [];
if ($this->isMobile()) {
$strClass = '.chapter-list';
} else {
$strClass = '.section-list';
}
$intDivIndex = 0;
if ($this->arrData['page'] == 0) {
$intDivIndex = 1;
}
$this->getQueryList()->find($strClass)->eq($intDivIndex)->find("a")->map(function ($A) {
$this->arrChapters[] = [
'name' => $A->text(),
'uri' => $A->attr('href'),
];
});
}
return $this->arrChapters;
}
protected $arrNextPageArgs = NULL;
/**
* Undocumented function
*
* @return array
*/
public function getNextPageArgs(): array
{
$this->arrNextPageArgs = [];
if ($this->arrData['page'] == 0) {
$this->getLatestChapterByNIdsList();
$this->getChapters();
$arrLatestChapterFirstBlock = count($this->arrLatestChapters) > 0 ? $this->arrLatestChapters[0] : null;
$arrLatestChapterSecondBlock = count($this->arrChapters) > 0 ? $this->arrChapters[count($this->arrChapters) - 1] : null;
if (
!empty($arrLatestChapterFirstBlock) && !empty($arrLatestChapterSecondBlock)
&& ($arrLatestChapterFirstBlock['uri'] != $arrLatestChapterSecondBlock['uri'])
) {
$this->arrNextPageArgs = [
'n_source_id' => $this->arrData['n_source_id'],
'page' => $this->arrData['page'] + 1,
];
}
} else {
$strLastUri = $this->getQueryList()->find('select:first option:last')->attr('value');
if ($strLastUri != '' && $strLastUri != $this->strUri) {
$this->arrNextPageArgs = [
'n_source_id' => $this->arrData['n_source_id'],
'page' => $this->arrData['page'] + 1,
];
}
}
return $this->arrNextPageArgs;
}
}

43
code/app/task/crawler/youzhi/page/VideoPagerPage.php Normal file
View File

@@ -0,0 +1,43 @@
<?php
declare(strict_types=1);
namespace app\task\crawler\youzhi\page;
use app\model\CategoryModel;
use app\model\CountersModel;
use app\model\NovelModel;
use storage\StorageCore;
class VideoPagerPage extends BasePage
{
/**
* Undocumented variable
*
* @var array
*/
protected $arrVideo = [];
public $arrOptions = [
'typeMap' => [
'root' => 'array',
'document' => 'array',
'array' => 'array',
],
];
/**
* Undocumented function
*
* @return array
*/
public function getVideoPager(): array
{
if (empty($this->arrVideo)) {
$this->arrVideo = json_decode($this->getContent(), true);
}
return $this->arrVideo;
}
}