admin/SEONexus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feature video craw

Browse Source
This commit is contained in:
Default
2025-04-22 17:43:48 +08:00
parent f21ece4de1
commit d771a5fa0b
4 changed files with 266 additions and 157 deletions
Download Patch File Download Diff File Expand all files Collapse all files

153
code/app/task/crawler/youzhi/Scheduler.php
View File

@@ -74,74 +74,111 @@ class Scheduler
*/
public function crawlFull()
{
$Site = $this->getSite();
$arrVSourceId = [];
$arrVSourceId[] = ['v_source_id' => 66281];
$arrAllVideoInfoPage = $Site->getVideoInfoPageList($arrVSourceId);
foreach ($arrAllVideoInfoPage as $VideoInfoPage) {
$arrVideo = $VideoInfoPage->getVideo();
print_r($arrVideo);
}
return;
$intLimit = 10;
// $Site = $this->getSite();
// $arrVSourceId = [];
// $arrVSourceId[] = ['v_source_id' => 14256];
// $arrAllVideoInfoPage = $Site->getVideoInfoPageList($arrVSourceId);
// foreach ($arrAllVideoInfoPage as $VideoInfoPage) {
// $arrVideo = $VideoInfoPage->getVideo();
// $VideoInfoPage->saveVideo();
// print_r($arrVideo);
// }
// return;
$Site = $this->getSite();
$arrAllCategory = $Site->getCategory();
$intLimit = 5;
$arrTask = [
'callback' => [static::class, 'fetchVSourceId'],
'data' => [],
];
foreach ($arrAllCategory as $arrCategory) {
$intTotalPage = ceil($arrCategory['total'] / $intLimit);
if ($arrCategory['total'] <= 0) {
continue;
}
$intTotalPage = $arrCategory['total_page'];
$arrFilter = [];
for ($intStartPage = 1; $intStartPage <= $intTotalPage; $intStartPage++) {
$arrFilter = [
[
'v_category_id' => $arrCategory['v_category_id'],
'page' => $intStartPage,
],
$arrFilter[] = [
'v_category_id' => $arrCategory['v_category_id'],
'page' => $intStartPage,
];
$arrVideoPageList = $Site->getVideoPageList($arrFilter);
$arrVideoPager = $arrVideoPageList[0]->getVideoPager();
$arrVSourceId = [];
foreach ($arrVideoPager['list'] as $arrVideoPager) {
$arrVSourceId[] = ['v_source_id' => (int)$arrVideoPager['vod_id']];
if (($intStartPage % $intLimit) == 0) {
$arrTask['data'] = $arrFilter;
$this->QueueManage->set($arrTask);
$arrFilter = [];
}
$arrAllVideoInfoPage = $Site->getVideoInfoPageList($arrVSourceId);
foreach ($arrAllVideoInfoPage as $VideoInfoPage) {
$arrVideo = $VideoInfoPage->getVideo();
print_r($arrVideo);
}
// foreach ($arrVideoPageList as $VideoPage) {
// $arrAllVideoPager = $VideoPage->getVideoPager();
// $arrVSourceId = [];
// foreach ($arrAllVideoPager['list'] as $arrVideoPager) {
// $arrVSourceId[] = ['v_source_id' => (int)$arrVideoPager['vod_id']];
// }
// $arrAllVideoInfoPage = $Site->getVideoInfoPageList($arrVSourceId);
// foreach ($arrAllVideoInfoPage as $VideoInfoPage) {
// $arrVideo = $VideoInfoPage->getVideo();
// print_r($arrVideo);
// }
// }
}
if (!empty($arrFilter)) {
$arrTask['data'] = $arrFilter;
$this->QueueManage->set($arrTask);
}
}
return;
}
/**
* Undocumented function
*
* @param array $arrFilter
* @return void
*/
public function fetchVSourceId(array $arrFilter)
{
$Site = $this->getSite();
$intLimit = 5;
$arrTask = [
'callback' => [static::class, 'fetchVideo'],
'data' => [],
];
# 访问分页，获取 v_source_id 集合
$arrAllVideoPageList = $Site->getVideoPageList($arrFilter);
$arrVSourceId = [];
foreach ($arrAllVideoPageList as $VideoPageList) {
$arrVideoPager = $VideoPageList->getVideoPager();
foreach ($arrVideoPager['list'] as $arrVideoPager) {
$arrVSourceId[] = ['v_source_id' => (int)$arrVideoPager['vod_id']];
if ((count($arrVSourceId) % $intLimit) == 0) {
$arrTask['data'] = $arrVSourceId;
$this->QueueManage->set($arrTask);
$arrVSourceId = [];
}
}
}
if (!empty($arrVSourceId)) {
$arrTask['data'] = $arrVSourceId;
$this->QueueManage->set($arrTask);
}
}
/**
* Undocumented function
*
* @param array $arrVSourceId
* @return void
*/
public function fetchVideo(array $arrVSourceId)
{
$Site = $this->getSite();
$arrAllVideoInfoPage = $Site->getVideoInfoPageList($arrVSourceId);
foreach ($arrAllVideoInfoPage as $VideoInfoPage) {
$VideoInfoPage->saveVideo();
}
}
}

3
code/app/task/crawler/youzhi/page/Site.php
View File

@@ -181,6 +181,7 @@ class Site
'v_category_name' => $arrFilter[$intKey]['type_name'],
'v_category_name_en' => zhToPinYin($arrFilter[$intKey]['type_name']),
'total' => $arrTemp['total'],
'total_page' => $arrTemp['pagecount'],
];
}
@@ -196,7 +197,7 @@ class Site
*/
public function getVideoPageUri(int $intVCategoryId, int $intPage): string
{
return sprintf('/inc/api_mac10.php?ac=list&t=%s&page=%s', $intVCategoryId, $intPage);
return sprintf('/inc/api_mac10.php?ac=list&t=%s&pg=%s', $intVCategoryId, $intPage);
}
/**

176
code/app/task/crawler/youzhi/page/VideoInfoPage.php
View File

@@ -4,6 +4,7 @@ declare(strict_types=1);
namespace app\task\crawler\youzhi\page;
use app\model\CountersModel;
use app\model\VideoModel;
use storage\StorageCore;
@@ -14,7 +15,7 @@ class VideoInfoPage extends BasePage
*
* @var string
*/
protected $strChapterContent = '';
protected $strVideoContent = '';
/**
* Undocumented variable
@@ -53,17 +54,49 @@ class VideoInfoPage extends BasePage
{
if (empty($this->arrVideo)) {
$this->arrVideo = json_decode($this->getContent(), true);
$this->arrVideo = $this->arrVideo['list'][0];
$this->arrVideo['vod_play_url'] = str_replace('#', '$', $this->arrVideo['vod_play_url']);
$this->arrVideo['vod_play_url'] = explode('$', $this->arrVideo['vod_play_url']);
$this->arrVideo['vod_play_url'] = array_chunk($this->arrVideo['vod_play_url'], 2);
foreach ($this->arrVideo['vod_play_url'] as $intKey => $arrVal) {
$this->arrVideo['vod_play_url'][$intKey] = [
$arrVideo = json_decode($this->getContent(), true);
$arrVideo = $arrVideo['list'][0];
// print_r($arrVideo);
$strPlayUrl = $arrVideo['vod_play_url'];
unset($arrVideo['vod_play_url']);
$arrPlayUrl = str_replace('#', '$', $strPlayUrl);
$arrPlayUrl = explode('$', $arrPlayUrl);
$arrPlayUrl = array_chunk($arrPlayUrl, 2);
foreach ($arrPlayUrl as $intKey => $arrVal) {
$arrPlayUrl[$intKey] = [
'name' => $arrVal[0],
'url' => $arrVal[1],
'url' => $arrVal[1]?? $arrVal[0],
];
}
$arrVideo['vod_play_url'][$this->getsite()->getSiteCode()] = $arrPlayUrl;
$arrVideo['vod_actor'] = explode(',', $arrVideo['vod_actor']);
$arrVideo['vod_director'] = explode(',', $arrVideo['vod_director']);
$arrVideo['vod_class'] = explode(',', $arrVideo['vod_class']);
$arrVideo['vod_content'] = trim($arrVideo['vod_content']);
foreach ($arrVideo as $strKey => $mixedVal) {
$strKey = str_replace('vod', 'v', $strKey);
if ($strKey == 'type_id') {
$strKey = 'v_category_id';
}
if ($strKey == 'type_name') {
$strKey = 'v_category';
}
if ($strKey == 'v_content') {
$strKey = 'v_description';
}
if ($strKey == 'v_enname') {
$strKey = 'v_name_en';
}
if ($strKey == 'v_time') {
$strKey = 'v_publish_date';
}
$this->arrVideo[$strKey] = $mixedVal;
}
}
return $this->arrVideo;
@@ -76,129 +109,76 @@ class VideoInfoPage extends BasePage
*/
protected $arrVideo;
/**
* Undocumented function
*
* @param integer $intNId
* @return boolean
*/
public function saveChapter(int $intNId): bool
public function saveVideo(): bool
{
$arrPageChapter = $this->getVideo();
$arrPageVideo = $this->getVideo();
if (
empty($arrPageChapter) ||
empty($arrPageChapter['c_page_title']) ||
empty($arrPageChapter['c_page_data']) ||
count($arrPageChapter['c_page_data']) < 5
empty($arrPageVideo) ||
empty($arrPageVideo['v_name']) ||
empty($arrPageVideo['v_category']) ||
empty($arrPageVideo['v_play_url']) ||
empty($arrPageVideo['v_play_url']['youzhi'])
) {
return false;
}
$strUUId = $this->getSite()->getSiteUUId('chapter-' . $this->arrData['c_source_id'] . '-' . $this->arrData['c_page']);
$arrExistsVideo = $this->getVideoModel()->findOne(['v_name' => $arrPageVideo['v_name']], $this->arrOptions);
$arrChapter = [
'n_id' => $intNId,
'c_name' => $arrPageChapter['c_page_data'][3],
'c_sort_num' => $this->arrData['c_sort_num'],
'c_page' => $this->arrData['c_page'],
'c_source_id' => $this->arrData['c_source_id'],
'c_site_code' => $this->getSite()->getSiteCode(),
'c_source_uuid' => $strUUId,
];
if ($arrExistsVideo) {
$arrExistsChapter = $this->getVideoModel()->findOne(['c_source_uuid' => $strUUId], $this->arrOptions);
if ($arrExistsChapter) {
$arrExistsChapter = (array) $arrExistsChapter;
$arrExistsVideo = (array) $arrExistsVideo;
$arrUpdate = [
'updated_at' => new \MongoDB\BSON\UTCDateTime(),
'v_play_url.youzhi' => $arrPageVideo['v_play_url']['youzhi'],
];
$this->getVideoModel()->updateOne(
['c_source_uuid' => $strUUId],
['v_name' => $arrPageVideo['v_name']],
['$set' => $arrUpdate],
);
$this->strCContentPath = $arrExistsChapter['c_content_path'];
} else {
$arrInsertNovel = $arrChapter;
$arrInsertNovel['c_content_path'] = '/novel/' . ($intNId % 100) . '/' . $intNId . '/' . $arrChapter['c_sort_num'] . '_' . $arrChapter['c_page'] . '.txt';
$arrInsertNovel['created_at'] = new \MongoDB\BSON\UTCDateTime();
$arrInsertVideo = [
'v_id' => CountersModel::getInstance()->getNextSequence('video'),
'v_name' => $arrPageVideo['v_name'],
'v_name_en' => zhToPinYin($arrPageVideo['v_name']),
'v_category' => $arrPageVideo['v_category'],
'v_category_en' => zhToPinYin($arrPageVideo['v_category']),
'v_actor' => $arrPageVideo['v_actor'],
'v_director' => $arrPageVideo['v_director'],
'v_class' => $arrPageVideo['v_class'],
'v_letter' => $arrPageVideo['v_letter'],
'v_pic' => $arrPageVideo['v_pic'],
'v_lang' => $arrPageVideo['v_lang'],
'v_area' => $arrPageVideo['v_area'],
'v_year' => $arrPageVideo['v_year'],
'v_remarks' => $arrPageVideo['v_remarks'],
'v_isend' => $arrPageVideo['v_isend'],
'v_score' => $arrPageVideo['v_score'],
'v_description' => $arrPageVideo['v_description'],
'v_play_url' => $arrPageVideo['v_play_url'],
'v_publish_date' => $arrPageVideo['v_publish_date'],
'created_at' => new \MongoDB\BSON\UTCDateTime(),
];
try {
$this->getVideoModel()->insert($arrInsertNovel);
$this->updateNovelHaveChapter($intNId);
$this->strCContentPath = $arrInsertNovel['c_content_path'];
$this->getVideoModel()->insert($arrInsertVideo);
} catch (\Exception $e) {
if ($e->getCode() === 11000) {
return $this->saveChapter($intNId);
return $this->saveVideo();
} else {
throw $e;
}
}
}
StorageCore::getInstance()->put($this->strCContentPath, $this->strChapterContent);
return true;
}
public $strCContentPath;
/**
* Undocumented function
*
* @param integer $intNId
* @return void
*/
protected function updateNovelHaveChapter(int $intNId)
{
$arrFilter = [
'n_id' => $intNId,
];
$VideoModel = VideoModel::getInstance();
$arrLasteChapter = $VideoModel->getLatestChapterByNId($intNId, false);
if (!empty($arrLasteChapter)) {
$arrUpdate['$set']['n_have_chapter'] = 1;
$arrUpdate['$set']['n_latest_chapter_name'] = $arrLasteChapter['c_name'];
$this->getMongoBase()->updateOne('novel', $arrFilter, $arrUpdate);
}
}
protected $arrNextPage;
/**
* Undocumented function
*
* @return array
*/
public function getNextPage(): array
{
if ($this->arrNextPage === NULL) {
$this->arrNextPage = [];
if ($this->isMobile()) {
$strPattern = "/var\s+hhekgsv\s*=\s*['\"](.*?)['\"]/";
preg_match_all($strPattern, $this->getContent(), $arrMatches);
$strNextPageUri = end($arrMatches[1]) ?: '';
} else {
$strHref = $this->getQueryList()->find('.read_btn a:contains("下一章")')->attr('href');
$strNextPageUri = $strHref ?: '';
}
if (is_numeric(strpos($strNextPageUri, '_'))) {
$this->arrNextPage = $this->arrData;
$this->arrNextPage['c_page'] += 1;
$this->arrNextPage['uri'] = $strNextPageUri;
}
}
return $this->arrNextPage;
}
}