From d771a5fa0b85ea903b9f5eeaa4ab7b08fa15dd11 Mon Sep 17 00:00:00 2001 From: Default Date: Tue, 22 Apr 2025 17:43:48 +0800 Subject: [PATCH] feature video craw --- code/app/task/crawler/youzhi/Scheduler.php | 153 +++++++++------ code/app/task/crawler/youzhi/page/Site.php | 3 +- .../crawler/youzhi/page/VideoInfoPage.php | 176 ++++++++---------- code/database/mongo/migrations/video.php | 91 +++++++++ 4 files changed, 266 insertions(+), 157 deletions(-) create mode 100644 code/database/mongo/migrations/video.php diff --git a/code/app/task/crawler/youzhi/Scheduler.php b/code/app/task/crawler/youzhi/Scheduler.php index 844f510..ea59121 100644 --- a/code/app/task/crawler/youzhi/Scheduler.php +++ b/code/app/task/crawler/youzhi/Scheduler.php @@ -74,74 +74,111 @@ class Scheduler */ public function crawlFull() { - $Site = $this->getSite(); - $arrVSourceId = []; - - $arrVSourceId[] = ['v_source_id' => 66281]; - - - $arrAllVideoInfoPage = $Site->getVideoInfoPageList($arrVSourceId); - foreach ($arrAllVideoInfoPage as $VideoInfoPage) { - $arrVideo = $VideoInfoPage->getVideo(); - print_r($arrVideo); - } - - return; - - - - - - - - $intLimit = 10; + // $Site = $this->getSite(); + // $arrVSourceId = []; + // $arrVSourceId[] = ['v_source_id' => 14256]; + // $arrAllVideoInfoPage = $Site->getVideoInfoPageList($arrVSourceId); + // foreach ($arrAllVideoInfoPage as $VideoInfoPage) { + // $arrVideo = $VideoInfoPage->getVideo(); + // $VideoInfoPage->saveVideo(); + // print_r($arrVideo); + // } + // return; $Site = $this->getSite(); $arrAllCategory = $Site->getCategory(); + $intLimit = 5; + + $arrTask = [ + 'callback' => [static::class, 'fetchVSourceId'], + 'data' => [], + ]; + foreach ($arrAllCategory as $arrCategory) { - $intTotalPage = ceil($arrCategory['total'] / $intLimit); + + if ($arrCategory['total'] <= 0) { + continue; + } + + $intTotalPage = $arrCategory['total_page']; + $arrFilter = []; for ($intStartPage = 1; $intStartPage <= $intTotalPage; $intStartPage++) { - $arrFilter = [ - [ - 'v_category_id' => $arrCategory['v_category_id'], - 'page' => $intStartPage, - ], + + $arrFilter[] = [ + 'v_category_id' => $arrCategory['v_category_id'], + 'page' => $intStartPage, ]; - - $arrVideoPageList = $Site->getVideoPageList($arrFilter); - $arrVideoPager = $arrVideoPageList[0]->getVideoPager(); - - $arrVSourceId = []; - - foreach ($arrVideoPager['list'] as $arrVideoPager) { - $arrVSourceId[] = ['v_source_id' => (int)$arrVideoPager['vod_id']]; + if (($intStartPage % $intLimit) == 0) { + $arrTask['data'] = $arrFilter; + $this->QueueManage->set($arrTask); + $arrFilter = []; } - - $arrAllVideoInfoPage = $Site->getVideoInfoPageList($arrVSourceId); - foreach ($arrAllVideoInfoPage as $VideoInfoPage) { - $arrVideo = $VideoInfoPage->getVideo(); - print_r($arrVideo); - } - - - // foreach ($arrVideoPageList as $VideoPage) { - // $arrAllVideoPager = $VideoPage->getVideoPager(); - - // $arrVSourceId = []; - // foreach ($arrAllVideoPager['list'] as $arrVideoPager) { - // $arrVSourceId[] = ['v_source_id' => (int)$arrVideoPager['vod_id']]; - // } - - // $arrAllVideoInfoPage = $Site->getVideoInfoPageList($arrVSourceId); - // foreach ($arrAllVideoInfoPage as $VideoInfoPage) { - // $arrVideo = $VideoInfoPage->getVideo(); - // print_r($arrVideo); - // } - // } + } + if (!empty($arrFilter)) { + $arrTask['data'] = $arrFilter; + $this->QueueManage->set($arrTask); } } - return; + } + + /** + * Undocumented function + * + * @param array $arrFilter + * @return void + */ + public function fetchVSourceId(array $arrFilter) + { + + $Site = $this->getSite(); + + $intLimit = 5; + + $arrTask = [ + 'callback' => [static::class, 'fetchVideo'], + 'data' => [], + ]; + + # 访问分页，获取 v_source_id 集合 + $arrAllVideoPageList = $Site->getVideoPageList($arrFilter); + + $arrVSourceId = []; + + foreach ($arrAllVideoPageList as $VideoPageList) { + $arrVideoPager = $VideoPageList->getVideoPager(); + + foreach ($arrVideoPager['list'] as $arrVideoPager) { + $arrVSourceId[] = ['v_source_id' => (int)$arrVideoPager['vod_id']]; + + if ((count($arrVSourceId) % $intLimit) == 0) { + $arrTask['data'] = $arrVSourceId; + $this->QueueManage->set($arrTask); + $arrVSourceId = []; + } + } + } + + if (!empty($arrVSourceId)) { + $arrTask['data'] = $arrVSourceId; + $this->QueueManage->set($arrTask); + } + } + + /** + * Undocumented function + * + * @param array $arrVSourceId + * @return void + */ + public function fetchVideo(array $arrVSourceId) + { + $Site = $this->getSite(); + + $arrAllVideoInfoPage = $Site->getVideoInfoPageList($arrVSourceId); + foreach ($arrAllVideoInfoPage as $VideoInfoPage) { + $VideoInfoPage->saveVideo(); + } } } diff --git a/code/app/task/crawler/youzhi/page/Site.php b/code/app/task/crawler/youzhi/page/Site.php index 8ca6692..218e48f 100644 --- a/code/app/task/crawler/youzhi/page/Site.php +++ b/code/app/task/crawler/youzhi/page/Site.php @@ -181,6 +181,7 @@ class Site 'v_category_name' => $arrFilter[$intKey]['type_name'], 'v_category_name_en' => zhToPinYin($arrFilter[$intKey]['type_name']), 'total' => $arrTemp['total'], + 'total_page' => $arrTemp['pagecount'], ]; } @@ -196,7 +197,7 @@ class Site */ public function getVideoPageUri(int $intVCategoryId, int $intPage): string { - return sprintf('/inc/api_mac10.php?ac=list&t=%s&page=%s', $intVCategoryId, $intPage); + return sprintf('/inc/api_mac10.php?ac=list&t=%s&pg=%s', $intVCategoryId, $intPage); } /** diff --git a/code/app/task/crawler/youzhi/page/VideoInfoPage.php b/code/app/task/crawler/youzhi/page/VideoInfoPage.php index 2c61d31..8892965 100644 --- a/code/app/task/crawler/youzhi/page/VideoInfoPage.php +++ b/code/app/task/crawler/youzhi/page/VideoInfoPage.php @@ -4,6 +4,7 @@ declare(strict_types=1); namespace app\task\crawler\youzhi\page; +use app\model\CountersModel; use app\model\VideoModel; use storage\StorageCore; @@ -14,7 +15,7 @@ class VideoInfoPage extends BasePage * * @var string */ - protected $strChapterContent = ''; + protected $strVideoContent = ''; /** * Undocumented variable @@ -53,17 +54,49 @@ class VideoInfoPage extends BasePage { if (empty($this->arrVideo)) { - $this->arrVideo = json_decode($this->getContent(), true); - $this->arrVideo = $this->arrVideo['list'][0]; - $this->arrVideo['vod_play_url'] = str_replace('#', '$', $this->arrVideo['vod_play_url']); - $this->arrVideo['vod_play_url'] = explode('$', $this->arrVideo['vod_play_url']); - $this->arrVideo['vod_play_url'] = array_chunk($this->arrVideo['vod_play_url'], 2); - foreach ($this->arrVideo['vod_play_url'] as $intKey => $arrVal) { - $this->arrVideo['vod_play_url'][$intKey] = [ + $arrVideo = json_decode($this->getContent(), true); + $arrVideo = $arrVideo['list'][0]; + // print_r($arrVideo); + + $strPlayUrl = $arrVideo['vod_play_url']; + unset($arrVideo['vod_play_url']); + + $arrPlayUrl = str_replace('#', '$', $strPlayUrl); + $arrPlayUrl = explode('$', $arrPlayUrl); + $arrPlayUrl = array_chunk($arrPlayUrl, 2); + foreach ($arrPlayUrl as $intKey => $arrVal) { + $arrPlayUrl[$intKey] = [ 'name' => $arrVal[0], - 'url' => $arrVal[1], + 'url' => $arrVal[1]?? $arrVal[0], ]; } + $arrVideo['vod_play_url'][$this->getsite()->getSiteCode()] = $arrPlayUrl; + $arrVideo['vod_actor'] = explode(',', $arrVideo['vod_actor']); + $arrVideo['vod_director'] = explode(',', $arrVideo['vod_director']); + $arrVideo['vod_class'] = explode(',', $arrVideo['vod_class']); + $arrVideo['vod_content'] = trim($arrVideo['vod_content']); + + foreach ($arrVideo as $strKey => $mixedVal) { + $strKey = str_replace('vod', 'v', $strKey); + + if ($strKey == 'type_id') { + $strKey = 'v_category_id'; + } + if ($strKey == 'type_name') { + $strKey = 'v_category'; + } + if ($strKey == 'v_content') { + $strKey = 'v_description'; + } + if ($strKey == 'v_enname') { + $strKey = 'v_name_en'; + } + if ($strKey == 'v_time') { + $strKey = 'v_publish_date'; + } + + $this->arrVideo[$strKey] = $mixedVal; + } } return $this->arrVideo; @@ -76,129 +109,76 @@ class VideoInfoPage extends BasePage */ protected $arrVideo; - /** * Undocumented function * - * @param integer $intNId * @return boolean */ - public function saveChapter(int $intNId): bool + public function saveVideo(): bool { - $arrPageChapter = $this->getVideo(); + $arrPageVideo = $this->getVideo(); if ( - empty($arrPageChapter) || - empty($arrPageChapter['c_page_title']) || - empty($arrPageChapter['c_page_data']) || - count($arrPageChapter['c_page_data']) < 5 + empty($arrPageVideo) || + empty($arrPageVideo['v_name']) || + empty($arrPageVideo['v_category']) || + empty($arrPageVideo['v_play_url']) || + empty($arrPageVideo['v_play_url']['youzhi']) ) { return false; } - $strUUId = $this->getSite()->getSiteUUId('chapter-' . $this->arrData['c_source_id'] . '-' . $this->arrData['c_page']); + $arrExistsVideo = $this->getVideoModel()->findOne(['v_name' => $arrPageVideo['v_name']], $this->arrOptions); - $arrChapter = [ - 'n_id' => $intNId, - 'c_name' => $arrPageChapter['c_page_data'][3], - 'c_sort_num' => $this->arrData['c_sort_num'], - 'c_page' => $this->arrData['c_page'], - 'c_source_id' => $this->arrData['c_source_id'], - 'c_site_code' => $this->getSite()->getSiteCode(), - 'c_source_uuid' => $strUUId, - ]; + if ($arrExistsVideo) { - $arrExistsChapter = $this->getVideoModel()->findOne(['c_source_uuid' => $strUUId], $this->arrOptions); - - if ($arrExistsChapter) { - - $arrExistsChapter = (array) $arrExistsChapter; + $arrExistsVideo = (array) $arrExistsVideo; $arrUpdate = [ 'updated_at' => new \MongoDB\BSON\UTCDateTime(), + 'v_play_url.youzhi' => $arrPageVideo['v_play_url']['youzhi'], ]; $this->getVideoModel()->updateOne( - ['c_source_uuid' => $strUUId], + ['v_name' => $arrPageVideo['v_name']], ['$set' => $arrUpdate], ); - $this->strCContentPath = $arrExistsChapter['c_content_path']; } else { - $arrInsertNovel = $arrChapter; - - $arrInsertNovel['c_content_path'] = '/novel/' . ($intNId % 100) . '/' . $intNId . '/' . $arrChapter['c_sort_num'] . '_' . $arrChapter['c_page'] . '.txt'; - $arrInsertNovel['created_at'] = new \MongoDB\BSON\UTCDateTime(); + $arrInsertVideo = [ + 'v_id' => CountersModel::getInstance()->getNextSequence('video'), + 'v_name' => $arrPageVideo['v_name'], + 'v_name_en' => zhToPinYin($arrPageVideo['v_name']), + 'v_category' => $arrPageVideo['v_category'], + 'v_category_en' => zhToPinYin($arrPageVideo['v_category']), + 'v_actor' => $arrPageVideo['v_actor'], + 'v_director' => $arrPageVideo['v_director'], + 'v_class' => $arrPageVideo['v_class'], + 'v_letter' => $arrPageVideo['v_letter'], + 'v_pic' => $arrPageVideo['v_pic'], + 'v_lang' => $arrPageVideo['v_lang'], + 'v_area' => $arrPageVideo['v_area'], + 'v_year' => $arrPageVideo['v_year'], + 'v_remarks' => $arrPageVideo['v_remarks'], + 'v_isend' => $arrPageVideo['v_isend'], + 'v_score' => $arrPageVideo['v_score'], + 'v_description' => $arrPageVideo['v_description'], + 'v_play_url' => $arrPageVideo['v_play_url'], + 'v_publish_date' => $arrPageVideo['v_publish_date'], + 'created_at' => new \MongoDB\BSON\UTCDateTime(), + ]; try { - $this->getVideoModel()->insert($arrInsertNovel); - $this->updateNovelHaveChapter($intNId); - $this->strCContentPath = $arrInsertNovel['c_content_path']; + $this->getVideoModel()->insert($arrInsertVideo); } catch (\Exception $e) { if ($e->getCode() === 11000) { - return $this->saveChapter($intNId); + return $this->saveVideo(); } else { throw $e; } } } - StorageCore::getInstance()->put($this->strCContentPath, $this->strChapterContent); - return true; } - - public $strCContentPath; - /** - * Undocumented function - * - * @param integer $intNId - * @return void - */ - protected function updateNovelHaveChapter(int $intNId) - { - $arrFilter = [ - 'n_id' => $intNId, - ]; - - $VideoModel = VideoModel::getInstance(); - $arrLasteChapter = $VideoModel->getLatestChapterByNId($intNId, false); - - if (!empty($arrLasteChapter)) { - $arrUpdate['$set']['n_have_chapter'] = 1; - $arrUpdate['$set']['n_latest_chapter_name'] = $arrLasteChapter['c_name']; - $this->getMongoBase()->updateOne('novel', $arrFilter, $arrUpdate); - } - } - - protected $arrNextPage; - - /** - * Undocumented function - * - * @return array - */ - public function getNextPage(): array - { - if ($this->arrNextPage === NULL) { - $this->arrNextPage = []; - - if ($this->isMobile()) { - $strPattern = "/var\s+hhekgsv\s*=\s*['\"](.*?)['\"]/"; - preg_match_all($strPattern, $this->getContent(), $arrMatches); - $strNextPageUri = end($arrMatches[1]) ?: ''; - } else { - $strHref = $this->getQueryList()->find('.read_btn a:contains("下一章")')->attr('href'); - $strNextPageUri = $strHref ?: ''; - } - - if (is_numeric(strpos($strNextPageUri, '_'))) { - $this->arrNextPage = $this->arrData; - $this->arrNextPage['c_page'] += 1; - $this->arrNextPage['uri'] = $strNextPageUri; - } - } - - return $this->arrNextPage; - } } diff --git a/code/database/mongo/migrations/video.php b/code/database/mongo/migrations/video.php new file mode 100644 index 0000000..6c6950f --- /dev/null +++ b/code/database/mongo/migrations/video.php @@ -0,0 +1,91 @@ + 'video', + 'indexes' => [ + [ + 'key' => ['v_id' => 1], + 'options' => ['unique' => true] + ], + [ + 'key' => ['v_name' => 1], + 'options' => ['unique' => true] + ], + [ + 'key' => ['v_category' => 1], + 'options' => ['unique' => false] + ], + [ + 'key' => ['v_category_en' => 1], + 'options' => ['unique' => false] + ], + [ + 'key' => ['v_play_url.v_source_code' => 1], + 'options' => ['unique' => false] + ], + + [ + 'key' => ['v_lang' => 1], + 'options' => ['unique' => false] + ], + [ + 'key' => ['v_area' => 1], + 'options' => ['unique' => false] + ], + [ + 'key' => ['v_year' => 1], + 'options' => ['unique' => false] + ], + [ + 'key' => ['v_actor' => 1], + 'options' => ['unique' => false] + ], + [ + 'key' => ['v_director' => 1], + 'options' => ['unique' => false] + ], + + ] +]; + +// 'v_publish_date', # 视频创建或更新时间，通常是时间戳或日期时间，表示视频记录的最后修改时间 +// 'v_id', # 视频的唯一标识符，可能是一个自定义的 ID 或 MongoDB 的 ObjectId +// 'v_name', # 视频的名称（中文或其他主要语言的标题） +// 'v_enname', # 视频的英文名称 +// 'v_sub', # 视频的字幕信息，可能存储字幕语言或字幕文件路径 +// 'v_letter', # 视频标题的首字母，用于排序或快速索引（如 A、B、C） +// 'v_color', # 视频的颜色标签，可能用于前端展示或分类（例如颜色编码） +// 'v_tag', # 视频的标签，字符串或数组，表示视频的关键词（如“动作”、“喜剧”） +// 'v_class', # 视频的分类，可能是一个更细分的类别（如“电视剧”、“电影”） +// 'v_category_id', # 视频所属类别的 ID，关联到分类表或集合 +// 'v_category_name', # 视频所属类别的名称，直接存储类别名称以便查询 +// 'v_pic', # 视频的封面图片 URL 或文件路径 +// 'v_lang', # 视频的语言（如“中文”、“英语”） +// 'v_area', # 视频的地区（如“中国”、“美国”） +// 'v_year', # 视频的发行年份（如 2023） +// 'v_remarks', # 视频的备注信息，可能用于存储额外说明或状态（如“完结”） +// 'v_actor', # 视频的演员列表，字符串或数组（如“张三, 李四”或 ["张三", "李四"]） +// 'v_director', # 视频的导演，字符串或数组 +// 'v_serial', # 视频的剧集编号或标识，用于表示系列剧的集数（如“第1集”） +// 'v_isend', # 视频是否完结，布尔值或整数（0=未完结，1=完结） +// 'v_lock', # 视频的锁定状态，布尔值或整数（0=未锁定，1=锁定，可能表示限制访问） +// 'v_level', # 视频的等级或权限级别（如 0=普通，1=会员专享） +// 'v_hits', # 视频的总点击量或播放量 +// 'v_hits_day', # 视频的日点击量或播放量 +// 'v_hits_week', # 视频的周点击量或播放量 +// 'v_hits_month', # 视频的月点击量或播放量 +// 'v_duration', # 视频的时长，通常以秒或格式化的字符串（如“120分钟”）存储 +// 'v_up', # 视频的点赞数或顶的数量 +// 'v_down', # 视频的踩数量或反对数量 +// 'v_score', # 视频的评分，可能是平均评分（如 8.5） +// 'v_score_all', # 视频的总评分累加值，用于计算平均评分 +// 'v_score_num', # 视频的评分人数 +// 'v_points_play', # 播放视频所需的积分（如果有积分系统） +// 'v_points_down', # 下载视频所需的积分 +// 'v_content', # 视频的详细描述或简介 +// 'v_play_from', # 视频播放的来源（如“优酷”、“腾讯”） +// 'v_play_note', # 视频播放的备注信息（如播放限制或提示） +// 'v_play_server', # 视频播放的服务器标识或地址 +// 'v_down_from', # 视频下载的来源（如“百度云”、“迅雷”） +// 'v_down_note', # 视频下载的备注信息（如下载限制） +// 'v_down_server', # 视频下载的服务器标识或地址 +// 'v_down_url', # 视频的下载地址，可能是一个 URL 或一组 URL \ No newline at end of file