feature video craw

code/app/task/crawler/youzhi/Scheduler.php

@@ -74,74 +74,111 @@ class Scheduler
      */
     public function crawlFull()
     {
-        $Site = $this->getSite();
+        // $Site = $this->getSite();
-        $arrVSourceId = [];
+        // $arrVSourceId = [];
-
+        // $arrVSourceId[] = ['v_source_id' => 14256];
-        $arrVSourceId[] = ['v_source_id' => 66281];
+        // $arrAllVideoInfoPage = $Site->getVideoInfoPageList($arrVSourceId);
-
+        // foreach ($arrAllVideoInfoPage as $VideoInfoPage) {
-
+        //     $arrVideo = $VideoInfoPage->getVideo();
-        $arrAllVideoInfoPage = $Site->getVideoInfoPageList($arrVSourceId);
+        //     $VideoInfoPage->saveVideo();
-        foreach ($arrAllVideoInfoPage as $VideoInfoPage) {
+        //     print_r($arrVideo);
-            $arrVideo = $VideoInfoPage->getVideo();
+        // }
-            print_r($arrVideo);
+        // return;
-        }
-
-        return;
-
-
-
-
-
-
-
-        $intLimit = 10;
 
         $Site = $this->getSite();
 
         $arrAllCategory = $Site->getCategory();
 
-        foreach ($arrAllCategory as $arrCategory) {
+        $intLimit = 5;
-            $intTotalPage = ceil($arrCategory['total'] / $intLimit);
+
-            for ($intStartPage = 1; $intStartPage <= $intTotalPage; $intStartPage++) {
+        $arrTask = [
-                $arrFilter = [
+            'callback' => [static::class, 'fetchVSourceId'],
-                    [
+            'data' => [],
-                        'v_category_id' => $arrCategory['v_category_id'],
-                        'page' => $intStartPage,
-                    ],
         ];
 
-                $arrVideoPageList = $Site->getVideoPageList($arrFilter);
+        foreach ($arrAllCategory as $arrCategory) {
-                $arrVideoPager = $arrVideoPageList[0]->getVideoPager();
+
+            if ($arrCategory['total'] <= 0) {
+                continue;
+            }
+
+            $intTotalPage = $arrCategory['total_page'];
+            $arrFilter = [];
+            for ($intStartPage = 1; $intStartPage <= $intTotalPage; $intStartPage++) {
+
+                $arrFilter[] = [
+                    'v_category_id' => $arrCategory['v_category_id'],
+                    'page' => $intStartPage,
+                ];
+                if (($intStartPage %  $intLimit) == 0) {
+                    $arrTask['data'] = $arrFilter;
+                    $this->QueueManage->set($arrTask);
+                    $arrFilter = [];
+                }
+            }
+            if (!empty($arrFilter)) {
+                $arrTask['data'] = $arrFilter;
+                $this->QueueManage->set($arrTask);
+            }
+        }
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @param array $arrFilter
+     * @return void
+     */
+    public function fetchVSourceId(array $arrFilter)
+    {
+
+        $Site = $this->getSite();
+
+        $intLimit = 5;
+
+        $arrTask = [
+            'callback' => [static::class, 'fetchVideo'],
+            'data' => [],
+        ];
+
+        # 访问分页，获取 v_source_id 集合
+        $arrAllVideoPageList = $Site->getVideoPageList($arrFilter);
 
         $arrVSourceId = [];
 
+        foreach ($arrAllVideoPageList as $VideoPageList) {
+            $arrVideoPager = $VideoPageList->getVideoPager();
+
             foreach ($arrVideoPager['list'] as $arrVideoPager) {
                 $arrVSourceId[] = ['v_source_id' => (int)$arrVideoPager['vod_id']];
+
+                if ((count($arrVSourceId) % $intLimit) == 0) {
+                    $arrTask['data'] = $arrVSourceId;
+                    $this->QueueManage->set($arrTask);
+                    $arrVSourceId = [];
                 }
+            }
+        }
+
+        if (!empty($arrVSourceId)) {
+            $arrTask['data'] = $arrVSourceId;
+            $this->QueueManage->set($arrTask);
+        }
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @param array $arrVSourceId
+     * @return void
+     */
+    public function fetchVideo(array $arrVSourceId)
+    {
+        $Site = $this->getSite();
 
         $arrAllVideoInfoPage = $Site->getVideoInfoPageList($arrVSourceId);
         foreach ($arrAllVideoInfoPage as $VideoInfoPage) {
-                    $arrVideo = $VideoInfoPage->getVideo();
+            $VideoInfoPage->saveVideo();
-                    print_r($arrVideo);
         }
-
-
-                // foreach ($arrVideoPageList as $VideoPage) {
-                //     $arrAllVideoPager = $VideoPage->getVideoPager();
-
-                //     $arrVSourceId = [];
-                //     foreach ($arrAllVideoPager['list'] as $arrVideoPager) {
-                //         $arrVSourceId[] = ['v_source_id' => (int)$arrVideoPager['vod_id']];
-                //     }
-
-                //     $arrAllVideoInfoPage = $Site->getVideoInfoPageList($arrVSourceId);
-                //     foreach ($arrAllVideoInfoPage as $VideoInfoPage) {
-                //         $arrVideo = $VideoInfoPage->getVideo();
-                //         print_r($arrVideo);
-                //     }
-                // }
-            }
-        }
-        return;
     }
 }

code/app/task/crawler/youzhi/page/Site.php

@@ -181,6 +181,7 @@ class Site
                 'v_category_name' =>  $arrFilter[$intKey]['type_name'],
                 'v_category_name_en' => zhToPinYin($arrFilter[$intKey]['type_name']),
                 'total' => $arrTemp['total'],
+                'total_page' => $arrTemp['pagecount'],
             ];
         }
 
@@ -196,7 +197,7 @@ class Site
      */
     public function getVideoPageUri(int $intVCategoryId, int $intPage): string
     {
-        return  sprintf('/inc/api_mac10.php?ac=list&t=%s&page=%s', $intVCategoryId, $intPage);
+        return  sprintf('/inc/api_mac10.php?ac=list&t=%s&pg=%s', $intVCategoryId, $intPage);
     }
 
     /**

code/app/task/crawler/youzhi/page/VideoInfoPage.php

@@ -4,6 +4,7 @@ declare(strict_types=1);
 
 namespace app\task\crawler\youzhi\page;
 
+use app\model\CountersModel;
 use app\model\VideoModel;
 use storage\StorageCore;
 
@@ -14,7 +15,7 @@ class VideoInfoPage extends BasePage
      *
      * @var string
      */
-    protected $strChapterContent = '';
+    protected $strVideoContent = '';
 
     /**
      * Undocumented variable
@@ -53,17 +54,49 @@ class VideoInfoPage extends BasePage
     {
         if (empty($this->arrVideo)) {
 
-            $this->arrVideo = json_decode($this->getContent(), true);
+            $arrVideo = json_decode($this->getContent(), true);
-            $this->arrVideo =   $this->arrVideo['list'][0];
+            $arrVideo = $arrVideo['list'][0];
-            $this->arrVideo['vod_play_url'] = str_replace('#', '$', $this->arrVideo['vod_play_url']);
+            // print_r($arrVideo);
-            $this->arrVideo['vod_play_url'] = explode('$', $this->arrVideo['vod_play_url']);
+
-            $this->arrVideo['vod_play_url'] = array_chunk($this->arrVideo['vod_play_url'], 2);
+            $strPlayUrl = $arrVideo['vod_play_url'];
-            foreach ($this->arrVideo['vod_play_url']  as $intKey => $arrVal) {
+            unset($arrVideo['vod_play_url']);
-                $this->arrVideo['vod_play_url'][$intKey] = [
+
+            $arrPlayUrl = str_replace('#', '$', $strPlayUrl);
+            $arrPlayUrl = explode('$', $arrPlayUrl);
+            $arrPlayUrl = array_chunk($arrPlayUrl, 2);
+            foreach ($arrPlayUrl  as $intKey => $arrVal) {
+                $arrPlayUrl[$intKey] = [
                     'name' => $arrVal[0],
-                    'url' => $arrVal[1],
+                    'url' => $arrVal[1]?? $arrVal[0],
                 ];
             }
+            $arrVideo['vod_play_url'][$this->getsite()->getSiteCode()] = $arrPlayUrl;
+            $arrVideo['vod_actor'] = explode(',', $arrVideo['vod_actor']);
+            $arrVideo['vod_director'] = explode(',', $arrVideo['vod_director']);
+            $arrVideo['vod_class'] = explode(',', $arrVideo['vod_class']);
+            $arrVideo['vod_content'] = trim($arrVideo['vod_content']);
+            
+            foreach ($arrVideo as $strKey => $mixedVal) {
+                $strKey = str_replace('vod', 'v', $strKey);
+
+                if ($strKey == 'type_id') {
+                    $strKey = 'v_category_id';
+                }
+                if ($strKey == 'type_name') {
+                    $strKey = 'v_category';
+                }
+                if ($strKey == 'v_content') {
+                    $strKey = 'v_description';
+                }
+                if ($strKey == 'v_enname') {
+                    $strKey = 'v_name_en';
+                }
+                if ($strKey == 'v_time') {
+                    $strKey = 'v_publish_date';
+                }
+
+                $this->arrVideo[$strKey] = $mixedVal;
+            }
         }
 
         return  $this->arrVideo;
@@ -76,129 +109,76 @@ class VideoInfoPage extends BasePage
      */
     protected $arrVideo;
 
-
     /**
      * Undocumented function
      *
-     * @param integer $intNId
      * @return boolean
      */
-    public function saveChapter(int $intNId): bool
+    public function saveVideo(): bool
     {
-        $arrPageChapter = $this->getVideo();
+        $arrPageVideo = $this->getVideo();
 
         if (
-            empty($arrPageChapter) ||
+            empty($arrPageVideo) ||
-            empty($arrPageChapter['c_page_title']) ||
+            empty($arrPageVideo['v_name']) ||
-            empty($arrPageChapter['c_page_data']) ||
+            empty($arrPageVideo['v_category']) ||
-            count($arrPageChapter['c_page_data']) < 5
+            empty($arrPageVideo['v_play_url']) ||
+            empty($arrPageVideo['v_play_url']['youzhi'])
         ) {
             return false;
         }
 
-        $strUUId = $this->getSite()->getSiteUUId('chapter-' . $this->arrData['c_source_id'] . '-' . $this->arrData['c_page']);
+        $arrExistsVideo = $this->getVideoModel()->findOne(['v_name' => $arrPageVideo['v_name']], $this->arrOptions);
 
-        $arrChapter = [
+        if ($arrExistsVideo) {
-            'n_id' => $intNId,
-            'c_name' => $arrPageChapter['c_page_data'][3],
-            'c_sort_num' => $this->arrData['c_sort_num'],
-            'c_page' => $this->arrData['c_page'],
-            'c_source_id' => $this->arrData['c_source_id'],
-            'c_site_code' => $this->getSite()->getSiteCode(),
-            'c_source_uuid' =>  $strUUId,
-        ];
 
-        $arrExistsChapter = $this->getVideoModel()->findOne(['c_source_uuid' => $strUUId], $this->arrOptions);
+            $arrExistsVideo = (array) $arrExistsVideo;
-
-        if ($arrExistsChapter) {
-
-            $arrExistsChapter = (array) $arrExistsChapter;
 
             $arrUpdate = [
                 'updated_at' => new \MongoDB\BSON\UTCDateTime(),
+                'v_play_url.youzhi' => $arrPageVideo['v_play_url']['youzhi'],
             ];
 
             $this->getVideoModel()->updateOne(
-                ['c_source_uuid' => $strUUId],
+                ['v_name' =>  $arrPageVideo['v_name']],
                 ['$set' =>   $arrUpdate],
             );
-            $this->strCContentPath = $arrExistsChapter['c_content_path'];
         } else {
 
-            $arrInsertNovel =  $arrChapter;
+            $arrInsertVideo = [
-
+                'v_id' =>   CountersModel::getInstance()->getNextSequence('video'),
-            $arrInsertNovel['c_content_path'] = '/novel/' . ($intNId % 100) . '/' . $intNId . '/' .  $arrChapter['c_sort_num'] . '_' .   $arrChapter['c_page'] . '.txt';
+                'v_name' => $arrPageVideo['v_name'],
-            $arrInsertNovel['created_at'] = new \MongoDB\BSON\UTCDateTime();
+                'v_name_en' => zhToPinYin($arrPageVideo['v_name']),
+                'v_category' => $arrPageVideo['v_category'],
+                'v_category_en' => zhToPinYin($arrPageVideo['v_category']),
+                'v_actor' => $arrPageVideo['v_actor'],
+                'v_director' => $arrPageVideo['v_director'],
+                'v_class' => $arrPageVideo['v_class'],
+                'v_letter' => $arrPageVideo['v_letter'],
+                'v_pic' => $arrPageVideo['v_pic'],
+                'v_lang' => $arrPageVideo['v_lang'],
+                'v_area' => $arrPageVideo['v_area'],
+                'v_year' => $arrPageVideo['v_year'],
+                'v_remarks' => $arrPageVideo['v_remarks'],
+                'v_isend' => $arrPageVideo['v_isend'],
+                'v_score' => $arrPageVideo['v_score'],
+                'v_description' => $arrPageVideo['v_description'],
+                'v_play_url' => $arrPageVideo['v_play_url'],
+                'v_publish_date' => $arrPageVideo['v_publish_date'],
+                'created_at' => new \MongoDB\BSON\UTCDateTime(),
+            ];
 
             try {
-                $this->getVideoModel()->insert($arrInsertNovel);
+                $this->getVideoModel()->insert($arrInsertVideo);
-                $this->updateNovelHaveChapter($intNId);
-                $this->strCContentPath = $arrInsertNovel['c_content_path'];
             } catch (\Exception $e) {
                 if ($e->getCode() === 11000) {
-                    return $this->saveChapter($intNId);
+                    return $this->saveVideo();
                 } else {
                     throw $e;
                 }
             }
         }
 
-        StorageCore::getInstance()->put($this->strCContentPath, $this->strChapterContent);
-
         return true;
     }
-
-    public $strCContentPath;
-    /**
-     * Undocumented function
-     *
-     * @param integer $intNId
-     * @return void
-     */
-    protected function updateNovelHaveChapter(int $intNId)
-    {
-        $arrFilter = [
-            'n_id' => $intNId,
-        ];
-
-        $VideoModel =  VideoModel::getInstance();
-        $arrLasteChapter =  $VideoModel->getLatestChapterByNId($intNId, false);
-
-        if (!empty($arrLasteChapter)) {
-            $arrUpdate['$set']['n_have_chapter'] = 1;
-            $arrUpdate['$set']['n_latest_chapter_name'] = $arrLasteChapter['c_name'];
-            $this->getMongoBase()->updateOne('novel', $arrFilter, $arrUpdate);
-        }
-    }
-
-    protected $arrNextPage;
-
-    /**
-     * Undocumented function
-     *
-     * @return array
-     */
-    public function getNextPage(): array
-    {
-        if ($this->arrNextPage === NULL) {
-            $this->arrNextPage = [];
-
-            if ($this->isMobile()) {
-                $strPattern = "/var\s+hhekgsv\s*=\s*['\"](.*?)['\"]/";
-                preg_match_all($strPattern, $this->getContent(), $arrMatches);
-                $strNextPageUri = end($arrMatches[1]) ?: '';
-            } else {
-                $strHref = $this->getQueryList()->find('.read_btn a:contains("下一章")')->attr('href');
-                $strNextPageUri = $strHref ?: '';
-            }
-
-            if (is_numeric(strpos($strNextPageUri, '_'))) {
-                $this->arrNextPage = $this->arrData;
-                $this->arrNextPage['c_page'] += 1;
-                $this->arrNextPage['uri'] = $strNextPageUri;
-            }
-        }
-
-        return $this->arrNextPage;
-    }
 }

code/database/mongo/migrations/video.php

@@ -0,0 +1,91 @@
+<?php
+return [
+    'collection' => 'video',
+    'indexes' => [
+        [
+            'key' => ['v_id' => 1],
+            'options' => ['unique' => true]
+        ],
+        [
+            'key' => ['v_name' => 1],
+            'options' => ['unique' => true]
+        ],
+        [
+            'key' => ['v_category' => 1],
+            'options' => ['unique' => false]
+        ],
+        [
+            'key' => ['v_category_en' => 1],
+            'options' => ['unique' => false]
+        ],
+        [
+            'key' => ['v_play_url.v_source_code' => 1],
+            'options' => ['unique' => false]
+        ],
+
+        [
+            'key' => ['v_lang' => 1],
+            'options' => ['unique' => false]
+        ],
+        [
+            'key' => ['v_area' => 1],
+            'options' => ['unique' => false]
+        ],
+        [
+            'key' => ['v_year' => 1],
+            'options' => ['unique' => false]
+        ],
+        [
+            'key' => ['v_actor' => 1],
+            'options' => ['unique' => false]
+        ],
+        [
+            'key' => ['v_director' => 1],
+            'options' => ['unique' => false]
+        ],
+
+    ]
+];
+
+// 'v_publish_date', # 视频创建或更新时间，通常是时间戳或日期时间，表示视频记录的最后修改时间
+// 'v_id', # 视频的唯一标识符，可能是一个自定义的 ID 或 MongoDB 的 ObjectId
+// 'v_name', # 视频的名称（中文或其他主要语言的标题）
+// 'v_enname', # 视频的英文名称
+// 'v_sub', # 视频的字幕信息，可能存储字幕语言或字幕文件路径
+// 'v_letter', # 视频标题的首字母，用于排序或快速索引（如 A、B、C）
+// 'v_color', # 视频的颜色标签，可能用于前端展示或分类（例如颜色编码）
+// 'v_tag', # 视频的标签，字符串或数组，表示视频的关键词（如“动作”、“喜剧”）
+// 'v_class', # 视频的分类，可能是一个更细分的类别（如“电视剧”、“电影”）
+// 'v_category_id', # 视频所属类别的 ID，关联到分类表或集合
+// 'v_category_name', # 视频所属类别的名称，直接存储类别名称以便查询
+// 'v_pic', # 视频的封面图片 URL 或文件路径
+// 'v_lang', # 视频的语言（如“中文”、“英语”）
+// 'v_area', # 视频的地区（如“中国”、“美国”）
+// 'v_year', # 视频的发行年份（如 2023）
+// 'v_remarks', # 视频的备注信息，可能用于存储额外说明或状态（如“完结”）
+// 'v_actor', # 视频的演员列表，字符串或数组（如“张三, 李四”或 ["张三", "李四"]）
+// 'v_director', # 视频的导演，字符串或数组
+// 'v_serial', # 视频的剧集编号或标识，用于表示系列剧的集数（如“第1集”）
+// 'v_isend', # 视频是否完结，布尔值或整数（0=未完结，1=完结）
+// 'v_lock', # 视频的锁定状态，布尔值或整数（0=未锁定，1=锁定，可能表示限制访问）
+// 'v_level', # 视频的等级或权限级别（如 0=普通，1=会员专享）
+// 'v_hits', # 视频的总点击量或播放量
+// 'v_hits_day', # 视频的日点击量或播放量
+// 'v_hits_week', # 视频的周点击量或播放量
+// 'v_hits_month', # 视频的月点击量或播放量
+// 'v_duration', # 视频的时长，通常以秒或格式化的字符串（如“120分钟”）存储
+// 'v_up', # 视频的点赞数或顶的数量
+// 'v_down', # 视频的踩数量或反对数量
+// 'v_score', # 视频的评分，可能是平均评分（如 8.5）
+// 'v_score_all', # 视频的总评分累加值，用于计算平均评分
+// 'v_score_num', # 视频的评分人数
+// 'v_points_play', # 播放视频所需的积分（如果有积分系统）
+// 'v_points_down', # 下载视频所需的积分
+// 'v_content', # 视频的详细描述或简介
+// 'v_play_from', # 视频播放的来源（如“优酷”、“腾讯”）
+// 'v_play_note', # 视频播放的备注信息（如播放限制或提示）
+// 'v_play_server', # 视频播放的服务器标识或地址
+// 'v_down_from', # 视频下载的来源（如“百度云”、“迅雷”）
+// 'v_down_note', # 视频下载的备注信息（如下载限制）
+// 'v_down_server', # 视频下载的服务器标识或地址
+// 'v_down_url', # 视频的下载地址，可能是一个 URL 或一组 URL