Merge branch 'dev' of ssh://git.bgbg.live:18150/root/SEONexus into dev

code/app/common.php

@@ -815,7 +815,8 @@ namespace {
      * @param integer $length
      * @return string
      */
-    function randomString(int $length): string {
+    function randomString(int $length): string
+    {
         $chars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ';
         $max = 51; // 字符集长度减1 (0-51)
         $result = '';
@@ -826,4 +827,39 @@ namespace {
 
         return $result;
     }
+
+    /**
+     * 图像转换函数
+     *
+     * @param string $strInputFile
+     * @param string $strOutputFile
+     * @param integer $intQuality
+     * @return boolean
+     */
+    function convertToWebP(string $strInputFile, string  $strOutputFile, int $intQuality = 80): bool
+    {
+        $strCwebpPath = root_path() . '/extend/tools/cwebp';
+
+        $strCMD = escapeshellcmd("$strCwebpPath -q $intQuality " . escapeshellarg($strInputFile) . " -o " . escapeshellarg($strOutputFile));
+
+        exec($strCMD . ' 2>&1', $strOutput, $intReturnCode);
+
+        if ($intReturnCode === 0) {
+            return true;
+        } else {
+            echo "Conversion failed: " . implode("\n", $strOutput);
+            return false;
+        }
+    }
+
+    function ensureUtf8(string $input): string
+    {
+        // 检查是否为有效的 UTF-8
+        if (mb_check_encoding($input, 'UTF-8')) {
+            return $input; // 已经是有效的 UTF-8
+        }
+
+        // 尝试将非 UTF-8 转换为 UTF-8，忽略无效字符
+        return iconv('UTF-8', 'UTF-8//IGNORE', $input);
+    }
 }

code/app/model/ConverterMovel.php

@@ -88,7 +88,7 @@ class ConverterMovel
      * Undocumented function
      *
      * @param string $strKey
-     * @return void
+     * @return string
      */
     static public function getVal(string $strKey)
     {

code/app/model/VideoClicksModel.php

@@ -70,7 +70,6 @@ class VideoClicksModel extends MongoModel
                 ],
                 [
                     '$inc' => ['vc_clicks' => 1],
-                    '$set' => ['v_category_id' => $Novel->v_category_id]
                 ],
                 [
                     'upsert' => true

code/app/model/VideoModel.php

@@ -43,9 +43,6 @@ class VideoModel extends MongoModel
         ],
         'projection' => [
             '_id' => 0,
-            'n_site_code' => 0,
-            'n_source_uuid' => 0,
-            'n_source_id' => 0,
         ],
     ];
 
@@ -63,12 +60,12 @@ class VideoModel extends MongoModel
 
         try {
             $arrConditions = array_map(function ($strKeyword) {
-                return ['n_name' => ['$regex' => trim($strKeyword), '$options' => 'i']];
+                return ['v_name' => ['$regex' => trim($strKeyword), '$options' => 'i']];
             }, $arrKeywords);
 
             $UpdateResult = $this->getCol()->updateMany(
                 ['$or' => $arrConditions],
-                ['$set' => ['n_level' => $intLevel]]
+                ['$set' => ['v_level' => $intLevel]]
             );
 
             $UpdateResult->getModifiedCount();
@@ -99,7 +96,6 @@ class VideoModel extends MongoModel
             $arrOptions['sort'] = $arrSort;
         }
 
-        $arrFilter['n_have_chapter'] = 1;
 
         $Cursor = $this->getCol()->find($arrFilter, $arrOptions);
 
@@ -179,7 +175,7 @@ class VideoModel extends MongoModel
      */
     public function getVideoByNId(int $intNId): null|array
     {
-        $arrFilter = ['n_id' => $intNId, 'n_have_chapter' => 1];
+        $arrFilter = ['v_id' => $intNId];
         return $this->findOne($arrFilter);
     }
 
@@ -202,7 +198,6 @@ class VideoModel extends MongoModel
                 return $arrResult;
             }
 
-            $arrFilter['n_have_chapter'] = 1;
 
             $arrPipeline = [
                 ['$match' => $arrFilter],
@@ -236,12 +231,11 @@ class VideoModel extends MongoModel
         array $arrFilter,
         int $intPage = 1,
         int $intLimit = 20,
-        array $arrSort = ['n_level' => -1],
+        array $arrSort = ['v_level' => -1],
         string $strKey = '',
         int $intLifeTime = 24 * 3600,
     ): array {
         try {
-            $arrFilter['n_have_chapter'] = 1;
             
             $arrResult = $this->checkCacheByKey($strKey);
             if ($arrResult !== NULL) {
@@ -295,23 +289,23 @@ class VideoModel extends MongoModel
         int $intLifeTime = 24 * 3600,
         int $intType = 0,
         $strStatus = NULL,
-        array $arrSort = ['n_level' => -1]
+        array $arrSort = ['v_level' => -1]
     ): array {
-        $arrFilter = ['n_have_chapter' => 1,];
+        $arrFilter = [];
 
         switch ($intType) {
             case 1:
                 $arrCategory = array_values(CategoryModel::getManCategory());
-                $arrFilter['n_category'] = ['$in' => $arrCategory];
+                $arrFilter['v_category'] = ['$in' => $arrCategory];
                 break;
             case 2:
                 $arrCategory = array_values(CategoryModel::getWomenCategory());
-                $arrFilter['n_category'] =  ['$in' => $arrCategory];
+                $arrFilter['v_category'] =  ['$in' => $arrCategory];
                 break;
         }
 
         if ($strStatus !== NULL) {
-            $arrFilter['n_status'] = $strStatus;
+            $arrFilter['v_status'] = $strStatus;
         }
         return $this->findPaginatedWithCache($arrFilter, $intPage, $intLimit,  $arrSort, $strKey, $intLifeTime,);
     }
@@ -340,7 +334,6 @@ class VideoModel extends MongoModel
                 return $arrResult;
             }
 
-            $arrFilter['n_have_chapter'] = 1;
 
             $arrOptions = self::$arrOptions;
 
@@ -385,17 +378,17 @@ class VideoModel extends MongoModel
         int $intCount = 10,
         int $intType = 0,
         $strStatus = NULL,
-        array $arrSort = ['n_level' => -1]
+        array $arrSort = ['v_level' => -1]
     ): array {
         try {
             $arrFilter = [];
 
             if (in_array($intType, [1, 2])) {
-                $arrFilter['n_category'] = ['$in' => CategoryModel::getCategoryByType($intType)];
+                $arrFilter['v_category'] = ['$in' => CategoryModel::getCategoryByType($intType)];
             }
 
             if ($strStatus !== NULL) {
-                $arrFilter['n_status'] = $strStatus;
+                $arrFilter['v_status'] = $strStatus;
             }
 
             return $this->findManyShuffledWithCache($arrFilter, $intCount, $arrSort, $strKey, $intLifeTime);
@@ -409,7 +402,6 @@ class VideoModel extends MongoModel
      *
      * @param string $strSign
      * @param integer $intCount
-     * @param integer $intType
      * @param string strDateType 
      * * daily     天
      * * weekly    周
@@ -417,7 +409,7 @@ class VideoModel extends MongoModel
      * * total     总
      * @return array
      */
-    public function getRankVideoOnCache(string $strKey, int $intCount = 10, int $intType = 0, $strDateType = 'daily', $strStatus = NULL): array
+    public function getRankVideoOnCache(string $strKey, int $intCount = 10,  $strDateType = 'daily', $strStatus = NULL): array
     {
         try {
             $intLifeTime = 24 * 3600;
@@ -431,18 +423,15 @@ class VideoModel extends MongoModel
             ];
 
             if ($strStatus !== NULL) {
-                $arrFilter['n_status'] = $strStatus;
+                $arrFilter['v_status'] = $strStatus;
             }
 
-            if (in_array($intType, [1, 2])) {
-                $arrFilter['n_category'] = ['$in' => CategoryModel::getCategoryByType($intType)];
-            }
 
             $arrOptions = [
                 'sort' => ['nc_clicks' => -1],
                 'limit' => $intCount * 3,
                 'projection' => [
-                    'n_id' => 1,
+                    'v_id' => 1,
                 ],
             ];
 
@@ -452,10 +441,10 @@ class VideoModel extends MongoModel
 
             $arrNId = [];
             foreach ($Cursor as $Doc) {
-                $arrNId[] = $Doc['n_id'];
+                $arrNId[] = $Doc['v_id'];
             }
 
-            $arrFilter =    ['n_id' => ['$in' => $arrNId]];
+            $arrFilter =    ['v_id' => ['$in' => $arrNId]];
             $arrOptions = self::$arrOptions;
             unset($arrOptions['typeMap']);
 

code/app/task/crawler/youzhi/Scheduler.php

@@ -74,15 +74,12 @@ class Scheduler
      */
     public function crawlFull()
     {
-        // $Site = $this->getSite();
+        // $arrVSourceId = [['v_source_id' => 14256]];
-        // $arrVSourceId = [];
+        // $arrTask = [
-        // $arrVSourceId[] = ['v_source_id' => 14256];
+        //     'callback' => [static::class, 'fetchVideo'],
-        // $arrAllVideoInfoPage = $Site->getVideoInfoPageList($arrVSourceId);
+        //     'data' => $arrVSourceId,
-        // foreach ($arrAllVideoInfoPage as $VideoInfoPage) {
+        // ];
-        //     $arrVideo = $VideoInfoPage->getVideo();
+        // $this->QueueManage->set($arrTask);
-        //     $VideoInfoPage->saveVideo();
-        //     print_r($arrVideo);
-        // }
         // return;
 
         $Site = $this->getSite();
@@ -141,13 +138,12 @@ class Scheduler
             'data' => [],
         ];
 
-        # 访问分页，获取 v_source_id 集合
+        $arrAllVideoPagerPage = $Site->getVideoPagerPageList($arrFilter);
-        $arrAllVideoPageList = $Site->getVideoPageList($arrFilter);
 
         $arrVSourceId = [];
 
-        foreach ($arrAllVideoPageList as $VideoPageList) {
+        foreach ($arrAllVideoPagerPage as $VideoPagerPage) {
-            $arrVideoPager = $VideoPageList->getVideoPager();
+            $arrVideoPager = $VideoPagerPage->getVideoPager();
 
             foreach ($arrVideoPager['list'] as $arrVideoPager) {
                 $arrVSourceId[] = ['v_source_id' => (int)$arrVideoPager['vod_id']];
@@ -177,8 +173,20 @@ class Scheduler
         $Site = $this->getSite();
 
         $arrAllVideoInfoPage = $Site->getVideoInfoPageList($arrVSourceId);
+
+        $arrPic = [];
         foreach ($arrAllVideoInfoPage as $VideoInfoPage) {
             $VideoInfoPage->saveVideo();
+
+            $arrPicItem = $VideoInfoPage->getDownPicInfo();
+
+            if (!StorageCore::getInstance()->has($arrPicItem['local_path'])) {
+                $arrPic[] = $arrPicItem;
+            }
+        }
+        
+        if ($arrPic) {
+            $Site->downRemoteImgToLocal($arrPic);
         }
     }
 }

code/app/task/crawler/youzhi/page/Site.php

@@ -142,7 +142,7 @@ class Site
      * retrieve multiple novel page model
      *
      * @param string $strUri
-     * @return VideoPage[]
+     * @return array
      */
     public function getCategory(): array
     {
@@ -204,13 +204,13 @@ class Site
      * retrieve multiple novel page model
      *
      * @param string $strUri
-     * @return VideoPage[]
+     * @return VideoPagerPage[]
      */
-    public function getVideoPageList(array $arrFilter): array
+    public function getVideoPagerPageList(array $arrFilter): array
     {
 
-        /** @var VideoPage[] */
+        /** @var VideoPagerPage[] */
-        $arrVideoPage = [];
+        $arrVideoPagerPage = [];
 
         foreach ($arrFilter as $intKey => $arrItem) {
 
@@ -232,15 +232,15 @@ class Site
             $strContent =  $Response->getBody()->getContents();
 
 
-            $VideoPage = $this->getVideoPage($this, $arrFilter[$intKey]['uri']);
+            $VideoPagerPage = $this->getVideoPagerPage($this, $arrFilter[$intKey]['uri']);
 
-            $VideoPage->setContent($strContent);
+            $VideoPagerPage->setContent($strContent);
-            $VideoPage->setData($arrFilter[$intKey]);
+            $VideoPagerPage->setData($arrFilter[$intKey]);
 
-            $arrVideoPage[] =  $VideoPage;
+            $arrVideoPagerPage[] =  $VideoPagerPage;
         }
 
-        return $arrVideoPage;
+        return $arrVideoPagerPage;
     }
 
     /**
@@ -306,25 +306,31 @@ class Site
     {
         $StorageCore = StorageCore::getInstance();
 
-        $arrPromises = [];
+        $Client = new Client();
 
-        foreach ($arrFilter as $intKey => $arrItem) {
+        foreach ($arrFilter as  $arrItem) {
             if (!$StorageCore->has($arrItem['local_path'])) {
-                $arrPromises[$intKey] = $this->getClient()->getAsync($arrItem['uri']);
+                try {
+                    $Response = $Client->get($arrItem['uri']);
+                    if ($Response->getStatusCode() === 200) {
+                        if (str_ends_with(strtolower($arrItem['local_path']), '.webp')) {
+                            $strTmpFile = str_replace('.webp', '.jpg', $arrItem['local_path']);
+                            $StorageCore->writeStream($strTmpFile, $Response->getBody()->detach());
+                            convertToWebP(
+                                $StorageCore->getRealPath($strTmpFile),
+                                $StorageCore->getRealPath($arrItem['local_path']),
+                                40
+                            );
+                            @unlink($StorageCore->getRealPath($strTmpFile));
+                        } else {
+                            $StorageCore->writeStream($arrItem['local_path'], $Response->getBody()->detach());
+                        }
+                    }
+                } catch (\Exception $e) {
+                    // echo $e->getMessage();
+                }
             }
         }
-
-        $arrResults = Promise\Utils::settle($arrPromises)->wait();
-
-        $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
-            return $arrResult['state'] === 'fulfilled';
-        });
-
-        foreach ($arrSuccessfulResults as $intKey => $arrResult) {
-            $Response = $arrResult['value'];
-
-            $StorageCore->writeStream($arrFilter[$intKey]['local_path'], $Response->getBody()->detach());;
-        }
     }
 
     /**
@@ -332,11 +338,11 @@ class Site
      *
      * @param object $Site
      * @param string $strUri
-     * @return VideoPage
+     * @return VideoPagerPage
      */
-    public function getVideoPage($Site, $strUri)
+    public function getVideoPagerPage($Site, $strUri)
     {
-        return new VideoPage($Site, $strUri);
+        return new VideoPagerPage($Site, $strUri);
     }
 
     /**

code/app/task/crawler/youzhi/page/VideoInfoPage.php

@@ -24,6 +24,26 @@ class VideoInfoPage extends BasePage
      */
     protected $VideoModel;
 
+    /**
+     * Undocumented variable
+     *
+     * @var array
+     */
+    protected $arrVideo;
+
+    /**
+     * Undocumented variable
+     *
+     * @var array
+     */
+    public $arrOptions = [
+        'typeMap' => [
+            'root' => 'array',
+            'document' => 'array',
+            'array' => 'array',
+        ],
+    ];
+
     /**
      * Undocumented function
      *
@@ -37,13 +57,6 @@ class VideoInfoPage extends BasePage
         return  $this->VideoModel;
     }
 
-    public $arrOptions = [
-        'typeMap' => [
-            'root' => 'array',
-            'document' => 'array',
-            'array' => 'array',
-        ],
-    ];
 
     /**
      * Undocumented function
@@ -67,7 +80,7 @@ class VideoInfoPage extends BasePage
             foreach ($arrPlayUrl  as $intKey => $arrVal) {
                 $arrPlayUrl[$intKey] = [
                     'name' => $arrVal[0],
-                    'url' => $arrVal[1]?? $arrVal[0],
+                    'url' => $arrVal[1] ?? $arrVal[0],
                 ];
             }
             $arrVideo['vod_play_url'][$this->getsite()->getSiteCode()] = $arrPlayUrl;
@@ -75,6 +88,8 @@ class VideoInfoPage extends BasePage
             $arrVideo['vod_director'] = explode(',', $arrVideo['vod_director']);
             $arrVideo['vod_class'] = explode(',', $arrVideo['vod_class']);
             $arrVideo['vod_content'] = trim($arrVideo['vod_content']);
+            $arrVideo['vod_content'] = trim($arrVideo['vod_content'], '\　');
+            $arrVideo['vod_content'] = ensureUtf8($arrVideo['vod_content']);
 
             foreach ($arrVideo as $strKey => $mixedVal) {
                 $strKey = str_replace('vod', 'v', $strKey);
@@ -102,12 +117,6 @@ class VideoInfoPage extends BasePage
         return  $this->arrVideo;
     }
 
-    /**
-     * Undocumented variable
-     *
-     * @var array
-     */
-    protected $arrVideo;
 
     /**
      * Undocumented function
@@ -143,6 +152,7 @@ class VideoInfoPage extends BasePage
                 ['v_name' =>  $arrPageVideo['v_name']],
                 ['$set' =>   $arrUpdate],
             );
+            $this->strPicLocalPath = $arrExistsVideo['pic'];
         } else {
 
             $arrInsertVideo = [
@@ -155,7 +165,7 @@ class VideoInfoPage extends BasePage
                 'v_director' => $arrPageVideo['v_director'],
                 'v_class' => $arrPageVideo['v_class'],
                 'v_letter' => $arrPageVideo['v_letter'],
-                'v_pic' => $arrPageVideo['v_pic'],
+                // 'v_pic' => $arrPageVideo['v_pic'],
                 'v_lang' => $arrPageVideo['v_lang'],
                 'v_area' => $arrPageVideo['v_area'],
                 'v_year' => $arrPageVideo['v_year'],
@@ -167,6 +177,9 @@ class VideoInfoPage extends BasePage
                 'v_publish_date' => $arrPageVideo['v_publish_date'],
                 'created_at' => new \MongoDB\BSON\UTCDateTime(),
             ];
+            $arrInsertVideo['v_pic'] = $this->generateVideoPath($arrInsertVideo['v_name_en']);
+
+            $this->strPicLocalPath =  $arrInsertVideo['v_pic'];
 
             try {
                 $this->getVideoModel()->insert($arrInsertVideo);
@@ -181,4 +194,33 @@ class VideoInfoPage extends BasePage
 
         return true;
     }
+
+    private $strPicLocalPath = '';
+
+    /**
+     * Undocumented function
+     *
+     * @param string $strName
+     * @return string
+     */
+    public function generateVideoPath($strName): string
+    {
+        $intTime = time() - mt_rand(1, 2 * 24 * 3600 * 360);
+
+        return sprintf('/img/video/%s/%s.webp', date('Y/m/d/H', $intTime), $strName);
+    }
+
+
+    /**
+     * Undocumented function
+     *
+     * @return array
+     */
+    public function getDownPicInfo(): array
+    {
+        return [
+            'uri' => $this->getVideo()['v_pic'],
+            'local_path' => $this->strPicLocalPath,
+        ];
+    }
 }

code/app/task/crawler/youzhi/page/VideoPage.php

@@ -1,417 +0,0 @@
-<?php
-
-declare(strict_types=1);
-
-namespace app\task\crawler\youzhi\page;
-
-use app\model\CategoryModel;
-use app\model\CountersModel;
-use app\model\NovelModel;
-use storage\StorageCore;
-
-class VideoPage extends BasePage
-{
-    /**
-     * Undocumented variable
-     *
-     * @var array
-     */
-    protected $arrVideo = [];
-
-    /**
-     * Undocumented variable
-     *
-     * @var array
-     */
-    protected $arrRelatedNovel = [];
-
-    /**
-     * Undocumented variable
-     *
-     * @var array
-     */
-    protected $arrForgeNovel = [];
-
-    /**
-     * Undocumented variable
-     *
-     * @var NovelModel
-     */
-    protected $NovelModel;
-
-    /**
-     * Undocumented function
-     *
-     * @return NovelModel
-     */
-    public function getNovelModel(): NovelModel
-    {
-        if ($this->NovelModel == NULL) {
-            $this->NovelModel = NovelModel::getInstance();
-        }
-        return  $this->NovelModel;
-    }
-
-    public $arrOptions = [
-        'typeMap' => [
-            'root' => 'array',
-            'document' => 'array',
-            'array' => 'array',
-        ],
-    ];
-
-    /**
-     * 查询看的小说
-     *
-     * @param integer $intNSourceId
-     * @param string $strName
-     * @return array
-     */
-    protected function findOrCreateNovel(int $intNSourceId, string $strName): array
-    {
-        $strUUId = $this->getSite()->getSiteUUId($intNSourceId);
-
-        $arrExistsNovel = $this->getNovelModel()->findOne(['n_source_uuid' => $strUUId], $this->arrOptions);
-
-        if ($arrExistsNovel) {
-            $arrVideo = (array) $arrExistsNovel;
-        } else {
-            $arrVideo = [
-                'n_id' =>  CountersModel::getInstance()->getNextSequence('novel'),
-                'n_name' => $strName,
-                'n_name_pinyin' => zhToPinYin($strName),
-                'n_site_code' =>  $this->getSite()->getSiteCode(),
-                'n_source_uuid' => $strUUId,
-                'n_source_id' => $intNSourceId,
-                'n_level' => 1,
-                'n_have_chapter' => 0,
-                'created_at' => new \MongoDB\BSON\UTCDateTime(),
-            ];
-            try {
-                $this->getNovelModel()->insert($arrVideo);
-            } catch (\Exception $e) {
-                if ($e->getCode() === 11000) {
-                    return $this->findOrCreateNovel($intNSourceId,  $strName);
-                } else {
-                    throw $e;
-                }
-            }
-        }
-        return $arrVideo;
-    }
-
-
-    /**
-     * 分析关联数据
-     *
-     * @return void
-     */
-    public function getReletedNovel() {}
-
-
-    /**
-     * Undocumented function
-     *
-     * @return array
-     */
-    public function getVideoPager(): array
-    {
-        if (empty($this->arrVideo)) {
-
-            $this->arrVideo = json_decode($this->getContent(), true);
-
-            $this->getReletedNovel();
-        }
-
-        return  $this->arrVideo;
-    }
-
-    /**
-     * Undocumented function
-     *
-     * @param string $strName
-     * @return string
-     */
-    public function generateCoverPath($strName): string
-    {
-        $intTime = time() - mt_rand(1, 2 * 24 * 3600 * 360);
-
-        return sprintf('/img/%s/%s.jpg', date('Y/m/d/H', $intTime), $strName);
-    }
-
-    public function filterOtherFields(&$arrNovel)
-    {
-        $arrFields = [
-            'n_id',
-            'n_name',
-            'n_name_pinyin',
-            'n_status',
-            'n_author',
-            'n_description',
-            'n_category',
-            'n_category_pinyin',
-            'n_category_sex_en',
-            'n_category_sex_zh',
-            'n_cover_path',
-            'n_latest_chapter_name',
-            'n_update_time',
-            'n_forge_novel',
-            'n_related_novel',
-            'n_site_code',
-            'n_source_uuid',
-            'n_source_id',
-            'n_have_chapter',
-            'created_at',
-            'updated_at',
-        ];
-
-        foreach ($arrNovel as $strKey => $strVal) {
-            if (!in_array($strKey, $arrFields)) {
-                unset($arrNovel[$strKey]);
-            }
-        }
-    }
-
-    /**
-     * Undocumented function
-     *
-     * @return bool
-     */
-    public function saveVideo(): bool
-    {
-        $arrPageInfo = $this->getVideo();
-
-        print_r($arrPageInfo);
-        return false;
-        if (
-            empty($arrPageInfo) ||
-            empty($arrPageInfo['og_title']) ||
-            empty($arrPageInfo['og_image']) ||
-            empty($arrPageInfo['og_novel_category'])
-        ) {
-            return false;
-        }
-
-        $arrVideo = [
-            // 'n_id'=> $arrPageInfo['xxxxx'],
-            'n_name' => $arrPageInfo['og_title'],
-            'n_name_pinyin' => zhToPinYin($arrPageInfo['og_title']),
-            'n_status' => $arrPageInfo['og_novel_status'],
-            'n_author' => $arrPageInfo['og_novel_author'],
-            'n_description' => $arrPageInfo['og_description'],
-            'n_category' => $arrPageInfo['og_novel_category'],
-            // 'n_category_pinyin' => $arrPageInfo['xxxxx'], 
-            // 'n_category_sex_en' => $arrPageInfo['xxxxx'], 
-            // 'n_category_sex_zh' => $arrPageInfo['xxxxx'], 
-            // 'n_cover_path' => $arrPageInfo['xxxxx'], 
-            'n_latest_chapter_name' => $arrPageInfo['og_novel_latest_chapter_name'],
-            'n_update_time' => $arrPageInfo['og_novel_update_time'],
-            'n_forge_novel' => $this->arrForgeNovel,
-            'n_related_novel' => $this->arrRelatedNovel,
-
-            'n_site_code' =>  $this->getSite()->getSiteCode(),
-            'n_source_id' => $this->arrData['n_source_id'],
-            'n_source_uuid' => $this->getSite()->getSiteUUId($this->arrData['n_source_id']),
-
-            // 'n_have_chapter' => $arrPageInfo['xxxxx'],
-            // 'created_at' => $arrPageInfo['xxxxx'],
-            // 'updated_at' => $arrPageInfo['xxxxx'],
-
-        ];
-
-        $arrExistsNovel = $this->getNovelModel()->findOne(['n_source_uuid' => $arrVideo['n_source_uuid']], $this->arrOptions);
-
-        if ($arrExistsNovel) {
-
-            $arrExistsNovel = (array) $arrExistsNovel;
-
-            $arrUpdate = [
-                'n_latest_chapter_name' => $arrVideo['n_latest_chapter_name'],
-                'n_update_time' => $arrVideo['n_update_time'],
-                'updated_at' => new \MongoDB\BSON\UTCDateTime(),
-            ];
-
-            # 如果没有分类，说明这条数据是最初从关联shu那里采集的，需要补全全部信息
-            if (empty($arrExistsNovel['n_category'])) {
-                $arrUpdate = array_merge($arrUpdate, $arrVideo);
-                $arrUpdate['n_related_novel'] = $this->arrRelatedNovel;
-                $arrUpdate['n_forge_novel'] = $this->arrForgeNovel;
-                $arrUpdate['n_category_pinyin'] = CategoryModel::getPinYin($this->arrVideo['og_novel_category']);
-                $arrUpdate['n_category_sex_en'] = CategoryModel::getSexEn($this->arrVideo['og_novel_category']);
-                $arrUpdate['n_category_sex_zh'] = CategoryModel::getSexZh($this->arrVideo['og_novel_category']);
-                $arrUpdate['n_related_novel'] = $this->arrRelatedNovel;
-                $arrUpdate['n_forge_novel'] = $this->arrForgeNovel;
-            }
-
-            # 图片不存在的时候，需要重新生成
-            if (
-                !isset($arrExistsNovel['n_cover_path']) ||
-                empty($arrExistsNovel['n_cover_path']) ||
-                !StorageCore::getInstance()->has($arrExistsNovel['n_cover_path']) ||
-                (basename($arrExistsNovel['n_cover_path']) == '.jpg')
-            ) {
-                $arrUpdate['n_cover_path'] = $this->generateCoverPath($arrVideo['n_name_pinyin']);
-                $this->strCoverLocalPath = $arrUpdate['n_cover_path'];
-            } else {
-                $this->strCoverLocalPath = $arrExistsNovel['n_cover_path'];
-            }
-
-            $this->getNovelModel()->updateOne(
-                ['n_source_uuid' =>  $arrVideo['n_source_uuid']],
-                ['$set' => $arrUpdate],
-            );
-            $this->intNId =       $arrExistsNovel['n_id'];
-        } else {
-
-            $arrVideo['n_category_pinyin'] = CategoryModel::getPinYin($this->arrVideo['og_novel_category']);
-            $arrVideo['n_category_sex_en'] = CategoryModel::getSexEn($this->arrVideo['og_novel_category']);
-            $arrVideo['n_category_sex_zh'] = CategoryModel::getSexZh($this->arrVideo['og_novel_category']);
-            $arrVideo['n_id'] =  CountersModel::getInstance()->getNextSequence('novel');
-            $arrVideo['n_level'] = 1;
-            $arrVideo['n_have_chapter'] = 0;
-            $arrVideo['n_cover_path'] =  $this->generateCoverPath($arrVideo['n_name_pinyin']);
-            $arrVideo['n_related_novel'] = $this->arrRelatedNovel;
-            $arrVideo['n_forge_novel'] = $this->arrForgeNovel;
-            $arrVideo['created_at'] = new \MongoDB\BSON\UTCDateTime();
-
-            try {
-                $this->getNovelModel()->insert($arrVideo);
-                $this->intNId =       $arrVideo['n_id'];
-                $this->strCoverLocalPath = $arrVideo['n_cover_path'];
-            } catch (\Exception $e) {
-                if ($e->getCode() === 11000) {
-                    return $this->saveVideo();
-                } else {
-                    throw $e;
-                }
-            }
-        }
-
-        return true;
-    }
-
-
-    public $strCoverLocalPath = '';
-
-    public $intNId;
-
-    /**
-     * Undocumented function
-     *
-     * @return array
-     */
-    public function getDownCoverInfo(): array
-    {
-        return [
-            'uri' => $this->arrVideo['og_image'],
-            'local_path' => $this->strCoverLocalPath,
-        ];
-    }
-
-    /**
-     * Undocumented variable
-     *
-     * @var array
-     */
-    protected $arrLatestChapters = NULL;
-
-    /**
-     * Undocumented variable
-     *
-     * @var array
-     */
-    protected $arrChapters = NULL;
-
-    public function getLatestChapterByNIdsList(): array
-    {
-        if ($this->arrLatestChapters === NULL) {
-            $this->arrLatestChapters = [];
-
-            if ($this->isMobile()) {
-                $strClass = '.chapter-list';
-            } else {
-                $strClass = '.section-list';
-            }
-
-            if ($this->arrData['page'] == 0) {
-                $this->getQueryList()->find($strClass)->eq(0)->find("a")->map(function ($A) {
-                    $this->arrLatestChapters[] = [
-                        'name' => $A->text(),
-                        'uri' => $A->attr('href'),
-                    ];
-                });
-            }
-        }
-        return $this->arrLatestChapters;
-    }
-
-    /**
-     * Undocumented function
-     *
-     * @return array
-     */
-    public function getChapters(): array
-    {
-        if ($this->arrChapters === NULL) {
-            $this->arrChapters = [];
-            if ($this->isMobile()) {
-                $strClass = '.chapter-list';
-            } else {
-                $strClass = '.section-list';
-            }
-
-            $intDivIndex = 0;
-            if ($this->arrData['page'] == 0) {
-                $intDivIndex = 1;
-            }
-
-            $this->getQueryList()->find($strClass)->eq($intDivIndex)->find("a")->map(function ($A) {
-                $this->arrChapters[] = [
-                    'name' => $A->text(),
-                    'uri' => $A->attr('href'),
-                ];
-            });
-        }
-        return $this->arrChapters;
-    }
-
-
-    protected $arrNextPageArgs = NULL;
-
-    /**
-     * Undocumented function
-     *
-     * @return array
-     */
-    public function getNextPageArgs(): array
-    {
-        $this->arrNextPageArgs = [];
-        if ($this->arrData['page'] == 0) {
-            $this->getLatestChapterByNIdsList();
-            $this->getChapters();
-
-            $arrLatestChapterFirstBlock =  count($this->arrLatestChapters) > 0 ? $this->arrLatestChapters[0] : null;
-            $arrLatestChapterSecondBlock = count($this->arrChapters) > 0 ?  $this->arrChapters[count($this->arrChapters) - 1] : null;
-
-            if (
-                !empty($arrLatestChapterFirstBlock) && !empty($arrLatestChapterSecondBlock)
-                && ($arrLatestChapterFirstBlock['uri'] != $arrLatestChapterSecondBlock['uri'])
-
-            ) {
-                $this->arrNextPageArgs =  [
-                    'n_source_id' => $this->arrData['n_source_id'],
-                    'page' => $this->arrData['page'] + 1,
-                ];
-            }
-        } else {
-            $strLastUri = $this->getQueryList()->find('select:first option:last')->attr('value');
-            if ($strLastUri != '' && $strLastUri != $this->strUri) {
-                $this->arrNextPageArgs = [
-                    'n_source_id' => $this->arrData['n_source_id'],
-                    'page' => $this->arrData['page'] + 1,
-                ];
-            }
-        }
-        return   $this->arrNextPageArgs;
-    }
-}

code/app/task/crawler/youzhi/page/VideoPagerPage.php

@@ -0,0 +1,43 @@
+<?php
+
+declare(strict_types=1);
+
+namespace app\task\crawler\youzhi\page;
+
+use app\model\CategoryModel;
+use app\model\CountersModel;
+use app\model\NovelModel;
+use storage\StorageCore;
+
+class VideoPagerPage extends BasePage
+{
+    /**
+     * Undocumented variable
+     *
+     * @var array
+     */
+    protected $arrVideo = [];
+
+    public $arrOptions = [
+        'typeMap' => [
+            'root' => 'array',
+            'document' => 'array',
+            'array' => 'array',
+        ],
+    ];
+
+    /**
+     * Undocumented function
+     *
+     * @return array
+     */
+    public function getVideoPager(): array
+    {
+        if (empty($this->arrVideo)) {
+
+            $this->arrVideo = json_decode($this->getContent(), true);
+        }
+
+        return  $this->arrVideo;
+    }
+}

code/database/mongo/migrations/video_clicks.php

@@ -0,0 +1,10 @@
+<?php
+return [
+    'collection' => 'video_clicks',
+    'indexes' => [
+        [
+            'key' => ['v_id' => 1, 'vc_date' => 1, 'vc_type' => 1],
+            'options' => ['unique' => true]
+        ],
+    ]
+];

code/extend/storage/StorageCore.php

@@ -95,4 +95,9 @@ class StorageCore
     {
         return $this->Filesystem->has($strKey);
     }
+
+    public function getRealPath($strKey): string
+    {
+        return $this->Adapter->applyPathPrefix($strKey);
+    }
 }