debug

code/app/model/DomainModel.php

@@ -103,7 +103,7 @@ class DomainModel extends BaseModel
     {
         
         if ( !array_key_exists($strKey, $this->t_cfg)) {
-            return  '';
+            return '';
             $strError = sprintf("t_cfg [%s] not found! %s ", $strKey , $strDomain . '-' . $strController . '-' . $strAction. '-' . $strDiff. '-' . $boolJoin.'-'.request()->url);
             throw new \Exception($strError);
         }

code/app/model/VideoCategoryModel.php

@@ -28,7 +28,7 @@ class VideoCategoryModel extends MongoModel
 
     /**
      * 自定义 总分类
-     */
+    */
     static $arrCategoryTypeONE = [
         [
             "v_category" => "电影",

code/app/task/logic/SiteMapLogic.php

@@ -156,6 +156,7 @@ EOF;
                 $strContent = sprintf($strTemplate, $strUrl, $strPriority);
                 file_put_contents($strMapBooksFile, $strContent, FILE_APPEND);
 
+                
                 # 2
                 $strKey = "NOVEL_CATALOG_URL";
                 $strDomain = $DomainModel->d_domain;

code/app/task/logic/VideoSiteMapLogic.php

@@ -0,0 +1,407 @@
+<?php
+
+declare(strict_types=1);
+
+namespace app\task\logic;
+
+use app\model\DomainModel;
+use app\model\VideoModel;
+use app\model\VideoCategoryModel;
+use app\services\StaticConfig;
+use microserver\ProcessSanitizer;
+use microserver\QueueManage;
+
+class VideoSiteMapLogic
+{
+    /**
+     * VideoModel
+     *
+     * @var VideoModel
+     */
+    public $VideoModel;
+
+    public $intPageSize = 40000;
+
+    public $strSiteMapPath = '';
+
+    public $intTotal = 0;
+
+    public $strDate;
+
+    public $arrVideoCategory;
+
+    public $arrVideoSortType;
+
+    public $arrVideoRankSortType;
+
+    public $arrVideoLang;
+
+    public $arrVideoYear;
+
+    public $arrVideoArea;
+
+    public $intBatchSize = 500;
+
+    /**
+     * Undocumented variable
+     *
+     * @var  DomainModel[]
+     */
+    public $arrDomainModel;
+
+    public function __construct()
+    {
+
+        $this->arrVideoCategory = VideoCategoryModel::getCustomCategory("ONE");
+
+        $this->arrVideoSortType = StaticConfig::$arrVideoSortType;
+
+        $this->arrVideoRankSortType = StaticConfig::$arrVideoRankSortType;
+
+        $this->arrVideoLang = StaticConfig::$arrVideoLang;
+
+        $this->arrVideoArea = StaticConfig::$arrVideoArea;
+
+        $this->arrVideoYear = StaticConfig::$arrVideoYear;
+
+        $this->strDate = date('Y-m-d');
+        $this->strSiteMapPath = root_path('storage/SiteMap');
+
+        $this->VideoModel = VideoModel::getInstance();
+
+        VideoModel::$arrOptions['projection'] = [
+            '_id' => 0,
+            'v_id' => 1,
+            'v_name' => 1,
+            'v_name_en' => 1,
+            'v_category' => 1,
+            'v_category_en' => 1,
+        ];
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @return  DomainModel[]
+     */
+    public function getDomain(): array
+    {
+        if (empty($this->arrDomain)) {
+            $this->arrDomainModel = DomainModel::getAllDomainOnCache();
+        }
+        return  $this->arrDomainModel;
+    }
+
+    /**
+     * Generate Site Map
+     *
+     * @return void
+     */
+    public function generateSiteMap()
+    {
+        $arrResult =  $this->VideoModel->findPaginatedWithCache([], 1, 1);
+        $this->intTotal = $arrResult['total'];
+
+        $this->generateMapIndex();
+
+        $this->generateMapMain();
+
+        $intTotalPage = ceil($this->intTotal / $this->intPageSize);
+        for ($intPage = 1; $intPage <= $intTotalPage; $intPage++) {
+            $this->generateMapInfo($intPage);
+        }
+
+        echo date('Y-m-d H:i:s') . PHP_EOL;
+    }
+
+
+    protected function generateMapInfo(int $intPage)
+    {
+
+        foreach ($this->getDomain() as $DomainModel) {
+
+            $strDomainDir = $this->strSiteMapPath . $DomainModel->d_domain;
+            if (is_dir($strDomainDir) == false) {
+                mkdir($strDomainDir);
+            }
+
+            $strHead = <<<EOF
+<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+EOF;
+            $strTemplate = <<<EOF
+<url>
+<loc>%s</loc>
+<lastmod>{$this->strDate}</lastmod>
+<changefreq>weekly</changefreq>
+<priority>%s</priority>
+</url>
+EOF;
+            $strEnd = <<<EOF
+</urlset>
+EOF;
+
+            $strMapVideosFile = sprintf('%s/sitemap-videos-%s.xml', $strDomainDir, $intPage);
+
+            file_put_contents($strMapVideosFile, $strHead);
+
+            # 取 intpage * limit  ~   intpage * (limit+1)  写入以上文件
+            foreach ($this->fetchVideosByBatch($intPage * $this->intPageSize, $this->intPageSize) as $Video) {
+
+                # 1 '/voddetail/{strPinYin}-{intVId}',
+                $strKey = "VIDEO_INFO_URL";
+                $strDomain = $DomainModel->d_domain;
+                $strController = "";
+                $strAction = "";
+                $arrArgs = [
+                    'intVId' => $Video->v_id,
+                    'strPinYin' =>  $Video->v_name_en,
+                ];
+
+                $strUri = $DomainModel->getFomartUrlEx($strKey, $strDomain, $strController, $strAction, $arrArgs);
+                $strUrl = sprintf("https://www.%s%s", $DomainModel->d_domain, $strUri);
+                $strPriority = '0.8';
+                $strContent = sprintf($strTemplate, $strUrl, $strPriority);
+                file_put_contents($strMapVideosFile, $strContent, FILE_APPEND);
+            }
+
+            file_put_contents($strMapVideosFile, $strEnd, FILE_APPEND);
+        }
+    }
+
+
+    /**
+     * 分批查询小说数据，并以生成器方式返回
+     *
+     * @param int $intStart  跳过的记录数（skip）
+     * @param int $intCount  需要查询的总记录数
+     * @return \Generator  逐条返回小说数据
+     * @throws \MongoDB\Exception\Exception
+     */
+    public function fetchVideosByBatch(int $intStart, int $intCount): \Generator
+    {
+        $intRecordsFetched = 0;
+        $intRemainingRecords = min($intCount, $this->VideoModel->getCol()->countDocuments() - $intStart);
+
+        while ($intRecordsFetched < $intRemainingRecords) {
+            $intLimit = min($this->intBatchSize, $intRemainingRecords - $intRecordsFetched);
+            $Cursor = $this->VideoModel->getCol()->find(
+                [
+                   
+                ],
+                [
+                    'skip' => $intStart + $intRecordsFetched,
+                    'limit' => $intLimit,
+                    'sort' => ['_id' => 1],
+                    'projection' => [
+                        '_id' => 1,
+                        'v_id' => 1,
+                        'v_name' => 1,
+                        'v_name_en' => 1,
+                        'v_category' => 1,
+                        'v_category_en' => 1,
+                    ],
+                ]
+            );
+
+            foreach ($Cursor as $Video) {
+                yield $Video;
+            }
+
+            $intRecordsFetched += $intLimit;
+            unset($Cursor);
+            gc_collect_cycles();
+        }
+    }
+
+
+    /**
+     * Generate Map Index
+     *
+     * @return void
+     */
+    protected function generateMapMain()
+    {
+
+        foreach ($this->getDomain() as $DomainModel) {
+
+            $strDomainDir = $this->strSiteMapPath . $DomainModel->d_domain;
+            if (is_dir($strDomainDir) == false) {
+                mkdir($strDomainDir);
+            }
+
+            $strMapIndexFile = $strDomainDir . '/sitemap-main.xml';
+
+            $strRankIndexUri = $DomainModel->getFomartUrlEx("VIDEO_RANK_INDEX_URL", $DomainModel->d_domain, "", "", []);
+            $strRankIndexUrl = sprintf("https://www.%s%s", $DomainModel->d_domain, $strRankIndexUri);
+
+            $strContent = <<<EOF
+<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+  <url>
+    <loc>https://www.{$DomainModel->d_domain}/</loc>
+    <lastmod>{$this->strDate}</lastmod>
+    <changefreq>daily</changefreq>
+    <priority>1.0</priority>
+  </url>
+  <url>
+    <loc>{$strRankIndexUrl}/</loc>
+    <lastmod>{$this->strDate}</lastmod>
+    <changefreq>daily</changefreq>
+    <priority>0.8</priority>
+  </url>
+EOF;
+            file_put_contents($strMapIndexFile, $strContent);
+
+            // 分类首页 '/vodtype/{strParentCategory}',
+            foreach ($this->arrVideoCategory as $strParentCategoryKey => $arrParentCategory) {
+
+                $strKey = "VIDEO_CATEGORY_INDEX_URL";
+                $strDomain = $DomainModel->d_domain;
+                $strController = "";
+                $strAction = "";
+                $arrArgs = [
+                    'strParentCategory' => $arrParentCategory['v_category_en'],
+                ];
+
+                $strUri = $DomainModel->getFomartUrlEx($strKey, $strDomain, $strController, $strAction, $arrArgs);
+
+                $strUrl = sprintf("https://www.%s%s", $DomainModel->d_domain, $strUri);
+
+                $strContent = <<<EOF
+<url>
+<loc>{$strUrl}</loc>
+<lastmod>{$this->strDate}</lastmod>
+<changefreq>weekly</changefreq>
+<priority>0.8</priority>
+</url>
+EOF;
+                file_put_contents($strMapIndexFile, $strContent, FILE_APPEND);
+            }
+
+
+            // 分类列表 '/vodtype-{strParentCategory}/{strCategory}-{strYear}-{strArea}-{strOrder}/{strLang}-page{intPage}',
+            foreach ($this->arrVideoCategory as $arrParentCategory) {
+                foreach ($arrParentCategory['children'] as $arrChildrenCategory) {
+                    foreach ($this->arrVideoSortType as $strSortKey => $strSortVal) {
+                        // foreach ($this->arrVideoLang as $strVideoLangKey => $strVideoLangVal) {
+                        //     foreach ($this->arrVideoArea as $strVideoAreaKey => $strVideoAreaVal) {
+                        //         foreach ($this->arrVideoYear as $strVideoYearKey => $strVideoYearVal) {
+
+                                    $strKey = "VIDEO_CATEGORY_URL";
+                                    $strDomain = $DomainModel->d_domain;
+                                    $strController = "";
+                                    $strAction = "";
+                                    $arrArgs = [
+                                        'strParentCategory' => $arrParentCategory['v_category_en'],
+                                        'strCategory' => $arrChildrenCategory['v_category_en'],
+                                        'strYear' => 'all',
+                                        'strArea' =>  'all',
+                                        'strOrder' =>  $strSortKey,
+                                        'strLang' =>  'all',
+                                        'intPage' => 1,
+                                    ];
+
+                                    $strUri = $DomainModel->getFomartUrlEx($strKey, $strDomain, $strController, $strAction, $arrArgs);
+                                    $strUrl = sprintf("https://www.%s%s", $DomainModel->d_domain, $strUri);
+                                    $strContent = <<<EOF
+<url>
+<loc>{$strUrl}</loc>
+<lastmod>{$this->strDate}</lastmod>
+<changefreq>weekly</changefreq>
+<priority>0.8</priority>
+</url>
+EOF;
+                                    file_put_contents($strMapIndexFile, $strContent, FILE_APPEND);
+                        //         }
+                        //     }
+                        // }
+                    }
+                }
+            }
+
+            // 排行榜列表 '/paihang/{strParentCategory}/{strCategory}/{strSortType}',
+            foreach ($this->arrVideoCategory as $arrParentCategory) {
+                foreach ($arrParentCategory['children'] as $arrChildrenCategory) {
+                    foreach ($this->arrVideoRankSortType as $strSortKey => $strSortVal) {
+
+                        $strKey = "VIDEO_RANK_LIST_URL";
+                        $strDomain = $DomainModel->d_domain;
+                        $strController = "";
+                        $strAction = "";
+                        $arrArgs = [
+                            'strParentCategory' => $arrParentCategory['v_category_en'],
+                            'strCategory' => $arrChildrenCategory['v_category_en'],
+                            'strSortType' =>  $strSortKey,
+                        ];
+
+                        $strUri = $DomainModel->getFomartUrlEx($strKey, $strDomain, $strController, $strAction, $arrArgs);
+
+                        $strUrl = sprintf("https://www.%s%s", $DomainModel->d_domain, $strUri);
+
+                        $strContent = <<<EOF
+<url>
+<loc>{$strUrl}</loc>
+<lastmod>{$this->strDate}</lastmod>
+<changefreq>weekly</changefreq>
+<priority>0.8</priority>
+</url>
+EOF;
+                        file_put_contents($strMapIndexFile, $strContent, FILE_APPEND);
+                    }
+                }
+            }
+
+
+
+            $strContent = <<<EOF
+</urlset>
+EOF;
+            file_put_contents($strMapIndexFile, $strContent, FILE_APPEND);
+        }
+    }
+
+    /**
+     * Generate Map Index
+     *
+     * @return void
+     */
+    protected function generateMapIndex()
+    {
+        $intTotalPage = ceil($this->intTotal / $this->intPageSize);
+
+        foreach ($this->getDomain() as $DomainModel) {
+
+            $strDomainDir = $this->strSiteMapPath . $DomainModel->d_domain;
+            if (is_dir($strDomainDir) == false) {
+                mkdir($strDomainDir);
+            }
+
+            $strMapIndexFile =   $strDomainDir . '/sitemap_index.xml';
+
+            $strContent = <<<EOF
+<?xml version="1.0" encoding="UTF-8"?>
+<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+<sitemap>
+<loc>https://www.{$DomainModel->d_domain}/sitemap-main.xml</loc>
+<lastmod>{$this->strDate}</lastmod>
+</sitemap>
+EOF;
+            file_put_contents($strMapIndexFile, $strContent);
+
+            for ($intPage = 1; $intPage <= $intTotalPage; $intPage++) {
+                $strContent = <<<EOF
+<sitemap>
+<loc>https://www.{$DomainModel->d_domain}/sitemap-videos-{$intPage}.xml</loc>
+<lastmod>{$this->strDate}</lastmod>
+</sitemap>
+EOF;
+                file_put_contents($strMapIndexFile, $strContent, FILE_APPEND);
+            }
+            $strContent = <<<EOF
+</sitemapindex>
+EOF;
+            file_put_contents($strMapIndexFile, $strContent, FILE_APPEND);
+        }
+    }
+}

code/app/task/module/PlanTask.php

@@ -6,6 +6,7 @@ namespace app\task\module;
 
 use app\model\PlanTaskModel;
 use app\task\logic\SiteMapLogic;
+use app\task\logic\VideoSiteMapLogic;
 use microserver\QueueManage;
 use microserver\ProcessSanitizer;
 
@@ -44,6 +45,9 @@ class PlanTask
                 case 'GENERATE_SITE_MAP':
                     self::generateSiteMap();
                     break;
+                case 'GENERATE_VIDEO_SITE_MAP':
+                    self::generateVideoSiteMap();
+                    break;
                 case 'PULL_630_ZW_NOVEL':
                     self::pull630ZwNovel();
                     break;
@@ -216,4 +220,20 @@ class PlanTask
         $QueueManage = new QueueManage(2);
         $QueueManage->set($arrTask);
     }
+
+    /**
+     * Undocumented function
+     *
+     * @return void
+     */
+    public static function generateVideoSiteMap()
+    {
+        $arrTask = [
+            'callback' => [VideoSiteMapLogic::class, 'generateSiteMap'],
+            'data' => []
+        ];
+
+        $QueueManage = new QueueManage(2);
+        $QueueManage->set($arrTask);
+    }
 }

code/database/seeders/PlanTaskSeeder.php

@@ -73,6 +73,12 @@ class PlanTaskSeeder
                 'pt_enable' => 0,
                 'pt_limit' => 1 * 24 * 3600,
             ],
+            [
+                'pt_name' => '生成视频网站地图',
+                'pt_code' => 'GENERATE_VIDEO_SITE_MAP',
+                'pt_enable' => 0,
+                'pt_limit' => 1 * 24 * 3600,
+            ],
             [
                 'pt_name' => '自定义爬虫任务触发',
                 'pt_code' => 'CUSTOM_CRAWLER',