diff --git a/code/app/model/DomainModel.php b/code/app/model/DomainModel.php index 8bc5d9d..6740657 100644 --- a/code/app/model/DomainModel.php +++ b/code/app/model/DomainModel.php @@ -103,7 +103,7 @@ class DomainModel extends BaseModel { if ( !array_key_exists($strKey, $this->t_cfg)) { - return ''; + return ''; $strError = sprintf("t_cfg [%s] not found! %s ", $strKey , $strDomain . '-' . $strController . '-' . $strAction. '-' . $strDiff. '-' . $boolJoin.'-'.request()->url); throw new \Exception($strError); } diff --git a/code/app/model/VideoCategoryModel.php b/code/app/model/VideoCategoryModel.php index 9f3a47f..bfaf0d1 100644 --- a/code/app/model/VideoCategoryModel.php +++ b/code/app/model/VideoCategoryModel.php @@ -28,7 +28,7 @@ class VideoCategoryModel extends MongoModel /** * 自定义 总分类 - */ + */ static $arrCategoryTypeONE = [ [ "v_category" => "电影", diff --git a/code/app/task/logic/SiteMapLogic.php b/code/app/task/logic/SiteMapLogic.php index e02d291..b811353 100644 --- a/code/app/task/logic/SiteMapLogic.php +++ b/code/app/task/logic/SiteMapLogic.php @@ -156,6 +156,7 @@ EOF; $strContent = sprintf($strTemplate, $strUrl, $strPriority); file_put_contents($strMapBooksFile, $strContent, FILE_APPEND); + # 2 $strKey = "NOVEL_CATALOG_URL"; $strDomain = $DomainModel->d_domain; diff --git a/code/app/task/logic/VideoSiteMapLogic.php b/code/app/task/logic/VideoSiteMapLogic.php new file mode 100644 index 0000000..863194a --- /dev/null +++ b/code/app/task/logic/VideoSiteMapLogic.php @@ -0,0 +1,407 @@ +arrVideoCategory = VideoCategoryModel::getCustomCategory("ONE"); + + $this->arrVideoSortType = StaticConfig::$arrVideoSortType; + + $this->arrVideoRankSortType = StaticConfig::$arrVideoRankSortType; + + $this->arrVideoLang = StaticConfig::$arrVideoLang; + + $this->arrVideoArea = StaticConfig::$arrVideoArea; + + $this->arrVideoYear = StaticConfig::$arrVideoYear; + + $this->strDate = date('Y-m-d'); + $this->strSiteMapPath = root_path('storage/SiteMap'); + + $this->VideoModel = VideoModel::getInstance(); + + VideoModel::$arrOptions['projection'] = [ + '_id' => 0, + 'v_id' => 1, + 'v_name' => 1, + 'v_name_en' => 1, + 'v_category' => 1, + 'v_category_en' => 1, + ]; + } + + /** + * Undocumented function + * + * @return DomainModel[] + */ + public function getDomain(): array + { + if (empty($this->arrDomain)) { + $this->arrDomainModel = DomainModel::getAllDomainOnCache(); + } + return $this->arrDomainModel; + } + + /** + * Generate Site Map + * + * @return void + */ + public function generateSiteMap() + { + $arrResult = $this->VideoModel->findPaginatedWithCache([], 1, 1); + $this->intTotal = $arrResult['total']; + + $this->generateMapIndex(); + + $this->generateMapMain(); + + $intTotalPage = ceil($this->intTotal / $this->intPageSize); + for ($intPage = 1; $intPage <= $intTotalPage; $intPage++) { + $this->generateMapInfo($intPage); + } + + echo date('Y-m-d H:i:s') . PHP_EOL; + } + + + protected function generateMapInfo(int $intPage) + { + + foreach ($this->getDomain() as $DomainModel) { + + $strDomainDir = $this->strSiteMapPath . $DomainModel->d_domain; + if (is_dir($strDomainDir) == false) { + mkdir($strDomainDir); + } + + $strHead = << + +EOF; + $strTemplate = << +%s +{$this->strDate} +weekly +%s + +EOF; + $strEnd = << +EOF; + + $strMapVideosFile = sprintf('%s/sitemap-videos-%s.xml', $strDomainDir, $intPage); + + file_put_contents($strMapVideosFile, $strHead); + + # 取 intpage * limit ~ intpage * (limit+1) 写入以上文件 + foreach ($this->fetchVideosByBatch($intPage * $this->intPageSize, $this->intPageSize) as $Video) { + + # 1 '/voddetail/{strPinYin}-{intVId}', + $strKey = "VIDEO_INFO_URL"; + $strDomain = $DomainModel->d_domain; + $strController = ""; + $strAction = ""; + $arrArgs = [ + 'intVId' => $Video->v_id, + 'strPinYin' => $Video->v_name_en, + ]; + + $strUri = $DomainModel->getFomartUrlEx($strKey, $strDomain, $strController, $strAction, $arrArgs); + $strUrl = sprintf("https://www.%s%s", $DomainModel->d_domain, $strUri); + $strPriority = '0.8'; + $strContent = sprintf($strTemplate, $strUrl, $strPriority); + file_put_contents($strMapVideosFile, $strContent, FILE_APPEND); + } + + file_put_contents($strMapVideosFile, $strEnd, FILE_APPEND); + } + } + + + /** + * 分批查询小说数据,并以生成器方式返回 + * + * @param int $intStart 跳过的记录数(skip) + * @param int $intCount 需要查询的总记录数 + * @return \Generator 逐条返回小说数据 + * @throws \MongoDB\Exception\Exception + */ + public function fetchVideosByBatch(int $intStart, int $intCount): \Generator + { + $intRecordsFetched = 0; + $intRemainingRecords = min($intCount, $this->VideoModel->getCol()->countDocuments() - $intStart); + + while ($intRecordsFetched < $intRemainingRecords) { + $intLimit = min($this->intBatchSize, $intRemainingRecords - $intRecordsFetched); + $Cursor = $this->VideoModel->getCol()->find( + [ + + ], + [ + 'skip' => $intStart + $intRecordsFetched, + 'limit' => $intLimit, + 'sort' => ['_id' => 1], + 'projection' => [ + '_id' => 1, + 'v_id' => 1, + 'v_name' => 1, + 'v_name_en' => 1, + 'v_category' => 1, + 'v_category_en' => 1, + ], + ] + ); + + foreach ($Cursor as $Video) { + yield $Video; + } + + $intRecordsFetched += $intLimit; + unset($Cursor); + gc_collect_cycles(); + } + } + + + /** + * Generate Map Index + * + * @return void + */ + protected function generateMapMain() + { + + foreach ($this->getDomain() as $DomainModel) { + + $strDomainDir = $this->strSiteMapPath . $DomainModel->d_domain; + if (is_dir($strDomainDir) == false) { + mkdir($strDomainDir); + } + + $strMapIndexFile = $strDomainDir . '/sitemap-main.xml'; + + $strRankIndexUri = $DomainModel->getFomartUrlEx("VIDEO_RANK_INDEX_URL", $DomainModel->d_domain, "", "", []); + $strRankIndexUrl = sprintf("https://www.%s%s", $DomainModel->d_domain, $strRankIndexUri); + + $strContent = << + + + https://www.{$DomainModel->d_domain}/ + {$this->strDate} + daily + 1.0 + + + {$strRankIndexUrl}/ + {$this->strDate} + daily + 0.8 + +EOF; + file_put_contents($strMapIndexFile, $strContent); + + // 分类首页 '/vodtype/{strParentCategory}', + foreach ($this->arrVideoCategory as $strParentCategoryKey => $arrParentCategory) { + + $strKey = "VIDEO_CATEGORY_INDEX_URL"; + $strDomain = $DomainModel->d_domain; + $strController = ""; + $strAction = ""; + $arrArgs = [ + 'strParentCategory' => $arrParentCategory['v_category_en'], + ]; + + $strUri = $DomainModel->getFomartUrlEx($strKey, $strDomain, $strController, $strAction, $arrArgs); + + $strUrl = sprintf("https://www.%s%s", $DomainModel->d_domain, $strUri); + + $strContent = << +{$strUrl} +{$this->strDate} +weekly +0.8 + +EOF; + file_put_contents($strMapIndexFile, $strContent, FILE_APPEND); + } + + + // 分类列表 '/vodtype-{strParentCategory}/{strCategory}-{strYear}-{strArea}-{strOrder}/{strLang}-page{intPage}', + foreach ($this->arrVideoCategory as $arrParentCategory) { + foreach ($arrParentCategory['children'] as $arrChildrenCategory) { + foreach ($this->arrVideoSortType as $strSortKey => $strSortVal) { + // foreach ($this->arrVideoLang as $strVideoLangKey => $strVideoLangVal) { + // foreach ($this->arrVideoArea as $strVideoAreaKey => $strVideoAreaVal) { + // foreach ($this->arrVideoYear as $strVideoYearKey => $strVideoYearVal) { + + $strKey = "VIDEO_CATEGORY_URL"; + $strDomain = $DomainModel->d_domain; + $strController = ""; + $strAction = ""; + $arrArgs = [ + 'strParentCategory' => $arrParentCategory['v_category_en'], + 'strCategory' => $arrChildrenCategory['v_category_en'], + 'strYear' => 'all', + 'strArea' => 'all', + 'strOrder' => $strSortKey, + 'strLang' => 'all', + 'intPage' => 1, + ]; + + $strUri = $DomainModel->getFomartUrlEx($strKey, $strDomain, $strController, $strAction, $arrArgs); + $strUrl = sprintf("https://www.%s%s", $DomainModel->d_domain, $strUri); + $strContent = << +{$strUrl} +{$this->strDate} +weekly +0.8 + +EOF; + file_put_contents($strMapIndexFile, $strContent, FILE_APPEND); + // } + // } + // } + } + } + } + + // 排行榜列表 '/paihang/{strParentCategory}/{strCategory}/{strSortType}', + foreach ($this->arrVideoCategory as $arrParentCategory) { + foreach ($arrParentCategory['children'] as $arrChildrenCategory) { + foreach ($this->arrVideoRankSortType as $strSortKey => $strSortVal) { + + $strKey = "VIDEO_RANK_LIST_URL"; + $strDomain = $DomainModel->d_domain; + $strController = ""; + $strAction = ""; + $arrArgs = [ + 'strParentCategory' => $arrParentCategory['v_category_en'], + 'strCategory' => $arrChildrenCategory['v_category_en'], + 'strSortType' => $strSortKey, + ]; + + $strUri = $DomainModel->getFomartUrlEx($strKey, $strDomain, $strController, $strAction, $arrArgs); + + $strUrl = sprintf("https://www.%s%s", $DomainModel->d_domain, $strUri); + + $strContent = << +{$strUrl} +{$this->strDate} +weekly +0.8 + +EOF; + file_put_contents($strMapIndexFile, $strContent, FILE_APPEND); + } + } + } + + + + $strContent = << +EOF; + file_put_contents($strMapIndexFile, $strContent, FILE_APPEND); + } + } + + /** + * Generate Map Index + * + * @return void + */ + protected function generateMapIndex() + { + $intTotalPage = ceil($this->intTotal / $this->intPageSize); + + foreach ($this->getDomain() as $DomainModel) { + + $strDomainDir = $this->strSiteMapPath . $DomainModel->d_domain; + if (is_dir($strDomainDir) == false) { + mkdir($strDomainDir); + } + + $strMapIndexFile = $strDomainDir . '/sitemap_index.xml'; + + $strContent = << + + +https://www.{$DomainModel->d_domain}/sitemap-main.xml +{$this->strDate} + +EOF; + file_put_contents($strMapIndexFile, $strContent); + + for ($intPage = 1; $intPage <= $intTotalPage; $intPage++) { + $strContent = << +https://www.{$DomainModel->d_domain}/sitemap-videos-{$intPage}.xml +{$this->strDate} + +EOF; + file_put_contents($strMapIndexFile, $strContent, FILE_APPEND); + } + $strContent = << +EOF; + file_put_contents($strMapIndexFile, $strContent, FILE_APPEND); + } + } +} diff --git a/code/app/task/module/PlanTask.php b/code/app/task/module/PlanTask.php index 88d4f04..9e85aed 100644 --- a/code/app/task/module/PlanTask.php +++ b/code/app/task/module/PlanTask.php @@ -6,6 +6,7 @@ namespace app\task\module; use app\model\PlanTaskModel; use app\task\logic\SiteMapLogic; +use app\task\logic\VideoSiteMapLogic; use microserver\QueueManage; use microserver\ProcessSanitizer; @@ -44,6 +45,9 @@ class PlanTask case 'GENERATE_SITE_MAP': self::generateSiteMap(); break; + case 'GENERATE_VIDEO_SITE_MAP': + self::generateVideoSiteMap(); + break; case 'PULL_630_ZW_NOVEL': self::pull630ZwNovel(); break; @@ -216,4 +220,20 @@ class PlanTask $QueueManage = new QueueManage(2); $QueueManage->set($arrTask); } + + /** + * Undocumented function + * + * @return void + */ + public static function generateVideoSiteMap() + { + $arrTask = [ + 'callback' => [VideoSiteMapLogic::class, 'generateSiteMap'], + 'data' => [] + ]; + + $QueueManage = new QueueManage(2); + $QueueManage->set($arrTask); + } } diff --git a/code/database/seeders/PlanTaskSeeder.php b/code/database/seeders/PlanTaskSeeder.php index 8edeceb..9e6b4ec 100644 --- a/code/database/seeders/PlanTaskSeeder.php +++ b/code/database/seeders/PlanTaskSeeder.php @@ -73,6 +73,12 @@ class PlanTaskSeeder 'pt_enable' => 0, 'pt_limit' => 1 * 24 * 3600, ], + [ + 'pt_name' => '生成视频网站地图', + 'pt_code' => 'GENERATE_VIDEO_SITE_MAP', + 'pt_enable' => 0, + 'pt_limit' => 1 * 24 * 3600, + ], [ 'pt_name' => '自定义爬虫任务触发', 'pt_code' => 'CUSTOM_CRAWLER',