admin/SEONexus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

FIX SITE MAP

Browse Source
This commit is contained in:
Default
2025-04-24 01:18:17 +08:00
parent 7494188846
commit 8754bbc58e
11 changed files with 571 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

369
code/app/task/logic/SiteMapLogic.php Normal file
View File

@@ -0,0 +1,369 @@
<?php
declare(strict_types=1);
namespace app\task\logic;
use app\model\DomainModel;
use app\model\NovelModel;
use app\services\StaticConfig;
use microserver\ProcessSanitizer;
use microserver\QueueManage;
class SiteMapLogic
{
/**
* NovelModel
*
* @var NovelModel
*/
public $NovelModel;
public $intPageSize = 3;
public $strSiteMapPath = '';
public $intTotal = 0;
public $strDate;
public $arrNovelCategory;
public $arrNovelSort;
public $arrNovelStatus;
public $intBatchSize = 500;
/**
* Undocumented variable
*
* @var DomainModel[]
*/
public $arrDomainModel;
public function __construct()
{
$this->arrNovelCategory = StaticConfig::getCategoryFilter();
$this->arrNovelSort = StaticConfig::$arrNovelSortType;
$this->arrNovelStatus = StaticConfig::$arrNovelStatus;
$this->strDate = date('Y-m-d');
$this->strSiteMapPath = root_path('storage/SiteMap');
$this->intPageSize = 3;
$this->NovelModel = NovelModel::getInstance();
NovelModel::$arrOptions['projection'] = [
'_id' => 0,
'n_id' => 1,
'n_name' => 1,
'n_name_pinyin' => 1,
'n_category' => 1,
'n_category_pinyin' => 1,
'n_category_sex_en' => 1,
];
}
/**
* Undocumented function
*
* @return DomainModel[]
*/
public function getDomain(): array
{
if (empty($this->arrDomain)) {
$this->arrDomainModel = DomainModel::getAllDomainOnCache();
}
return $this->arrDomainModel;
}
/**
* Generate Site Map
*
* @return void
*/
public function generateSiteMap()
{
$arrResult = $this->NovelModel->findPaginatedWithCache([], 1, 1);
$this->intTotal = $arrResult['total'];
$this->generateMapIndex();
$this->generateMapMain();
$intTotalPage = ceil($this->intTotal / $this->intPageSize);
for ($intPage = 1; $intPage <= $intTotalPage; $intPage++) {
$this->generateMapInfo($intPage);
}
echo date('Y-m-d H:i:s') . PHP_EOL;
}
protected function generateMapInfo(int $intPage)
{
foreach ($this->getDomain() as $DomainModel) {
$strDomainDir = $this->strSiteMapPath . $DomainModel->d_domain;
if (is_dir($strDomainDir) == false) {
mkdir($strDomainDir);
}
$strHead = <<<EOF
<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
EOF;
$strTemplate = <<<EOF
<url>
<loc>%s</loc>
<lastmod>{$this->strDate}</lastmod>
<changefreq>weekly</changefreq>
<priority>%s</priority>
</url>
EOF;
$strEnd = <<<EOF
</urlset>
EOF;
$strMapBooksFile = sprintf('%s/sitemap-books-catalog-%s.xml', $strDomainDir, $intPage);
$strMapCatalogFile = sprintf('%s/sitemap-books-%s.xml', $strDomainDir, $intPage);
$strMapChaptersFile = sprintf('%s/sitemap-chapters-%s.xml', $strDomainDir, $intPage);
file_put_contents($strMapBooksFile, $strHead);
file_put_contents($strMapCatalogFile, $strHead);
file_put_contents($strMapChaptersFile, $strHead);
# 取 intpage * limit ~ intpage * (limit+1) 写入以上文件
foreach ($this->fetchNovelsByBatch($intPage * $this->intPageSize, $this->intPageSize) as $Novel) {
# 1
$strKey = "NOVEL_INFO_URL";
$strDomain = $DomainModel->d_domain;
$strController = "";
$strAction = "";
$arrArgs = [
'intNId' => $Novel->n_id,
'strPinYin' => $Novel->n_name_pinyin,
];
$strUri = $DomainModel->getFomartUrlEx($strKey, $strDomain, $strController, $strAction, $arrArgs);
$strUrl = sprintf("https://www.%s%s", $DomainModel->d_domain, $strUri);
$strPriority = '0.8';
$strContent = sprintf($strTemplate, $strUrl, $strPriority);
file_put_contents($strMapBooksFile, $strContent, FILE_APPEND);
# 2
$strKey = "NOVEL_CATALOG_URL";
$strDomain = $DomainModel->d_domain;
$strController = "";
$strAction = "";
$arrArgs = [
'intNId' => $Novel->n_id,
'strPinYin' => $Novel->n_name_pinyin,
'strOrder' => 'zheng',
'intPage' => 1,
];
$strUri = $DomainModel->getFomartUrlEx($strKey, $strDomain, $strController, $strAction, $arrArgs);
$strUrl = sprintf("https://www.%s%s", $DomainModel->d_domain, $strUri);
$strPriority = '0.8';
$strContent = sprintf($strTemplate, $strUrl, $strPriority);
file_put_contents($strMapCatalogFile, $strContent, FILE_APPEND);
# 3
$strKey = "NEWS_CONTENT_INFO_URL";
$strDomain = $DomainModel->d_domain;
$strController = "";
$strAction = "";
$arrArgs = [
'intNId' => $Novel->n_id,
'strPinYin' => $Novel->n_name_pinyin,
];
$strUri = $DomainModel->getFomartUrlEx($strKey, $strDomain, $strController, $strAction, $arrArgs);
$strUrl = sprintf("https://www.%s%s", $DomainModel->d_domain, $strUri);
$strPriority = '0.6';
$strContent = sprintf($strTemplate, $strUrl, $strPriority);
file_put_contents($strMapChaptersFile, $strContent, FILE_APPEND);
}
file_put_contents($strMapBooksFile, $strEnd, FILE_APPEND);
file_put_contents($strMapCatalogFile, $strEnd, FILE_APPEND);
file_put_contents($strMapChaptersFile, $strEnd, FILE_APPEND);
}
}
/**
* 分批查询小说数据，并以生成器方式返回
*
* @param int $intStart 跳过的记录数（skip）
* @param int $intCount 需要查询的总记录数
* @return \Generator 逐条返回小说数据
* @throws \MongoDB\Exception\Exception
*/
public function fetchNovelsByBatch(int $intStart, int $intCount): \Generator
{
$intRecordsFetched = 0;
$intRemainingRecords = min($intCount, $this->NovelModel->getCol()->countDocuments() - $intStart);
while ($intRecordsFetched < $intRemainingRecords) {
$intLimit = min($this->intBatchSize, $intRemainingRecords - $intRecordsFetched);
$Cursor = $this->NovelModel->getCol()->find(
[
'n_have_chapter' => 1,
],
[
'skip' => $intStart + $intRecordsFetched,
'limit' => $intLimit,
'sort' => ['_id' => 1],
'projection' => [
'_id' => 1,
'n_id' => 1,
'n_name' => 1,
'n_name_pinyin' => 1,
'n_category' => 1,
'n_category_pinyin' => 1,
'n_category_sex_en' => 1,
],
]
);
foreach ($Cursor as $novel) {
yield $novel;
}
$intRecordsFetched += $intLimit;
unset($Cursor);
gc_collect_cycles();
}
}
/**
* Generate Map Index
*
* @return void
*/
protected function generateMapMain()
{
foreach ($this->getDomain() as $DomainModel) {
$strDomainDir = $this->strSiteMapPath . $DomainModel->d_domain;
if (is_dir($strDomainDir) == false) {
mkdir($strDomainDir);
}
$strMapIndexFile = $strDomainDir . '/sitemap-main.xml';
$strContent = <<<EOF
<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
<url>
<loc>https://www.{$DomainModel->d_domain}/</loc>
<lastmod>{$this->strDate}</lastmod>
<changefreq>daily</changefreq>
<priority>1.0</priority>
</url>
EOF;
file_put_contents($strMapIndexFile, $strContent);
foreach ($this->arrNovelCategory as $strCategoryKey => $strCategoryVal) {
foreach ($this->arrNovelSort as $strSortKey => $strSortVal) {
foreach ($this->arrNovelStatus as $strStatusKey => $strStatusVal) {
$strKey = "CATEGORY_INFO_URL";
$strDomain = $DomainModel->d_domain;
$strController = "";
$strAction = "";
$arrArgs = [
'strCategory' => $strCategoryKey,
'strOrder' => $strSortKey,
'strStatus' => $strStatusKey,
'strGender' => '',
'intPage' => 1,
];
$strUri = $DomainModel->getFomartUrlEx($strKey, $strDomain, $strController, $strAction, $arrArgs);
$strUrl = sprintf("https://www.%s%s", $DomainModel->d_domain, $strUri);
$strContent = <<<EOF
<url>
<loc>{$strUrl}</loc>
<lastmod>{$this->strDate}</lastmod>
<changefreq>weekly</changefreq>
<priority>0.8</priority>
</url>
EOF;
file_put_contents($strMapIndexFile, $strContent, FILE_APPEND);
}
}
}
$strContent = <<<EOF
</urlset>
EOF;
file_put_contents($strMapIndexFile, $strContent, FILE_APPEND);
}
}
/**
* Generate Map Index
*
* @return void
*/
protected function generateMapIndex()
{
$intTotalPage = ceil($this->intTotal / $this->intPageSize);
foreach ($this->getDomain() as $DomainModel) {
$strDomainDir = $this->strSiteMapPath . $DomainModel->d_domain;
if (is_dir($strDomainDir) == false) {
mkdir($strDomainDir);
}
$strMapIndexFile = $strDomainDir . '/sitemap_index.xml';
$strContent = <<<EOF
<?xml version="1.0" encoding="UTF-8"?>
<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
<sitemap>
<loc>https://www.{$DomainModel->d_domain}/sitemap-main.xml</loc>
<lastmod>{$this->strDate}</lastmod>
</sitemap>
EOF;
file_put_contents($strMapIndexFile, $strContent);
for ($intPage = 1; $intPage <= $intTotalPage; $intPage++) {
$strContent = <<<EOF
<sitemap>
<loc>https://www.{$DomainModel->d_domain}/sitemap-books-{$intPage}.xml</loc>
<lastmod>{$this->strDate}</lastmod>
</sitemap>
<sitemap>
<loc>https://www.{$DomainModel->d_domain}/sitemap-books-catalog-{$intPage}.xml</loc>
<lastmod>{$this->strDate}</lastmod>
</sitemap>
<sitemap>
<loc>https://www.{$DomainModel->d_domain}/sitemap-chapters-{$intPage}.xml</loc>
<lastmod>{$this->strDate}</lastmod>
</sitemap>
EOF;
file_put_contents($strMapIndexFile, $strContent, FILE_APPEND);
}
$strContent = <<<EOF
</sitemapindex>
EOF;
file_put_contents($strMapIndexFile, $strContent, FILE_APPEND);
}
}
}

25
code/app/task/module/PlanTask.php
View File

@@ -8,7 +8,7 @@ use app\model\PlanTaskModel;
use app\task\crawler\zw630\Scheduler as Zw630Scheduler;
use app\task\crawler\zw74\Scheduler as Zw74Scheduler;
use app\task\crawler\youzhi\Scheduler as YouZhiScheduler;
use app\task\logic\SiteMapLogic;
use microserver\QueueManage;
use microserver\ProcessSanitizer;
@@ -48,10 +48,31 @@ class PlanTask
case 'PULL_YOU_ZHI_SHI_PIN':
self::pullYouZhiShiPin();
break;
case 'GENERATE_SITE_MAP':
self::generateSiteMap();
break;
}
}
}
/**
* Undocumented function
*
* @return void
*/
public static function generateSiteMap()
{
$arrTask = [
'callback' => [SiteMapLogic::class, 'generateSiteMap'],
'data' => []
];
$QueueManage = new QueueManage(2);
$QueueManage->set($arrTask);
}
/**
* Undocumented function
*