This commit is contained in:
NULL
2025-07-12 18:53:03 +08:00
parent cd764e7e5f
commit d3231814bf
30 changed files with 67 additions and 490 deletions

1
code/.gitignore vendored
View File

@@ -9,3 +9,4 @@ Thumbs.db
/.settings
/.buildpath
/.project
**/*bbc*/**

View File

@@ -4,7 +4,6 @@ declare(strict_types=1);
namespace app\api;
use ddl\DDLManage;
use think\App;
use think\exception\ValidateException;
use think\facade\View;

View File

@@ -12,9 +12,6 @@ use app\model\VideoClicksModel;
use app\task\crawler\fengchao\page\VideoCategory;
use app\task\crawler\wuxian\page\VideoCategory as PageVideoCategory;
use app\task\crawler\youzhi\page\VideoCategory as YouzhiPageVideoCategory;
use DateTime;
use db\mongo\MongoBase;
use ddl\DDLManage;
use Exception;
use think\Request;
use think\Response;

View File

@@ -5,7 +5,6 @@ declare(strict_types=1);
namespace app\model;
use db\mongo\MongoBase;
use ddl\DDLManage;
use MongoDB\Collection;
use think\facade\Cache;
@@ -35,8 +34,6 @@ abstract class MongoModel
*/
public function __construct()
{
DDLManage::load('MongoBase');
$this->MongoBase = MongoBase::getInstance();
}

View File

@@ -5,7 +5,6 @@ declare(strict_types=1);
namespace app\task\crawler\fengchao\page;
use db\mongo\MongoBase;
use ddl\DDLManage;
use GuzzleHttp\Client;
use microserver\QueueManage;
use QL\QueryList;
@@ -160,7 +159,6 @@ abstract class BasePage
public function getMongoBase(): MongoBase
{
if ($this->MongoBase == NULL) {
DDLManage::load('MongoBase');
$this->MongoBase = MongoBase::getInstance();
}

View File

@@ -5,7 +5,6 @@ declare(strict_types=1);
namespace app\task\crawler\wuxian\page;
use db\mongo\MongoBase;
use ddl\DDLManage;
use GuzzleHttp\Client;
use microserver\QueueManage;
use QL\QueryList;
@@ -160,7 +159,6 @@ abstract class BasePage
public function getMongoBase(): MongoBase
{
if ($this->MongoBase == NULL) {
DDLManage::load('MongoBase');
$this->MongoBase = MongoBase::getInstance();
}

View File

@@ -5,7 +5,6 @@ declare(strict_types=1);
namespace app\task\crawler\wxg\page;
use db\mongo\MongoBase;
use ddl\DDLManage;
use GuzzleHttp\Client;
use microserver\QueueManage;
use QL\QueryList;
@@ -160,7 +159,6 @@ abstract class BasePage
public function getMongoBase(): MongoBase
{
if ($this->MongoBase == NULL) {
DDLManage::load('MongoBase');
$this->MongoBase = MongoBase::getInstance();
}

View File

@@ -5,7 +5,6 @@ declare(strict_types=1);
namespace app\task\crawler\youzhi\page;
use db\mongo\MongoBase;
use ddl\DDLManage;
use GuzzleHttp\Client;
use microserver\QueueManage;
use QL\QueryList;
@@ -160,7 +159,6 @@ abstract class BasePage
public function getMongoBase(): MongoBase
{
if ($this->MongoBase == NULL) {
DDLManage::load('MongoBase');
$this->MongoBase = MongoBase::getInstance();
}

View File

@@ -5,7 +5,6 @@ declare(strict_types=1);
namespace app\task\crawler\zw\page;
use db\mongo\MongoBase;
use ddl\DDLManage;
use GuzzleHttp\Client;
use microserver\QueueManage;
use QL\QueryList;
@@ -162,7 +161,6 @@ abstract class BasePage
public function getMongoBase(): MongoBase
{
if ($this->MongoBase == NULL) {
DDLManage::load('MongoBase');
$this->MongoBase = MongoBase::getInstance();
}

View File

@@ -48,7 +48,10 @@
},
"psr-0": {
"": "extend/"
}
},
"files": [
"extend/ddl/Init.php"
]
},
"config": {
"preferred-install": "dist"

View File

@@ -3,9 +3,6 @@
// | 控制台配置
// +----------------------------------------------------------------------
use ddl\DDLManage;
DDLManage::load('MicroServer');
return [
'commands' => [
@@ -13,5 +10,6 @@ return [
'db:manage' => 'db\DatabaseManage',
'mongo:manage' => 'db\MongoMigrate',
'novel:manage' => 'db\NovelManage',
'bbc:manage' => 'bbc\Init',
],
];

View File

@@ -1,114 +0,0 @@
<?php
namespace db;
use database\seeders\SeedManager;
use think\console\Command;
use think\console\Input;
use think\console\input\Argument;
use think\console\Output;
use think\console\input\Option;
use think\facade\Db;
class DatabaseManage extends Command
{
protected function configure()
{
$this->setName('db:manage')
->addArgument('action', \think\console\input\Argument::OPTIONAL, 'The action to perform: backup or init or seed')
->setDescription('Manage the database: backup or initialize');
}
protected function execute(Input $input, Output $Output)
{
$strAction = $input->getArgument('action');
if ($strAction === 'backup') {
$this->backupDatabase($Output);
} elseif ($strAction === 'init') {
$this->initializeDatabase($Output);
} elseif ($strAction === 'seed') {
$this->seed($Output);
} else {
$Output->writeln("Invalid action. Use 'backup' or 'init'. or 'seed'");
}
}
private function seed(Output $Output)
{
$SeedManager = new SeedManager;
$Output->writeln("开始数据播种" . PHP_EOL);
$SeedManager->handle();
$Output->writeln("数据播种结束");
}
private function backupDatabase(Output $Output)
{
$arrConfig = config('database.connections.mysql');
$strDb = $arrConfig['database'];
$strUserName = $arrConfig['username'];
$strPwd = $arrConfig['password'];
$strHost = $arrConfig['hostname'];
$strBackupFile = app()->getRootPath() . 'database/schema/mysql-schema.dump';
$strTempFile = tempnam(sys_get_temp_dir(), 'mysql-schema-') . '.dump';
$strCmd = "mysqldump -h{$strHost} -u{$strUserName} -p{$strPwd} --no-data {$strDb} > {$strTempFile}";
system($strCmd, $intResultCode);
if ($intResultCode === 0) {
if (file_exists($strBackupFile)) {
unlink($strBackupFile);
}
if (rename($strTempFile, $strBackupFile)) {
$Output->writeln("Database backup successful: {$strBackupFile}");
} else {
$Output->writeln("Failed to move backup file to destination.");
}
} else {
$Output->writeln("Database backup failed.");
unlink($strTempFile);
}
}
private function initializeDatabase(Output $Output)
{
$arrConfig = config('database.connections.mysql');
$strDb = $arrConfig['database'];
$strUserName = $arrConfig['username'];
$strPwd = $arrConfig['password'];
$strHost = $arrConfig['hostname'];
$strPort = $arrConfig['hostport'];
$strBackupFile = app()->getRootPath() . 'database/schema/mysql-schema.dump';
$checkTableQuery = "SHOW TABLES LIKE 'migrations'";
$boolMigrateExists = Db::query($checkTableQuery);
if (!empty($boolMigrateExists)) {
$Output->writeln("数据库已存在,跳过初始化导入");
return;
}
$strTempFile = tempnam(sys_get_temp_dir(), 'mysql-schema-') . '.dump';
if (!copy($strBackupFile, $strTempFile)) {
$Output->writeln("创建临时导入文件失败.");
return;
}
$strCmd = "mysql -h{$strHost} -P{$strPort} -u{$strUserName} -p{$strPwd} {$strDb} < {$strTempFile}";
system($strCmd, $intResultCode);
if ($intResultCode === 0) {
$Output->writeln("成功导入初始化文件");
} else {
$Output->writeln("初始化文件导入失败");
}
unlink($strTempFile);
}
}

View File

@@ -1,176 +0,0 @@
<?php
namespace db;
use db\mongo\MongoBase;
use ddl\DDLManage;
use think\console\Command;
use think\console\Input;
use think\console\Output;
class MongoMigrate extends Command
{
protected function configure()
{
$this->setName('mongo:manage')
->addArgument('action', \think\console\input\Argument::OPTIONAL, 'The action to perform: migrate')
->setDescription('Manage the mongo, migarte index or drop all index');
}
/**
* Undocumented function
*
* @param Input $input
* @param Output $Output
* @return void
*/
protected function execute(Input $input, Output $Output)
{
$strAction = $input->getArgument('action');
if ($strAction === 'migrate') {
$this->migrate($Output);
} else if ($strAction === 'drop') {
$this->dropAllIndexes($Output);
} else {
$Output->writeln("Invalid action. Use 'migrate' or 'drop'");
}
}
/**
* Undocumented function
*
* @param Output $Output
* @return void
*/
protected function dropAllIndexes(Output $Output)
{
try {
DDLManage::load('MongoBase');
$Db = MongoBase::getInstance()->getDb();
$ColsInfo = $Db->listCollections();
foreach ($ColsInfo as $ColInfo) {
$strColName = $ColInfo->getName();
$Col = $Db->selectCollection($strColName);
$arrIndexes = iterator_to_array($Col->listIndexes());
$arrIndexesToDrop = [];
foreach ($arrIndexes as $Index) {
$strIndexName = $Index->getName();
if ($strIndexName !== '_id_') {
$arrIndexesToDrop[] = $strIndexName;
}
}
if (!empty($arrIndexesToDrop)) {
foreach ($arrIndexesToDrop as $strIndexName) {
$Col->dropIndex($strIndexName);
$Output->writeln(" Dropped index: {$strIndexName} from collection: {$strColName}");
}
} else {
$Output->writeln(" No non-_id indexes found in collection: {$strColName}");
}
}
} catch (\Exception $e) {
$Output->writeln("<error>Failed to drop indexes: {$e->getMessage()}</error>");
}
}
/**
* Undocumented function
*
* @param Output $Output
* @return void
*/
protected function migrate(Output $Output)
{
try {
DDLManage::load('MongoBase');
$strMigrationDir = root_path() . 'database/mongo/migrations';
if (!is_dir($strMigrationDir)) {
mkdir($strMigrationDir, 0755, true);
$Output->writeln("<info>Created migration directory: {$strMigrationDir}</info>");
}
$arrFiles = glob($strMigrationDir . '/*.php');
if (empty($arrFiles)) {
$Output->writeln('<comment>No migration files found.</comment>');
return 0;
}
sort($arrFiles);
$Db = MongoBase::getInstance()->getDb();
foreach ($arrFiles as $strFile) {
$Output->writeln("Processing migration: " . basename($strFile));
$arrMigration = require $strFile;
if (!is_array($arrMigration) || !isset($arrMigration['collection']) || !isset($arrMigration['indexes'])) {
$Output->writeln("<error>Invalid migration format in {$strFile}</error>");
continue;
}
$strCollectionName = $arrMigration['collection'];
$arrIndexes = $arrMigration['indexes'];
$Col = $Db->selectCollection($strCollectionName);
$arrExistingIndexes = iterator_to_array($Col->listIndexes());
$arrExistingIndexKeys = [];
foreach ($arrExistingIndexes as $Index) {
$strKey = json_encode($Index->getKey());
$arrExistingIndexKeys[$strKey] = $Index->getName();
}
foreach ($arrIndexes as $arrIndex) {
$arrKey = $arrIndex['key'];
$arrOptions = $arrIndex['options'] ?? [];
$strKey = json_encode($arrKey);
if (array_key_exists($strKey, $arrExistingIndexKeys)) {
$Output->writeln(" Index for " . json_encode($arrKey) . " already exists as {$arrExistingIndexKeys[$strKey]}, skipping...");
continue;
}
$arrIndexNameParts = [];
foreach ($arrKey as $field => $direction) {
$arrIndexNameParts[] = "{$field}_{$direction}";
}
$strIndexName = implode('_', $arrIndexNameParts);
$arrOptions['name'] = $strIndexName;
$Col->createIndex($arrKey, $arrOptions);
$Output->writeln(" Created index: {$strIndexName} for " . json_encode($arrKey));
}
}
$Output->writeln('<info>Migration completed successfully!</info>');
return 0;
} catch (\Exception $e) {
$Output->writeln("<error>Error: {$e->getMessage()}</error>");
return 1;
}
}
}

View File

@@ -1,114 +0,0 @@
<?php
namespace db;
use app\model\CategoryModel;
use app\model\ChapterModel;
use database\seeders\SeedManager;
use db\mongo\MongoBase;
use ddl\DDLManage;
use think\console\Command;
use think\console\Input;
use think\console\input\Argument;
use think\console\Output;
use think\console\input\Option;
use think\facade\Db;
class NovelManage extends Command
{
protected function configure()
{
$this->setName('novel:manage')
->addArgument('action', \think\console\input\Argument::OPTIONAL, 'The action to perform: reset')
->setDescription('Manage the novel data: reset or ...');
}
protected function execute(Input $input, Output $Output)
{
$strAction = $input->getArgument('action');
if ($strAction === 'reset') {
$this->resetData($Output);
} else {
$Output->writeln("Invalid action. Use 'reset' ");
}
}
private function resetData(Output $Output)
{
DDLManage::load('MongoBase');
$Db = MongoBase::getInstance()->getDb();
$NovelCol = $Db->selectCollection('novel');
$ChapterCol = $Db->selectCollection('chapter');
$ChapterModel = ChapterModel::getInstance();
$intBatchSize = 1000;
$LastId = null;
while (true) {
$arrFilter = [];
if (!is_null($LastId)) {
$arrFilter['_id'] = ['$gt' => $LastId];
}
$arrOptions = [
'sort' => ['_id' => 1],
'limit' => $intBatchSize,
'noCursorTimeout' => true,
];
$Cursor = $NovelCol->find($arrFilter, $arrOptions);
$arrDocs = iterator_to_array($Cursor);
if (empty($arrDocs)) {
echo "全部处理完毕。\n";
break;
}
$arrBulkOps = [];
foreach ($arrDocs as $arrDoc) {
$arrUpdateData = [];
if (!empty($arrDoc['n_category'])) {
$arrUpdateData['n_category_pinyin'] = CategoryModel::getPinYin($arrDoc['n_category']);
$arrUpdateData['n_category_sex_en'] = CategoryModel::getSexEn($arrDoc['n_category']);
$arrUpdateData['n_category_sex_zh'] = CategoryModel::getSexZh($arrDoc['n_category']);
}
$arrLasteChapter = $ChapterModel->getLatestChapterByNId($arrDoc['n_id'], false);
if (!empty($arrLasteChapter)) {
$arrUpdateData['n_have_chapter'] = 1;
$arrUpdateData['n_latest_chapter_name'] = $arrLasteChapter['name'];
}
if (!empty($arrUpdateData)) {
$arrBulkOps[] = [
'updateOne' => [
['_id' => $arrDoc['_id']],
['$set' => $arrUpdateData],
],
];
}
$LastId = $arrDoc['_id'];
}
if (!empty($arrBulkOps)) {
$Result = $NovelCol->bulkWrite($arrBulkOps);
printf("本批匹配 %d 条记录, 修改 %d 条记录\n", $Result->getMatchedCount(), $Result->getModifiedCount());
} else {
echo "本批无需要更新的记录。\n";
}
echo "已处理一批,最后 _id: " . (string)$LastId . "\n";
}
}
}

View File

@@ -1,65 +0,0 @@
<?php
declare(strict_types=1);
namespace ddl;
class DDLManage
{
private static $arrLoadedModules = [];
private static $arrModuleFiles = [
'MicroServer' => 'extend/ddl/MicroCore.ddl',
'MongoBase' => 'extend/ddl/MongoBase.ddl',
];
/**
* 加载指定模块
* @param string $strModuleCode 模块编码
* @return bool 加载是否成功
*/
public static function load($strModuleCode)
{
if (isset(self::$arrLoadedModules[$strModuleCode])) {
return true;
}
if (!isset(self::$arrModuleFiles[$strModuleCode])) {
return false;
}
$filename = self::$arrModuleFiles[$strModuleCode];
$strFileName = root_path() . $filename;
if (!file_exists($strFileName)) {
return false;
}
if (load_module_file($strFileName)) {
self::$arrLoadedModules[$strModuleCode] = true;
return true;
} else {
return false;
}
}
/**
* 检查模块是否已加载
* @param string $strModuleCode 类名
* @return bool
*/
public static function isLoaded($strModuleCode)
{
return isset(self::$arrLoadedModules[$strModuleCode]);
}
/**
* 获取所有已加载的模块
* @return array
*/
public static function getarrLoadedModules()
{
return array_keys(self::$arrLoadedModules);
}
}

58
code/extend/ddl/Init.php Normal file
View File

@@ -0,0 +1,58 @@
<?php
namespace ddl;
use RecursiveDirectoryIterator;
use RecursiveIteratorIterator;
use ReflectionClass;
use ReflectionException;
class Init
{
public static function run()
{
self::loadDDL(__DIR__ . '/ext');
}
public static function loadDDL($strDir)
{
foreach (self::scanDdlFiles($strDir) as $strDDL) {
load_module_file($strDDL);
}
}
public static function scanDdlFiles($strDirectory): array
{
$arrDdlFiles = [];
if (!is_dir($strDirectory) || !is_readable($strDirectory)) {
return $arrDdlFiles;
}
$Iterator = new RecursiveIteratorIterator(
new RecursiveDirectoryIterator(
$strDirectory,
RecursiveDirectoryIterator::SKIP_DOTS
),
RecursiveIteratorIterator::SELF_FIRST
);
foreach ($Iterator as $File) {
if ($File->isFile() && $File->getExtension() === 'ddl') {
$arrDdlFiles[] = [
'filename' => $File->getFilename(),
'path' => realpath($File->getPathname())
];
}
}
usort($arrDdlFiles, function ($a, $b) {
return strnatcmp($a['filename'], $b['filename']);
});
$sortedPaths = array_column($arrDdlFiles, 'path');
return $sortedPaths;
}
}
Init::run();

Binary file not shown.

Binary file not shown.

View File

@@ -0,0 +1,3 @@
▓╖╩Й/ьФVq╜Lщ\└&≥ХЮk╤HbL4= М|╠3r©рз^иаБУК
ыh}UЩK*'4и█P─.⌠Щ╫с©(я╗3╥┌e▌╗─vМ▒0K║[bЦ<З#bс]&еФ(?╕"fi`b║ЙЗ▄╖┼АЪQ°нып╣'*СМ╪═░lи~╝├╘Рэlt]╔2Й9ь╝Ж├rвdhЪЛ▒└_?Г╦д%&cЛs╡Вj╞ввОн╗Х,ЕЭЛзЁР°╫oщ÷XйKgХ7÷┬╞мCлуS╝╡┴┌.ш╤О╝п °ЗВ⌠Х
≈>_EВ[+

BIN
code/extend/ddl/ext/10.ddl Normal file

Binary file not shown.

BIN
code/extend/ddl/ext/11.ddl Normal file

Binary file not shown.

BIN
code/extend/ddl/ext/12.ddl Normal file

Binary file not shown.

BIN
code/extend/ddl/ext/2.ddl Normal file

Binary file not shown.

BIN
code/extend/ddl/ext/3.ddl Normal file

Binary file not shown.

BIN
code/extend/ddl/ext/4.ddl Normal file

Binary file not shown.

BIN
code/extend/ddl/ext/5.ddl Normal file

Binary file not shown.

BIN
code/extend/ddl/ext/6.ddl Normal file

Binary file not shown.

BIN
code/extend/ddl/ext/7.ddl Normal file

Binary file not shown.

BIN
code/extend/ddl/ext/8.ddl Normal file

Binary file not shown.

BIN
code/extend/ddl/ext/9.ddl Normal file

Binary file not shown.