admin/SEONexus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add github spider md push and async task logging

Browse Source
This commit is contained in:
www
2026-04-19 19:37:53 +08:00
parent e8688443e5
commit fccd2bfe5b
12 changed files with 970 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

247
code/app/common/helper/DomainSpiderMdRunHelper.php
View File

@@ -23,7 +23,9 @@ class DomainSpiderMdRunHelper
$runs = [];
SpiderMdConfigHelper::ensureSystemConfigDefaults();
$gitee = self::resolveGiteeConfig();
$github = self::resolveGithubConfig();
unset($gitee['token']);
unset($github['token']);
$matches = glob(self::baseRoot() . '/*/*/summary.json');
if (is_array($matches)) {
rsort($matches);
@@ -39,6 +41,7 @@ class DomainSpiderMdRunHelper
'generated_at' => date(DATE_ATOM),
'run_root' => self::baseRoot(),
'gitee' => $gitee,
'github' => $github,
'runtime_config' => self::resolveRuntimeConfig(),
'config_summary' => SpiderMdConfigHelper::buildConfigSummary(),
'latest_run' => $runs[0] ?? null,
@@ -50,19 +53,24 @@ class DomainSpiderMdRunHelper
{
$config = self::resolveRuntimeConfig($options);
$giteeConfig = self::resolveGiteeConfig();
$githubConfig = self::resolveGithubConfig();
self::ensureDir(self::baseRoot());
$forceRegenerate = !empty($options['force_regenerate']);
$previous = self::findLatestSuccessfulRunSummary();
$today = date('Ymd');
$giteeTargetChanged = !empty($previous) ? self::giteeTargetChanged($previous, $giteeConfig) : false;
$remoteTargetChanged = !empty($previous)
? (self::giteeTargetChanged($previous, $giteeConfig) || self::githubTargetChanged($previous, $githubConfig))
: false;
$previousGiteePushStatus = trim((string)($previous['gitee_push_status'] ?? ''));
$previousGithubPushStatus = trim((string)($previous['github_push_status'] ?? ''));
$allowRetryToday = in_array($previousGiteePushStatus, ['failed', 'partial_failed'], true);
$allowRetryToday = $allowRetryToday || in_array($previousGithubPushStatus, ['failed', 'partial_failed'], true);
if (
!$forceRegenerate
&& !empty($previous)
&& substr((string)($previous['generated_at'] ?? ''), 0, 10) === date('Y-m-d')
&& !$giteeTargetChanged
&& !$remoteTargetChanged
&& !$allowRetryToday
) {
self::log($logger, '今日已生成过蜘蛛池 MD，跳过本轮；如需全新生成请使用强制重生成');
@@ -72,11 +80,11 @@ class DomainSpiderMdRunHelper
return $previous;
}
if ($forceRegenerate || empty($previous) || $giteeTargetChanged) {
if ($forceRegenerate || empty($previous) || $remoteTargetChanged) {
if ($forceRegenerate) {
self::clearAllRuns();
} elseif ($giteeTargetChanged) {
self::log($logger, '检测到 Gitee 推送目标已变更，本轮按全新生成处理');
} elseif ($remoteTargetChanged) {
self::log($logger, '检测到远端推送目标已变更，本轮按全新生成处理');
}
$dateDir = self::baseRoot() . '/' . $today;
self::ensureDir($dateDir);
@@ -216,11 +224,17 @@ class DomainSpiderMdRunHelper
],
'files' => $writtenFiles,
'gitee' => array_diff_key($giteeConfig, ['token' => true]),
'github' => array_diff_key($githubConfig, ['token' => true]),
'gitee_files' => [],
'gitee_pushed' => 0,
'gitee_push_attempted' => 0,
'gitee_push_status' => 'pending',
'gitee_push_error' => '',
'github_files' => [],
'github_pushed' => 0,
'github_push_attempted' => 0,
'github_push_status' => 'pending',
'github_push_error' => '',
];
file_put_contents(
@@ -235,6 +249,12 @@ class DomainSpiderMdRunHelper
$summary['gitee_push_attempted'] = (int)($giteePush['attempted'] ?? 0);
$summary['gitee_push_status'] = (string)($giteePush['status'] ?? 'skipped');
$summary['gitee_push_error'] = (string)($giteePush['error'] ?? '');
$githubPush = self::pushRunToGithub($summary, $logger);
$summary['github_files'] = (array)($githubPush['items'] ?? []);
$summary['github_pushed'] = !empty($summary['github_files']) ? 1 : 0;
$summary['github_push_attempted'] = (int)($githubPush['attempted'] ?? 0);
$summary['github_push_status'] = (string)($githubPush['status'] ?? 'skipped');
$summary['github_push_error'] = (string)($githubPush['error'] ?? '');
file_put_contents(
$runRoot . '/gitee-links.json',
@@ -246,6 +266,16 @@ class DomainSpiderMdRunHelper
'items' => $summary['gitee_files'],
], JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES) . PHP_EOL
);
file_put_contents(
$runRoot . '/github-links.json',
json_encode([
'run_id' => $runId,
'generated_at' => date(DATE_ATOM),
'status' => $summary['github_push_status'],
'error' => $summary['github_push_error'],
'items' => $summary['github_files'],
], JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES) . PHP_EOL
);
file_put_contents(
$runRoot . '/summary.json',
json_encode($summary, JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES) . PHP_EOL
@@ -1094,6 +1124,213 @@ class DomainSpiderMdRunHelper
return SpiderMdConfigHelper::resolveGiteeConfig();
}
protected static function resolveGithubConfig(): array
{
return SpiderMdConfigHelper::resolveGithubConfig();
}
protected static function githubTargetChanged(array $previous, array $currentConfig): bool
{
$previousGithub = (array)($previous['github'] ?? []);
foreach (['enabled', 'owner', 'repo', 'branch', 'root'] as $key) {
if ((string)($previousGithub[$key] ?? '') !== (string)($currentConfig[$key] ?? '')) {
return true;
}
}
return false;
}
protected static function pushRunToGithub(array $summary, ?callable $logger = null): array
{
$config = self::resolveGithubConfig();
if (empty($config['enabled']) || empty($config['configured'])) {
self::log($logger, 'GitHub 未启用或未配置，本轮跳过 GitHub 推送');
return [
'status' => 'skipped',
'attempted' => 0,
'error' => '',
'items' => [],
];
}
$items = [];
$dateDir = basename(dirname((string)($summary['run_root'] ?? '')));
if (!preg_match('/^\d{8}$/', $dateDir)) {
$dateDir = date('Ymd', strtotime((string)($summary['generated_at'] ?? 'now')));
}
$baseRemoteDir = trim((string)$config['root'], '/');
$baseRemoteDir = trim($baseRemoteDir . '/' . $dateDir . '/' . (string)($summary['run_id'] ?? ''), '/');
$localFiles = array_merge(
array_map(static function (array $file): string {
return (string)($file['file_path'] ?? '');
}, (array)($summary['files'] ?? [])),
[
(string)($summary['run_root'] ?? '') . '/summary.json',
(string)($summary['run_root'] ?? '') . '/summary.html',
(string)($summary['run_root'] ?? '') . '/github-links.json',
]
);
try {
foreach ($localFiles as $filePath) {
if (!is_file($filePath)) {
continue;
}
$fileName = basename($filePath);
$remotePath = $baseRemoteDir . '/' . $fileName;
$response = self::upsertGithubFile($config, $remotePath, (string)file_get_contents($filePath), 'spider md update ' . $fileName);
$items[] = [
'file_name' => $fileName,
'remote_path' => $remotePath,
'html_url' => (string)($response['html_url'] ?? self::buildGithubHtmlUrl($config, $remotePath)),
'download_url' => (string)($response['download_url'] ?? self::buildGithubRawUrl($config, $remotePath)),
'local_path' => $filePath,
];
self::log($logger, '已推送 GitHub：' . $fileName);
}
} catch (\Throwable $throwable) {
$message = trim($throwable->getMessage());
self::log($logger, 'GitHub 推送失败：' . $message);
return [
'status' => !empty($items) ? 'partial_failed' : 'failed',
'attempted' => 1,
'error' => $message,
'items' => $items,
];
}
return [
'status' => 'success',
'attempted' => 1,
'error' => '',
'items' => $items,
];
}
protected static function upsertGithubFile(array $config, string $remotePath, string $content, string $message): array
{
$existing = self::githubRequest(
'GET',
sprintf(
'https://api.github.com/repos/%s/%s/contents/%s?ref=%s',
rawurlencode((string)$config['owner']),
rawurlencode((string)$config['repo']),
str_replace('%2F', '/', rawurlencode($remotePath)),
rawurlencode((string)$config['branch'])
),
$config,
[],
true
);
$payload = [
'message' => $message,
'content' => base64_encode($content),
'branch' => (string)$config['branch'],
];
if (!empty($existing['sha'])) {
$payload['sha'] = (string)$existing['sha'];
}
return self::githubRequest(
'PUT',
sprintf(
'https://api.github.com/repos/%s/%s/contents/%s',
rawurlencode((string)$config['owner']),
rawurlencode((string)$config['repo']),
str_replace('%2F', '/', rawurlencode($remotePath))
),
$config,
$payload
);
}
protected static function githubRequest(string $method, string $url, array $config, array $payload = [], bool $allowNotFound = false): array
{
if (!function_exists('curl_init')) {
throw new \RuntimeException('当前 PHP 环境缺少 curl 扩展，无法推送 GitHub');
}
$curl = curl_init();
if ($curl === false) {
throw new \RuntimeException('初始化 GitHub 请求失败');
}
$headers = [
'Accept: application/vnd.github+json',
'Authorization: Bearer ' . (string)$config['token'],
'X-GitHub-Api-Version: 2022-11-28',
'User-Agent: SEONexus-SpiderMd',
];
curl_setopt_array($curl, [
CURLOPT_URL => $url,
CURLOPT_RETURNTRANSFER => true,
CURLOPT_TIMEOUT => 45,
CURLOPT_CUSTOMREQUEST => $method,
CURLOPT_HTTPHEADER => $headers,
]);
if (!empty($payload)) {
curl_setopt($curl, CURLOPT_POSTFIELDS, json_encode($payload, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES));
}
$response = curl_exec($curl);
$errno = curl_errno($curl);
$error = curl_error($curl);
$status = (int)curl_getinfo($curl, CURLINFO_HTTP_CODE);
curl_close($curl);
if ($errno !== 0) {
throw new \RuntimeException('GitHub 请求失败：' . $error);
}
$decoded = json_decode((string)$response, true);
if ($allowNotFound && $status === 404) {
return [];
}
if ($status >= 400) {
$message = is_array($decoded) ? (string)($decoded['message'] ?? json_encode($decoded, JSON_UNESCAPED_UNICODE)) : (string)$response;
throw new \RuntimeException('GitHub 请求返回异常：' . $message);
}
if (is_array($decoded) && isset($decoded['content']) && is_array($decoded['content'])) {
$decoded['html_url'] = (string)($decoded['content']['html_url'] ?? '');
$decoded['download_url'] = (string)($decoded['content']['download_url'] ?? '');
$decoded['sha'] = (string)($decoded['content']['sha'] ?? ($decoded['sha'] ?? ''));
}
return is_array($decoded) ? $decoded : [];
}
protected static function buildGithubHtmlUrl(array $config, string $remotePath): string
{
return sprintf(
'https://github.com/%s/%s/blob/%s/%s',
rawurlencode((string)$config['owner']),
rawurlencode((string)$config['repo']),
rawurlencode((string)$config['branch']),
str_replace('%2F', '/', rawurlencode($remotePath))
);
}
protected static function buildGithubRawUrl(array $config, string $remotePath): string
{
return sprintf(
'https://raw.githubusercontent.com/%s/%s/%s/%s',
rawurlencode((string)$config['owner']),
rawurlencode((string)$config['repo']),
rawurlencode((string)$config['branch']),
str_replace('%2F', '/', rawurlencode($remotePath))
);
}
protected static function readJsonFile(string $path): array
{
if (!is_file($path)) {

79
code/app/common/helper/SpiderMdConfigHelper.php
View File

@@ -36,32 +36,62 @@ class SpiderMdConfigHelper
],
'SPIDER_MD_GITEE_ENABLED' => [
'default' => '0',
'description' => '蜘蛛池MD是否启用 Gitee 推送',
'description' => '蜘蛛池MD是否启用 Gitee 推送（0=关闭，1=开启；开启后会把生成的 markdown 与 links.json 同步到 Gitee）',
'secret' => false,
],
'SPIDER_MD_GITEE_OWNER' => [
'default' => '',
'description' => '蜘蛛池MD Gitee 仓库 owner',
'description' => '蜘蛛池MD Gitee 仓库 owner（仓库所属用户名或组织名，例如 yourname）',
'secret' => false,
],
'SPIDER_MD_GITEE_REPO' => [
'default' => '',
'description' => '蜘蛛池MD Gitee 仓库 repo',
'description' => '蜘蛛池MD Gitee 仓库 repo（仓库名，不带 owner，例如 seo-spider-md）',
'secret' => false,
],
'SPIDER_MD_GITEE_BRANCH' => [
'default' => 'master',
'description' => '蜘蛛池MD Gitee 分支',
'description' => '蜘蛛池MD Gitee 分支（默认 master；推送文件会写入这个分支）',
'secret' => false,
],
'SPIDER_MD_GITEE_ROOT' => [
'default' => 'seo-spider-md',
'description' => '蜘蛛池MD Gitee 远程根目录',
'description' => '蜘蛛池MD Gitee 远程根目录（仓库内子目录，例如 seo-spider-md；留空表示直接写仓库根目录）',
'secret' => false,
],
'SPIDER_MD_GITEE_TOKEN' => [
'default' => '',
'description' => '蜘蛛池MD Gitee Token',
'description' => '蜘蛛池MD Gitee Token（需要仓库 contents / push 权限；后台保存后会参与远程推送）',
'secret' => true,
],
'SPIDER_MD_GITHUB_ENABLED' => [
'default' => '0',
'description' => '蜘蛛池MD是否启用 GitHub 推送（0=关闭，1=开启；开启后会把生成的 markdown 与 links.json 同步到 GitHub）',
'secret' => false,
],
'SPIDER_MD_GITHUB_OWNER' => [
'default' => '',
'description' => '蜘蛛池MD GitHub 仓库 owner（仓库所属用户名或组织名，例如 yourname）',
'secret' => false,
],
'SPIDER_MD_GITHUB_REPO' => [
'default' => '',
'description' => '蜘蛛池MD GitHub 仓库 repo（仓库名，不带 owner，例如 seo-spider-md）',
'secret' => false,
],
'SPIDER_MD_GITHUB_BRANCH' => [
'default' => 'main',
'description' => '蜘蛛池MD GitHub 分支（默认 main；推送文件会写入这个分支）',
'secret' => false,
],
'SPIDER_MD_GITHUB_ROOT' => [
'default' => 'seo-spider-md',
'description' => '蜘蛛池MD GitHub 远程根目录（仓库内子目录，例如 seo-spider-md；留空表示直接写仓库根目录）',
'secret' => false,
],
'SPIDER_MD_GITHUB_TOKEN' => [
'default' => '',
'description' => '蜘蛛池MD GitHub Token（建议使用细粒度 Token，并授予当前仓库 Contents: Read and write 权限）',
'secret' => true,
],
'SPIDER_MD_RETENTION_DAYS' => [
@@ -197,6 +227,43 @@ class SpiderMdConfigHelper
];
}
public static function resolveGithubConfig(): array
{
self::ensureSystemConfigDefaults();
$enabledMeta = self::readValueMeta('SPIDER_MD_GITHUB_ENABLED');
$ownerMeta = self::readValueMeta('SPIDER_MD_GITHUB_OWNER');
$repoMeta = self::readValueMeta('SPIDER_MD_GITHUB_REPO');
$branchMeta = self::readValueMeta('SPIDER_MD_GITHUB_BRANCH');
$rootMeta = self::readValueMeta('SPIDER_MD_GITHUB_ROOT');
$tokenMeta = self::readValueMeta('SPIDER_MD_GITHUB_TOKEN');
$enabled = (int)($enabledMeta['value'] ?? '0') === 1;
$owner = trim((string)($ownerMeta['value'] ?? ''));
$repo = trim((string)($repoMeta['value'] ?? ''));
$branch = trim((string)($branchMeta['value'] ?? 'main'));
$root = trim((string)($rootMeta['value'] ?? 'seo-spider-md'));
$token = trim((string)($tokenMeta['value'] ?? ''));
return [
'enabled' => $enabled ? 1 : 0,
'configured' => ($enabled && $owner !== '' && $repo !== '' && $token !== '') ? 1 : 0,
'owner' => $owner,
'repo' => $repo,
'branch' => $branch !== '' ? $branch : 'main',
'root' => $root,
'token' => $token,
'sources' => [
'enabled' => (string)($enabledMeta['source'] ?? 'env'),
'owner' => (string)($ownerMeta['source'] ?? 'env'),
'repo' => (string)($repoMeta['source'] ?? 'env'),
'branch' => (string)($branchMeta['source'] ?? 'env'),
'root' => (string)($rootMeta['source'] ?? 'env'),
'token' => (string)($tokenMeta['source'] ?? 'env'),
],
];
}
protected static function readOperationalValue(string $code): string
{
return (string)(self::readValueMeta($code)['value'] ?? '');