diff --git a/code/app/common/helper/DomainSpiderMdRunHelper.php b/code/app/common/helper/DomainSpiderMdRunHelper.php index c51ee64..400b513 100644 --- a/code/app/common/helper/DomainSpiderMdRunHelper.php +++ b/code/app/common/helper/DomainSpiderMdRunHelper.php @@ -17,6 +17,11 @@ class DomainSpiderMdRunHelper return dirname(__DIR__, 3) . '/storage/spider_md_runs'; } + public static function activeRunRoot(): string + { + return self::baseRoot() . '/active'; + } + public static function buildSummary(int $limit = 20): array { $limit = max(1, min(100, $limit)); @@ -26,13 +31,19 @@ class DomainSpiderMdRunHelper $github = self::resolveGithubConfig(); unset($gitee['token']); unset($github['token']); - $matches = glob(self::baseRoot() . '/*/*/summary.json'); - if (is_array($matches)) { - rsort($matches); - foreach (array_slice($matches, 0, $limit) as $summaryPath) { - $payload = self::readJsonFile((string)$summaryPath); - if (!empty($payload)) { - $runs[] = $payload; + $activeSummaryPath = self::activeRunRoot() . '/summary.json'; + $activePayload = self::readJsonFile($activeSummaryPath); + if (!empty($activePayload)) { + $runs[] = $activePayload; + } else { + $matches = glob(self::baseRoot() . '/*/*/summary.json'); + if (is_array($matches)) { + rsort($matches); + foreach (array_slice($matches, 0, $limit) as $summaryPath) { + $payload = self::readJsonFile((string)$summaryPath); + if (!empty($payload)) { + $runs[] = $payload; + } } } } @@ -55,54 +66,36 @@ class DomainSpiderMdRunHelper $giteeConfig = self::resolveGiteeConfig(); $githubConfig = self::resolveGithubConfig(); self::ensureDir(self::baseRoot()); + self::ensureDir(self::activeRunRoot()); $forceRegenerate = !empty($options['force_regenerate']); $previous = self::findLatestSuccessfulRunSummary(); - $today = date('Ymd'); $remoteTargetChanged = !empty($previous) ? (self::giteeTargetChanged($previous, $giteeConfig) || self::githubTargetChanged($previous, $githubConfig)) : false; $previousGiteePushStatus = trim((string)($previous['gitee_push_status'] ?? '')); $previousGithubPushStatus = trim((string)($previous['github_push_status'] ?? '')); - $allowRetryToday = in_array($previousGiteePushStatus, ['failed', 'partial_failed'], true); - $allowRetryToday = $allowRetryToday || in_array($previousGithubPushStatus, ['failed', 'partial_failed'], true); + $allowRetryPush = in_array($previousGiteePushStatus, ['failed', 'partial_failed'], true) + || in_array($previousGithubPushStatus, ['failed', 'partial_failed'], true); + $forceFullRemoteSync = $forceRegenerate || empty($previous) || $remoteTargetChanged || $allowRetryPush; - if ( - !$forceRegenerate - && !empty($previous) - && substr((string)($previous['generated_at'] ?? ''), 0, 10) === date('Y-m-d') - && !$remoteTargetChanged - && !$allowRetryToday - ) { - self::log($logger, '今日已生成过蜘蛛池 MD，跳过本轮；如需全新生成请使用强制重生成'); - $previous['status'] = 'skipped'; - $previous['skipped_reason'] = 'already_generated_today'; - $previous['runtime_config'] = $config; - return $previous; - } - - if ($forceRegenerate || empty($previous) || $remoteTargetChanged) { + if ($forceRegenerate || empty($previous)) { if ($forceRegenerate) { self::clearAllRuns(); - } elseif ($remoteTargetChanged) { - self::log($logger, '检测到远端推送目标已变更，本轮按全新生成处理'); } - $dateDir = self::baseRoot() . '/' . $today; - self::ensureDir($dateDir); $runId = date('His') . '_spider_md_' . substr(md5(uniqid('', true)), 0, 6); - $runRoot = $dateDir . '/' . $runId; + $runRoot = self::activeRunRoot(); self::ensureDir($runRoot); self::log($logger, $forceRegenerate ? '开始强制重生成蜘蛛池 MD' : '开始首次生成蜘蛛池 MD'); } else { - $runId = (string)($previous['run_id'] ?? ''); - $runRoot = (string)($previous['run_root'] ?? ''); - if ($runId === '' || $runRoot === '') { - throw new \RuntimeException('历史蜘蛛池MD运行目录无效，无法续写'); - } + $runId = date('His') . '_spider_md_' . substr(md5(uniqid('', true)), 0, 6); + $runRoot = self::activeRunRoot(); self::ensureDir($runRoot); - if ($allowRetryToday) { - self::log($logger, '检测到上一轮 Gitee 推送失败，本轮继续沿用当前目录并重试推送'); - } else { - self::log($logger, '开始增量续写蜘蛛池 MD'); + self::log($logger, '开始增量续写蜘蛛池 MD'); + if ($remoteTargetChanged) { + self::log($logger, '检测到远端推送目标已变更，本轮保留现有分片，仅执行全量远端同步'); + } + if ($allowRetryPush) { + self::log($logger, '检测到上一轮远端推送失败，本轮保留现有分片并重试远端同步'); } } @@ -113,6 +106,7 @@ class DomainSpiderMdRunHelper $existingState = self::loadExistingRunState($runRoot, (int)$config['max_links_per_file']); $aggregateLinks = []; $writtenFiles = (array)($existingState['files'] ?? []); + $changedFiles = []; $chunkState = [ 'records' => (array)($existingState['pending_records'] ?? []), 'max_links_per_file' => (int)$config['max_links_per_file'], @@ -120,6 +114,7 @@ class DomainSpiderMdRunHelper 'generated_at' => date('Y-m-d H:i:s'), 'run_root' => $runRoot, 'files' => &$writtenFiles, + 'changed_files' => &$changedFiles, 'known_urls' => (array)($existingState['known_urls'] ?? []), 'reused_last_file' => (string)($existingState['reused_last_file'] ?? ''), ]; @@ -155,10 +150,14 @@ class DomainSpiderMdRunHelper /** @var DomainModel $domain */ $domain = $domainRow['domain']; $forgePolicy = $domain->getForgeSeoCfg(); + $existingDetailCount = (int)(($existingState['detail_count_by_host'] ?? [])[$host] ?? 0); + $existingForgeCount = (int)(($existingState['forge_count_by_host'] ?? [])[$host] ?? 0); + $detailLimit = (int)$config['detail_limit_per_host']; + $forgeLimit = (int)$config['forge_limit_per_host']; $domainProgress[$host] = [ 'row' => $domainRow, - 'detail_remaining' => (int)$config['detail_limit_per_host'] > 0 ? (int)$config['detail_limit_per_host'] : null, - 'forge_remaining' => (int)$config['forge_limit_per_host'] > 0 ? (int)$config['forge_limit_per_host'] : null, + 'detail_remaining' => $detailLimit > 0 ? max(0, $detailLimit - $existingDetailCount) : null, + 'forge_remaining' => $forgeLimit > 0 ? max(0, $forgeLimit - $existingForgeCount) : null, 'forge_per_video' => max(0, (int)($forgePolicy['sitemap_count'] ?? 0)), ]; } @@ -216,6 +215,7 @@ class DomainSpiderMdRunHelper 'max_links_per_file' => (int)$config['max_links_per_file'], 'config' => $config, 'mode' => $forceRegenerate || empty($previous) ? 'rebuild' : 'append', + 'remote_mode' => 'fixed-root-incremental', 'new_links' => [ 'aggregate_count' => $newAggregateCount, 'detail_count' => $newDetailCount, @@ -223,6 +223,8 @@ class DomainSpiderMdRunHelper 'total_links' => $newAggregateCount + $newDetailCount + $newForgeCount, ], 'files' => $writtenFiles, + 'changed_files' => array_values(array_unique($changedFiles)), + 'full_remote_sync' => $forceFullRemoteSync ? 1 : 0, 'gitee' => array_diff_key($giteeConfig, ['token' => true]), 'github' => array_diff_key($githubConfig, ['token' => true]), 'gitee_files' => [], @@ -237,11 +239,7 @@ class DomainSpiderMdRunHelper 'github_push_error' => '', ]; - file_put_contents( - $runRoot . '/summary.json', - json_encode($summary, JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES) . PHP_EOL - ); - file_put_contents($runRoot . '/summary.html', self::renderSummaryHtml($summary)); + self::writeSummaryArtifacts($runRoot, $summary); $giteePush = self::pushRunToGitee($summary, $logger); $summary['gitee_files'] = (array)($giteePush['items'] ?? []); @@ -256,31 +254,13 @@ class DomainSpiderMdRunHelper $summary['github_push_status'] = (string)($githubPush['status'] ?? 'skipped'); $summary['github_push_error'] = (string)($githubPush['error'] ?? ''); - file_put_contents( - $runRoot . '/gitee-links.json', - json_encode([ - 'run_id' => $runId, - 'generated_at' => date(DATE_ATOM), - 'status' => $summary['gitee_push_status'], - 'error' => $summary['gitee_push_error'], - 'items' => $summary['gitee_files'], - ], JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES) . PHP_EOL - ); - file_put_contents( - $runRoot . '/github-links.json', - json_encode([ - 'run_id' => $runId, - 'generated_at' => date(DATE_ATOM), - 'status' => $summary['github_push_status'], - 'error' => $summary['github_push_error'], - 'items' => $summary['github_files'], - ], JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES) . PHP_EOL - ); + self::writeLinkArtifacts($runRoot, $summary); file_put_contents( $runRoot . '/summary.json', json_encode($summary, JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES) . PHP_EOL ); file_put_contents($runRoot . '/summary.html', self::renderSummaryHtml($summary)); + self::syncMetadataToRemotes($summary, $logger); self::log($logger, '蜘蛛池 MD 生成完成，共 ' . count($writtenFiles) . ' 个文件'); @@ -609,6 +589,7 @@ class DomainSpiderMdRunHelper return (string)($item['section_label'] ?? ''); }, $state['records']))), ]; + $state['changed_files'][] = $fileName; $state['records'] = []; $state['reused_last_file'] = ''; @@ -616,6 +597,11 @@ class DomainSpiderMdRunHelper protected static function findLatestSuccessfulRunSummary(): array { + $activeSummary = self::readJsonFile(self::activeRunRoot() . '/summary.json'); + if (!empty($activeSummary) && (string)($activeSummary['status'] ?? '') === 'success') { + return $activeSummary; + } + $matches = glob(self::baseRoot() . '/*/*/summary.json'); if (!is_array($matches) || empty($matches)) { return []; @@ -674,9 +660,45 @@ class DomainSpiderMdRunHelper 'aggregate_count' => (int)($summary['aggregate_count'] ?? 0), 'detail_count' => (int)($summary['detail_count'] ?? 0), 'forge_count' => (int)($summary['forge_count'] ?? 0), + 'detail_count_by_host' => self::countExistingSectionByHost($files, $pendingRecords, 'detail'), + 'forge_count_by_host' => self::countExistingSectionByHost($files, $pendingRecords, 'forge'), ]; } + protected static function countExistingSectionByHost(array $files, array $pendingRecords, string $section): array + { + $counts = []; + foreach ($files as $fileMeta) { + $filePath = (string)($fileMeta['file_path'] ?? ''); + if ($filePath === '' || !is_file($filePath)) { + continue; + } + foreach (self::parseMarkdownChunkRecords($filePath) as $record) { + if ((string)($record['section'] ?? '') !== $section) { + continue; + } + $host = trim((string)($record['host'] ?? '')); + if ($host === '') { + continue; + } + $counts[$host] = (int)($counts[$host] ?? 0) + 1; + } + } + + foreach ($pendingRecords as $record) { + if ((string)($record['section'] ?? '') !== $section) { + continue; + } + $host = trim((string)($record['host'] ?? '')); + if ($host === '') { + continue; + } + $counts[$host] = (int)($counts[$host] ?? 0) + 1; + } + + return $counts; + } + protected static function parseMarkdownChunkRecords(string $filePath): array { $lines = @file($filePath, FILE_IGNORE_NEW_LINES); @@ -822,6 +844,8 @@ class DomainSpiderMdRunHelper foreach (glob(self::baseRoot() . '/*') ?: [] as $dateDir) { self::deletePath((string)$dateDir); } + self::deletePath(self::activeRunRoot()); + self::ensureDir(self::activeRunRoot()); } protected static function deletePath(string $path): void @@ -955,23 +979,8 @@ class DomainSpiderMdRunHelper } $items = []; - $dateDir = basename(dirname((string)($summary['run_root'] ?? ''))); - if (!preg_match('/^\d{8}$/', $dateDir)) { - $dateDir = date('Ymd', strtotime((string)($summary['generated_at'] ?? 'now'))); - } $baseRemoteDir = trim((string)$config['root'], '/'); - $baseRemoteDir = trim($baseRemoteDir . '/' . $dateDir . '/' . (string)($summary['run_id'] ?? ''), '/'); - - $localFiles = array_merge( - array_map(static function (array $file): string { - return (string)($file['file_path'] ?? ''); - }, (array)($summary['files'] ?? [])), - [ - (string)($summary['run_root'] ?? '') . '/summary.json', - (string)($summary['run_root'] ?? '') . '/summary.html', - (string)($summary['run_root'] ?? '') . '/gitee-links.json', - ] - ); + $localFiles = self::resolveIncrementalLocalFiles($summary); try { foreach ($localFiles as $filePath) { @@ -1007,7 +1016,7 @@ class DomainSpiderMdRunHelper 'status' => 'success', 'attempted' => 1, 'error' => '', - 'items' => $items, + 'items' => self::buildRemoteItems($summary, $config, 'gitee'), ]; } @@ -1155,23 +1164,8 @@ class DomainSpiderMdRunHelper } $items = []; - $dateDir = basename(dirname((string)($summary['run_root'] ?? ''))); - if (!preg_match('/^\d{8}$/', $dateDir)) { - $dateDir = date('Ymd', strtotime((string)($summary['generated_at'] ?? 'now'))); - } $baseRemoteDir = trim((string)$config['root'], '/'); - $baseRemoteDir = trim($baseRemoteDir . '/' . $dateDir . '/' . (string)($summary['run_id'] ?? ''), '/'); - - $localFiles = array_merge( - array_map(static function (array $file): string { - return (string)($file['file_path'] ?? ''); - }, (array)($summary['files'] ?? [])), - [ - (string)($summary['run_root'] ?? '') . '/summary.json', - (string)($summary['run_root'] ?? '') . '/summary.html', - (string)($summary['run_root'] ?? '') . '/github-links.json', - ] - ); + $localFiles = self::resolveIncrementalLocalFiles($summary); try { foreach ($localFiles as $filePath) { @@ -1207,10 +1201,161 @@ class DomainSpiderMdRunHelper 'status' => 'success', 'attempted' => 1, 'error' => '', - 'items' => $items, + 'items' => self::buildRemoteItems($summary, $config, 'github'), ]; } + protected static function resolveIncrementalLocalFiles(array $summary): array + { + $changedFileNames = array_values(array_unique(array_filter(array_map(static function ($item): string { + return trim((string)$item); + }, (array)($summary['changed_files'] ?? []))))); + $allFiles = array_values(array_filter(array_map(static function (array $file): string { + return (string)($file['file_path'] ?? ''); + }, (array)($summary['files'] ?? [])))); + + if (!empty($summary['full_remote_sync'])) { + return $allFiles; + } + + if (empty($changedFileNames)) { + return []; + } + + $selected = []; + foreach ((array)($summary['files'] ?? []) as $file) { + $fileName = trim((string)($file['file_name'] ?? '')); + $filePath = (string)($file['file_path'] ?? ''); + if ($fileName === '' || $filePath === '') { + continue; + } + if (in_array($fileName, $changedFileNames, true)) { + $selected[] = $filePath; + } + } + + return array_values(array_unique($selected)); + } + + protected static function buildRemoteItems(array $summary, array $config, string $provider): array + { + $baseRemoteDir = trim((string)($config['root'] ?? ''), '/'); + $items = []; + $fileNames = []; + foreach ((array)($summary['files'] ?? []) as $file) { + $fileName = trim((string)($file['file_name'] ?? '')); + $filePath = (string)($file['file_path'] ?? ''); + if ($fileName === '') { + continue; + } + $fileNames[] = [ + 'file_name' => $fileName, + 'local_path' => $filePath, + ]; + } + $fileNames[] = ['file_name' => 'summary.json', 'local_path' => (string)($summary['run_root'] ?? '') . '/summary.json']; + $fileNames[] = ['file_name' => 'summary.html', 'local_path' => (string)($summary['run_root'] ?? '') . '/summary.html']; + $fileNames[] = ['file_name' => $provider . '-links.json', 'local_path' => (string)($summary['run_root'] ?? '') . '/' . $provider . '-links.json']; + + foreach ($fileNames as $fileMeta) { + $fileName = (string)($fileMeta['file_name'] ?? ''); + if ($fileName === '') { + continue; + } + $remotePath = trim($baseRemoteDir . '/' . $fileName, '/'); + $items[] = [ + 'file_name' => $fileName, + 'remote_path' => $remotePath, + 'html_url' => $provider === 'github' + ? self::buildGithubHtmlUrl($config, $remotePath) + : self::buildGiteeHtmlUrl($config, $remotePath), + 'download_url' => $provider === 'github' + ? self::buildGithubRawUrl($config, $remotePath) + : self::buildGiteeRawUrl($config, $remotePath), + 'local_path' => (string)($fileMeta['local_path'] ?? ''), + ]; + } + + return $items; + } + + protected static function writeSummaryArtifacts(string $runRoot, array $summary): void + { + file_put_contents( + $runRoot . '/summary.json', + json_encode($summary, JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES) . PHP_EOL + ); + file_put_contents($runRoot . '/summary.html', self::renderSummaryHtml($summary)); + } + + protected static function writeLinkArtifacts(string $runRoot, array $summary): void + { + file_put_contents( + $runRoot . '/gitee-links.json', + json_encode([ + 'run_id' => (string)($summary['run_id'] ?? ''), + 'generated_at' => date(DATE_ATOM), + 'status' => (string)($summary['gitee_push_status'] ?? 'pending'), + 'error' => (string)($summary['gitee_push_error'] ?? ''), + 'items' => (array)($summary['gitee_files'] ?? []), + ], JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES) . PHP_EOL + ); + file_put_contents( + $runRoot . '/github-links.json', + json_encode([ + 'run_id' => (string)($summary['run_id'] ?? ''), + 'generated_at' => date(DATE_ATOM), + 'status' => (string)($summary['github_push_status'] ?? 'pending'), + 'error' => (string)($summary['github_push_error'] ?? ''), + 'items' => (array)($summary['github_files'] ?? []), + ], JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES) . PHP_EOL + ); + } + + protected static function syncMetadataToRemotes(array $summary, ?callable $logger = null): void + { + self::syncMetadataToGitee($summary, $logger); + self::syncMetadataToGithub($summary, $logger); + } + + protected static function syncMetadataToGitee(array $summary, ?callable $logger = null): void + { + $config = self::resolveGiteeConfig(); + if (empty($config['enabled']) || empty($config['configured'])) { + return; + } + + $baseRemoteDir = trim((string)$config['root'], '/'); + foreach (['summary.json', 'summary.html', 'gitee-links.json'] as $fileName) { + $filePath = (string)($summary['run_root'] ?? '') . '/' . $fileName; + if (!is_file($filePath)) { + continue; + } + $remotePath = trim($baseRemoteDir . '/' . $fileName, '/'); + self::upsertGiteeFile($config, $remotePath, (string)file_get_contents($filePath), 'spider md metadata update ' . $fileName); + self::log($logger, '已更新 Gitee 元数据：' . $fileName); + } + } + + protected static function syncMetadataToGithub(array $summary, ?callable $logger = null): void + { + $config = self::resolveGithubConfig(); + if (empty($config['enabled']) || empty($config['configured'])) { + return; + } + + $baseRemoteDir = trim((string)$config['root'], '/'); + foreach (['summary.json', 'summary.html', 'github-links.json'] as $fileName) { + $filePath = (string)($summary['run_root'] ?? '') . '/' . $fileName; + if (!is_file($filePath)) { + continue; + } + $remotePath = trim($baseRemoteDir . '/' . $fileName, '/'); + self::upsertGithubFile($config, $remotePath, (string)file_get_contents($filePath), 'spider md metadata update ' . $fileName); + self::log($logger, '已更新 GitHub 元数据：' . $fileName); + } + } + protected static function upsertGithubFile(array $config, string $remotePath, string $content, string $message): array { $existing = self::githubRequest( @@ -1255,12 +1400,6 @@ class DomainSpiderMdRunHelper if (!function_exists('curl_init')) { throw new \RuntimeException('当前 PHP 环境缺少 curl 扩展，无法推送 GitHub'); } - - $curl = curl_init(); - if ($curl === false) { - throw new \RuntimeException('初始化 GitHub 请求失败'); - } - $headers = [ 'Accept: application/vnd.github+json', 'Authorization: Bearer ' . (string)$config['token'], @@ -1268,45 +1407,80 @@ class DomainSpiderMdRunHelper 'User-Agent: SEONexus-SpiderMd', ]; - curl_setopt_array($curl, [ - CURLOPT_URL => $url, - CURLOPT_RETURNTRANSFER => true, - CURLOPT_TIMEOUT => 45, - CURLOPT_CUSTOMREQUEST => $method, - CURLOPT_HTTPHEADER => $headers, - ]); + $maxAttempts = 3; + $lastError = ''; + for ($attempt = 1; $attempt <= $maxAttempts; $attempt++) { + $curl = curl_init(); + if ($curl === false) { + throw new \RuntimeException('初始化 GitHub 请求失败'); + } - if (!empty($payload)) { - curl_setopt($curl, CURLOPT_POSTFIELDS, json_encode($payload, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES)); + curl_setopt_array($curl, [ + CURLOPT_URL => $url, + CURLOPT_RETURNTRANSFER => true, + CURLOPT_TIMEOUT => 90, + CURLOPT_CUSTOMREQUEST => $method, + CURLOPT_HTTPHEADER => $headers, + ]); + + if (!empty($payload)) { + curl_setopt($curl, CURLOPT_POSTFIELDS, json_encode($payload, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES)); + } + + $response = curl_exec($curl); + $errno = curl_errno($curl); + $error = curl_error($curl); + $status = (int)curl_getinfo($curl, CURLINFO_HTTP_CODE); + curl_close($curl); + + if ($errno !== 0) { + $lastError = 'GitHub 请求失败：' . $error; + if ($attempt < $maxAttempts) { + usleep(300000 * $attempt); + continue; + } + throw new \RuntimeException($lastError); + } + + $decoded = json_decode((string)$response, true); + if ($allowNotFound && $status === 404) { + return []; + } + + if ($status >= 400) { + $message = ''; + if (is_array($decoded)) { + $message = trim((string)($decoded['message'] ?? '')); + if ($message === '') { + $message = json_encode($decoded, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES) ?: ''; + } + } else { + $message = trim((string)$response); + } + if ($message === '') { + $message = 'HTTP ' . $status . '，响应体为空'; + } else { + $message = 'HTTP ' . $status . '，' . $message; + } + + $lastError = 'GitHub 请求返回异常：' . $message; + if (in_array($status, [429, 500, 502, 503, 504], true) && $attempt < $maxAttempts) { + usleep(500000 * $attempt); + continue; + } + throw new \RuntimeException($lastError); + } + + if (is_array($decoded) && isset($decoded['content']) && is_array($decoded['content'])) { + $decoded['html_url'] = (string)($decoded['content']['html_url'] ?? ''); + $decoded['download_url'] = (string)($decoded['content']['download_url'] ?? ''); + $decoded['sha'] = (string)($decoded['content']['sha'] ?? ($decoded['sha'] ?? '')); + } + + return is_array($decoded) ? $decoded : []; } - $response = curl_exec($curl); - $errno = curl_errno($curl); - $error = curl_error($curl); - $status = (int)curl_getinfo($curl, CURLINFO_HTTP_CODE); - curl_close($curl); - - if ($errno !== 0) { - throw new \RuntimeException('GitHub 请求失败：' . $error); - } - - $decoded = json_decode((string)$response, true); - if ($allowNotFound && $status === 404) { - return []; - } - - if ($status >= 400) { - $message = is_array($decoded) ? (string)($decoded['message'] ?? json_encode($decoded, JSON_UNESCAPED_UNICODE)) : (string)$response; - throw new \RuntimeException('GitHub 请求返回异常：' . $message); - } - - if (is_array($decoded) && isset($decoded['content']) && is_array($decoded['content'])) { - $decoded['html_url'] = (string)($decoded['content']['html_url'] ?? ''); - $decoded['download_url'] = (string)($decoded['content']['download_url'] ?? ''); - $decoded['sha'] = (string)($decoded['content']['sha'] ?? ($decoded['sha'] ?? '')); - } - - return is_array($decoded) ? $decoded : []; + throw new \RuntimeException($lastError !== '' ? $lastError : 'GitHub 请求失败：未知异常'); } protected static function buildGithubHtmlUrl(array $config, string $remotePath): string diff --git a/docs/2026-04-18-GPT模板SEO公共主线汇总.md b/docs/2026-04-18-GPT模板SEO公共主线汇总.md index ec412f6..0c9d9f3 100644 --- a/docs/2026-04-18-GPT模板SEO公共主线汇总.md +++ b/docs/2026-04-18-GPT模板SEO公共主线汇总.md @@ -100,3 +100,16 @@ GPT 模板专属工作区： 6. 今天补充的后台体验修正： - `蜘蛛池MD` 弹窗现在支持主表右侧固定 `一键复制MD` 按钮 - 复制内容已恢复为旧习惯：只复制 `.md` 直链，不复制 `summary.json / summary.html` +7. 今天补充的 GitHub 推送稳定性修复： + - 已确认 `Spider MD` 后台显示的“GitHub 部分失败”并不等于全量失败，之前真实原因是 GitHub 中途短暂异常，旧日志未带出 `HTTP 状态码` + - `DomainSpiderMdRunHelper` 已补强 GitHub 请求层： + - 失败时记录 `HTTP 状态码` + - 空响应时明确显示“响应体为空” + - 对 `429 / 500 / 502 / 503 / 504` 增加自动重试 + - 单次 GitHub 请求超时已放宽到 `90s` + - 本地 `storage/spider_md_runs` 目录权限已统一修正为 `www:www`，避免 `part-047.md` 这类文件因属主错误导致“还没推 GitHub 就先本地写失败” + - 当前已实测： + - GitHub 远端目录可一键清空 + - 本地活动缓存可同步清空 + - 重跑后 `part-001.md ~ part-047.md + summary.json + summary.html + github-links.json` 已完整成功推送 + - 当前 Spider MD GitHub 推送的稳定状态应以 `run_id = 202353_spider_md_d28664` 这轮为准 diff --git a/docs/2026-04-19-GPT模板蜘蛛日志复盘与代码判断.md b/docs/2026-04-19-GPT模板蜘蛛日志复盘与代码判断.md index 78b0a50..0721fab 100644 --- a/docs/2026-04-19-GPT模板蜘蛛日志复盘与代码判断.md +++ b/docs/2026-04-19-GPT模板蜘蛛日志复盘与代码判断.md @@ -238,6 +238,70 @@ 3. 今天先不动 GPT 模板业务代码 4. 如果明天 `googlebot / bingbot` 仍大面积只打 `robots/category` 且持续 `444`，再单独开一轮 `Nginx 蜘蛛放行策略` 优化 +## 10. 今日新增工程修复说明 + +今天除了日志判断，还额外处理了 `Spider MD -> GitHub` 这条工程链路，避免后续误把“推送工具异常”当成 SEO 或模板代码异常。 + +### 已确认的问题 + +1. 前一轮 `GitHub：部分失败` 不等于“没有推上” +2. 真实现象是： + - `part-001.md ~ part-022.md` 已上传 + - 后续某次 GitHub 接口中途异常 + - 旧代码只留下了空报错：`GitHub 请求返回异常：` + - 但 `summary.json / summary.html / github-links.json` 元数据又成功更新 +3. 所以前端会表现成： + - GitHub 上已经有一批 md 文件 + - 后台状态却还是 `partial_failed` + +### 已完成的修复 + +1. GitHub 请求层现在会明确记录： + - `HTTP 状态码` + - 空响应体 + - 原始异常信息 +2. 对 `429 / 500 / 502 / 503 / 504` 已增加自动重试 +3. GitHub 单次请求超时已从 `45s` 放宽到 `90s` +4. `storage/spider_md_runs` 目录属主已修正为 `www:www` + - 避免出现 `part-047.md: Permission denied` 这种“本地先失败，远端还没开始”的误判 + +### 最终实测结果 + +修复后再次重跑，成功 run： + +- `run_id = 202353_spider_md_d28664` +- `github_push_status = success` +- `github_files_count = 50` + +对应含义： + +1. `part-001.md ~ part-047.md` +2. `summary.json` +3. `summary.html` +4. `github-links.json` + +已经全部完整推送成功。 + +### 对 SEO 判断的影响 + +这条修复属于： + +- 工程稳定性修复 +- 运维链路修复 +- 后台日志可读性修复 + +不属于： + +- GPT 模板业务层 SEO 结构调整 +- 影视页面内容策略变化 +- 蜘蛛放行策略变化 + +所以今天的 SEO 主判断不变： + +1. GPT 模板仍以“继续观察蜘蛛回访”为主 +2. 今天不需要因为 Spider MD 这次异常去大改模板业务代码 +3. Spider MD 问题已经单独收口，不应再干扰明天蜘蛛日志判断 + ## 10. 一句话总结 今天的 GPT 模板 SEO 进度总体仍符合“修复后进入正向回访期”的预期，但还没有进入理想放量阶段；当前更该继续看日志和入口链路，而不是急着再改一轮模板业务代码。