123

code/app/task/crawler/zw630/Scheduler.php

@@ -0,0 +1,213 @@
+<?php
+
+declare(strict_types=1);
+
+namespace app\task\crawler\zw630;
+
+use app\task\crawler\zw630\page\NovelPage;
+use app\task\crawler\zw630\page\Site;
+use microserver\QueueManage;
+
+/**
+ * @mixin think\Model
+ */
+class Scheduler
+{
+    /**
+     * Undocumented variable
+     *
+     * @var \microserver\QueueManage
+     */
+    protected $QueueManage = NULL;
+
+    public function __construct()
+    {
+        $this->QueueManage = new QueueManage();
+    }
+
+    /**
+     * 全局扫描任务
+     *
+     * @return void
+     */
+    public function crawlFull()
+    {
+        $intSliceSize = 6;
+
+        $arrFilter = [];
+        // for ($intNId = 1; $intNId <= 531208; $intNId++) {
+        // for ($intNId = 1; $intNId <= 10000; $intNId++) {
+        for ($intNId = 1; $intNId <= 20; $intNId++) {
+
+            $arrFilter[] = [
+                'n_source_id' => $intNId,
+                'page' => 0,
+            ];
+
+            if ($intNId % $intSliceSize == 0) {
+                $this->fetchNovel($arrFilter);
+                $arrFilter = [];
+            }
+        }
+        if (!empty($arrFilter)) {
+            $this->fetchNovel($arrFilter);
+        }
+        return;
+
+
+
+        // $arrTest =  [
+        //     'n_id' => 9973,
+        //     'filter' => [
+        //         [
+        //             'name' => '第十四章 神兽空青蛇妖',
+        //             'uri' => '/shu/1/33.html',
+        //             'c_sort_num' => 32,
+        //             'c_source_id' => 33,
+        //             'c_page' => 0,
+        //         ],
+        //     ]
+        // ];
+
+        // $arrTask = [
+        //     'callback' => [self::class, 'getChapterInfo'],
+        //     'data' => $arrTest
+        // ];
+        // $this->QueueManage->set($arrTask);
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @param array $arrFilter
+     * @return void
+     */
+    public function fetchNovel(array $arrFilter) //(int $intNSourceId, int $intPage = 0)
+    {
+        $arrTask = [
+            'callback' => [self::class, 'getNovelInfo'],
+            'data' => $arrFilter,
+            // 'data' => [
+            //     'n_source_id' => $intNSourceId,
+            //     'page' => $intPage,
+            // ]
+        ];
+        $this->QueueManage->set($arrTask);
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @param integer $intNId
+     * @param integer $intPage
+     * @param array $arrChapterList
+     * @return void
+     */
+    public function fetchChapter(int $intNId, int $intPage, array $arrChapterList = [])
+    {
+        $intSliceSize = 6;
+        $intPageSize = 100;
+
+        $arrTask = [
+            'callback' => [self::class, 'getChapterInfo'],
+            'data' => [
+                'n_id' => $intNId,
+                // 'filter' => $arrFilter,
+            ]
+        ];
+        $arrFilter = [];
+        foreach ($arrChapterList as $intKey => $arrChapter) {
+            $arrChapter['c_sort_num'] = $intPage * $intPageSize + $intKey + 1;
+            $arrChapter['c_source_id'] = supperExtractFilename($arrChapter['uri']);
+            $arrChapter['c_page'] = 0;
+
+            $arrFilter[] = $arrChapter;
+            if ($intKey % $intSliceSize == 0) {
+                $arrTask['data']['filter'] =  $arrFilter;
+                $this->QueueManage->set($arrTask);
+                $arrFilter = [];
+            }
+        }
+
+        if (!empty($arrFilter)) {
+            $arrTask['data']['filter'] =  $arrFilter;
+            $this->QueueManage->set($arrTask);
+        }
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @param array $arrFilter
+     * @return void
+     */
+    public function getNovelInfo(array $arrFilter)
+    {
+        $Site = new Site;
+
+        $arrNovelPageList = $Site->getNovelPageList($arrFilter);
+
+        $arrCover = [];
+        foreach ($arrNovelPageList as $NovelPage) {
+            $arrNovel = $NovelPage->getNovelInfo();
+            if (empty($arrNovel)) {
+                continue;
+            }
+
+
+            $arrNextPageData = $NovelPage->getNextPageArgs();
+            if (!empty($arrNextPageData)) {
+                $this->fetchNovel([
+                    [
+                        'n_source_id' => $arrNextPageData['n_source_id'],
+                        'page' =>  $arrNextPageData['page'],
+                    ]
+                ]);
+            }
+
+            // $arrCover[] = $NovelPage->getDownCoverInfo();
+            // $arrChapterList = $NovelPage->getChapters();
+
+            // $this->fetchChapter($arrNovel['n_id'], $NovelPage->getData()['page'], $arrChapterList);
+        }
+        if ($arrCover) {
+            $Site->downRemoteImgToLocal($arrCover);
+        }
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @param array $arrData
+     * @return void
+     */
+    public function getChapterInfo(array $arrData)
+    {
+        $Site = new Site;
+
+        $arrChapterPageList = $Site->getChapterPageList($arrData['filter']);
+
+        $arrNextPageFilterList = [];
+        foreach ($arrChapterPageList as $ChapterPage) {
+
+            $ChapterPage->saveChapter($arrData['n_id']);
+
+            $arrNextPageFilter = $ChapterPage->getNextPage();
+
+            if (!empty($arrNextPageFilter)) {
+                $arrNextPageFilterList[] = $arrNextPageFilter;
+            }
+        }
+
+        if (!empty($arrNextPageFilterList)) {
+
+            $arrTask = $arrData;
+            $arrTask['filter'] = $arrNextPageFilterList;
+            $arrTask = [
+                'callback' => [self::class, 'getChapterInfo'],
+                'data' => $arrTask
+            ];
+            $this->QueueManage->set($arrTask);
+        }
+    }
+}

code/app/task/crawler/zw630/page/BasePage.php

@@ -0,0 +1,258 @@
+<?php
+
+declare(strict_types=1);
+
+namespace app\task\crawler\zw630\page;
+
+use db\mongo\MongoBase;
+use ddl\DDLManage;
+use GuzzleHttp\Client;
+use microserver\QueueManage;
+use QL\QueryList;
+use Psr\Http\Message\ResponseInterface;
+
+/**
+ * @mixin think\Model
+ */
+abstract class BasePage
+{
+    /**
+     * Undocumented variable
+     *
+     * @var string
+     */
+    // protected $strDomain;
+
+    /**
+     * Undocumented variable
+     *
+     * @var string
+     */
+    protected $strUri;
+
+    /**
+     * Undocumented variable
+     *
+     * @var MongoBase
+     */
+    protected $MongoBase;
+
+    /**
+     * Undocumented variable
+     *
+     * @var \GuzzleHttp\Client
+     */
+    protected $Client = NULL;
+
+    /**
+     * Undocumented variable
+     *
+     * @var \microserver\QueueManage
+     */
+    protected $QueueManage = NULL;
+
+
+    /**
+     * Undocumented variable
+     *
+     * @var array
+     */
+    protected $arrCaiJiPeiZhi = [];
+
+    /**
+     * Undocumented variable
+     *
+     * @var string
+     */
+    protected $strContent;
+
+    /**
+     * Undocumented variable
+     *
+     * @var QueryList
+     */
+    protected $QueryList;
+
+    /**
+     * Undocumented variable
+     *
+     * @var array
+     */
+    protected $arrData = [];
+
+    /**
+     * Undocumented variable
+     *
+     * @var bool
+     */
+    protected $boolIsMobile;
+
+    /**
+     * Undocumented variable
+     *
+     * @var ResponseInterface
+     */
+    private $Response;
+
+
+    /**
+     * Undocumented function
+     *
+     * @param string $strUri
+     */
+    public function __construct(Site $Site, string $strUri)
+    {
+        $this->setSite($Site);
+
+        $this->strUri = $strUri;
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @param array $arrData
+     * @return void
+     */
+    public function setData(array $arrData)
+    {
+        $this->arrData = $arrData;
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @return array
+     */
+    public function getData(): array
+    {
+        return $this->arrData;
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @return QueueManage
+     */
+    public function getQueueManage(): QueueManage
+    {
+        if ($this->QueueManage == NULL) {
+            $this->QueueManage = QueueManage::getInstance();
+        }
+
+        return $this->QueueManage;
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @return MongoBase
+     */
+    public function getMongoBase(): MongoBase
+    {
+        if ($this->MongoBase == NULL) {
+            DDLManage::load('MongoBase');
+            $this->MongoBase = MongoBase::getInstance();
+        }
+
+        return $this->MongoBase;
+    }
+
+    public function getQueryList(): QueryList
+    {
+        if ($this->QueryList == NULL) {
+            $this->QueryList = QueryList::html($this->getContent());
+        }
+
+        return $this->QueryList;
+    }
+
+
+    /**
+     * Undocumented function
+     *
+     * @return string
+     */
+    public function getContent(): string
+    {
+        if ($this->strContent == NULL) {
+            $this->Response =  $this->getClient()->get($this->strUri);
+            $this->strContent =  $this->Response->getBody()->getContents();
+        }
+
+        return $this->strContent;
+    }
+
+    /**
+     * force set page content
+     *
+     * @param string $strContent
+     * @return void
+     */
+    public function setContent($strContent)
+    {
+        $this->strContent = $strContent;
+    }
+
+
+    /**
+     * Undocumented function
+     *
+     * @return boolean
+     */
+    public function isMobile($strMark = 'content="mobile"'): bool
+    {
+        if ($this->boolIsMobile === NULL) {
+            $this->boolIsMobile = is_numeric(strpos($this->getContent(), $strMark));
+        }
+        return $this->boolIsMobile;
+    }
+
+
+    /**
+     * Undocumented function
+     *
+     * @return Client
+     */
+    protected function getClient(): Client
+    {
+        if ($this->Client == NULL) {
+            $this->Client = $this->getSite()->getClient();
+        }
+        return $this->Client;
+    }
+
+
+    /**
+     * Undocumented variable
+     *
+     * @var Site
+     */
+    protected  $Site;
+
+    public function setSite(Site $Site)
+    {
+        $this->Site = $Site;
+    }
+
+    public function getSite()
+    {
+        return  $this->Site;
+    }
+
+    /**
+     * Undocumented variable
+     *
+     * @var boolean
+     */
+    protected $boolIsFirstSave = false;
+
+    /**
+     * Undocumented function
+     *
+     * @return boolean
+     */
+    public function getIsFirstSave(): bool
+    {
+        return $this->boolIsFirstSave;
+    }
+}

code/app/task/crawler/zw630/page/ChapterPage.php

@@ -0,0 +1,139 @@
+<?php
+
+declare(strict_types=1);
+
+namespace app\task\crawler\zw630\page;
+
+use storage\StorageCore;
+
+class ChapterPage extends BasePage
+{
+    /**
+     * Undocumented variable
+     *
+     * @var string
+     */
+    protected $strChapterContent = '';
+
+    /**
+     * Undocumented function
+     *
+     * @return array
+     */
+    public function getChapterInfo(): array
+    {
+        if ($this->arrChapterInfo === NULL) {
+
+            $this->arrChapterInfo = [];
+
+            $strPattern = "/qsbs\.bb\('([^']*)'\)/";
+
+            preg_match_all($strPattern,  $this->getContent(), $arrMatches);
+
+            $strContent = '';
+
+            foreach ($arrMatches[1] as $str) {
+                $strContent  .= base64_decode($str);
+            }
+
+            // $this->arrChapterInfo['c_content'] = $strContent;
+            $this->strChapterContent =  $strContent;
+
+            $this->arrChapterInfo['c_page_title'] = $this->getQueryList()->find('title')->text();
+            $this->arrChapterInfo['c_page_keywords'] = $this->getQueryList()->find('meta[name="keywords"]')->attr('content');
+            $this->arrChapterInfo['c_page_description'] = $this->getQueryList()->find('meta[name="description"]')->attr('content');
+        }
+
+        return $this->arrChapterInfo;
+    }
+
+    /**
+     * Undocumented variable
+     *
+     * @var array
+     */
+    protected $arrChapterInfo;
+
+    public function saveChapter(int $intNId): array
+    {
+
+        try {
+            $this->getChapterInfo();
+
+            $strUUId = 'zw630-chapter-' . $this->arrData['c_source_id'] . '-' . $this->arrData['c_page'];
+            $Col = $this->getMongoBase()->getDb()->selectCollection('chapter');
+
+            $arrExisting = $Col->findOne(['c_site_uuid' => $strUUId]);
+
+            if ($arrExisting) {
+
+                $arrExisting = (array) $arrExisting;
+                $this->arrChapterInfo = array_merge($arrExisting,   $this->getChapterInfo());
+
+                $this->arrChapterInfo['updated_at'] = new \MongoDB\BSON\UTCDateTime(); // 添加更新时间
+                $Col->updateOne(
+                    ['c_site_uuid' => $strUUId],
+                    ['$set' =>   $this->arrChapterInfo],
+                    // ['upsert' => false]
+                );
+            } else {
+
+                $this->arrChapterInfo['n_id'] =  $intNId;
+
+                $this->arrChapterInfo['name'] =  $this->arrData['name'];
+
+                $this->arrChapterInfo['c_sort_num'] =  $this->arrData['c_sort_num'];
+                $this->arrChapterInfo['c_page'] =  $this->arrData['c_page'];
+
+                $this->arrChapterInfo['c_source_id'] = $this->arrData['c_source_id'];
+                $this->arrChapterInfo['c_source_uri'] =  $this->arrData['uri'];
+                $this->arrChapterInfo['c_site_name'] = '恋上你中文';
+                $this->arrChapterInfo['c_site_code'] = 'zw630';
+                $this->arrChapterInfo['c_site_uuid'] = $strUUId;
+                $this->arrChapterInfo['c_content_path'] = '/novel/' . ($intNId % 100) . '/' . $intNId . '/' . $this->arrChapterInfo['c_sort_num'] . '_' .  $this->arrChapterInfo['c_page'] . '.txt';
+
+                $this->getMongoBase()->insertOne('chapter',   $this->arrChapterInfo);
+
+                if (!StorageCore::getInstance()->has($this->arrChapterInfo['c_content_path'])) {
+                    StorageCore::getInstance()->set($this->arrChapterInfo['c_content_path'], $this->strChapterContent);
+                }
+
+                $this->boolIsFirstSave = true;
+            }
+            return   $this->arrChapterInfo;
+        } catch (\MongoDB\Exception\Exception $e) {
+            throw new \Exception("Upsert failed: " . $e->getMessage());
+        }
+    }
+
+    protected $arrNextPage;
+
+    /**
+     * Undocumented function
+     *
+     * @return array
+     */
+    public function getNextPage(): array
+    {
+        if ($this->arrNextPage === NULL) {
+            $this->arrNextPage = [];
+
+            if ($this->isMobile()) {
+                $strPattern = "/var\s+hhekgsv\s*=\s*['\"](.*?)['\"]/";
+                preg_match_all($strPattern, $this->getContent(), $arrMatches);
+                $strNextPageUri = end($arrMatches[1]) ?: '';
+            } else {
+                $strHref = $this->getQueryList()->find('.read_btn a:contains("下一章")')->attr('href');
+                $strNextPageUri = $strHref ?: '';
+            }
+
+            if (is_numeric(strpos($strNextPageUri, '_'))) {
+                $this->arrNextPage = $this->arrData;
+                $this->arrNextPage['c_page'] += 1;
+                $this->arrNextPage['uri'] = $strNextPageUri;
+            }
+        }
+
+        return $this->arrNextPage;
+    }
+}

code/app/task/crawler/zw630/page/NovelPage.php

@@ -0,0 +1,304 @@
+<?php
+
+declare(strict_types=1);
+
+namespace app\task\crawler\zw630\page;
+
+use ddl\DDLManage;
+use QL\QueryList;
+
+
+class NovelPage extends BasePage
+{
+
+    /**
+     * Undocumented variable
+     *
+     * @var array
+     */
+    protected $arrNovelInfo = [];
+
+    /**
+     * Undocumented variable
+     *
+     * @var array
+     */
+    protected $arrRelatedNovel = [];
+
+    /**
+     * Undocumented variable
+     *
+     * @var array
+     */
+    protected $arrForgeNovel = [];
+
+    private function findOrCreateNovel(int $intNSourceId, string $strName): array
+    {
+
+        $strUUId = 'zw630-' . $intNSourceId;
+        $Col = $this->getMongoBase()->getDb()->selectCollection('novel');
+
+        $arrExisting = $Col->findOne(['n_site_uuid' => $strUUId]);
+
+        if ($arrExisting) {
+            $arrNovelInfo = (array) $arrExisting;
+        } else {
+            $arrNovelInfo['og_title'] = $strName;
+            $arrNovelInfo['og_novel_book_name'] = $strName;
+            $arrNovelInfo['og_novel_pin_yin'] = zhToPinYin($strName);
+            $arrNovelInfo['n_site_name'] = '恋上你中文';
+            $arrNovelInfo['n_site_code'] = 'zw630';
+            $arrNovelInfo['n_site_uuid'] = $strUUId;
+            $arrNovelInfo['n_source_id'] = $intNSourceId;
+            $arrNovelInfo['n_level'] = 1;
+            $arrNovelInfo['n_id'] =  $this->getMongoBase()->getNextSequence('novel');
+            $this->getMongoBase()->insertOne('novel',   $arrNovelInfo);
+        }
+        return $arrNovelInfo;
+    }
+
+    /**
+     * Find or create a category by name, handling concurrency safely.
+     *
+     * @param string $strCategory
+     * @return array|null The category document (existing or newly created)
+     */
+    public function findOrCreateCategory(string $strCategory)
+    {
+        if (empty($strCategory)) {
+            return false;
+        }
+        $Col = $this->getMongoBase()->getDb()->selectCollection('category');
+
+        $arrCategory = ['ca_name' => $strCategory];
+        try {
+            $Result = $Col->findOneAndUpdate(
+                $arrCategory,
+                ['$set' => $arrCategory],
+                [
+                    'upsert' => true,
+                    'returnDocument' => \MongoDB\Operation\FindOneAndUpdate::RETURN_DOCUMENT_AFTER
+                ]
+            );
+            return $Result;
+        } catch (\MongoDB\Driver\Exception\Exception $e) {
+            return null;
+        }
+    }
+
+
+    /**
+     * Undocumented function
+     *
+     * @return array
+     */
+    public function getNovelInfo($boolSave = true): array
+    {
+        if (empty($this->arrNovelInfo)) {
+            $this->getQueryList()->find('meta[property]')->each(function ($Item) {
+                $strKey = str_replace(':', '_', $Item->attr('property'));
+                $this->arrNovelInfo[$strKey] = $Item->attr('content');
+            });
+
+            $this->arrNovelInfo['n_page_title'] = $this->getQueryList()->find('title')->text();
+            $this->arrNovelInfo['n_page_keywords'] = $this->getQueryList()->find('meta[name="keywords"]')->attr('content');
+            $this->arrNovelInfo['n_page_description'] = $this->getQueryList()->find('meta[name="description"]')->attr('content');
+
+            $this->getQueryList()->find('.first_txt')->eq(0)->find("a")->map(function ($A) {
+                $strName =  $A->text();
+                $strUri =  $A->attr('href');
+                if (is_numeric(strpos($strUri, 'shu'))) {
+                    $intNSourceId = (int)supperExtractFilename($strUri);
+                    $arrNovelInfo = $this->findOrCreateNovel($intNSourceId,  $strName);
+                    $this->arrRelatedNovel[] = [
+                        "n_id" => $arrNovelInfo["n_id"],
+                        "n_source_id" => $arrNovelInfo["n_source_id"],
+                        "og_title" => $arrNovelInfo["og_title"],
+                    ];
+                } elseif (is_numeric(strpos($strUri, 'kan'))) {
+                    $intNForgeId = (int)supperExtractFilename($strUri);
+                    $this->arrForgeNovel[] = [
+                        "n_forge_id" => $intNForgeId,
+                        "n_forge_title" => $strName,
+                    ];
+                }
+            });
+            $this->arrNovelInfo['n_related_novel'] = $this->arrRelatedNovel;
+            $this->arrNovelInfo['n_forge_novel'] = $this->arrForgeNovel;
+        }
+
+
+        if ($boolSave && !empty($this->arrNovelInfo)) {
+            $this->saveNovelInfo();
+
+            $this->findOrCreateCategory($this->arrNovelInfo['og_novel_category']);
+        }
+        return  $this->arrNovelInfo;
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @return void
+     */
+    public function saveNovelInfo()
+    {
+        try {
+            $strUUId = 'zw630-' . $this->arrData['n_source_id'];
+            $Col = $this->getMongoBase()->getDb()->selectCollection('novel');
+
+            $arrExisting = $Col->findOne(['n_site_uuid' => $strUUId]);
+
+            if ($arrExisting) {
+
+                $arrExisting = (array) $arrExisting;
+                $this->arrNovelInfo = array_merge($arrExisting,   $this->arrNovelInfo);
+
+                if (!isset($this->arrNovelInfo['og_novel_pin_yin']) || empty($this->arrNovelInfo['og_novel_pin_yin'])) {
+                    $this->arrNovelInfo['og_novel_pin_yin'] = zhToPinYin($this->arrNovelInfo['og_novel_book_name'] ?? '');
+                }
+
+                if (!isset($this->arrNovelInfo['n_cover_path']) || empty($this->arrNovelInfo['n_cover_path'])) {
+                    $this->arrNovelInfo['n_cover_path'] = 'img/' . date('Y/m/d', time() - 24 * 3600 * 14) . '/' . $this->arrNovelInfo['n_id'] . '.jpg';
+                }
+
+                $this->arrNovelInfo['updated_at'] = new \MongoDB\BSON\UTCDateTime();
+                $Col->updateOne(
+                    ['n_site_uuid' => $strUUId],
+                    ['$set' =>   $this->arrNovelInfo],
+                    // ['upsert' => false]
+                );
+            } else {
+                $this->arrNovelInfo['og_novel_pin_yin'] = zhToPinYin($this->arrNovelInfo['og_novel_book_name'] ?? '');
+                $this->arrNovelInfo['n_site_name'] = '恋上你中文';
+                $this->arrNovelInfo['n_site_code'] = 'zw630';
+                $this->arrNovelInfo['n_site_uuid'] = $strUUId;
+                $this->arrNovelInfo['n_source_id'] = $this->arrData['n_source_id'];
+                $this->arrNovelInfo['n_id'] =  $this->getMongoBase()->getNextSequence('novel');
+                $this->arrNovelInfo['n_level'] = 1;
+                $this->arrNovelInfo['n_cover_path'] = '/img/' . date('Y/m/d', time() - 24 * 3600 * 14) . '/' . $this->arrNovelInfo['n_id'] . '.jpg';
+                $this->getMongoBase()->insertOne('novel',   $this->arrNovelInfo);
+
+                $this->boolIsFirstSave = true;
+            }
+            return   $this->arrNovelInfo;
+        } catch (\MongoDB\Exception\Exception $e) {
+            throw new \Exception("Upsert failed: " . $e->getMessage());
+        }
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @return array
+     */
+    public function getDownCoverInfo(): array
+    {
+        return [
+            'uri' => $this->arrNovelInfo['og_image'],
+            'local_path' => $this->arrNovelInfo['n_cover_path'],
+        ];
+    }
+
+    /**
+     * Undocumented variable
+     *
+     * @var array
+     */
+    protected $arrLatestChapters = NULL;
+
+    /**
+     * Undocumented variable
+     *
+     * @var array
+     */
+    protected $arrChapters = NULL;
+
+    public function getLatestChaptersList(): array
+    {
+        if ($this->arrLatestChapters === NULL) {
+            $this->arrLatestChapters = [];
+
+            if ($this->isMobile()) {
+                $strClass = '.chapter-list';
+            } else {
+                $strClass = '.section-list';
+            }
+
+            if ($this->arrData['page'] == 0) {
+                $this->getQueryList()->find($strClass)->eq(0)->find("a")->map(function ($A) {
+                    $this->arrLatestChapters[] = [
+                        'name' => $A->text(),
+                        'uri' => $A->attr('href'),
+                    ];
+                });
+            }
+        }
+        return $this->arrLatestChapters;
+    }
+
+    public function getChapters(): array
+    {
+        if ($this->arrChapters === NULL) {
+            $this->arrChapters = [];
+            if ($this->isMobile()) {
+                $strClass = '.chapter-list';
+            } else {
+                $strClass = '.section-list';
+            }
+
+            $intDivIndex = 0;
+            if ($this->arrData['page'] == 0) {
+                $intDivIndex = 1;
+            }
+
+            $this->getQueryList()->find($strClass)->eq($intDivIndex)->find("a")->map(function ($A) {
+                $this->arrChapters[] = [
+                    'name' => $A->text(),
+                    'uri' => $A->attr('href'),
+                ];
+            });
+        }
+        return $this->arrChapters;
+    }
+
+
+    protected $arrNextPageArgs = NULL;
+
+    /**
+     * Undocumented function
+     *
+     * @return array
+     */
+    public function getNextPageArgs(): array
+    {
+        $this->arrNextPageArgs = [];
+        if ($this->arrData['page'] == 0) {
+            $this->getLatestChaptersList();
+            $this->getChapters();
+
+            $arrLatestChapterFirstBlock =  count($this->arrLatestChapters) > 0 ? $this->arrLatestChapters[0] : null;
+            $arrLatestChapterSecondBlock = count($this->arrChapters) > 0 ?  $this->arrChapters[count($this->arrChapters) - 1] : null;
+
+            if (
+                !empty($arrLatestChapterFirstBlock) && !empty($arrLatestChapterSecondBlock)
+                && ($arrLatestChapterFirstBlock['uri'] != $arrLatestChapterSecondBlock['uri'])
+
+            ) {
+                $this->arrNextPageArgs =  [
+                    'n_source_id' => $this->arrData['n_source_id'],
+                    'page' => $this->arrData['page'] + 1,
+                ];
+            }
+        } else {
+            $strLastUri = $this->getQueryList()->find('select:first option:last')->attr('value');
+            if ($strLastUri != '' && $strLastUri != $this->strUri) {
+                $this->arrNextPageArgs = [
+                    'n_source_id' => $this->arrData['n_source_id'],
+                    'page' => $this->arrData['page'] + 1,
+                ];
+            }
+        }
+        return   $this->arrNextPageArgs;
+    }
+}

code/app/task/crawler/zw630/page/Site.php

@@ -0,0 +1,257 @@
+<?php
+
+declare(strict_types=1);
+
+namespace app\task\crawler\zw630\page;
+
+use GuzzleHttp\Client;
+use GuzzleHttp\Promise;
+use storage\StorageCore;
+
+/**
+ * @mixin think\Model
+ */
+class Site
+{
+    /**
+     * Undocumented variable
+     *
+     * @var Client
+     */
+    private $Client;
+
+    /**
+     * Undocumented variable
+     *
+     * @var string
+     */
+    protected $strDomain = 'https://www.630zw.org';
+
+    /**
+     * Undocumented function
+     */
+    public function __construct()
+    {
+        // $this->Client = new Client([
+        //     'http_version' => '1.1', // 强制用 HTTP/1.1
+        //     'base_uri' => $this->strDomain,
+        //     'http_errors' => false,
+        //     // 'version' => 2.0,
+        //     'timeout' => 160,
+        //     'connect_timeout' => 30,
+        //     'curl' => [
+        //         // CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_2_0,
+        //         CURLOPT_TCP_KEEPALIVE => 1,
+        //         CURLOPT_SSL_VERIFYPEER => true,
+        //     ],
+        //     'headers' => [
+        //         'User-Agent' => 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Mobile Safari/537.36 Edg/134.0.0.0',
+        //         'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
+        //         'Accept-Language' => 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
+        //         'Referer' => $this->strDomain,
+        //         'sec-ch-ua-mobile' => '?1',
+        //         'sec-ch-ua-platform' => '"Android"',
+        //         'sec-fetch-dest' => 'document',
+        //         'sec-fetch-mode' => 'navigate',
+        //         'sec-fetch-site' => 'same-origin',
+        //         'sec-fetch-user' => '?1',
+        //         'upgrade-insecure-requests' => '1',
+        //     ],
+        // ]);
+
+        $this->Client = new Client([
+            'http_version' => '1.1', // 强制用 HTTP/1.1，已有，保持
+            'base_uri' => $this->strDomain, // 基础域名，已有，保持
+            'http_errors' => false, // 不抛 HTTP 错误，已有，保持
+            'timeout' => 60, // 从 160 秒改为 60 秒
+            'connect_timeout' => 10, // 从 30 秒改为 10 秒
+            'curl' => [
+                CURLOPT_TCP_KEEPALIVE => 1, // 启用 TCP Keep-Alive，已有，保持
+                CURLOPT_TCP_KEEPIDLE => 120, // Keep-Alive 空闲时间
+                CURLOPT_TCP_KEEPINTVL => 60, // Keep-Alive 探测间隔
+                CURLOPT_SSL_VERIFYPEER => true, // 验证 SSL，已有，保持
+                CURLOPT_SSL_VERIFYHOST => 2, // 确保主机名匹配
+            ],
+            'headers' => [
+                // Mobile
+                // 'User-Agent' => 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Mobile Safari/537.36 Edg/134.0.0.0',
+                // 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
+
+                //PC
+                'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36 Edg/134.0.0.0',
+                'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
+
+                'Accept-Language' => 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
+                'Referer' => $this->strDomain,
+                'Connection' => 'keep-alive', // 显式启用 Keep-Alive
+
+
+                // Mobile
+                // 'sec-ch-ua-mobile' => '?1',
+                // 'sec-ch-ua-platform' => '"Android"',
+                // 'sec-fetch-dest' => 'document',
+                // 'sec-fetch-mode' => 'navigate',
+                // 'sec-fetch-site' => 'same-origin',
+                // 'sec-fetch-user' => '?1',
+          
+
+                // PC
+                'sec-ch-ua' => '"Chromium";v="134", "Not:A-Brand";v="24", "Microsoft Edge";v="134"',
+                'sec-ch-ua-mobile' => '?0',
+                'sec-ch-ua-platform' => '"Windows"',
+                'sec-fetch-dest' => 'document',
+                'sec-fetch-mode' => 'navigate',
+                'sec-fetch-site' => 'same-origin',
+                'sec-fetch-user' => '?1',
+
+                'upgrade-insecure-requests' => '1',
+            ],
+            'pool_size' => 40, // 并发连接池大小（视需求调整）
+        ]);
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @return Client
+     */
+    public function getClient(): Client
+    {
+        return $this->Client;
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @param string $strUri
+     * @return NovelPage
+     */
+    public function getNovelPage(string $strUri): NovelPage
+    {
+        return new NovelPage($this, $strUri);
+    }
+
+
+    /**
+     * Undocumented function
+     *
+     * @param string $strUri
+     * @return NovelPage[]
+     */
+    public function getNovelPageList(array $arrFilter): array
+    {
+
+        /** @var NovelPage[] */
+        $arrNovelPage = [];
+
+        foreach ($arrFilter as $intKey => $arrItem) {
+
+            if ($arrItem['page'] == 0) {
+                $arrItem['uri'] = sprintf('/shu/%s.html', $arrItem['n_source_id']);
+            } else {
+                $arrItem['uri'] = sprintf('/shu/%s_%s.html', $arrItem['n_source_id'], $arrItem['page']);
+            }
+            $arrFilter[$intKey]['uri'] = $arrItem['uri'];
+
+            $arrPromises[$intKey] = $this->getClient()->getAsync($arrItem['uri']);
+        }
+
+        $arrResults = Promise\Utils::settle($arrPromises)->wait();
+
+        $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
+            return $arrResult['state'] === 'fulfilled';
+        });
+
+        foreach ($arrSuccessfulResults as $intKey => $arrResult) {
+            $Response = $arrResult['value'];
+
+            $strContent =  $Response->getBody()->getContents();
+            if ($strContent == '索引文件不存在！') {
+                continue;
+            }
+
+            $NovelPage = new NovelPage($this, $arrItem['uri']);
+
+            $NovelPage->setContent($strContent);
+            $NovelPage->setData($arrFilter[$intKey]);
+
+            $arrNovelPage[] =  $NovelPage;
+        }
+
+
+        return $arrNovelPage;
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @param string $strUri
+     * @return ChapterPage[]
+     */
+    public function getChapterPageList(array $arrFilter): array
+    {
+
+        /** @var ChapterPage[] */
+        $arrChapterPage = [];
+
+        $arrPromises = [];
+        foreach ($arrFilter as $intKey => $arrItem) {
+            $arrPromises[$intKey] = $this->getClient()->getAsync($arrItem['uri']);
+        }
+
+        $arrResults = Promise\Utils::settle($arrPromises)->wait();
+
+        $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
+            return $arrResult['state'] === 'fulfilled';
+        });
+
+        foreach ($arrSuccessfulResults as $intKey => $arrResult) {
+            $Response = $arrResult['value'];
+
+            $strContent =  $Response->getBody()->getContents();;
+
+            $ChapterPage = new ChapterPage($this, $arrItem['uri']);
+
+            $ChapterPage->setContent($strContent);
+            $ChapterPage->setData($arrFilter[$intKey]);
+
+            $arrChapterPage[] =  $ChapterPage;
+        }
+
+
+        return $arrChapterPage;
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @param array $arrFilter
+     * @return void
+     */
+    public function downRemoteImgToLocal(array $arrFilter)
+    {
+        $StorageCore = StorageCore::getInstance();
+
+        $arrPromises = [];
+
+        foreach ($arrFilter as $intKey => $arrItem) {
+            if (!$StorageCore->has($arrItem['local_path'])) {
+                $arrPromises[$intKey] = $this->getClient()->getAsync($arrItem['uri']);
+            }
+        }
+
+        $arrResults = Promise\Utils::settle($arrPromises)->wait();
+
+        $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
+            return $arrResult['state'] === 'fulfilled';
+        });
+
+        foreach ($arrSuccessfulResults as $intKey => $arrResult) {
+            $Response = $arrResult['value'];
+
+            // $strContent =  $Response->getBody()->getContents();
+
+            $StorageCore->writeStream($arrFilter[$intKey]['local_path'], $Response->getBody()->detach());;
+        }
+    }
+}