debug

code/app/model/VideoClicksModel.php

@@ -74,7 +74,7 @@ class VideoClicksModel extends MongoModel
                         'v_category' => $Video->v_category,
                         'v_parent_category' => $Video->v_parent_category,
                         'v_category_en' => $Video->v_category_en,
-                        'v_parent_categoryen' => $Video->v_parent_category_en,
+                        'v_parent_category_en' => $Video->v_parent_category_en,
                     ],
                 ],
                 [

code/app/task/crawler/custom/Scheduler.php

@@ -6,7 +6,9 @@ namespace app\task\crawler\custom;
 
 use app\model\ChapterModel;
 use app\model\CustomCrawlerModel;
+use app\model\NovelModel;
 use app\task\crawler\custom\page\Site;
+use GuzzleHttp\Client;
 use microserver\QueueManage;
 use storage\StorageCore;
 
@@ -22,12 +24,20 @@ class Scheduler
      */
     protected $QueueManage = NULL;
 
+    /**
+     * Undocumented variable
+     *
+     * @var NovelModel
+     */
+    protected $NovelModel = NULL;
+
     /**
      * Undocumented function
      */
     public function __construct()
     {
-        $this->QueueManage = new QueueManage();
+        $this->QueueManage = QueueManage::getInstance();
+        $this->NovelModel =    NovelModel::getInstance();
     }
 
     /**
@@ -55,7 +65,7 @@ class Scheduler
             switch ($CustomCrawlerModel->cc_code) {
                 case 'PULL_WEN_XUE_GUAN':
                     $arrTask = [
-                        'callback' => [self::class, 'createCrawlWenXueGuanTask'],
+                        'callback' => [WenXueWang::class, 'createCrawlWenXueGuanTask'],
                         'data' => [
                             'cc_data' => $CustomCrawlerModel->cc_data
                         ],
@@ -67,10 +77,4 @@ class Scheduler
             $this->QueueManage->set($arrTask);
         }
     }
-
-    public function createCrawlWenXueGuanTask(array $arrData)
-    {
-        $arrCcData = json_decode($arrData['cc_data'], true);
-        print_r($arrCcData);
-    }
 }

code/app/task/crawler/custom/WenXueWang.php

@@ -0,0 +1,62 @@
+<?php
+
+declare(strict_types=1);
+
+namespace app\task\crawler\custom;
+
+use app\model\ChapterModel;
+use app\model\CustomCrawlerModel;
+use app\model\NovelModel;
+use app\task\crawler\custom\page\Site;
+use app\task\crawler\wenxuewang\Scheduler  as WxwScheduler;
+use app\task\crawler\zw\page\NovelPage;
+use GuzzleHttp\Client;
+use microserver\QueueManage;
+use storage\StorageCore;
+
+/**
+ * @mixin think\Model
+ */
+class WenXueWang
+{
+    /**
+     * Undocumented variable
+     *
+     * @var \microserver\QueueManage
+     */
+    protected $QueueManage = NULL;
+
+    /**
+     * Undocumented variable
+     *
+     * @var NovelModel
+     */
+    protected $NovelModel = NULL;
+
+    /**
+     * Undocumented function
+     */
+    public function __construct()
+    {
+        $this->QueueManage = QueueManage::getInstance();
+        $this->NovelModel =    NovelModel::getInstance();
+    }
+
+
+    public function createCrawlWenXueGuanTask(array $arrData)
+    {
+        $arrCcData = json_decode($arrData['cc_data'], true);
+        foreach ($arrCcData as $intNSourceId) {
+            $arrTask = [
+                'callback' => [WxwScheduler::class, 'getNovelInfo'],
+                'data' => [
+                    [
+                        'n_source_id' => $intNSourceId,
+                        'page' => 0,
+                    ]
+                ],
+            ];
+            $this->QueueManage->set($arrTask);
+        }
+    }
+}

code/app/task/crawler/wenxuewang/Scheduler.php

@@ -0,0 +1,343 @@
+<?php
+
+declare(strict_types=1);
+
+namespace app\task\crawler\wenxuewang;
+
+use app\model\ChapterModel;
+use app\task\crawler\wenxuewang\page\Site;
+use microserver\QueueManage;
+use storage\StorageCore;
+
+/**
+ * @mixin think\Model
+ */
+class Scheduler
+{
+    protected $intSlice = 2;
+
+    protected $intStartNSourceId = 1;
+
+    protected $intEndNSourceId = 500000;
+
+    /**
+     * Undocumented variable
+     *
+     * @var \microserver\QueueManage
+     */
+    protected $QueueManage = NULL;
+
+    /**
+     * Undocumented function
+     */
+    public function __construct()
+    {
+        $this->QueueManage = new QueueManage();
+        $this->init();
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @return void
+     */
+    public function init()
+    {
+        $this->intSlice = 2;
+    }
+
+    /**
+     * create a site-wide craw task
+     *
+     * @return void
+     */
+    public function crawlFull()
+    {
+        $intSliceSize = $this->intSlice;
+
+        $arrFilter = [];
+        for ($intNId = $this->intStartNSourceId; $intNId <= $this->intEndNSourceId; $intNId++) {
+            $arrFilter[] = [
+                'n_source_id' => $intNId,
+                'page' => 0,
+            ];
+
+            if ($intNId % $intSliceSize == 0) {
+                $this->fetchNovel($arrFilter);
+                $arrFilter = [];
+            }
+        }
+        if (!empty($arrFilter)) {
+            $this->fetchNovel($arrFilter);
+        }
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @param array $arrFilter
+     * @return void
+     */
+    public function fetchNovel(array $arrFilter)
+    {
+        $arrTask = [
+            'callback' => [static::class, 'getNovelInfo'],
+            'data' => $arrFilter,
+        ];
+
+        $this->QueueManage->set($arrTask);
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @param array $arrFilter
+     * @return void
+     */
+    public function getNovelInfo(array $arrFilter)
+    {
+        $Site = $this->getSite();
+
+        $arrNovelPageList = $Site->getNovelPageList($arrFilter);
+
+        $arrCover = [];
+        foreach ($arrNovelPageList as $NovelPage) {
+            // $arrNovel = $NovelPage->getNovelInfo();
+            // print_r($arrNovel);
+            // return;
+
+            $boolResult = $NovelPage->saveNovelInfo();
+            // return;
+
+            if ($boolResult == false) {
+                continue;
+            }
+
+            // $arrNextPageData = $NovelPage->getNextPageArgs();
+            // if (!empty($arrNextPageData)) {
+            //     $this->fetchNovel([
+            //         [
+            //             'n_source_id' => $arrNextPageData['n_source_id'],
+            //             'page' =>  $arrNextPageData['page'],
+            //         ]
+            //     ]);
+            // }
+
+            $arrCoverItem = $NovelPage->getDownCoverInfo();
+
+            if (!StorageCore::getInstance()->has($arrCoverItem['local_path'])) {
+                $arrCover[] = $arrCoverItem;
+            }
+
+            $arrChapterList = $NovelPage->getChapters();
+            $this->fetchChapter($NovelPage->intNId, $NovelPage->getData()['page'], $arrChapterList);
+        }
+
+        if ($arrCover) {
+            $Site->downRemoteImgToLocal($arrCover);
+        }
+    }
+
+    /**
+     * create a task to craw the latest chapter
+     *
+     * @return void
+     */
+    public function crawLatest()
+    {
+        $intSliceSize =  $this->intSlice;
+        $arrUri = [];
+        $strUri = '/list/lastupdate_%s_0_0_%s.html';
+
+        $intEndCategory = $intEndPage = 10;
+
+        for ($intCategoryKey = 0; $intCategoryKey < $intEndCategory; $intCategoryKey++) {
+            for ($intPage = 1; $intPage <= $intEndPage; $intPage++) {
+                $arrUri[] = sprintf($strUri, $intCategoryKey, $intPage);
+            }
+        }
+
+        $arrUri = array_chunk($arrUri, $intSliceSize);
+
+        $arrTask = [
+            'callback' => [static::class, 'getLatestChapter'],
+            'data' => [],
+        ];
+
+        foreach ($arrUri as $arrUriSlice) {
+            $arrTask['data'] = $arrUriSlice;
+            $this->QueueManage->set($arrTask);
+        }
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @param array $arrData
+     * @return void
+     */
+    public function getLatestChapter(array $arrFilter)
+    {
+        $Site = $this->getSite();
+
+        $arrLatestListPageList = $Site->getLatestListPageList($arrFilter);
+
+        $arrNSourceId = [];
+
+        foreach ($arrLatestListPageList as $LatestListPage) {
+            $arrResult = $LatestListPage->getNovelFilter();
+            $arrNSourceId = array_merge($arrNSourceId, $arrResult);
+        }
+
+        $arrNSourceId = array_unique($arrNSourceId);
+        $arrNSourceId  = array_values($arrNSourceId);
+
+        $intSliceSize =  $this->intSlice;
+        $arrFilter = [];
+
+        foreach ($arrNSourceId as $intKey => $intNSourceId) {
+            $arrFilter[] = [
+                'n_source_id' => $intNSourceId,
+                'page' => 0,
+            ];
+
+            if ($intKey % $intSliceSize == 0) {
+                $this->fetchNovel($arrFilter);
+                $arrFilter = [];
+            }
+        }
+
+        if (!empty($arrFilter)) {
+            $this->fetchNovel($arrFilter);
+        }
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @param integer $intNId
+     * @param integer $intPage
+     * @param array $arrChapterList
+     * @return void
+     */
+    public function fetchChapter(int $intNId, int $intPage, array $arrChapterList = [])
+    {
+        $intSliceSize =  $this->intSlice;
+        $intPageSize = 100;
+
+        $arrTask = [
+            'callback' => [static::class, 'getChapterInfo'],
+            'data' => [
+                'n_id' => $intNId,
+            ]
+        ];
+        $arrFilter = [];
+        foreach ($arrChapterList as $intKey => $arrChapter) {
+            $arrChapter['c_sort_num'] = $intPage * $intPageSize + $intKey + 1;
+            $arrChapter['c_source_id'] = supperExtractFilename($arrChapter['uri']);
+            $arrChapter['c_page'] = 0;
+
+            $arrFilter[] = $arrChapter;
+            if ($intKey % $intSliceSize == 0) {
+                $arrTask['data']['filter'] =  $arrFilter;
+                $this->QueueManage->set($arrTask);
+                $arrFilter = [];
+            }
+        }
+
+        if (!empty($arrFilter)) {
+            $arrTask['data']['filter'] =  $arrFilter;
+            $this->QueueManage->set($arrTask);
+        }
+    }
+
+    /**
+     * Undocumented variable
+     *
+     * @var Site
+     */
+    public $Site;
+
+    /**
+     * Undocumented function
+     *
+     * @return Site
+     */
+    public function getSite()
+    {
+        if ($this->Site == NULL) {
+            $this->Site = new Site;
+        }
+        return  $this->Site;
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @param array $arrData
+     * @return void
+     */
+    public function getChapterInfo(array $arrData)
+    {
+
+        $Site = $this->getSite();
+
+        $ChapterModel = ChapterModel::getInstance();
+
+        #FIXED 这里增加临时代码，将章节任务拆分为一个章节用一个进程请求，防止代理同时请求多个异常问题。
+        $arrTask = [
+            'callback' => [static::class, 'getChapterInfo'],
+            'data' => [
+                'n_id' => $arrData['n_id'],
+                'filter' => [],
+            ]
+        ];
+
+        foreach ($arrData['filter'] as $intKey => $arrChapterFilter) {
+            $strUUId = sprintf('%s-chapter-%s-%s',  $Site->getSiteCode(),  $arrChapterFilter['c_source_id'],  $arrChapterFilter['c_page']);
+            $arrChapter = $ChapterModel->findOne(['c_source_uuid' => $strUUId]);
+
+            if (!empty($arrChapter) && !empty($arrChapter['content'])) {
+                unset($arrData['filter'][$intKey]);
+            } else {
+                #FIXED 这里增加临时代码，将章节任务拆分为一个章节用一个进程请求，防止代理同时请求多个异常问题。
+
+                $arrFilter = [$arrChapterFilter];
+                $arrTask['data']['filter'] =  $arrFilter;
+                $this->QueueManage->set($arrTask);
+            }
+        }
+        if (count($arrData['filter']) > 1 || count($arrData['filter']) <= 0) {
+            #FIXED 这里增加临时代码，将章节任务拆分为一个章节用一个进程请求，防止代理同时请求多个异常问题。
+            return false;
+        }
+
+
+        $arrData['filter'] = array_values($arrData['filter']);
+
+        $arrChapterPageList = $Site->getChapterPageList($arrData['filter']);
+
+        $arrNextPageFilterList = [];
+        foreach ($arrChapterPageList as $ChapterPage) {
+
+            $ChapterPage->saveChapter($arrData['n_id']);
+
+            $arrNextPageFilter = $ChapterPage->getNextPage();
+
+            if (!empty($arrNextPageFilter)) {
+                $arrNextPageFilterList[] = $arrNextPageFilter;
+            }
+        }
+
+        if (!empty($arrNextPageFilterList)) {
+
+            $arrTask = $arrData;
+            $arrTask['filter'] = $arrNextPageFilterList;
+            $arrTask = [
+                'callback' => [static::class, 'getChapterInfo'],
+                'data' => $arrTask
+            ];
+            $this->QueueManage->set($arrTask);
+        }
+    }
+}

code/app/task/crawler/wenxuewang/page/BasePage.php

@@ -0,0 +1,251 @@
+<?php
+
+declare(strict_types=1);
+
+namespace app\task\crawler\wenxuewang\page;
+
+use db\mongo\MongoBase;
+use ddl\DDLManage;
+use GuzzleHttp\Client;
+use microserver\QueueManage;
+use QL\QueryList;
+use Psr\Http\Message\ResponseInterface;
+
+/**
+ * @mixin think\Model
+ */
+abstract class BasePage
+{
+    /**
+     * Undocumented variable
+     *
+     * @var string
+     */
+    // protected $strDomain;
+
+    /**
+     * Undocumented variable
+     *
+     * @var string
+     */
+    protected $strUri;
+
+    /**
+     * Undocumented variable
+     *
+     * @var MongoBase
+     */
+    protected $MongoBase;
+
+    /**
+     * Undocumented variable
+     *
+     * @var \GuzzleHttp\Client
+     */
+    protected $Client = NULL;
+
+    /**
+     * Undocumented variable
+     *
+     * @var \microserver\QueueManage
+     */
+    protected $QueueManage = NULL;
+
+
+    /**
+     * Undocumented variable
+     *
+     * @var array
+     */
+    protected $arrCaiJiPeiZhi = [];
+
+    /**
+     * Undocumented variable
+     *
+     * @var string
+     */
+    protected $strContent;
+
+    /**
+     * Undocumented variable
+     *
+     * @var QueryList
+     */
+    protected $QueryList;
+
+    /**
+     * Undocumented variable
+     *
+     * @var array
+     */
+    protected $arrData = [];
+
+    /**
+     * Undocumented variable
+     *
+     * @var bool
+     */
+    protected $boolIsMobile;
+
+    /**
+     * Undocumented variable
+     *
+     * @var ResponseInterface
+     */
+    private $Response;
+
+
+    /**
+     * Undocumented function
+     *
+     * @param string $strUri
+     */
+    public function __construct($Site, string $strUri)
+    {
+        $this->setSite($Site);
+
+        $this->strUri = $strUri;
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @return string
+     */
+    public function getUri(): string
+    {
+        return $this->strUri;
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @param array $arrData
+     * @return void
+     */
+    public function setData(array $arrData)
+    {
+        $this->arrData = $arrData;
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @return array
+     */
+    public function getData(): array
+    {
+        return $this->arrData;
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @return QueueManage
+     */
+    public function getQueueManage(): QueueManage
+    {
+        if ($this->QueueManage == NULL) {
+            $this->QueueManage = QueueManage::getInstance();
+        }
+
+        return $this->QueueManage;
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @return MongoBase
+     */
+    public function getMongoBase(): MongoBase
+    {
+        if ($this->MongoBase == NULL) {
+            DDLManage::load('MongoBase');
+            $this->MongoBase = MongoBase::getInstance();
+        }
+
+        return $this->MongoBase;
+    }
+
+    public function getQueryList(): QueryList
+    {
+        if ($this->QueryList == NULL) {
+            $this->QueryList = QueryList::html($this->getContent());
+        }
+
+        return $this->QueryList;
+    }
+
+
+    /**
+     * Undocumented function
+     *
+     * @return string
+     */
+    public function getContent(): string
+    {
+        if ($this->strContent == NULL) {
+            $this->Response =  $this->getClient()->get($this->strUri);
+            $this->strContent =  $this->Response->getBody()->getContents();
+        }
+
+        return $this->strContent;
+    }
+
+    /**
+     * force set page content
+     *
+     * @param string $strContent
+     * @return void
+     */
+    public function setContent($strContent)
+    {
+        $this->strContent = $strContent;
+    }
+
+
+    /**
+     * Undocumented function
+     *
+     * @return boolean
+     */
+    public function isMobile($strMark = 'content="mobile"'): bool
+    {
+        if ($this->boolIsMobile === NULL) {
+            $this->boolIsMobile = is_numeric(strpos($this->getContent(), $strMark));
+        }
+        return $this->boolIsMobile;
+    }
+
+
+    /**
+     * Undocumented function
+     *
+     * @return Client
+     */
+    protected function getClient(): Client
+    {
+        if ($this->Client == NULL) {
+            $this->Client = $this->getSite()->getClient();
+        }
+        return $this->Client;
+    }
+
+
+    /**
+     * Undocumented variable
+     *
+     * @var Site
+     */
+    protected  $Site;
+
+    public function setSite(Site $Site)
+    {
+        $this->Site = $Site;
+    }
+
+    public function getSite()
+    {
+        return  $this->Site;
+    }
+}

code/app/task/crawler/wenxuewang/page/ChapterPage.php

@@ -0,0 +1,226 @@
+<?php
+
+declare(strict_types=1);
+
+namespace app\task\crawler\wenxuewang\page;
+
+use app\model\ChapterModel;
+use storage\StorageCore;
+
+class ChapterPage extends BasePage
+{
+    /**
+     * Undocumented variable
+     *
+     * @var string
+     */
+    protected $strChapterContent = '';
+
+    /**
+     * Undocumented variable
+     *
+     * @var ChapterModel
+     */
+    protected $ChapterModel;
+
+    /**
+     * Undocumented function
+     *
+     * @return ChapterModel
+     */
+    public function getChapterModel(): ChapterModel
+    {
+        if ($this->ChapterModel == NULL) {
+            $this->ChapterModel = ChapterModel::getInstance();
+        }
+        return  $this->ChapterModel;
+    }
+
+    public $arrOptions = [
+        'typeMap' => [
+            'root' => 'array',
+            'document' => 'array',
+            'array' => 'array',
+        ],
+    ];
+
+    /**
+     * Undocumented function
+     *
+     * @return array
+     */
+    public function getChapterInfo(): array
+    {
+        if ($this->arrChapterInfo === NULL) {
+
+            $this->arrChapterInfo = [];
+
+            $strPattern = "/qsbs\.bb\('([^']*)'\)/";
+
+            preg_match_all($strPattern,  $this->getContent(), $arrMatches);
+
+            $strContent = '';
+
+            foreach ($arrMatches[1] as $str) {
+                $strContent  .= base64_decode($str);
+            }
+
+            $this->strChapterContent =  $strContent;
+
+            $this->arrChapterInfo['c_page_title'] = $this->getQueryList()->find('title')->text();
+            $this->arrChapterInfo['c_page_keywords'] = $this->getQueryList()->find('meta[name="keywords"]')->attr('content');
+            $this->arrChapterInfo['c_page_description'] = $this->getQueryList()->find('meta[name="description"]')->attr('content');
+
+            $strPattern = "/lastread\.set\((.*?)\);/";
+
+            if (preg_match($strPattern,  $this->getContent(), $arrMatches)) {
+                $strMatch = $arrMatches[1]; // 提取括号内的内容
+
+                $arrData = array_map('trim', explode(",", $strMatch));
+
+                $arrData = array_map(function ($item) {
+                    return trim($item, "'");
+                }, $arrData);
+
+                $this->arrChapterInfo['c_page_data'] = $arrData;
+            } else {
+                $this->arrChapterInfo['c_page_data'] = NULL;
+            }
+        }
+
+        return $this->arrChapterInfo;
+    }
+
+    /**
+     * Undocumented variable
+     *
+     * @var array
+     */
+    protected $arrChapterInfo;
+
+
+    /**
+     * Undocumented function
+     *
+     * @param integer $intNId
+     * @return boolean
+     */
+    public function saveChapter(int $intNId): bool
+    {
+        $arrPageChapter = $this->getChapterInfo();
+
+        if (
+            empty($arrPageChapter) ||
+            empty($arrPageChapter['c_page_title']) ||
+            empty($arrPageChapter['c_page_data']) ||
+            count($arrPageChapter['c_page_data']) < 5
+        ) {
+            return false;
+        }
+
+        $strUUId = $this->getSite()->getSiteUUId('chapter-' . $this->arrData['c_source_id'] . '-' . $this->arrData['c_page']);
+
+        $arrChapter = [
+            'n_id' => $intNId,
+            'c_name' => $arrPageChapter['c_page_data'][3],
+            'c_sort_num' => $this->arrData['c_sort_num'],
+            'c_page' => $this->arrData['c_page'],
+            'c_source_id' => $this->arrData['c_source_id'],
+            'c_site_code' => $this->getSite()->getSiteCode(),
+            'c_source_uuid' =>  $strUUId,
+        ];
+
+        $arrExistsChapter = $this->getChapterModel()->findOne(['c_source_uuid' => $strUUId], $this->arrOptions);
+
+        if ($arrExistsChapter) {
+
+            $arrExistsChapter = (array) $arrExistsChapter;
+
+            $arrUpdate = [
+                'updated_at' => new \MongoDB\BSON\UTCDateTime(),
+            ];
+
+            $this->getChapterModel()->updateOne(
+                ['c_source_uuid' => $strUUId],
+                ['$set' =>   $arrUpdate],
+            );
+            $this->strCContentPath = $arrExistsChapter['c_content_path'];
+        } else {
+
+            $arrInsertNovel =  $arrChapter;
+
+            $arrInsertNovel['c_content_path'] = '/novel/' . ($intNId % 100) . '/' . $intNId . '/' .  $arrChapter['c_sort_num'] . '_' .   $arrChapter['c_page'] . '.txt';
+            $arrInsertNovel['created_at'] = new \MongoDB\BSON\UTCDateTime();
+
+            try {
+                $this->getChapterModel()->insert($arrInsertNovel);
+                $this->updateNovelHaveChapter($intNId);
+                $this->strCContentPath = $arrInsertNovel['c_content_path'];
+            } catch (\Exception $e) {
+                if ($e->getCode() === 11000) {
+                    return $this->saveChapter($intNId);
+                } else {
+                    throw $e;
+                }
+            }
+        }
+
+        StorageCore::getInstance()->put($this->strCContentPath, $this->strChapterContent);
+
+        return true;
+    }
+
+    public $strCContentPath;
+    /**
+     * Undocumented function
+     *
+     * @param integer $intNId
+     * @return void
+     */
+    protected function updateNovelHaveChapter(int $intNId)
+    {
+        $arrFilter = [
+            'n_id' => $intNId,
+        ];
+
+        $ChapterModel =  ChapterModel::getInstance();
+        $arrLasteChapter =  $ChapterModel->getLatestChapterByNId($intNId, false);
+
+        if (!empty($arrLasteChapter)) {
+            $arrUpdate['$set']['n_have_chapter'] = 1;
+            $arrUpdate['$set']['n_latest_chapter_name'] = $arrLasteChapter['c_name'];
+            $this->getMongoBase()->updateOne('novel', $arrFilter, $arrUpdate);
+        }
+    }
+
+    protected $arrNextPage;
+
+    /**
+     * Undocumented function
+     *
+     * @return array
+     */
+    public function getNextPage(): array
+    {
+        if ($this->arrNextPage === NULL) {
+            $this->arrNextPage = [];
+
+            if ($this->isMobile()) {
+                $strPattern = "/var\s+hhekgsv\s*=\s*['\"](.*?)['\"]/";
+                preg_match_all($strPattern, $this->getContent(), $arrMatches);
+                $strNextPageUri = end($arrMatches[1]) ?: '';
+            } else {
+                $strHref = $this->getQueryList()->find('.read_btn a:contains("下一章")')->attr('href');
+                $strNextPageUri = $strHref ?: '';
+            }
+
+            if (is_numeric(strpos($strNextPageUri, '_'))) {
+                $this->arrNextPage = $this->arrData;
+                $this->arrNextPage['c_page'] += 1;
+                $this->arrNextPage['uri'] = $strNextPageUri;
+            }
+        }
+
+        return $this->arrNextPage;
+    }
+}

code/app/task/crawler/wenxuewang/page/LatestListPage.php

@@ -0,0 +1,64 @@
+<?php
+
+declare(strict_types=1);
+
+namespace app\task\crawler\wenxuewang\page;
+
+use app\model\NovelModel;
+
+
+class LatestListPage extends BasePage
+{
+
+    /**
+     * Undocumented variable
+     *
+     * @var NovelModel
+     */
+    protected $NovelModel;
+
+    /**
+     * Undocumented function
+     *
+     * @return NovelModel
+     */
+    public function getNovelModel(): NovelModel
+    {
+        if ($this->NovelModel == NULL) {
+            $this->NovelModel = NovelModel::getInstance();
+        }
+        return  $this->NovelModel;
+    }
+
+    static public $arrOptions = [
+        'typeMap' => [
+            'root' => 'array',
+            'document' => 'array',
+            'array' => 'array',
+        ],
+    ];
+
+    /**
+     * Undocumented function
+     *
+     * @return array
+     */
+    public function getNovelFilter(): array
+    {
+        $arrFilter = [];
+        if ($this->isMobile()) {
+            $arrFilter =  $this->getQueryList()->find('div.wrap-box')->eq(0)->find('ul.sort_list li')->map(function ($li) {
+                $strUri = $li->find('a')->eq(0)->attr('href');
+                return supperExtractFilename($strUri);
+            })->filter()->all();
+        } else {
+
+            $arrFilter =  $this->getQueryList()->find('div.layout.layout2.layout-col2.fl ul.txt-list li')->map(function ($li) {
+                $strUri = $li->find('a')->eq(0)->attr('href');
+                return supperExtractFilename($strUri);
+            })->filter()->all();
+        }
+
+        return $arrFilter;
+    }
+}

code/app/task/crawler/wenxuewang/page/NovelPage.php

@@ -0,0 +1,423 @@
+<?php
+
+declare(strict_types=1);
+
+namespace app\task\crawler\wenxuewang\page;
+
+use app\model\CategoryModel;
+use app\model\CountersModel;
+use app\model\NovelModel;
+use storage\StorageCore;
+
+class NovelPage extends BasePage
+{
+    /**
+     * Undocumented variable
+     *
+     * @var array
+     */
+    protected $arrNovelInfo = [];
+
+    /**
+     * Undocumented variable
+     *
+     * @var array
+     */
+    protected $arrRelatedNovel = [];
+
+    /**
+     * Undocumented variable
+     *
+     * @var array
+     */
+    protected $arrForgeNovel = [];
+
+    /**
+     * Undocumented variable
+     *
+     * @var NovelModel
+     */
+    protected $NovelModel;
+
+    /**
+     * Undocumented function
+     *
+     * @return NovelModel
+     */
+    public function getNovelModel(): NovelModel
+    {
+        if ($this->NovelModel == NULL) {
+            $this->NovelModel = NovelModel::getInstance();
+        }
+        return  $this->NovelModel;
+    }
+
+    public $arrOptions = [
+        'typeMap' => [
+            'root' => 'array',
+            'document' => 'array',
+            'array' => 'array',
+        ],
+    ];
+
+    /**
+     * 查询看的小说
+     *
+     * @param integer $intNSourceId
+     * @param string $strName
+     * @return array
+     */
+    protected function findOrCreateNovel(int $intNSourceId, string $strName): array
+    {
+        $strUUId = $this->getSite()->getSiteUUId($intNSourceId);
+
+        $arrExistsNovel = $this->getNovelModel()->findOne(['n_source_uuid' => $strUUId], $this->arrOptions);
+
+        if ($arrExistsNovel) {
+            $arrNovelInfo = (array) $arrExistsNovel;
+        } else {
+            $arrNovelInfo = [
+                'n_id' =>  CountersModel::getInstance()->getNextSequence('novel'),
+                'n_name' => $strName,
+                'n_name_pinyin' => zhToPinYin($strName),
+                'n_site_code' =>  $this->getSite()->getSiteCode(),
+                'n_source_uuid' => $strUUId,
+                'n_source_id' => $intNSourceId,
+                'n_level' => 1,
+                'n_have_chapter' => 0,
+                'created_at' => new \MongoDB\BSON\UTCDateTime(),
+            ];
+            try {
+                $this->getNovelModel()->insert($arrNovelInfo);
+            } catch (\Exception $e) {
+                if ($e->getCode() === 11000) {
+                    return $this->findOrCreateNovel($intNSourceId,  $strName);
+                } else {
+                    throw $e;
+                }
+            }
+        }
+        return $arrNovelInfo;
+    }
+
+
+    /**
+     * 分析关联数据
+     *
+     * @return void
+     */
+    public function getReletedNovel() {}
+
+
+    /**
+     * Undocumented function
+     *
+     * @return array
+     */
+    public function getNovelInfo(): array
+    {
+        if (empty($this->arrNovelInfo)) {
+
+            # 获取head字段
+            $this->getQueryList()->find('meta[property]')->each(function ($Item) {
+                $strKey = str_replace(':', '_', $Item->attr('property'));
+                $this->arrNovelInfo[$strKey] = $Item->attr('content');
+            });
+
+            if (!empty($this->arrNovelInfo['og_novel_category']) && $this->arrNovelInfo['og_novel_category'] == '玄幻小说') {
+                $this->arrNovelInfo['og_novel_category'] = '玄幻修真';
+            }
+
+
+            $this->getReletedNovel();
+        }
+
+        return  $this->arrNovelInfo;
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @param string $strName
+     * @return string
+     */
+    public function generateCoverPath($strName): string
+    {
+        $intTime = time() - mt_rand(1, 2 * 24 * 3600 * 360);
+
+        return sprintf('/img/%s/%s.jpg', date('Y/m/d/H', $intTime), $strName);
+    }
+
+    public function filterOtherFields(&$arrNovel)
+    {
+        $arrFields = [
+            'n_id',
+            'n_name',
+            'n_name_pinyin',
+            'n_status',
+            'n_author',
+            'n_description',
+            'n_category',
+            'n_category_pinyin',
+            'n_category_sex_en',
+            'n_category_sex_zh',
+            'n_cover_path',
+            'n_latest_chapter_name',
+            'n_update_time',
+            'n_forge_novel',
+            'n_related_novel',
+            'n_site_code',
+            'n_source_uuid',
+            'n_source_id',
+            'n_have_chapter',
+            'created_at',
+            'updated_at',
+        ];
+
+        foreach ($arrNovel as $strKey => $strVal) {
+            if (!in_array($strKey, $arrFields)) {
+                unset($arrNovel[$strKey]);
+            }
+        }
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @return bool
+     */
+    public function saveNovelInfo(): bool
+    {
+        $arrPageInfo = $this->getNovelInfo();
+        if (
+            empty($arrPageInfo) ||
+            empty($arrPageInfo['og_title']) ||
+            empty($arrPageInfo['og_image']) ||
+            empty($arrPageInfo['og_novel_category'])
+        ) {
+            return false;
+        }
+
+        $arrNovelInfo = [
+            // 'n_id'=> $arrPageInfo['xxxxx'],
+            'n_name' => $arrPageInfo['og_title'],
+            'n_name_pinyin' => zhToPinYin($arrPageInfo['og_title']),
+            'n_status' => $arrPageInfo['og_novel_status'],
+            'n_author' => $arrPageInfo['og_novel_author'],
+            'n_description' => $arrPageInfo['og_description'],
+            'n_category' => $arrPageInfo['og_novel_category'],
+            // 'n_category_pinyin' => $arrPageInfo['xxxxx'], 
+            // 'n_category_sex_en' => $arrPageInfo['xxxxx'], 
+            // 'n_category_sex_zh' => $arrPageInfo['xxxxx'], 
+            // 'n_cover_path' => $arrPageInfo['xxxxx'], 
+            'n_latest_chapter_name' => $arrPageInfo['og_novel_latest_chapter_name'],
+            'n_update_time' => $arrPageInfo['og_novel_update_time'],
+            'n_forge_novel' => $this->arrForgeNovel,
+            'n_related_novel' => $this->arrRelatedNovel,
+
+            'n_site_code' =>  $this->getSite()->getSiteCode(),
+            'n_source_id' => $this->arrData['n_source_id'],
+            'n_source_uuid' => $this->getSite()->getSiteUUId($this->arrData['n_source_id']),
+
+            // 'n_have_chapter' => $arrPageInfo['xxxxx'],
+            // 'created_at' => $arrPageInfo['xxxxx'],
+            // 'updated_at' => $arrPageInfo['xxxxx'],
+
+        ];
+
+        $arrExistsNovel = $this->getNovelModel()->findOne(['n_source_uuid' => $arrNovelInfo['n_source_uuid']], $this->arrOptions);
+
+        if ($arrExistsNovel) {
+
+            $arrExistsNovel = (array) $arrExistsNovel;
+
+            $arrUpdate = [
+                'n_latest_chapter_name' => $arrNovelInfo['n_latest_chapter_name'],
+                'n_update_time' => $arrNovelInfo['n_update_time'],
+                'updated_at' => new \MongoDB\BSON\UTCDateTime(),
+            ];
+
+            # 如果没有分类，说明这条数据是最初从关联shu那里采集的，需要补全全部信息
+            if (empty($arrExistsNovel['n_category'])) {
+                $arrUpdate = array_merge($arrUpdate, $arrNovelInfo);
+                $arrUpdate['n_related_novel'] = $this->arrRelatedNovel;
+                $arrUpdate['n_forge_novel'] = $this->arrForgeNovel;
+                $arrUpdate['n_category_pinyin'] = CategoryModel::getPinYin($this->arrNovelInfo['og_novel_category']);
+                $arrUpdate['n_category_sex_en'] = CategoryModel::getSexEn($this->arrNovelInfo['og_novel_category']);
+                $arrUpdate['n_category_sex_zh'] = CategoryModel::getSexZh($this->arrNovelInfo['og_novel_category']);
+                $arrUpdate['n_related_novel'] = $this->arrRelatedNovel;
+                $arrUpdate['n_forge_novel'] = $this->arrForgeNovel;
+            }
+
+            # 图片不存在的时候，需要重新生成
+            if (
+                !isset($arrExistsNovel['n_cover_path']) ||
+                empty($arrExistsNovel['n_cover_path']) ||
+                !StorageCore::getInstance()->has($arrExistsNovel['n_cover_path']) ||
+                (basename($arrExistsNovel['n_cover_path']) == '.jpg')
+            ) {
+                $arrUpdate['n_cover_path'] = $this->generateCoverPath($arrNovelInfo['n_name_pinyin']);
+                $this->strCoverLocalPath = $arrUpdate['n_cover_path'];
+            } else {
+                $this->strCoverLocalPath = $arrExistsNovel['n_cover_path'];
+            }
+
+            $this->getNovelModel()->updateOne(
+                ['n_source_uuid' =>  $arrNovelInfo['n_source_uuid']],
+                ['$set' => $arrUpdate],
+            );
+            $this->intNId =       $arrExistsNovel['n_id'];
+        } else {
+
+            $arrNovelInfo['n_category_pinyin'] = CategoryModel::getPinYin($this->arrNovelInfo['og_novel_category']);
+            $arrNovelInfo['n_category_sex_en'] = CategoryModel::getSexEn($this->arrNovelInfo['og_novel_category']);
+            $arrNovelInfo['n_category_sex_zh'] = CategoryModel::getSexZh($this->arrNovelInfo['og_novel_category']);
+            $arrNovelInfo['n_id'] =  CountersModel::getInstance()->getNextSequence('novel');
+            $arrNovelInfo['n_level'] = 1;
+            $arrNovelInfo['n_have_chapter'] = 0;
+            $arrNovelInfo['n_cover_path'] =  $this->generateCoverPath($arrNovelInfo['n_name_pinyin']);
+            $arrNovelInfo['n_related_novel'] = $this->arrRelatedNovel;
+            $arrNovelInfo['n_forge_novel'] = $this->arrForgeNovel;
+            $arrNovelInfo['created_at'] = new \MongoDB\BSON\UTCDateTime();
+
+            try {
+                $this->getNovelModel()->insert($arrNovelInfo);
+                $this->intNId =       $arrNovelInfo['n_id'];
+                $this->strCoverLocalPath = $arrNovelInfo['n_cover_path'];
+            } catch (\Exception $e) {
+                if ($e->getCode() === 11000) {
+                    return $this->saveNovelInfo();
+                } else {
+                    throw $e;
+                }
+            }
+        }
+
+        return true;
+    }
+
+
+    public $strCoverLocalPath = '';
+
+    public $intNId;
+
+    /**
+     * Undocumented function
+     *
+     * @return array
+     */
+    public function getDownCoverInfo(): array
+    {
+        return [
+            'uri' => $this->arrNovelInfo['og_image'],
+            'local_path' => $this->strCoverLocalPath,
+        ];
+    }
+
+    /**
+     * Undocumented variable
+     *
+     * @var array
+     */
+    protected $arrLatestChapters = NULL;
+
+    /**
+     * Undocumented variable
+     *
+     * @var array
+     */
+    protected $arrChapters = NULL;
+
+    public function getLatestChapterByNIdsList(): array
+    {
+        if ($this->arrLatestChapters === NULL) {
+            $this->arrLatestChapters = [];
+
+            if ($this->isMobile()) {
+                $strClass = '.chapter-list';
+            } else {
+                $strClass = '.section-list';
+            }
+
+            if ($this->arrData['page'] == 0) {
+                $this->getQueryList()->find($strClass)->eq(0)->find("a")->map(function ($A) {
+                    $this->arrLatestChapters[] = [
+                        'name' => $A->text(),
+                        'uri' => $A->attr('href'),
+                    ];
+                });
+            }
+        }
+        return $this->arrLatestChapters;
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @return array
+     */
+    public function getChapters(): array
+    {
+        if ($this->arrChapters === NULL) {
+            $this->arrChapters = [];
+            if ($this->isMobile()) {
+                $strClass = '.chapter-list';
+            } else {
+                $strClass = '.section-list';
+            }
+
+            $intDivIndex = 0;
+            if ($this->arrData['page'] == 0) {
+                $intDivIndex = 1;
+            }
+
+            $this->getQueryList()->find($strClass)->eq($intDivIndex)->find("a")->map(function ($A) {
+                $this->arrChapters[] = [
+                    'name' => $A->text(),
+                    'uri' => $A->attr('href'),
+                ];
+            });
+        }
+        return $this->arrChapters;
+    }
+
+
+    protected $arrNextPageArgs = NULL;
+
+    /**
+     * Undocumented function
+     *
+     * @return array
+     */
+    public function getNextPageArgs(): array
+    {
+        $this->arrNextPageArgs = [];
+        if ($this->arrData['page'] == 0) {
+            $this->getLatestChapterByNIdsList();
+            $this->getChapters();
+
+            $arrLatestChapterFirstBlock =  count($this->arrLatestChapters) > 0 ? $this->arrLatestChapters[0] : null;
+            $arrLatestChapterSecondBlock = count($this->arrChapters) > 0 ?  $this->arrChapters[count($this->arrChapters) - 1] : null;
+
+            if (
+                !empty($arrLatestChapterFirstBlock) && !empty($arrLatestChapterSecondBlock)
+                && ($arrLatestChapterFirstBlock['uri'] != $arrLatestChapterSecondBlock['uri'])
+
+            ) {
+                $this->arrNextPageArgs =  [
+                    'n_source_id' => $this->arrData['n_source_id'],
+                    'page' => $this->arrData['page'] + 1,
+                ];
+            }
+        } else {
+            $strLastUri = $this->getQueryList()->find('select:first option:last')->attr('value');
+            if ($strLastUri != '' && $strLastUri != $this->strUri) {
+                $this->arrNextPageArgs = [
+                    'n_source_id' => $this->arrData['n_source_id'],
+                    'page' => $this->arrData['page'] + 1,
+                ];
+            }
+        }
+        return   $this->arrNextPageArgs;
+    }
+}

code/app/task/crawler/wenxuewang/page/Site.php

@@ -0,0 +1,352 @@
+<?php
+
+declare(strict_types=1);
+
+namespace app\task\crawler\wenxuewang\page;
+
+use GuzzleHttp\Client;
+use GuzzleHttp\Promise;
+use storage\StorageCore;
+
+/**
+ * @mixin think\Model
+ */
+class Site
+{
+    /**
+     * Undocumented variable
+     *
+     * @var Client
+     */
+    protected $Client;
+
+    /**
+     * Undocumented variable
+     *
+     * @var string
+     */
+    protected $strDomain = 'https://www.bqvvxg8.cc/';
+
+    /**
+     * Undocumented variable
+     *
+     * @var string
+     */
+    protected $strSiteCode = 'wenxuewang';
+
+    /**
+     * Undocumented function
+     */
+    public function __construct()
+    {
+        $this->initClient();
+    }
+
+    /**
+     * Undocumented variable
+     *
+     * @var boolean
+     */
+    public $boolReInitClient = false;
+
+    /**
+     * Undocumented function
+     */
+    public function initClient()
+    {
+        $arrConfig = [
+            'http_version' => '1.1', // 强制用 HTTP/1.1，已有，保持
+            'base_uri' => $this->strDomain, // 基础域名，已有，保持
+            'http_errors' => false, // 不抛 HTTP 错误，已有，保持
+            'timeout' => 60, // 从 160 秒改为 60 秒
+            'connect_timeout' => 10, // 从 30 秒改为 10 秒
+            'curl' => [
+                CURLOPT_TCP_KEEPALIVE => 1, // 启用 TCP Keep-Alive，已有，保持
+                CURLOPT_TCP_KEEPIDLE => 120, // Keep-Alive 空闲时间
+                CURLOPT_TCP_KEEPINTVL => 60, // Keep-Alive 探测间隔
+                // CURLOPT_SSL_VERIFYPEER => true, // 验证 SSL，已有，保持
+            ],
+            'headers' => [
+                'User-Agent' => 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36',
+                'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
+                'Accept-Language' => 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
+                'Referer' => $this->strDomain,
+                'Connection' => 'keep-alive',
+                'sec-ch-ua' => '"Not/A)Brand";v="8", "Chromium";v="130", "Google Chrome";v="130"',
+                'sec-ch-ua-mobile' => '?1',
+                'sec-ch-ua-platform' => '"Android"',
+                'sec-fetch-dest' => 'document',
+                'sec-fetch-mode' => 'navigate',
+                'sec-fetch-site' => 'same-origin',
+                'sec-fetch-user' => '?1',
+                'upgrade-insecure-requests' => '1',
+            ],
+            'pool_size' => 10,
+            'verify' => false,
+        ];
+
+        $arrProxy = config('proxy');
+        if ($arrProxy['proxy_status']) {
+
+            $strCode = $arrProxy['proxy_code'];
+            $strProxyHost = $arrProxy['proxy_host'];
+            $strKey = $arrProxy['proxy_key'];
+            $strSecret = $arrProxy['proxy_secret'];
+
+
+            switch ($strCode) {
+                case 'XIONGMAO':
+                    $intTime = time();
+                    $strTxt = "orderno=" . $strKey . ",secret=" . $strSecret . ",timestamp=" . $intTime;
+                    $strSign = strtoupper(md5($strTxt));
+                    $strAuth = 'sign=' . $strSign . '&orderno=' . $strKey . '&timestamp=' . $intTime;
+                    $arrConfig['proxy'] = $strProxyHost;
+                    $arrConfig['headers']['Proxy-Authorization'] =  $strAuth;
+                    break;
+                case 'KUAI':
+                    $arrConfig['proxy'] = $strProxyHost;
+                    $arrConfig['curl'][CURLOPT_PROXYAUTH] = CURLAUTH_BASIC;
+                    $arrConfig['curl'][CURLOPT_PROXYUSERPWD] = "$strKey:$strSecret";
+                    break;
+            }
+        }
+
+
+        $this->Client = new Client($arrConfig);
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @return string
+     */
+    public function getSiteCode(): string
+    {
+        return $this->strSiteCode;
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @param integer|string $strSign
+     * @return string
+     */
+    public function getSiteUUId(int|string $strSign): string
+    {
+        return $this->getSiteCode() . '-' . $strSign;
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @return Client
+     */
+    public function getClient(): Client
+    {
+        if ($this->boolReInitClient) {
+            $this->initClient();
+        }
+
+        return $this->Client;
+    }
+
+    public function getNovelPageUri($intNSourceId, $intPage): string
+    {
+        $intUriFirst = intdiv($intNSourceId, 1000);
+
+        $strUri = sprintf('/wenzhang/%s/%s/', $intUriFirst, $intNSourceId);
+
+        return  $strUri;
+    }
+
+    /**
+     * retrieve multiple novel page model
+     *
+     * @param string $strUri
+     * @return NovelPage[]
+     */
+    public function getNovelPageList(array $arrFilter): array
+    {
+
+        /** @var NovelPage[] */
+        $arrNovelPage = [];
+
+        foreach ($arrFilter as $intKey => $arrItem) {
+
+            $arrFilter[$intKey]['uri'] = $this->getNovelPageUri($arrItem['n_source_id'], $arrItem['page']);
+
+            $arrPromises[$intKey] = $this->getClient()->getAsync($arrFilter[$intKey]['uri']);
+        }
+
+        $arrResults = Promise\Utils::settle($arrPromises)->wait();
+
+
+        $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
+            return $arrResult['state'] === 'fulfilled';
+        });
+
+        foreach ($arrSuccessfulResults as $intKey => $arrResult) {
+            $Response = $arrResult['value'];
+
+            $strContent =  $Response->getBody()->getContents();
+            if ($strContent == '索引文件不存在！') {
+                continue;
+            }
+
+            $NovelPage = $this->getNovelPage($this, $arrFilter[$intKey]['uri']);
+
+            $NovelPage->setContent($strContent);
+            $NovelPage->setData($arrFilter[$intKey]);
+
+            $arrNovelPage[] =  $NovelPage;
+        }
+
+        return $arrNovelPage;
+    }
+
+    /**
+     * retrieve multiple chapter page model
+     *
+     * @param string $strUri
+     * @return ChapterPage[]
+     */
+    public function getChapterPageList(array $arrFilter): array
+    {
+
+        /** @var ChapterPage[] */
+        $arrChapterPage = [];
+
+        $arrPromises = [];
+        foreach ($arrFilter as $intKey => $arrItem) {
+            $arrPromises[$intKey] = $this->getClient()->getAsync($arrItem['uri']);
+        }
+
+        $arrResults = Promise\Utils::settle($arrPromises)->wait();
+
+        $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
+            return $arrResult['state'] === 'fulfilled';
+        });
+
+        foreach ($arrSuccessfulResults as $intKey => $arrResult) {
+            $Response = $arrResult['value'];
+
+            $strContent =  $Response->getBody()->getContents();;
+
+            $ChapterPage = $this->getChapterPage($this, $arrFilter[$intKey]['uri']);
+
+            $ChapterPage->setContent($strContent);
+            $ChapterPage->setData($arrFilter[$intKey]);
+
+            $arrChapterPage[] =  $ChapterPage;
+        }
+
+        return $arrChapterPage;
+    }
+
+
+    /**
+     * retrieve multiple chapter page model
+     *
+     * @param string $strUri
+     * @return LatestListPage[]
+     */
+    public function getLatestListPageList(array $arrFilter): array
+    {
+
+        /** @var LatestListPage[] */
+        $arrLatestListPage = [];
+
+        $arrPromises = [];
+        foreach ($arrFilter as $intKey => $strUri) {
+            $arrPromises[$intKey] = $this->getClient()->getAsync($strUri);
+        }
+
+        $arrResults = Promise\Utils::settle($arrPromises)->wait();
+
+        $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
+            return $arrResult['state'] === 'fulfilled';
+        });
+
+        foreach ($arrSuccessfulResults as $intKey => $arrResult) {
+            $Response = $arrResult['value'];
+
+            $strContent =  $Response->getBody()->getContents();;
+
+            $LatestListPage = $this->getLatestListPage($this, $arrFilter[$intKey]);
+
+            $LatestListPage->setContent($strContent);
+
+            $arrLatestListPage[] =  $LatestListPage;
+        }
+
+
+        return $arrLatestListPage;
+    }
+
+    /**
+     * download the image to the local system
+     *
+     * @param array $arrFilter
+     * @return void
+     */
+    public function downRemoteImgToLocal(array $arrFilter)
+    {
+        $StorageCore = StorageCore::getInstance();
+
+        $arrPromises = [];
+
+        foreach ($arrFilter as $intKey => $arrItem) {
+            if (!$StorageCore->has($arrItem['local_path'])) {
+                $arrPromises[$intKey] = $this->getClient()->getAsync($arrItem['uri']);
+            }
+        }
+
+        $arrResults = Promise\Utils::settle($arrPromises)->wait();
+
+        $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
+            return $arrResult['state'] === 'fulfilled';
+        });
+
+        foreach ($arrSuccessfulResults as $intKey => $arrResult) {
+            $Response = $arrResult['value'];
+
+            $StorageCore->writeStream($arrFilter[$intKey]['local_path'], $Response->getBody()->detach());;
+        }
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @param object $Site
+     * @param string $strUri
+     * @return NovelPage
+     */
+    public function getNovelPage($Site, $strUri)
+    {
+        return new NovelPage($Site, $strUri);
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @param object $Site
+     * @param string $strUri
+     * @return ChapterPage
+     */
+    public function getChapterPage($Site, $strUri)
+    {
+        return new ChapterPage($Site, $strUri);
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @param object $Site
+     * @param string $strUri
+     * @return LatestListPage
+     */
+    public function getLatestListPage($Site, $strUri)
+    {
+        return new LatestListPage($Site, $strUri);
+    }
+}

code/app/task/crawler/zw/Scheduler.php

@@ -103,6 +103,9 @@ class Scheduler
         $arrCover = [];
         foreach ($arrNovelPageList as $NovelPage) {
             // $arrNovel = $NovelPage->getNovelInfo();
+            // print_r($arrNovel);
+            // return;
+
             $boolResult = $NovelPage->saveNovelInfo();
             // return;
 

code/app/task/module/PlanTask.php

@@ -115,7 +115,7 @@ class PlanTask
             'data' => []
         ];
 
-        $QueueManage = QueueManage::getInstance(2);
+        $QueueManage = QueueManage::getInstance();
         $QueueManage->set($arrTask);
     }