debug

code/app/task/crawler/zw/page/BasePage.php

@@ -0,0 +1,251 @@
+<?php
+
+declare(strict_types=1);
+
+namespace app\task\crawler\zw\page;
+
+use db\mongo\MongoBase;
+use ddl\DDLManage;
+use GuzzleHttp\Client;
+use microserver\QueueManage;
+use QL\QueryList;
+use Psr\Http\Message\ResponseInterface;
+
+/**
+ * @mixin think\Model
+ */
+abstract class BasePage
+{
+    /**
+     * Undocumented variable
+     *
+     * @var string
+     */
+    // protected $strDomain;
+
+    /**
+     * Undocumented variable
+     *
+     * @var string
+     */
+    protected $strUri;
+
+    /**
+     * Undocumented variable
+     *
+     * @var MongoBase
+     */
+    protected $MongoBase;
+
+    /**
+     * Undocumented variable
+     *
+     * @var \GuzzleHttp\Client
+     */
+    protected $Client = NULL;
+
+    /**
+     * Undocumented variable
+     *
+     * @var \microserver\QueueManage
+     */
+    protected $QueueManage = NULL;
+
+
+    /**
+     * Undocumented variable
+     *
+     * @var array
+     */
+    protected $arrCaiJiPeiZhi = [];
+
+    /**
+     * Undocumented variable
+     *
+     * @var string
+     */
+    protected $strContent;
+
+    /**
+     * Undocumented variable
+     *
+     * @var QueryList
+     */
+    protected $QueryList;
+
+    /**
+     * Undocumented variable
+     *
+     * @var array
+     */
+    protected $arrData = [];
+
+    /**
+     * Undocumented variable
+     *
+     * @var bool
+     */
+    protected $boolIsMobile;
+
+    /**
+     * Undocumented variable
+     *
+     * @var ResponseInterface
+     */
+    private $Response;
+
+
+    /**
+     * Undocumented function
+     *
+     * @param string $strUri
+     */
+    public function __construct($Site, string $strUri)
+    {
+        $this->setSite($Site);
+
+        $this->strUri = $strUri;
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @return string
+     */
+    public function getUri(): string
+    {
+        return $this->strUri;
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @param array $arrData
+     * @return void
+     */
+    public function setData(array $arrData)
+    {
+        $this->arrData = $arrData;
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @return array
+     */
+    public function getData(): array
+    {
+        return $this->arrData;
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @return QueueManage
+     */
+    public function getQueueManage(): QueueManage
+    {
+        if ($this->QueueManage == NULL) {
+            $this->QueueManage = QueueManage::getInstance();
+        }
+
+        return $this->QueueManage;
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @return MongoBase
+     */
+    public function getMongoBase(): MongoBase
+    {
+        if ($this->MongoBase == NULL) {
+            DDLManage::load('MongoBase');
+            $this->MongoBase = MongoBase::getInstance();
+        }
+
+        return $this->MongoBase;
+    }
+
+    public function getQueryList(): QueryList
+    {
+        if ($this->QueryList == NULL) {
+            $this->QueryList = QueryList::html($this->getContent());
+        }
+
+        return $this->QueryList;
+    }
+
+
+    /**
+     * Undocumented function
+     *
+     * @return string
+     */
+    public function getContent(): string
+    {
+        if ($this->strContent == NULL) {
+            $this->Response =  $this->getClient()->get($this->strUri);
+            $this->strContent =  $this->Response->getBody()->getContents();
+        }
+
+        return $this->strContent;
+    }
+
+    /**
+     * force set page content
+     *
+     * @param string $strContent
+     * @return void
+     */
+    public function setContent($strContent)
+    {
+        $this->strContent = $strContent;
+    }
+
+
+    /**
+     * Undocumented function
+     *
+     * @return boolean
+     */
+    public function isMobile($strMark = 'content="mobile"'): bool
+    {
+        if ($this->boolIsMobile === NULL) {
+            $this->boolIsMobile = is_numeric(strpos($this->getContent(), $strMark));
+        }
+        return $this->boolIsMobile;
+    }
+
+
+    /**
+     * Undocumented function
+     *
+     * @return Client
+     */
+    protected function getClient(): Client
+    {
+        if ($this->Client == NULL) {
+            $this->Client = $this->getSite()->getClient();
+        }
+        return $this->Client;
+    }
+
+
+    /**
+     * Undocumented variable
+     *
+     * @var Site
+     */
+    protected  $Site;
+
+    public function setSite(Site $Site)
+    {
+        $this->Site = $Site;
+    }
+
+    public function getSite()
+    {
+        return  $this->Site;
+    }
+}

code/app/task/crawler/zw/page/ChapterPage.php

@@ -0,0 +1,217 @@
+<?php
+
+declare(strict_types=1);
+
+namespace app\task\crawler\zw\page;
+
+use app\model\ChapterModel;
+use storage\StorageCore;
+
+class ChapterPage extends BasePage
+{
+    /**
+     * Undocumented variable
+     *
+     * @var string
+     */
+    protected $strChapterContent = '';
+
+    /**
+     * Undocumented variable
+     *
+     * @var ChapterModel
+     */
+    protected $ChapterModel;
+
+    /**
+     * Undocumented function
+     *
+     * @return ChapterModel
+     */
+    public function getChapterModel(): ChapterModel
+    {
+        if ($this->ChapterModel == NULL) {
+            $this->ChapterModel = ChapterModel::getInstance();
+        }
+        return  $this->ChapterModel;
+    }
+
+    public $arrOptions = [
+        'typeMap' => [
+            'root' => 'array',
+            'document' => 'array',
+            'array' => 'array',
+        ],
+    ];
+
+    /**
+     * Undocumented function
+     *
+     * @return array
+     */
+    public function getChapterInfo(): array
+    {
+        if ($this->arrChapterInfo === NULL) {
+
+            $this->arrChapterInfo = [];
+
+            $strPattern = "/qsbs\.bb\('([^']*)'\)/";
+
+            preg_match_all($strPattern,  $this->getContent(), $arrMatches);
+
+            $strContent = '';
+
+            foreach ($arrMatches[1] as $str) {
+                $strContent  .= base64_decode($str);
+            }
+
+            $this->strChapterContent =  $strContent;
+
+            $this->arrChapterInfo['c_page_title'] = $this->getQueryList()->find('title')->text();
+            $this->arrChapterInfo['c_page_keywords'] = $this->getQueryList()->find('meta[name="keywords"]')->attr('content');
+            $this->arrChapterInfo['c_page_description'] = $this->getQueryList()->find('meta[name="description"]')->attr('content');
+        }
+
+        return $this->arrChapterInfo;
+    }
+
+    /**
+     * Undocumented variable
+     *
+     * @var array
+     */
+    protected $arrChapterInfo;
+
+
+    public function filterOtherFields(&$arrChapter)
+    {
+        $arrFields = [
+            'n_id',
+            'name',
+            'c_sort_num',
+            'c_page',
+            'c_source_id',
+            'c_site_code',
+            'c_site_uuid',
+            'c_content_path',
+            'created_at',
+            'updated_at',
+
+        ];
+
+        foreach ($arrChapter as $strKey => $strVal) {
+            if (!in_array($strKey, $arrFields)) {
+                unset($arrChapter[$strKey]);
+            }
+        }
+    }
+
+    public function saveChapter(int $intNId): array
+    {
+
+        try {
+            $this->getChapterInfo();
+
+            $strUUId = $this->getSite()->getSiteUUId('chapter-' . $this->arrData['c_source_id'] . '-' . $this->arrData['c_page']);
+            $arrExisting = $this->getChapterModel()->findOne(['c_site_uuid' => $strUUId], $this->arrOptions);
+
+            if ($arrExisting) {
+
+                $arrExisting = (array) $arrExisting;
+                $this->arrChapterInfo = array_merge($arrExisting,   $this->getChapterInfo());
+
+                $this->arrChapterInfo['updated_at'] = new \MongoDB\BSON\UTCDateTime();
+
+                $arrUpdate = $this->arrChapterInfo;
+
+                $this->filterOtherFields($arrUpdate);
+
+                $this->getChapterModel()->updateOne(
+                    ['c_site_uuid' => $strUUId],
+                    ['$set' =>   $arrUpdate],
+                );
+            } else {
+
+                $this->arrChapterInfo['n_id'] =  $intNId;
+                $this->arrChapterInfo['name'] =  $this->arrData['name'];
+                $this->arrChapterInfo['c_sort_num'] =  $this->arrData['c_sort_num'];
+                $this->arrChapterInfo['c_page'] =  $this->arrData['c_page'];
+                $this->arrChapterInfo['c_source_id'] = $this->arrData['c_source_id'];
+                $this->arrChapterInfo['c_site_code'] =  $this->getSite()->getSiteCode();
+                $this->arrChapterInfo['c_site_uuid'] = $strUUId;
+                $this->arrChapterInfo['c_content_path'] = '/novel/' . ($intNId % 100) . '/' . $intNId . '/' . $this->arrChapterInfo['c_sort_num'] . '_' .  $this->arrChapterInfo['c_page'] . '.txt';
+                $this->arrChapterInfo['created_at'] = new \MongoDB\BSON\UTCDateTime();
+                $arrInsert = $this->arrChapterInfo;
+
+                $this->filterOtherFields($arrInsert);
+
+                $this->getChapterModel()->insert($arrInsert);
+
+                $this->updateNovelHaveChapter($intNId);
+            }
+
+            if (mb_strlen($this->strChapterContent) > 20) {
+                if (!StorageCore::getInstance()->has($this->arrChapterInfo['c_content_path'])) {
+                    StorageCore::getInstance()->set($this->arrChapterInfo['c_content_path'], $this->strChapterContent);
+                }
+            }
+            return   $this->arrChapterInfo;
+        } catch (\Exception $e) {
+            throw new \Exception("Upsert failed: " . $e->getMessage());
+        }
+    }
+
+
+    /**
+     * Undocumented function
+     *
+     * @param integer $intNId
+     * @return void
+     */
+    protected function updateNovelHaveChapter(int $intNId)
+    {
+        $arrFilter = [
+            'n_id' => $intNId,
+        ];
+
+        $ChapterModel =  ChapterModel::getInstance();
+        $arrLasteChapter =  $ChapterModel->getLatestChapterByNId($intNId, false);
+
+        if (!empty($arrLasteChapter)) {
+            $arrUpdate['$set']['n_have_chapter'] = 1;
+            $arrUpdate['$set']['og_novel_latest_chapter_name'] = $arrLasteChapter['name'];
+            $this->getMongoBase()->updateOne('novel', $arrFilter, $arrUpdate);
+        }
+    }
+
+    protected $arrNextPage;
+
+    /**
+     * Undocumented function
+     *
+     * @return array
+     */
+    public function getNextPage(): array
+    {
+        if ($this->arrNextPage === NULL) {
+            $this->arrNextPage = [];
+
+            if ($this->isMobile()) {
+                $strPattern = "/var\s+hhekgsv\s*=\s*['\"](.*?)['\"]/";
+                preg_match_all($strPattern, $this->getContent(), $arrMatches);
+                $strNextPageUri = end($arrMatches[1]) ?: '';
+            } else {
+                $strHref = $this->getQueryList()->find('.read_btn a:contains("下一章")')->attr('href');
+                $strNextPageUri = $strHref ?: '';
+            }
+
+            if (is_numeric(strpos($strNextPageUri, '_'))) {
+                $this->arrNextPage = $this->arrData;
+                $this->arrNextPage['c_page'] += 1;
+                $this->arrNextPage['uri'] = $strNextPageUri;
+            }
+        }
+
+        return $this->arrNextPage;
+    }
+}

code/app/task/crawler/zw/page/LatestListPage.php

@@ -0,0 +1,64 @@
+<?php
+
+declare(strict_types=1);
+
+namespace app\task\crawler\zw\page;
+
+use app\model\NovelModel;
+
+
+class LatestListPage extends BasePage
+{
+
+    /**
+     * Undocumented variable
+     *
+     * @var NovelModel
+     */
+    protected $NovelModel;
+
+    /**
+     * Undocumented function
+     *
+     * @return NovelModel
+     */
+    public function getNovelModel(): NovelModel
+    {
+        if ($this->NovelModel == NULL) {
+            $this->NovelModel = NovelModel::getInstance();
+        }
+        return  $this->NovelModel;
+    }
+
+    static public $arrOptions = [
+        'typeMap' => [
+            'root' => 'array',
+            'document' => 'array',
+            'array' => 'array',
+        ],
+    ];
+
+    /**
+     * Undocumented function
+     *
+     * @return array
+     */
+    public function getNovelFilter(): array
+    {
+        $arrFilter = [];
+        if ($this->isMobile()) {
+            $arrFilter =  $this->getQueryList()->find('div.wrap-box')->eq(0)->find('ul.sort_list li')->map(function ($li) {
+                $strUri = $li->find('a')->eq(0)->attr('href');
+                return supperExtractFilename($strUri);
+            })->filter()->all();
+        } else {
+
+            $arrFilter =  $this->getQueryList()->find('div.layout.layout2.layout-col2.fl ul.txt-list li')->map(function ($li) {
+                $strUri = $li->find('a')->eq(0)->attr('href');
+                return supperExtractFilename($strUri);
+            })->filter()->all();
+        }
+
+        return $arrFilter;
+    }
+}

code/app/task/crawler/zw/page/NovelPage.php

@@ -0,0 +1,370 @@
+<?php
+
+declare(strict_types=1);
+
+namespace app\task\crawler\zw\page;
+
+use app\model\CategoryModel;
+use app\model\CountersModel;
+use app\model\NovelModel;
+
+
+class NovelPage extends BasePage
+{
+    /**
+     * Undocumented variable
+     *
+     * @var array
+     */
+    protected $arrNovelInfo = [];
+
+    /**
+     * Undocumented variable
+     *
+     * @var array
+     */
+    protected $arrRelatedNovel = [];
+
+    /**
+     * Undocumented variable
+     *
+     * @var array
+     */
+    protected $arrForgeNovel = [];
+
+    /**
+     * Undocumented variable
+     *
+     * @var NovelModel
+     */
+    protected $NovelModel;
+
+    /**
+     * Undocumented function
+     *
+     * @return NovelModel
+     */
+    public function getNovelModel(): NovelModel
+    {
+        if ($this->NovelModel == NULL) {
+            $this->NovelModel = NovelModel::getInstance();
+        }
+        return  $this->NovelModel;
+    }
+
+    public $arrOptions = [
+        'typeMap' => [
+            'root' => 'array',
+            'document' => 'array',
+            'array' => 'array',
+        ],
+    ];
+
+    /**
+     * 查询看的小说
+     *
+     * @param integer $intNSourceId
+     * @param string $strName
+     * @return array
+     */
+    protected function findOrCreateNovel(int $intNSourceId, string $strName): array
+    {
+        $strUUId = $this->getSite()->getSiteUUId($intNSourceId);
+
+        $arrExisting = $this->getNovelModel()->findOne(['n_site_uuid' => $strUUId], $this->arrOptions);
+
+        if ($arrExisting) {
+            $arrNovelInfo = (array) $arrExisting;
+        } else {
+            $arrNovelInfo = [
+                'og_novel_book_name' => $strName,
+                'og_novel_pin_yin' => zhToPinYin($strName),
+                'n_site_code' =>  $this->getSite()->getSiteCode(),
+                'n_site_uuid' => $strUUId,
+                'n_source_id' => $intNSourceId,
+                'n_level' => 1,
+                'n_have_chapter' => 0,
+                'n_id' =>  CountersModel::getInstance()->getNextSequence('novel'),
+            ];
+            $this->getNovelModel()->insert($arrNovelInfo);
+        }
+        return $arrNovelInfo;
+    }
+
+
+    /**
+     * 分析关联数据
+     *
+     * @return void
+     */
+    public function getReletedNovel() {}
+
+
+    /**
+     * Undocumented function
+     *
+     * @return array
+     */
+    public function getNovelInfo(): array
+    {
+        if (empty($this->arrNovelInfo)) {
+
+            # 获取head字段
+            $this->getQueryList()->find('meta[property]')->each(function ($Item) {
+                $strKey = str_replace(':', '_', $Item->attr('property'));
+                $this->arrNovelInfo[$strKey] = $Item->attr('content');
+            });
+
+            $this->getReletedNovel();
+
+            $this->arrNovelInfo['n_related_novel'] = $this->arrRelatedNovel;
+            $this->arrNovelInfo['n_forge_novel'] = $this->arrForgeNovel;
+
+            // $this->arrNovelInfo['og_novel_pin_yin'] = zhToPinYin($this->arrNovelInfo['og_novel_book_name'] ?? '');
+
+            // if (!empty($this->arrNovelInfo['og_novel_category'])) {
+            //     $this->arrNovelInfo['category_pinyin'] = CategoryModel::getPinYinByZh($this->arrNovelInfo['og_novel_category']);
+            //     $this->arrNovelInfo['category_sex'] = CategoryModel::checkSex($this->arrNovelInfo['og_novel_category']);
+            //     $this->arrNovelInfo['category_sex_zh'] = CategoryModel::checkSexZh($this->arrNovelInfo['og_novel_category']);
+            // }
+
+            // $this->arrNovelInfo['n_site_code'] =  $this->getSite()->getSiteCode();
+            // $this->arrNovelInfo['n_source_id'] = $this->arrData['n_source_id'];
+        }
+
+        print_r($this->arrNovelInfo);
+
+        return  $this->arrNovelInfo;
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @param string $strName
+     * @return string
+     */
+    public function generateCoverPath($strName): string
+    {
+        return sprintf('/img/%s/%s.jpg', date('Y/m/d', time() - 24 * 3600 * 14), $strName);
+    }
+
+    public function filterOtherFields(&$arrNovel)
+    {
+        $arrFields = [
+            'n_id',
+            'og_novel_book_name',
+            'og_novel_pin_yin',
+            'og_novel_status',
+            'og_novel_author',
+            'og_description',
+            'og_novel_category',
+            'category_pinyin',
+            'category_sex',
+            'category_sex_zh',
+            'n_cover_path',
+            'og_novel_latest_chapter_name',
+            'og_novel_update_time',
+            'n_forge_novel',
+            'n_related_novel',
+            'og_site_code',
+            'n_site_uuid',
+            'n_source_id',
+            'n_have_chapter',
+            'created_at',
+            'updated_at',
+        ];
+
+        foreach ($arrNovel as $strKey => $strVal) {
+            if (!in_array($strKey, $arrFields)) {
+                unset($arrNovel[$strKey]);
+            }
+        }
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @return void
+     */
+    public function saveNovelInfo()
+    {
+        try {
+
+            $this->getNovelInfo();
+
+            if (!isset($this->arrNovelInfo['og_novel_book_name'])) {
+                return false;
+            }
+
+
+            $strUUId = $this->getSite()->getSiteUUId($this->arrData['n_source_id']);
+
+            $arrExisting = $this->getNovelModel()->findOne(['n_site_uuid' => $strUUId], $this->arrOptions);
+
+            if ($arrExisting) {
+
+                $arrExisting = (array) $arrExisting;
+                $this->arrNovelInfo = array_merge($arrExisting,   $this->arrNovelInfo);
+
+                if (
+                    !isset($this->arrNovelInfo['n_cover_path']) ||
+                    empty($this->arrNovelInfo['n_cover_path']) ||
+                    !file_exists($this->arrNovelInfo['n_cover_path'])  ||
+
+                    (basename($this->arrNovelInfo['n_cover_path']) == '.jpg')
+                ) {
+                    $this->arrNovelInfo['n_cover_path'] = $this->generateCoverPath($this->arrNovelInfo['og_novel_pin_yin']);
+                }
+
+                $this->arrNovelInfo['updated_at'] = new \MongoDB\BSON\UTCDateTime();
+
+                $arrUpdate =  $this->arrNovelInfo;
+                $this->filterOtherFields($arrUpdate);
+
+                $this->getNovelModel()->updateOne(
+                    ['n_site_uuid' => $strUUId],
+                    ['$set' => $arrUpdate],
+                );
+            } else {
+                $this->arrNovelInfo['n_site_uuid'] = $strUUId;
+                $this->arrNovelInfo['n_id'] =  CountersModel::getInstance()->getNextSequence('novel');
+                $this->arrNovelInfo['n_level'] = 1;
+                $this->arrNovelInfo['n_have_chapter'] = 0;
+                $this->arrNovelInfo['n_cover_path'] =  $this->generateCoverPath($this->arrNovelInfo['og_novel_pin_yin']);
+
+                $arrInsert =  $this->arrNovelInfo;
+                $this->filterOtherFields($arrInsert);
+
+                $this->getNovelModel()->insert($arrInsert);
+            }
+            return $this->arrNovelInfo;
+        } catch (\Exception $e) {
+            throw new \Exception("Upsert failed: " . $e->getMessage());
+        }
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @return array
+     */
+    public function getDownCoverInfo(): array
+    {
+        if (!key_exists('og_image', $this->arrNovelInfo)) {
+            print_r($this->arrNovelInfo);
+            echo $this->getContent();
+        }
+
+        return [
+            'uri' => $this->arrNovelInfo['og_image'],
+            'local_path' => $this->arrNovelInfo['n_cover_path'],
+        ];
+    }
+
+    /**
+     * Undocumented variable
+     *
+     * @var array
+     */
+    protected $arrLatestChapters = NULL;
+
+    /**
+     * Undocumented variable
+     *
+     * @var array
+     */
+    protected $arrChapters = NULL;
+
+    public function getLatestChapterByNIdsList(): array
+    {
+        if ($this->arrLatestChapters === NULL) {
+            $this->arrLatestChapters = [];
+
+            if ($this->isMobile()) {
+                $strClass = '.chapter-list';
+            } else {
+                $strClass = '.section-list';
+            }
+
+            if ($this->arrData['page'] == 0) {
+                $this->getQueryList()->find($strClass)->eq(0)->find("a")->map(function ($A) {
+                    $this->arrLatestChapters[] = [
+                        'name' => $A->text(),
+                        'uri' => $A->attr('href'),
+                    ];
+                });
+            }
+        }
+        return $this->arrLatestChapters;
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @return array
+     */
+    public function getChapters(): array
+    {
+        if ($this->arrChapters === NULL) {
+            $this->arrChapters = [];
+            if ($this->isMobile()) {
+                $strClass = '.chapter-list';
+            } else {
+                $strClass = '.section-list';
+            }
+
+            $intDivIndex = 0;
+            if ($this->arrData['page'] == 0) {
+                $intDivIndex = 1;
+            }
+
+            $this->getQueryList()->find($strClass)->eq($intDivIndex)->find("a")->map(function ($A) {
+                $this->arrChapters[] = [
+                    'name' => $A->text(),
+                    'uri' => $A->attr('href'),
+                ];
+            });
+        }
+        return $this->arrChapters;
+    }
+
+
+    protected $arrNextPageArgs = NULL;
+
+    /**
+     * Undocumented function
+     *
+     * @return array
+     */
+    public function getNextPageArgs(): array
+    {
+        $this->arrNextPageArgs = [];
+        if ($this->arrData['page'] == 0) {
+            $this->getLatestChapterByNIdsList();
+            $this->getChapters();
+
+            $arrLatestChapterFirstBlock =  count($this->arrLatestChapters) > 0 ? $this->arrLatestChapters[0] : null;
+            $arrLatestChapterSecondBlock = count($this->arrChapters) > 0 ?  $this->arrChapters[count($this->arrChapters) - 1] : null;
+
+            if (
+                !empty($arrLatestChapterFirstBlock) && !empty($arrLatestChapterSecondBlock)
+                && ($arrLatestChapterFirstBlock['uri'] != $arrLatestChapterSecondBlock['uri'])
+
+            ) {
+                $this->arrNextPageArgs =  [
+                    'n_source_id' => $this->arrData['n_source_id'],
+                    'page' => $this->arrData['page'] + 1,
+                ];
+            }
+        } else {
+            $strLastUri = $this->getQueryList()->find('select:first option:last')->attr('value');
+            if ($strLastUri != '' && $strLastUri != $this->strUri) {
+                $this->arrNextPageArgs = [
+                    'n_source_id' => $this->arrData['n_source_id'],
+                    'page' => $this->arrData['page'] + 1,
+                ];
+            }
+        }
+        return   $this->arrNextPageArgs;
+    }
+}

code/app/task/crawler/zw/page/Site.php

@@ -0,0 +1,283 @@
+<?php
+
+declare(strict_types=1);
+
+namespace app\task\crawler\zw\page;
+
+use GuzzleHttp\Client;
+use GuzzleHttp\Promise;
+use storage\StorageCore;
+
+/**
+ * @mixin think\Model
+ */
+class Site
+{
+    /**
+     * Undocumented variable
+     *
+     * @var Client
+     */
+    protected $Client;
+
+    /**
+     * Undocumented variable
+     *
+     * @var string
+     */
+    protected $strDomain = '';
+
+    /**
+     * Undocumented variable
+     *
+     * @var string
+     */
+    protected $strSiteCode = '';
+
+    /**
+     * Undocumented function
+     */
+    public function __construct()
+    {
+        $this->initClient();
+    }
+
+  
+
+    /**
+     * Undocumented function
+     *
+     * @return string
+     */
+    public function getSiteCode(): string
+    {
+        return $this->strSiteCode;
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @param integer|string $strSign
+     * @return string
+     */
+    public function getSiteUUId(int|string $strSign): string
+    {
+        return $this->getSiteCode() . '-' . $strSign;
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @return Client
+     */
+    public function getClient(): Client
+    {
+        return $this->Client;
+    }
+
+    public function getNovelPageUri($intNSourceId, $intPage): string
+    {
+        $intUriFirst = intdiv($intNSourceId, 1000);
+
+        $strUri = '';
+
+        if ($intPage == 0) {
+            $strUri = sprintf('/%s/%s/', $intUriFirst, $intNSourceId);
+        } else {
+            $strUri = sprintf('/book/%s/list%s.html', $intNSourceId,  $intPage);
+        }
+
+        return  $strUri;
+    }
+
+    /**
+     * retrieve multiple novel page model
+     *
+     * @param string $strUri
+     * @return NovelPage[]
+     */
+    public function getNovelPageList(array $arrFilter): array
+    {
+
+        /** @var NovelPage[] */
+        $arrNovelPage = [];
+
+        foreach ($arrFilter as $intKey => $arrItem) {
+
+            $arrFilter[$intKey]['uri'] = $this->getNovelPageUri($arrItem['n_source_id'], $arrItem['page']);
+
+            $arrPromises[$intKey] = $this->getClient()->getAsync($arrFilter[$intKey]['uri']);
+        }
+
+        $arrResults = Promise\Utils::settle($arrPromises)->wait();
+
+
+        $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
+            return $arrResult['state'] === 'fulfilled';
+        });
+
+        foreach ($arrSuccessfulResults as $intKey => $arrResult) {
+            $Response = $arrResult['value'];
+
+            $strContent =  $Response->getBody()->getContents();
+            if ($strContent == '索引文件不存在！') {
+                continue;
+            }
+
+            $NovelPage = $this->getNovelPage($this, $arrFilter[$intKey]['uri']);
+
+            $NovelPage->setContent($strContent);
+            $NovelPage->setData($arrFilter[$intKey]);
+
+            $arrNovelPage[] =  $NovelPage;
+        }
+
+        return $arrNovelPage;
+    }
+
+    /**
+     * retrieve multiple chapter page model
+     *
+     * @param string $strUri
+     * @return ChapterPage[]
+     */
+    public function getChapterPageList(array $arrFilter): array
+    {
+
+        /** @var ChapterPage[] */
+        $arrChapterPage = [];
+
+        $arrPromises = [];
+        foreach ($arrFilter as $intKey => $arrItem) {
+            $arrPromises[$intKey] = $this->getClient()->getAsync($arrItem['uri']);
+        }
+
+        $arrResults = Promise\Utils::settle($arrPromises)->wait();
+
+        $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
+            return $arrResult['state'] === 'fulfilled';
+        });
+
+        foreach ($arrSuccessfulResults as $intKey => $arrResult) {
+            $Response = $arrResult['value'];
+
+            $strContent =  $Response->getBody()->getContents();;
+
+            $ChapterPage = $this->getChapterPage($this, $arrFilter[$intKey]['uri']);
+
+            $ChapterPage->setContent($strContent);
+            $ChapterPage->setData($arrFilter[$intKey]);
+
+            $arrChapterPage[] =  $ChapterPage;
+        }
+
+        return $arrChapterPage;
+    }
+
+
+    /**
+     * retrieve multiple chapter page model
+     *
+     * @param string $strUri
+     * @return LatestListPage[]
+     */
+    public function getLatestListPageList(array $arrFilter): array
+    {
+
+        /** @var LatestListPage[] */
+        $arrLatestListPage = [];
+
+        $arrPromises = [];
+        foreach ($arrFilter as $intKey => $strUri) {
+            $arrPromises[$intKey] = $this->getClient()->getAsync($strUri);
+        }
+
+        $arrResults = Promise\Utils::settle($arrPromises)->wait();
+
+        $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
+            return $arrResult['state'] === 'fulfilled';
+        });
+
+        foreach ($arrSuccessfulResults as $intKey => $arrResult) {
+            $Response = $arrResult['value'];
+
+            $strContent =  $Response->getBody()->getContents();;
+
+            $LatestListPage = $this->getLatestListPage($this, $arrFilter[$intKey]);
+
+            $LatestListPage->setContent($strContent);
+
+            $arrLatestListPage[] =  $LatestListPage;
+        }
+
+
+        return $arrLatestListPage;
+    }
+
+    /**
+     * download the image to the local system
+     *
+     * @param array $arrFilter
+     * @return void
+     */
+    public function downRemoteImgToLocal(array $arrFilter)
+    {
+        $StorageCore = StorageCore::getInstance();
+
+        $arrPromises = [];
+
+        foreach ($arrFilter as $intKey => $arrItem) {
+            if (!$StorageCore->has($arrItem['local_path'])) {
+                $arrPromises[$intKey] = $this->getClient()->getAsync($arrItem['uri']);
+            }
+        }
+
+        $arrResults = Promise\Utils::settle($arrPromises)->wait();
+
+        $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
+            return $arrResult['state'] === 'fulfilled';
+        });
+
+        foreach ($arrSuccessfulResults as $intKey => $arrResult) {
+            $Response = $arrResult['value'];
+
+            $StorageCore->writeStream($arrFilter[$intKey]['local_path'], $Response->getBody()->detach());;
+        }
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @param object $Site
+     * @param string $strUri
+     * @return NovelPage
+     */
+    public function getNovelPage($Site, $strUri)
+    {
+        return new NovelPage($Site, $strUri);
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @param object $Site
+     * @param string $strUri
+     * @return ChapterPage
+     */
+    public function getChapterPage($Site, $strUri)
+    {
+        return new ChapterPage($Site, $strUri);
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @param object $Site
+     * @param string $strUri
+     * @return LatestListPage
+     */
+    public function getLatestListPage($Site, $strUri)
+    {
+        return new LatestListPage($Site, $strUri);
+    }
+}