DEBUG

code/app/task/crawler/youzhi/Scheduler.php

@@ -0,0 +1,122 @@
+<?php
+
+declare(strict_types=1);
+
+namespace app\task\crawler\youzhi;
+
+use app\model\ChapterModel;
+use app\task\crawler\youzhi\page\Site;
+use microserver\QueueManage;
+use storage\StorageCore;
+
+/**
+ * @mixin think\Model
+ */
+class Scheduler
+{
+    protected $intSlice = 8;
+
+    protected $intStartNSourceId = 1;
+
+    protected $intEndNSourceId = 500000;
+
+    /**
+     * Undocumented variable
+     *
+     * @var \microserver\QueueManage
+     */
+    protected $QueueManage = NULL;
+
+    /**
+     * Undocumented variable
+     *
+     * @var Site
+     */
+    public $Site;
+
+    /**
+     * Undocumented function
+     */
+    public function __construct()
+    {
+        $this->QueueManage = new QueueManage();
+        $this->init();
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @return void
+     */
+    public function init()
+    {
+        $this->intSlice = 8;
+    }
+
+    /**
+     * get site
+     *
+     * @return Site
+     */
+    public function getSite()
+    {
+        if ($this->Site == NULL) {
+            $this->Site = new Site;
+        }
+        return  $this->Site;
+    }
+
+
+    /**
+     * create a site-wide craw task
+     *
+     * @return void
+     */
+    public function crawlFull()
+    {
+        $intLimit = 10;
+
+        $Site = $this->getSite();
+
+        $arrAllCategory = $Site->getCategory();
+
+        foreach ($arrAllCategory as $arrCategory) {
+            $intTotalPage = ceil($arrCategory['total'] / $intLimit);
+            for ($intStartPage = 1; $intStartPage <= $intTotalPage; $intStartPage++) {
+                $arrFilter = [
+                    [
+                        'v_category_id' => $arrCategory['v_category_id'],
+                        'page' => $intStartPage,
+                    ],
+                ];
+
+                $arrVideoPageList = $Site->getVideoPageList($arrFilter);
+
+                foreach ($arrVideoPageList as $VideoPage) {
+                    $arrAllVideoPager = $VideoPage->getVideoPager();
+
+                    $arrVSourceId = [];
+                    foreach ($arrAllVideoPager['list'] as $arrVideoPager) {
+                        $arrVSourceId[] = ['v_source_id' => (int)$arrVideoPager['vod_id']];
+                    }
+
+                    $arrAllVideoInfoPage = $Site->getVideoInfoPageList($arrVSourceId);
+                    foreach ($arrAllVideoInfoPage as $VideoInfoPage) {
+                        $arrVideo = $VideoInfoPage->getVideo();
+                        print_r($arrVideo);
+                    }
+                }
+            }
+        }
+
+
+        $arrCover = [];
+
+
+        if ($arrCover) {
+            // $Site->downRemoteImgToLocal($arrCover);
+        }
+
+        return;
+    }
+}

code/app/task/crawler/youzhi/page/BasePage.php

@@ -0,0 +1,251 @@
+<?php
+
+declare(strict_types=1);
+
+namespace app\task\crawler\youzhi\page;
+
+use db\mongo\MongoBase;
+use ddl\DDLManage;
+use GuzzleHttp\Client;
+use microserver\QueueManage;
+use QL\QueryList;
+use Psr\Http\Message\ResponseInterface;
+
+/**
+ * @mixin think\Model
+ */
+abstract class BasePage
+{
+    /**
+     * Undocumented variable
+     *
+     * @var string
+     */
+    // protected $strDomain;
+
+    /**
+     * Undocumented variable
+     *
+     * @var string
+     */
+    protected $strUri;
+
+    /**
+     * Undocumented variable
+     *
+     * @var MongoBase
+     */
+    protected $MongoBase;
+
+    /**
+     * Undocumented variable
+     *
+     * @var \GuzzleHttp\Client
+     */
+    protected $Client = NULL;
+
+    /**
+     * Undocumented variable
+     *
+     * @var \microserver\QueueManage
+     */
+    protected $QueueManage = NULL;
+
+
+    /**
+     * Undocumented variable
+     *
+     * @var array
+     */
+    protected $arrCaiJiPeiZhi = [];
+
+    /**
+     * Undocumented variable
+     *
+     * @var string
+     */
+    protected $strContent;
+
+    /**
+     * Undocumented variable
+     *
+     * @var QueryList
+     */
+    protected $QueryList;
+
+    /**
+     * Undocumented variable
+     *
+     * @var array
+     */
+    protected $arrData = [];
+
+    /**
+     * Undocumented variable
+     *
+     * @var bool
+     */
+    protected $boolIsMobile;
+
+    /**
+     * Undocumented variable
+     *
+     * @var ResponseInterface
+     */
+    private $Response;
+
+
+    /**
+     * Undocumented function
+     *
+     * @param string $strUri
+     */
+    public function __construct($Site, string $strUri)
+    {
+        $this->setSite($Site);
+
+        $this->strUri = $strUri;
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @return string
+     */
+    public function getUri(): string
+    {
+        return $this->strUri;
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @param array $arrData
+     * @return void
+     */
+    public function setData(array $arrData)
+    {
+        $this->arrData = $arrData;
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @return array
+     */
+    public function getData(): array
+    {
+        return $this->arrData;
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @return QueueManage
+     */
+    public function getQueueManage(): QueueManage
+    {
+        if ($this->QueueManage == NULL) {
+            $this->QueueManage = QueueManage::getInstance();
+        }
+
+        return $this->QueueManage;
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @return MongoBase
+     */
+    public function getMongoBase(): MongoBase
+    {
+        if ($this->MongoBase == NULL) {
+            DDLManage::load('MongoBase');
+            $this->MongoBase = MongoBase::getInstance();
+        }
+
+        return $this->MongoBase;
+    }
+
+    public function getQueryList(): QueryList
+    {
+        if ($this->QueryList == NULL) {
+            $this->QueryList = QueryList::html($this->getContent());
+        }
+
+        return $this->QueryList;
+    }
+
+
+    /**
+     * Undocumented function
+     *
+     * @return string
+     */
+    public function getContent(): string
+    {
+        if ($this->strContent == NULL) {
+            $this->Response =  $this->getClient()->get($this->strUri);
+            $this->strContent =  $this->Response->getBody()->getContents();
+        }
+
+        return $this->strContent;
+    }
+
+    /**
+     * force set page content
+     *
+     * @param string $strContent
+     * @return void
+     */
+    public function setContent($strContent)
+    {
+        $this->strContent = $strContent;
+    }
+
+
+    /**
+     * Undocumented function
+     *
+     * @return boolean
+     */
+    public function isMobile($strMark = 'content="mobile"'): bool
+    {
+        if ($this->boolIsMobile === NULL) {
+            $this->boolIsMobile = is_numeric(strpos($this->getContent(), $strMark));
+        }
+        return $this->boolIsMobile;
+    }
+
+
+    /**
+     * Undocumented function
+     *
+     * @return Client
+     */
+    protected function getClient(): Client
+    {
+        if ($this->Client == NULL) {
+            $this->Client = $this->getSite()->getClient();
+        }
+        return $this->Client;
+    }
+
+
+    /**
+     * Undocumented variable
+     *
+     * @var Site
+     */
+    protected  $Site;
+
+    public function setSite(Site $Site)
+    {
+        $this->Site = $Site;
+    }
+
+    public function getSite()
+    {
+        return  $this->Site;
+    }
+}

code/app/task/crawler/youzhi/page/Site.php

@@ -0,0 +1,369 @@
+<?php
+
+declare(strict_types=1);
+
+namespace app\task\crawler\youzhi\page;
+
+use GuzzleHttp\Client;
+use GuzzleHttp\Promise;
+use storage\StorageCore;
+
+/**
+ * @mixin think\Model
+ */
+class Site
+{
+    /**
+     * Undocumented variable
+     *
+     * @var Client
+     */
+    protected $Client;
+
+    /**
+     * Undocumented variable
+     *
+     * @var string
+     */
+    protected $strDomain = 'https://api.yzzy-api.com';
+
+    /**
+     * Undocumented variable
+     *
+     * @var string
+     */
+    protected $strSiteCode = 'youzhi';
+
+    /**
+     * use proxy
+     *
+     * @var boolean
+     */
+    protected $boolProxy = false;
+
+    /**
+     * Undocumented function
+     */
+    public function __construct()
+    {
+        $this->initClient();
+    }
+
+    /**
+     * Undocumented function
+     */
+    public function initClient()
+    {
+        $arrConfig = [
+            'http_version' => '1.1', // 强制用 HTTP/1.1，已有，保持
+            'base_uri' => $this->strDomain, // 基础域名，已有，保持
+            'http_errors' => false, // 不抛 HTTP 错误，已有，保持
+            'timeout' => 60, // 从 160 秒改为 60 秒
+            'connect_timeout' => 10, // 从 30 秒改为 10 秒
+            'curl' => [
+                CURLOPT_TCP_KEEPALIVE => 1, // 启用 TCP Keep-Alive，已有，保持
+                CURLOPT_TCP_KEEPIDLE => 120, // Keep-Alive 空闲时间
+                CURLOPT_TCP_KEEPINTVL => 60, // Keep-Alive 探测间隔
+                // CURLOPT_SSL_VERIFYPEER => true, // 验证 SSL，已有，保持
+            ],
+            'headers' => [
+                'User-Agent' => 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36',
+                'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
+                'Accept-Language' => 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
+                'Referer' => $this->strDomain,
+                'Connection' => 'keep-alive',
+                'sec-ch-ua' => '"Not/A)Brand";v="8", "Chromium";v="130", "Google Chrome";v="130"',
+                'sec-ch-ua-mobile' => '?1',
+                'sec-ch-ua-platform' => '"Android"',
+                'sec-fetch-dest' => 'document',
+                'sec-fetch-mode' => 'navigate',
+                'sec-fetch-site' => 'same-origin',
+                'sec-fetch-user' => '?1',
+                'upgrade-insecure-requests' => '1',
+            ],
+            'pool_size' => 20,
+            'verify' => false,
+        ];
+
+        $arrProxy = config('proxy');
+
+
+        if ($this->boolProxy && $arrProxy['proxy_status']) {
+
+            $strCode = $arrProxy['proxy_code'];
+            $strProxyHost = $arrProxy['proxy_host'];
+            $strKey = $arrProxy['proxy_key'];
+            $strSecret = $arrProxy['proxy_secret'];
+
+
+            switch ($strCode) {
+                case 'XIONGMAO':
+                    $intTime = time();
+                    $strTxt = "orderno=" . $strKey . ",secret=" . $strSecret . ",timestamp=" . $intTime;
+                    $strSign = strtoupper(md5($strTxt));
+                    $strAuth = 'sign=' . $strSign . '&orderno=' . $strKey . '&timestamp=' . $intTime;
+                    $arrConfig['proxy'] = $strProxyHost;
+                    $arrConfig['headers']['Proxy-Authorization'] =  $strAuth;
+                    break;
+                case 'KUAI':
+                    $arrConfig['proxy'] = $strProxyHost;
+                    $arrConfig['curl'][CURLOPT_PROXYAUTH] = CURLAUTH_BASIC;
+                    $arrConfig['curl'][CURLOPT_PROXYUSERPWD] = "$strKey:$strSecret";
+                    break;
+            }
+        }
+
+
+        $this->Client = new Client($arrConfig);
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @return string
+     */
+    public function getSiteCode(): string
+    {
+        return $this->strSiteCode;
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @param integer|string $strSign
+     * @return string
+     */
+    public function getSiteUUId(int|string $strSign): string
+    {
+        return $this->getSiteCode() . '-' . $strSign;
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @return Client
+     */
+    public function getClient(): Client
+    {
+        return $this->Client;
+    }
+
+    /**
+     * retrieve multiple novel page model
+     *
+     * @param string $strUri
+     * @return VideoPage[]
+     */
+    public function getCategory(): array
+    {
+        $strUri = '/inc/api_mac10.php?ac=list&t=1&page=1';
+
+        $Response = $this->getClient()->get($strUri);
+        $strContent =  $Response->getBody()->getContents();
+
+        $arrContent = json_decode($strContent, true);
+        $arrFilter = $arrContent['class'];
+
+        $arrCategory = [];
+
+        foreach ($arrFilter as $intKey => $arrItem) {
+
+            $arrFilter[$intKey]['uri'] = $this->getVideoPageUri((int)$arrItem['type_id'], 1);
+
+            $arrPromises[$intKey] = $this->getClient()->getAsync($arrFilter[$intKey]['uri']);
+        }
+
+        $arrResults = Promise\Utils::settle($arrPromises)->wait();
+
+        $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
+            return $arrResult['state'] === 'fulfilled';
+        });
+
+        foreach ($arrSuccessfulResults as $intKey => $arrResult) {
+            $Response = $arrResult['value'];
+
+            $strContent =  $Response->getBody()->getContents();
+
+            $arrTemp = json_decode($strContent, true);
+
+            $arrCategory[] = [
+                'v_category_id' =>  (int)$arrFilter[$intKey]['type_id'],
+                'v_category_name' =>  $arrFilter[$intKey]['type_name'],
+                'v_category_name_en' => zhToPinYin($arrFilter[$intKey]['type_name']),
+                'total' => $arrTemp['total'],
+            ];
+        }
+
+        return $arrCategory;
+    }
+
+    /**
+     * generate video list uri
+     *
+     * @param integer $intVCategoryId
+     * @param integer $intPage
+     * @return string
+     */
+    public function getVideoPageUri(int $intVCategoryId, int $intPage): string
+    {
+        return  sprintf('/inc/api_mac10.php?ac=list&t=%s&page=%s', $intVCategoryId, $intPage);
+    }
+
+    /**
+     * retrieve multiple novel page model
+     *
+     * @param string $strUri
+     * @return VideoPage[]
+     */
+    public function getVideoPageList(array $arrFilter): array
+    {
+
+        /** @var VideoPage[] */
+        $arrVideoPage = [];
+
+        foreach ($arrFilter as $intKey => $arrItem) {
+
+            $arrFilter[$intKey]['uri'] = $this->getVideoPageUri($arrItem['v_category_id'], $arrItem['page']);
+
+            $arrPromises[$intKey] = $this->getClient()->getAsync($arrFilter[$intKey]['uri']);
+        }
+
+        $arrResults = Promise\Utils::settle($arrPromises)->wait();
+
+
+        $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
+            return $arrResult['state'] === 'fulfilled';
+        });
+
+        foreach ($arrSuccessfulResults as $intKey => $arrResult) {
+            $Response = $arrResult['value'];
+
+            $strContent =  $Response->getBody()->getContents();
+
+
+            $VideoPage = $this->getVideoPage($this, $arrFilter[$intKey]['uri']);
+
+            $VideoPage->setContent($strContent);
+            $VideoPage->setData($arrFilter[$intKey]);
+
+            $arrVideoPage[] =  $VideoPage;
+        }
+
+        return $arrVideoPage;
+    }
+
+    /**
+     * generate video list uri
+     *
+     * @param integer $intVCategoryId
+     * @param integer $intPage
+     * @return string
+     */
+    public function getVideoUri(int $intVSourceId): string
+    {
+        return  sprintf('/inc/api_mac10.php?ac=detail&tds=%s', $intVSourceId);
+    }
+
+    /**
+     * retrieve multiple chapter page model
+     *
+     * @param string $strUri
+     * @return VideoInfoPage[]
+     */
+    public function getVideoInfoPageList(array $arrFilter): array
+    {
+
+        /** @var VideoInfoPage[] */
+        $arrVideoInfoPage = [];
+
+        $arrPromises = [];
+        foreach ($arrFilter as $intKey => $arrItem) {
+            $arrFilter[$intKey]['uri'] = $this->getVideoUri($arrItem['v_source_id']);
+            $arrPromises[$intKey] = $arrFilter[$intKey]['uri'];
+        }
+
+        print_r($arrFilter);
+
+        $arrResults = Promise\Utils::settle($arrPromises)->wait();
+
+
+        var_dump($arrResults);
+        return [];
+
+        $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
+            return $arrResult['state'] === 'fulfilled';
+        });
+
+        foreach ($arrSuccessfulResults as $intKey => $arrResult) {
+            $Response = $arrResult['value'];
+
+            var_dump($arrResult);
+
+            $strContent =  $Response->getBody()->getContents();;
+
+            $VideoInfoPage = $this->getVideoInfoPage($this, $arrFilter[$intKey]['uri']);
+
+            $VideoInfoPage->setContent($strContent);
+            $VideoInfoPage->setData($arrFilter[$intKey]);
+
+            $arrVideoInfoPage[] =  $VideoInfoPage;
+        }
+
+        return $arrVideoInfoPage;
+    }
+
+    /**
+     * download the image to the local system
+     *
+     * @param array $arrFilter
+     * @return void
+     */
+    public function downRemoteImgToLocal(array $arrFilter)
+    {
+        $StorageCore = StorageCore::getInstance();
+
+        $arrPromises = [];
+
+        foreach ($arrFilter as $intKey => $arrItem) {
+            if (!$StorageCore->has($arrItem['local_path'])) {
+                $arrPromises[$intKey] = $this->getClient()->getAsync($arrItem['uri']);
+            }
+        }
+
+        $arrResults = Promise\Utils::settle($arrPromises)->wait();
+
+        $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
+            return $arrResult['state'] === 'fulfilled';
+        });
+
+        foreach ($arrSuccessfulResults as $intKey => $arrResult) {
+            $Response = $arrResult['value'];
+
+            $StorageCore->writeStream($arrFilter[$intKey]['local_path'], $Response->getBody()->detach());;
+        }
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @param object $Site
+     * @param string $strUri
+     * @return VideoPage
+     */
+    public function getVideoPage($Site, $strUri)
+    {
+        return new VideoPage($Site, $strUri);
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @param object $Site
+     * @param string $strUri
+     * @return VideoInfoPage
+     */
+    public function getVideoInfoPage($Site, $strUri)
+    {
+        return new VideoInfoPage($Site, $strUri);
+    }
+}

code/app/task/crawler/youzhi/page/VideoInfoPage.php

@@ -0,0 +1,194 @@
+<?php
+
+declare(strict_types=1);
+
+namespace app\task\crawler\youzhi\page;
+
+use app\model\VideoModel;
+use storage\StorageCore;
+
+class VideoInfoPage extends BasePage
+{
+    /**
+     * Undocumented variable
+     *
+     * @var string
+     */
+    protected $strChapterContent = '';
+
+    /**
+     * Undocumented variable
+     *
+     * @var VideoModel
+     */
+    protected $VideoModel;
+
+    /**
+     * Undocumented function
+     *
+     * @return VideoModel
+     */
+    public function getVideoModel(): VideoModel
+    {
+        if ($this->VideoModel == NULL) {
+            $this->VideoModel = VideoModel::getInstance();
+        }
+        return  $this->VideoModel;
+    }
+
+    public $arrOptions = [
+        'typeMap' => [
+            'root' => 'array',
+            'document' => 'array',
+            'array' => 'array',
+        ],
+    ];
+
+      /**
+     * Undocumented function
+     *
+     * @return array
+     */
+    public function getVideo(): array
+    {
+        if (empty($this->arrVideo)) {
+
+            $this->arrVideo = json_decode($this->getContent(), true);
+        }
+
+        return  $this->arrVideo;
+    }
+
+    /**
+     * Undocumented variable
+     *
+     * @var array
+     */
+    protected $arrVideo;
+
+
+    /**
+     * Undocumented function
+     *
+     * @param integer $intNId
+     * @return boolean
+     */
+    public function saveChapter(int $intNId): bool
+    {
+        $arrPageChapter = $this->getVideo();
+
+        if (
+            empty($arrPageChapter) ||
+            empty($arrPageChapter['c_page_title']) ||
+            empty($arrPageChapter['c_page_data']) ||
+            count($arrPageChapter['c_page_data']) < 5
+        ) {
+            return false;
+        }
+
+        $strUUId = $this->getSite()->getSiteUUId('chapter-' . $this->arrData['c_source_id'] . '-' . $this->arrData['c_page']);
+
+        $arrChapter = [
+            'n_id' => $intNId,
+            'c_name' => $arrPageChapter['c_page_data'][3],
+            'c_sort_num' => $this->arrData['c_sort_num'],
+            'c_page' => $this->arrData['c_page'],
+            'c_source_id' => $this->arrData['c_source_id'],
+            'c_site_code' => $this->getSite()->getSiteCode(),
+            'c_source_uuid' =>  $strUUId,
+        ];
+
+        $arrExistsChapter = $this->getVideoModel()->findOne(['c_source_uuid' => $strUUId], $this->arrOptions);
+
+        if ($arrExistsChapter) {
+
+            $arrExistsChapter = (array) $arrExistsChapter;
+
+            $arrUpdate = [
+                'updated_at' => new \MongoDB\BSON\UTCDateTime(),
+            ];
+
+            $this->getVideoModel()->updateOne(
+                ['c_source_uuid' => $strUUId],
+                ['$set' =>   $arrUpdate],
+            );
+            $this->strCContentPath = $arrExistsChapter['c_content_path'];
+        } else {
+
+            $arrInsertNovel =  $arrChapter;
+
+            $arrInsertNovel['c_content_path'] = '/novel/' . ($intNId % 100) . '/' . $intNId . '/' .  $arrChapter['c_sort_num'] . '_' .   $arrChapter['c_page'] . '.txt';
+            $arrInsertNovel['created_at'] = new \MongoDB\BSON\UTCDateTime();
+
+            try {
+                $this->getVideoModel()->insert($arrInsertNovel);
+                $this->updateNovelHaveChapter($intNId);
+                $this->strCContentPath = $arrInsertNovel['c_content_path'];
+            } catch (\Exception $e) {
+                if ($e->getCode() === 11000) {
+                    return $this->saveChapter($intNId);
+                } else {
+                    throw $e;
+                }
+            }
+        }
+
+        StorageCore::getInstance()->put($this->strCContentPath, $this->strChapterContent);
+
+        return true;
+    }
+
+    public $strCContentPath;
+    /**
+     * Undocumented function
+     *
+     * @param integer $intNId
+     * @return void
+     */
+    protected function updateNovelHaveChapter(int $intNId)
+    {
+        $arrFilter = [
+            'n_id' => $intNId,
+        ];
+
+        $VideoModel =  VideoModel::getInstance();
+        $arrLasteChapter =  $VideoModel->getLatestChapterByNId($intNId, false);
+
+        if (!empty($arrLasteChapter)) {
+            $arrUpdate['$set']['n_have_chapter'] = 1;
+            $arrUpdate['$set']['n_latest_chapter_name'] = $arrLasteChapter['c_name'];
+            $this->getMongoBase()->updateOne('novel', $arrFilter, $arrUpdate);
+        }
+    }
+
+    protected $arrNextPage;
+
+    /**
+     * Undocumented function
+     *
+     * @return array
+     */
+    public function getNextPage(): array
+    {
+        if ($this->arrNextPage === NULL) {
+            $this->arrNextPage = [];
+
+            if ($this->isMobile()) {
+                $strPattern = "/var\s+hhekgsv\s*=\s*['\"](.*?)['\"]/";
+                preg_match_all($strPattern, $this->getContent(), $arrMatches);
+                $strNextPageUri = end($arrMatches[1]) ?: '';
+            } else {
+                $strHref = $this->getQueryList()->find('.read_btn a:contains("下一章")')->attr('href');
+                $strNextPageUri = $strHref ?: '';
+            }
+
+            if (is_numeric(strpos($strNextPageUri, '_'))) {
+                $this->arrNextPage = $this->arrData;
+                $this->arrNextPage['c_page'] += 1;
+                $this->arrNextPage['uri'] = $strNextPageUri;
+            }
+        }
+
+        return $this->arrNextPage;
+    }
+}

code/app/task/crawler/youzhi/page/VideoPage.php

@@ -0,0 +1,417 @@
+<?php
+
+declare(strict_types=1);
+
+namespace app\task\crawler\youzhi\page;
+
+use app\model\CategoryModel;
+use app\model\CountersModel;
+use app\model\NovelModel;
+use storage\StorageCore;
+
+class VideoPage extends BasePage
+{
+    /**
+     * Undocumented variable
+     *
+     * @var array
+     */
+    protected $arrVideo = [];
+
+    /**
+     * Undocumented variable
+     *
+     * @var array
+     */
+    protected $arrRelatedNovel = [];
+
+    /**
+     * Undocumented variable
+     *
+     * @var array
+     */
+    protected $arrForgeNovel = [];
+
+    /**
+     * Undocumented variable
+     *
+     * @var NovelModel
+     */
+    protected $NovelModel;
+
+    /**
+     * Undocumented function
+     *
+     * @return NovelModel
+     */
+    public function getNovelModel(): NovelModel
+    {
+        if ($this->NovelModel == NULL) {
+            $this->NovelModel = NovelModel::getInstance();
+        }
+        return  $this->NovelModel;
+    }
+
+    public $arrOptions = [
+        'typeMap' => [
+            'root' => 'array',
+            'document' => 'array',
+            'array' => 'array',
+        ],
+    ];
+
+    /**
+     * 查询看的小说
+     *
+     * @param integer $intNSourceId
+     * @param string $strName
+     * @return array
+     */
+    protected function findOrCreateNovel(int $intNSourceId, string $strName): array
+    {
+        $strUUId = $this->getSite()->getSiteUUId($intNSourceId);
+
+        $arrExistsNovel = $this->getNovelModel()->findOne(['n_source_uuid' => $strUUId], $this->arrOptions);
+
+        if ($arrExistsNovel) {
+            $arrVideo = (array) $arrExistsNovel;
+        } else {
+            $arrVideo = [
+                'n_id' =>  CountersModel::getInstance()->getNextSequence('novel'),
+                'n_name' => $strName,
+                'n_name_pinyin' => zhToPinYin($strName),
+                'n_site_code' =>  $this->getSite()->getSiteCode(),
+                'n_source_uuid' => $strUUId,
+                'n_source_id' => $intNSourceId,
+                'n_level' => 1,
+                'n_have_chapter' => 0,
+                'created_at' => new \MongoDB\BSON\UTCDateTime(),
+            ];
+            try {
+                $this->getNovelModel()->insert($arrVideo);
+            } catch (\Exception $e) {
+                if ($e->getCode() === 11000) {
+                    return $this->findOrCreateNovel($intNSourceId,  $strName);
+                } else {
+                    throw $e;
+                }
+            }
+        }
+        return $arrVideo;
+    }
+
+
+    /**
+     * 分析关联数据
+     *
+     * @return void
+     */
+    public function getReletedNovel() {}
+
+
+    /**
+     * Undocumented function
+     *
+     * @return array
+     */
+    public function getVideoPager(): array
+    {
+        if (empty($this->arrVideo)) {
+
+            $this->arrVideo = json_decode($this->getContent(), true);
+
+            $this->getReletedNovel();
+        }
+
+        return  $this->arrVideo;
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @param string $strName
+     * @return string
+     */
+    public function generateCoverPath($strName): string
+    {
+        $intTime = time() - mt_rand(1, 2 * 24 * 3600 * 360);
+
+        return sprintf('/img/%s/%s.jpg', date('Y/m/d/H', $intTime), $strName);
+    }
+
+    public function filterOtherFields(&$arrNovel)
+    {
+        $arrFields = [
+            'n_id',
+            'n_name',
+            'n_name_pinyin',
+            'n_status',
+            'n_author',
+            'n_description',
+            'n_category',
+            'n_category_pinyin',
+            'n_category_sex_en',
+            'n_category_sex_zh',
+            'n_cover_path',
+            'n_latest_chapter_name',
+            'n_update_time',
+            'n_forge_novel',
+            'n_related_novel',
+            'n_site_code',
+            'n_source_uuid',
+            'n_source_id',
+            'n_have_chapter',
+            'created_at',
+            'updated_at',
+        ];
+
+        foreach ($arrNovel as $strKey => $strVal) {
+            if (!in_array($strKey, $arrFields)) {
+                unset($arrNovel[$strKey]);
+            }
+        }
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @return bool
+     */
+    public function saveVideo(): bool
+    {
+        $arrPageInfo = $this->getVideo();
+
+        print_r($arrPageInfo);
+        return false;
+        if (
+            empty($arrPageInfo) ||
+            empty($arrPageInfo['og_title']) ||
+            empty($arrPageInfo['og_image']) ||
+            empty($arrPageInfo['og_novel_category'])
+        ) {
+            return false;
+        }
+
+        $arrVideo = [
+            // 'n_id'=> $arrPageInfo['xxxxx'],
+            'n_name' => $arrPageInfo['og_title'],
+            'n_name_pinyin' => zhToPinYin($arrPageInfo['og_title']),
+            'n_status' => $arrPageInfo['og_novel_status'],
+            'n_author' => $arrPageInfo['og_novel_author'],
+            'n_description' => $arrPageInfo['og_description'],
+            'n_category' => $arrPageInfo['og_novel_category'],
+            // 'n_category_pinyin' => $arrPageInfo['xxxxx'], 
+            // 'n_category_sex_en' => $arrPageInfo['xxxxx'], 
+            // 'n_category_sex_zh' => $arrPageInfo['xxxxx'], 
+            // 'n_cover_path' => $arrPageInfo['xxxxx'], 
+            'n_latest_chapter_name' => $arrPageInfo['og_novel_latest_chapter_name'],
+            'n_update_time' => $arrPageInfo['og_novel_update_time'],
+            'n_forge_novel' => $this->arrForgeNovel,
+            'n_related_novel' => $this->arrRelatedNovel,
+
+            'n_site_code' =>  $this->getSite()->getSiteCode(),
+            'n_source_id' => $this->arrData['n_source_id'],
+            'n_source_uuid' => $this->getSite()->getSiteUUId($this->arrData['n_source_id']),
+
+            // 'n_have_chapter' => $arrPageInfo['xxxxx'],
+            // 'created_at' => $arrPageInfo['xxxxx'],
+            // 'updated_at' => $arrPageInfo['xxxxx'],
+
+        ];
+
+        $arrExistsNovel = $this->getNovelModel()->findOne(['n_source_uuid' => $arrVideo['n_source_uuid']], $this->arrOptions);
+
+        if ($arrExistsNovel) {
+
+            $arrExistsNovel = (array) $arrExistsNovel;
+
+            $arrUpdate = [
+                'n_latest_chapter_name' => $arrVideo['n_latest_chapter_name'],
+                'n_update_time' => $arrVideo['n_update_time'],
+                'updated_at' => new \MongoDB\BSON\UTCDateTime(),
+            ];
+
+            # 如果没有分类，说明这条数据是最初从关联shu那里采集的，需要补全全部信息
+            if (empty($arrExistsNovel['n_category'])) {
+                $arrUpdate = array_merge($arrUpdate, $arrVideo);
+                $arrUpdate['n_related_novel'] = $this->arrRelatedNovel;
+                $arrUpdate['n_forge_novel'] = $this->arrForgeNovel;
+                $arrUpdate['n_category_pinyin'] = CategoryModel::getPinYin($this->arrVideo['og_novel_category']);
+                $arrUpdate['n_category_sex_en'] = CategoryModel::getSexEn($this->arrVideo['og_novel_category']);
+                $arrUpdate['n_category_sex_zh'] = CategoryModel::getSexZh($this->arrVideo['og_novel_category']);
+                $arrUpdate['n_related_novel'] = $this->arrRelatedNovel;
+                $arrUpdate['n_forge_novel'] = $this->arrForgeNovel;
+            }
+
+            # 图片不存在的时候，需要重新生成
+            if (
+                !isset($arrExistsNovel['n_cover_path']) ||
+                empty($arrExistsNovel['n_cover_path']) ||
+                !StorageCore::getInstance()->has($arrExistsNovel['n_cover_path']) ||
+                (basename($arrExistsNovel['n_cover_path']) == '.jpg')
+            ) {
+                $arrUpdate['n_cover_path'] = $this->generateCoverPath($arrVideo['n_name_pinyin']);
+                $this->strCoverLocalPath = $arrUpdate['n_cover_path'];
+            } else {
+                $this->strCoverLocalPath = $arrExistsNovel['n_cover_path'];
+            }
+
+            $this->getNovelModel()->updateOne(
+                ['n_source_uuid' =>  $arrVideo['n_source_uuid']],
+                ['$set' => $arrUpdate],
+            );
+            $this->intNId =       $arrExistsNovel['n_id'];
+        } else {
+
+            $arrVideo['n_category_pinyin'] = CategoryModel::getPinYin($this->arrVideo['og_novel_category']);
+            $arrVideo['n_category_sex_en'] = CategoryModel::getSexEn($this->arrVideo['og_novel_category']);
+            $arrVideo['n_category_sex_zh'] = CategoryModel::getSexZh($this->arrVideo['og_novel_category']);
+            $arrVideo['n_id'] =  CountersModel::getInstance()->getNextSequence('novel');
+            $arrVideo['n_level'] = 1;
+            $arrVideo['n_have_chapter'] = 0;
+            $arrVideo['n_cover_path'] =  $this->generateCoverPath($arrVideo['n_name_pinyin']);
+            $arrVideo['n_related_novel'] = $this->arrRelatedNovel;
+            $arrVideo['n_forge_novel'] = $this->arrForgeNovel;
+            $arrVideo['created_at'] = new \MongoDB\BSON\UTCDateTime();
+
+            try {
+                $this->getNovelModel()->insert($arrVideo);
+                $this->intNId =       $arrVideo['n_id'];
+                $this->strCoverLocalPath = $arrVideo['n_cover_path'];
+            } catch (\Exception $e) {
+                if ($e->getCode() === 11000) {
+                    return $this->saveVideo();
+                } else {
+                    throw $e;
+                }
+            }
+        }
+
+        return true;
+    }
+
+
+    public $strCoverLocalPath = '';
+
+    public $intNId;
+
+    /**
+     * Undocumented function
+     *
+     * @return array
+     */
+    public function getDownCoverInfo(): array
+    {
+        return [
+            'uri' => $this->arrVideo['og_image'],
+            'local_path' => $this->strCoverLocalPath,
+        ];
+    }
+
+    /**
+     * Undocumented variable
+     *
+     * @var array
+     */
+    protected $arrLatestChapters = NULL;
+
+    /**
+     * Undocumented variable
+     *
+     * @var array
+     */
+    protected $arrChapters = NULL;
+
+    public function getLatestChapterByNIdsList(): array
+    {
+        if ($this->arrLatestChapters === NULL) {
+            $this->arrLatestChapters = [];
+
+            if ($this->isMobile()) {
+                $strClass = '.chapter-list';
+            } else {
+                $strClass = '.section-list';
+            }
+
+            if ($this->arrData['page'] == 0) {
+                $this->getQueryList()->find($strClass)->eq(0)->find("a")->map(function ($A) {
+                    $this->arrLatestChapters[] = [
+                        'name' => $A->text(),
+                        'uri' => $A->attr('href'),
+                    ];
+                });
+            }
+        }
+        return $this->arrLatestChapters;
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @return array
+     */
+    public function getChapters(): array
+    {
+        if ($this->arrChapters === NULL) {
+            $this->arrChapters = [];
+            if ($this->isMobile()) {
+                $strClass = '.chapter-list';
+            } else {
+                $strClass = '.section-list';
+            }
+
+            $intDivIndex = 0;
+            if ($this->arrData['page'] == 0) {
+                $intDivIndex = 1;
+            }
+
+            $this->getQueryList()->find($strClass)->eq($intDivIndex)->find("a")->map(function ($A) {
+                $this->arrChapters[] = [
+                    'name' => $A->text(),
+                    'uri' => $A->attr('href'),
+                ];
+            });
+        }
+        return $this->arrChapters;
+    }
+
+
+    protected $arrNextPageArgs = NULL;
+
+    /**
+     * Undocumented function
+     *
+     * @return array
+     */
+    public function getNextPageArgs(): array
+    {
+        $this->arrNextPageArgs = [];
+        if ($this->arrData['page'] == 0) {
+            $this->getLatestChapterByNIdsList();
+            $this->getChapters();
+
+            $arrLatestChapterFirstBlock =  count($this->arrLatestChapters) > 0 ? $this->arrLatestChapters[0] : null;
+            $arrLatestChapterSecondBlock = count($this->arrChapters) > 0 ?  $this->arrChapters[count($this->arrChapters) - 1] : null;
+
+            if (
+                !empty($arrLatestChapterFirstBlock) && !empty($arrLatestChapterSecondBlock)
+                && ($arrLatestChapterFirstBlock['uri'] != $arrLatestChapterSecondBlock['uri'])
+
+            ) {
+                $this->arrNextPageArgs =  [
+                    'n_source_id' => $this->arrData['n_source_id'],
+                    'page' => $this->arrData['page'] + 1,
+                ];
+            }
+        } else {
+            $strLastUri = $this->getQueryList()->find('select:first option:last')->attr('value');
+            if ($strLastUri != '' && $strLastUri != $this->strUri) {
+                $this->arrNextPageArgs = [
+                    'n_source_id' => $this->arrData['n_source_id'],
+                    'page' => $this->arrData['page'] + 1,
+                ];
+            }
+        }
+        return   $this->arrNextPageArgs;
+    }
+}