Merge branch 'dev' of ssh://git.bgbg.live:18150/root/SEONexus into dev

code/app/api/controller/Statis.php

@@ -70,6 +70,26 @@ class Statis extends BaseController
 
     public function test0909()
     {
+
+
+        $a = '[{"type_id":1,"type_pid":0,"type_name":"电影"},{"type_id":2,"type_pid":0,"type_name":"连续剧"},{"type_id":3,"type_pid":0,"type_name":"综艺片"},{"type_id":4,"type_pid":0,"type_name":"动漫"},{"type_id":5,"type_pid":1,"type_name":"纪录片"},{"type_id":6,"type_pid":1,"type_name":"动作片"},{"type_id":7,"type_pid":1,"type_name":"爱情片"},{"type_id":8,"type_pid":1,"type_name":"喜剧片"},{"type_id":9,"type_pid":1,"type_name":"科幻片"},{"type_id":10,"type_pid":1,"type_name":"恐怖片"},{"type_id":11,"type_pid":1,"type_name":"剧情片"},{"type_id":12,"type_pid":1,"type_name":"战争片"},{"type_id":13,"type_pid":2,"type_name":"国产剧"},{"type_id":14,"type_pid":2,"type_name":"香港剧"},{"type_id":15,"type_pid":2,"type_name":"欧美剧"},{"type_id":16,"type_pid":2,"type_name":"韩剧"},{"type_id":20,"type_pid":1,"type_name":"动漫电影"},{"type_id":21,"type_pid":2,"type_name":"泰国剧"},{"type_id":22,"type_pid":2,"type_name":"日本剧"},{"type_id":23,"type_pid":2,"type_name":"台湾剧"},{"type_id":24,"type_pid":2,"type_name":"海外剧"},{"type_id":25,"type_pid":3,"type_name":"大陆综艺"},{"type_id":26,"type_pid":3,"type_name":"日韩综艺"},{"type_id":27,"type_pid":3,"type_name":"港台综艺"},{"type_id":28,"type_pid":3,"type_name":"欧美综艺"},{"type_id":29,"type_pid":3,"type_name":"演唱会"},{"type_id":30,"type_pid":4,"type_name":"国产动漫"},{"type_id":31,"type_pid":4,"type_name":"日本动漫"},{"type_id":32,"type_pid":4,"type_name":"欧美动漫"},{"type_id":33,"type_pid":4,"type_name":"海外动漫"},{"type_id":34,"type_pid":0,"type_name":"伦理片"},{"type_id":35,"type_pid":0,"type_name":"新闻资讯"},{"type_id":36,"type_pid":0,"type_name":"体育赛事"},{"type_id":37,"type_pid":0,"type_name":"短剧大全"},{"type_id":38,"type_pid":36,"type_name":"篮球"},{"type_id":39,"type_pid":36,"type_name":"足球"},{"type_id":40,"type_pid":36,"type_name":"网球"},{"type_id":41,"type_pid":36,"type_name":"斯诺克"},{"type_id":42,"type_pid":36,"type_name":"LPL"},{"type_id":43,"type_pid":37,"type_name":"重生民国"},{"type_id":44,"type_pid":37,"type_name":"穿越现代"},{"type_id":45,"type_pid":37,"type_name":"反转爽剧"},{"type_id":46,"type_pid":37,"type_name":"言情总裁"},{"type_id":47,"type_pid":37,"type_name":"现代都市"},{"type_id":48,"type_pid":37,"type_name":"古装仙侠"},{"type_id":49,"type_pid":37,"type_name":"悬疑烧脑"},{"type_id":50,"type_pid":1,"type_name":"惊悚片"},{"type_id":51,"type_pid":35,"type_name":"电影资讯"},{"type_id":52,"type_pid":35,"type_name":"娱乐新闻"}]';
+        $b = json_decode($a, true);
+
+        $c = [];
+        foreach ($b as $val) {
+            $c[$val['type_id']] = [
+                'name' => $val['type_name'],
+                'pinyin' => zhToPinYin($val['type_name']),
+            ];
+            if (empty($c[$val['type_id']]['pinyin'])) {
+                $c[$val['type_id']]['pinyin'] =  $c[$val['type_id']]['name'];
+            }
+        }
+        echo '<pre>';
+        print_r($c);
+        exit;
+
+
         $a =  VideoCategory::$arrCategory;
         $b = PageVideoCategory::$arrCategory;
         $c = YouzhiPageVideoCategory::$arrCategory;

code/app/task/crawler/douban/Scheduler.php

@@ -0,0 +1,206 @@
+<?php
+
+declare(strict_types=1);
+
+namespace app\task\crawler\douban;
+
+use app\model\ChapterModel;
+use app\task\crawler\CrawlerConfig;
+use app\task\crawler\douban\page\Site;
+use microserver\QueueManage;
+use storage\StorageCore;
+
+/**
+ * @mixin think\Model
+ */
+class Scheduler
+{
+    protected $intSlice = 8;
+
+    protected $intStartNSourceId = 1;
+
+    protected $intEndNSourceId = 500000;
+
+    /**
+     * Undocumented variable
+     *
+     * @var \microserver\QueueManage
+     */
+    protected $QueueManage = NULL;
+
+    /**
+     * Undocumented variable
+     *
+     * @var Site
+     */
+    public $Site;
+
+    /**
+     * Undocumented function
+     */
+    public function __construct()
+    {
+        $this->QueueManage = new QueueManage();
+        $this->init();
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @return void
+     */
+    public function init()
+    {
+        $this->intSlice = 8;
+    }
+
+    /**
+     * get site
+     *
+     * @return Site
+     */
+    public function getSite()
+    {
+        if ($this->Site == NULL) {
+            $this->Site = new Site;
+        }
+        return  $this->Site;
+    }
+
+
+    /**
+     * create a site-wide craw task
+     *
+     * @return void
+     */
+    public function crawlFull()
+    {
+        // $arrVSourceId = [['v_source_id' => 14256]];
+        // $arrTask = [
+        //     'callback' => [static::class, 'fetchVideo'],
+        //     'data' => $arrVSourceId,
+        // ];
+        // $this->QueueManage->set($arrTask);
+
+        // $this->fetchVideo($arrVSourceId);
+
+        // return;
+
+        $Site = $this->getSite();
+
+        $arrAllCategory = $Site->getCategory();
+
+        print_r($arrAllCategory);
+
+        return;
+
+        $intLimit = 5;
+
+        $arrTask = [
+            'callback' => [static::class, 'fetchVSourceId'],
+            'data' => [],
+        ];
+
+        foreach ($arrAllCategory as $arrCategory) {
+
+            if ($arrCategory['total'] <= 0) {
+                continue;
+            }
+
+            if (in_array($arrCategory['v_category_name'], CrawlerConfig::$arrExcludeVideoCategory)) {
+                continue;
+            }
+
+
+            $intTotalPage = $arrCategory['total_page'];
+            $arrFilter = [];
+            for ($intStartPage = 1; $intStartPage <= $intTotalPage; $intStartPage++) {
+
+                $arrFilter[] = [
+                    'v_category_id' => $arrCategory['v_category_id'],
+                    'page' => $intStartPage,
+                ];
+                if (($intStartPage %  $intLimit) == 0) {
+                    $arrTask['data'] = $arrFilter;
+                    $this->QueueManage->set($arrTask);
+                    $arrFilter = [];
+                }
+            }
+            if (!empty($arrFilter)) {
+                $arrTask['data'] = $arrFilter;
+                $this->QueueManage->set($arrTask);
+            }
+        }
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @param array $arrFilter
+     * @return void
+     */
+    public function fetchVSourceId(array $arrFilter)
+    {
+
+        $Site = $this->getSite();
+
+        $intLimit = 5;
+
+        $arrTask = [
+            'callback' => [static::class, 'fetchVideo'],
+            'data' => [],
+        ];
+
+        $arrAllVideoPagerPage = $Site->getVideoPagerPageList($arrFilter);
+
+        $arrVSourceId = [];
+
+        foreach ($arrAllVideoPagerPage as $VideoPagerPage) {
+            $arrVideoPager = $VideoPagerPage->getVideoPager();
+
+            foreach ($arrVideoPager['list'] as $arrVideoPager) {
+                $arrVSourceId[] = ['v_source_id' => (int)$arrVideoPager['vod_id']];
+
+                if ((count($arrVSourceId) % $intLimit) == 0) {
+                    $arrTask['data'] = $arrVSourceId;
+                    $this->QueueManage->set($arrTask);
+                    $arrVSourceId = [];
+                }
+            }
+        }
+
+        if (!empty($arrVSourceId)) {
+            $arrTask['data'] = $arrVSourceId;
+            $this->QueueManage->set($arrTask);
+        }
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @param array $arrVSourceId
+     * @return void
+     */
+    public function fetchVideo(array $arrVSourceId)
+    {
+        $Site = $this->getSite();
+
+        $arrAllVideoInfoPage = $Site->getVideoInfoPageList($arrVSourceId);
+
+        $arrPic = [];
+        foreach ($arrAllVideoInfoPage as $VideoInfoPage) {
+
+            $VideoInfoPage->saveVideo();
+
+            $arrPicItem = $VideoInfoPage->getDownPicInfo();
+
+            if (!StorageCore::getInstance()->has($arrPicItem['local_path'])) {
+                $arrPic[] = $arrPicItem;
+            }
+        }
+
+        if ($arrPic) {
+            $Site->downRemoteImgToLocal($arrPic);
+        }
+    }
+}

code/app/task/crawler/douban/page/BasePage.php

@@ -0,0 +1,249 @@
+<?php
+
+declare(strict_types=1);
+
+namespace app\task\crawler\douban\page;
+
+use db\mongo\MongoBase;
+use GuzzleHttp\Client;
+use microserver\QueueManage;
+use QL\QueryList;
+use Psr\Http\Message\ResponseInterface;
+
+/**
+ * @mixin think\Model
+ */
+abstract class BasePage
+{
+    /**
+     * Undocumented variable
+     *
+     * @var string
+     */
+    // protected $strDomain;
+
+    /**
+     * Undocumented variable
+     *
+     * @var string
+     */
+    protected $strUri;
+
+    /**
+     * Undocumented variable
+     *
+     * @var MongoBase
+     */
+    protected $MongoBase;
+
+    /**
+     * Undocumented variable
+     *
+     * @var \GuzzleHttp\Client
+     */
+    protected $Client = NULL;
+
+    /**
+     * Undocumented variable
+     *
+     * @var \microserver\QueueManage
+     */
+    protected $QueueManage = NULL;
+
+
+    /**
+     * Undocumented variable
+     *
+     * @var array
+     */
+    protected $arrCaiJiPeiZhi = [];
+
+    /**
+     * Undocumented variable
+     *
+     * @var string
+     */
+    protected $strContent;
+
+    /**
+     * Undocumented variable
+     *
+     * @var QueryList
+     */
+    protected $QueryList;
+
+    /**
+     * Undocumented variable
+     *
+     * @var array
+     */
+    protected $arrData = [];
+
+    /**
+     * Undocumented variable
+     *
+     * @var bool
+     */
+    protected $boolIsMobile;
+
+    /**
+     * Undocumented variable
+     *
+     * @var ResponseInterface
+     */
+    private $Response;
+
+
+    /**
+     * Undocumented function
+     *
+     * @param string $strUri
+     */
+    public function __construct($Site, string $strUri)
+    {
+        $this->setSite($Site);
+
+        $this->strUri = $strUri;
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @return string
+     */
+    public function getUri(): string
+    {
+        return $this->strUri;
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @param array $arrData
+     * @return void
+     */
+    public function setData(array $arrData)
+    {
+        $this->arrData = $arrData;
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @return array
+     */
+    public function getData(): array
+    {
+        return $this->arrData;
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @return QueueManage
+     */
+    public function getQueueManage(): QueueManage
+    {
+        if ($this->QueueManage == NULL) {
+            $this->QueueManage = QueueManage::getInstance();
+        }
+
+        return $this->QueueManage;
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @return MongoBase
+     */
+    public function getMongoBase(): MongoBase
+    {
+        if ($this->MongoBase == NULL) {
+            $this->MongoBase = MongoBase::getInstance();
+        }
+
+        return $this->MongoBase;
+    }
+
+    public function getQueryList(): QueryList
+    {
+        if ($this->QueryList == NULL) {
+            $this->QueryList = QueryList::html($this->getContent());
+        }
+
+        return $this->QueryList;
+    }
+
+
+    /**
+     * Undocumented function
+     *
+     * @return string
+     */
+    public function getContent(): string
+    {
+        if ($this->strContent == NULL) {
+            $this->Response =  $this->getClient()->get($this->strUri);
+            $this->strContent =  $this->Response->getBody()->getContents();
+        }
+
+        return $this->strContent;
+    }
+
+    /**
+     * force set page content
+     *
+     * @param string $strContent
+     * @return void
+     */
+    public function setContent($strContent)
+    {
+        $this->strContent = $strContent;
+    }
+
+
+    /**
+     * Undocumented function
+     *
+     * @return boolean
+     */
+    public function isMobile($strMark = 'content="mobile"'): bool
+    {
+        if ($this->boolIsMobile === NULL) {
+            $this->boolIsMobile = is_numeric(strpos($this->getContent(), $strMark));
+        }
+        return $this->boolIsMobile;
+    }
+
+
+    /**
+     * Undocumented function
+     *
+     * @return Client
+     */
+    protected function getClient(): Client
+    {
+        if ($this->Client == NULL) {
+            $this->Client = $this->getSite()->getClient();
+        }
+        return $this->Client;
+    }
+
+
+    /**
+     * Undocumented variable
+     *
+     * @var Site
+     */
+    protected  $Site;
+
+    public function setSite(Site $Site)
+    {
+        $this->Site = $Site;
+    }
+
+    public function getSite()
+    {
+        return  $this->Site;
+    }
+}

code/app/task/crawler/douban/page/Site.php

@@ -0,0 +1,389 @@
+<?php
+
+declare(strict_types=1);
+
+namespace app\task\crawler\douban\page;
+
+use GuzzleHttp\Client;
+use GuzzleHttp\Promise;
+use storage\StorageCore;
+
+/**
+ * @mixin think\Model
+ */
+class Site
+{
+    /**
+     * Undocumented variable
+     *
+     * @var Client
+     */
+    protected $Client;
+
+    /**
+     * Undocumented variable
+     *
+     * @var string
+     */
+    protected $strDomain = 'https://caiji.dbzy5.com';
+
+    /**
+     * Undocumented variable
+     *
+     * @var string
+     */
+    protected $strSiteCode = 'douban';
+
+    /**
+     * use proxy
+     *
+     * @var boolean
+     */
+    protected $boolProxy = false;
+
+    /**
+     * Undocumented function
+     */
+    public function __construct()
+    {
+        $this->initClient();
+    }
+
+    /**
+     * Undocumented function
+     */
+    public function initClient()
+    {
+        $arrConfig = [
+            'http_version' => '1.1', // 强制用 HTTP/1.1，已有，保持
+            'base_uri' => $this->strDomain, // 基础域名，已有，保持
+            'http_errors' => false, // 不抛 HTTP 错误，已有，保持
+            'timeout' => 60, // 从 160 秒改为 60 秒
+            'connect_timeout' => 10, // 从 30 秒改为 10 秒
+            'curl' => [
+                CURLOPT_TCP_KEEPALIVE => 1, // 启用 TCP Keep-Alive，已有，保持
+                CURLOPT_TCP_KEEPIDLE => 120, // Keep-Alive 空闲时间
+                CURLOPT_TCP_KEEPINTVL => 60, // Keep-Alive 探测间隔
+                // CURLOPT_SSL_VERIFYPEER => true, // 验证 SSL，已有，保持
+            ],
+            'headers' => [
+                'User-Agent' => 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36',
+                'Referer' => $this->strDomain,
+                'Connection' => 'keep-alive',
+            ],
+            'pool_size' => 20,
+            'verify' => false,
+        ];
+
+        $arrProxy = config('proxy');
+
+
+        if ($this->boolProxy && $arrProxy['proxy_status']) {
+
+            $strCode = $arrProxy['proxy_code'];
+            $strProxyHost = $arrProxy['proxy_host'];
+            $strKey = $arrProxy['proxy_key'];
+            $strSecret = $arrProxy['proxy_secret'];
+
+
+            switch ($strCode) {
+                case 'XIONGMAO':
+                    $intTime = time();
+                    $strTxt = "orderno=" . $strKey . ",secret=" . $strSecret . ",timestamp=" . $intTime;
+                    $strSign = strtoupper(md5($strTxt));
+                    $strAuth = 'sign=' . $strSign . '&orderno=' . $strKey . '&timestamp=' . $intTime;
+                    $arrConfig['proxy'] = $strProxyHost;
+                    $arrConfig['headers']['Proxy-Authorization'] =  $strAuth;
+                    break;
+                case 'KUAI':
+                    $arrConfig['proxy'] = $strProxyHost;
+                    $arrConfig['curl'][CURLOPT_PROXYAUTH] = CURLAUTH_BASIC;
+                    $arrConfig['curl'][CURLOPT_PROXYUSERPWD] = "$strKey:$strSecret";
+                    break;
+                case 'QING':
+                    $arrConfig['proxy'] = $strProxyHost;
+                    $strProxyAuth = base64_encode($strKey . ":" . $strSecret);
+                    $arrConfig['headers']['Proxy-Authorization'] =  "Basic " . $strProxyAuth;
+                    break;
+                case 'JULIANG':
+                    // 隧道代理信息
+                    // $strProxy = 'http://username:password@XXX.juliangip.com:23456';
+                    $strProxy = "http://" . $strKey . ":" . $strSecret . "@" . $strProxyHost;
+                    $arrConfig['proxy'] = $strProxy;
+                    break;
+            }
+        }
+
+
+        $this->Client = new Client($arrConfig);
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @return string
+     */
+    public function getSiteCode(): string
+    {
+        return $this->strSiteCode;
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @param integer|string $strSign
+     * @return string
+     */
+    public function getSiteUUId(int|string $strSign): string
+    {
+        return $this->getSiteCode() . '-' . $strSign;
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @return Client
+     */
+    public function getClient(): Client
+    {
+        return $this->Client;
+    }
+
+    /**
+     * retrieve multiple novel page model
+     *
+     * @param string $strUri
+     * @return array
+     */
+    public function getCategory(): array
+    {
+        $strUri = '/api.php/provide/vod/at/josn?ac=list';
+
+        $Response = $this->getClient()->get($strUri);
+        $strContent =  $Response->getBody()->getContents();
+
+        $arrContent = json_decode($strContent, true);
+        $arrFilter = $arrContent['class'];
+
+        // print_r($arrFilter);
+
+        // $arrA = [];
+        // foreach ($arrFilter as $b) {
+        //     $arrA[$b['type_id']] = [
+        //         'name' => $b['type_name'],
+        //         'pinyin' => zhToPinYin($b['type_name']),
+        //     ];
+        // }
+        // print_r($arrA);
+
+
+        $arrCategory = [];
+
+        foreach ($arrFilter as $intKey => $arrItem) {
+
+            $arrFilter[$intKey]['uri'] = $this->getVideoPageUri((int)$arrItem['type_id'], 1);
+
+            $arrPromises[$intKey] = $this->getClient()->getAsync($arrFilter[$intKey]['uri']);
+        }
+
+        $arrResults = Promise\Utils::settle($arrPromises)->wait();
+
+        $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
+            return $arrResult['state'] === 'fulfilled';
+        });
+
+        foreach ($arrSuccessfulResults as $intKey => $arrResult) {
+            $Response = $arrResult['value'];
+
+            $strContent =  $Response->getBody()->getContents();
+
+            $arrTemp = json_decode($strContent, true);
+
+            $arrCategory[] = [
+                'v_category_id' =>  (int)$arrFilter[$intKey]['type_id'],
+                'v_category_name' =>  $arrFilter[$intKey]['type_name'],
+                'v_category_name_en' => zhToPinYin($arrFilter[$intKey]['type_name']),
+                'total' => $arrTemp['total'],
+                'total_page' => $arrTemp['pagecount'],
+            ];
+        }
+
+        return $arrCategory;
+    }
+
+    /**
+     * generate video list uri
+     *
+     * @param integer $intVCategoryId
+     * @param integer $intPage
+     * @return string
+     */
+    public function getVideoPageUri(int $intVCategoryId, int $intPage): string
+    {
+        return  sprintf('/api.php/provide/vod/at/josn/?ac=list&t=%s&pg=%s', $intVCategoryId, $intPage);
+    }
+
+    /**
+     * retrieve multiple novel page model
+     *
+     * @param string $strUri
+     * @return VideoPagerPage[]
+     */
+    public function getVideoPagerPageList(array $arrFilter): array
+    {
+
+        /** @var VideoPagerPage[] */
+        $arrVideoPagerPage = [];
+
+        foreach ($arrFilter as $intKey => $arrItem) {
+
+            $arrFilter[$intKey]['uri'] = $this->getVideoPageUri($arrItem['v_category_id'], $arrItem['page']);
+
+            $arrPromises[$intKey] = $this->getClient()->getAsync($arrFilter[$intKey]['uri']);
+        }
+
+        $arrResults = Promise\Utils::settle($arrPromises)->wait();
+
+
+        $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
+            return $arrResult['state'] === 'fulfilled';
+        });
+
+        foreach ($arrSuccessfulResults as $intKey => $arrResult) {
+            $Response = $arrResult['value'];
+
+            $strContent =  $Response->getBody()->getContents();
+
+            if (is_numeric(strpos($strContent, 'System Error'))) {
+                continue;
+            }
+
+            $VideoPagerPage = $this->getVideoPagerPage($this, $arrFilter[$intKey]['uri']);
+
+            $VideoPagerPage->setContent($strContent);
+            $VideoPagerPage->setData($arrFilter[$intKey]);
+
+            $arrVideoPagerPage[] =  $VideoPagerPage;
+        }
+
+        return $arrVideoPagerPage;
+    }
+
+    /**
+     * generate video list uri
+     *
+     * @param integer $intVCategoryId
+     * @param integer $intPage
+     * @return string
+     */
+    public function getVideoUri(int $intVSourceId): string
+    {
+        return  sprintf('/api.php/provide/vod/at/josn?ac=detail&ids=%s', $intVSourceId);
+    }
+
+    /**
+     * retrieve multiple chapter page model
+     *
+     * @param string $strUri
+     * @return VideoInfoPage[]
+     */
+    public function getVideoInfoPageList(array $arrFilter): array
+    {
+
+        /** @var VideoInfoPage[] */
+        $arrVideoInfoPage = [];
+
+        $arrPromises = [];
+        foreach ($arrFilter as $intKey => $arrItem) {
+            $arrFilter[$intKey]['uri'] = $this->getVideoUri($arrItem['v_source_id']);
+
+            $arrPromises[$intKey] = $this->getClient()->getAsync($arrFilter[$intKey]['uri']);
+        }
+
+        $arrResults = Promise\Utils::settle($arrPromises)->wait();
+
+        $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
+            return $arrResult['state'] === 'fulfilled';
+        });
+
+        foreach ($arrSuccessfulResults as $intKey => $arrResult) {
+            $Response = $arrResult['value'];
+
+            $strContent =  $Response->getBody()->getContents();;
+
+            if (is_numeric(strpos($strContent, 'System Error'))) {
+                continue;
+            }
+
+            $VideoInfoPage = $this->getVideoInfoPage($this, $arrFilter[$intKey]['uri']);
+
+            $VideoInfoPage->setContent($strContent);
+            $VideoInfoPage->setData($arrFilter[$intKey]);
+
+            $arrVideoInfoPage[] =  $VideoInfoPage;
+        }
+
+        return $arrVideoInfoPage;
+    }
+
+    /**
+     * download the image to the local system
+     *
+     * @param array $arrFilter
+     * @return void
+     */
+    public function downRemoteImgToLocal(array $arrFilter)
+    {
+        $StorageCore = StorageCore::getInstance();
+
+        $Client = new Client();
+
+        foreach ($arrFilter as  $arrItem) {
+            if (!$StorageCore->has($arrItem['local_path'])) {
+                try {
+                    $Response = $Client->get($arrItem['uri']);
+                    if ($Response->getStatusCode() === 200) {
+                        if (str_ends_with(strtolower($arrItem['local_path']), '.webp')) {
+                            $strTmpFile = str_replace('.webp', '.jpg', $arrItem['local_path']);
+                            $StorageCore->writeStream($strTmpFile, $Response->getBody()->detach());
+                            convertToWebP(
+                                $StorageCore->getRealPath($strTmpFile),
+                                $StorageCore->getRealPath($arrItem['local_path']),
+                                40
+                            );
+                            @unlink($StorageCore->getRealPath($strTmpFile));
+                        } else {
+                            $StorageCore->writeStream($arrItem['local_path'], $Response->getBody()->detach());
+                        }
+                    }
+                } catch (\Exception $e) {
+                    // echo $e->getMessage();
+                }
+            }
+        }
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @param object $Site
+     * @param string $strUri
+     * @return VideoPagerPage
+     */
+    public function getVideoPagerPage($Site, $strUri)
+    {
+        return new VideoPagerPage($Site, $strUri);
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @param object $Site
+     * @param string $strUri
+     * @return VideoInfoPage
+     */
+    public function getVideoInfoPage($Site, $strUri)
+    {
+        return new VideoInfoPage($Site, $strUri);
+    }
+}

code/app/task/crawler/douban/page/VideoCategory.php

@@ -0,0 +1,304 @@
+<?php
+
+declare(strict_types=1);
+
+namespace app\task\crawler\douban\page;
+
+class VideoCategory
+{
+    static $arrCategory = [
+        1 =>
+        [
+            'name' => '电影',
+            'pinyin' => 'dian-ying',
+        ],
+
+        2 =>
+        [
+            'name' => '连续剧',
+            'pinyin' => 'lian-xu-ju',
+        ],
+
+        3 =>
+        [
+            'name' => '综艺片',
+            'pinyin' => 'zong-yi-pian',
+        ],
+
+        4 =>
+        [
+            'name' => '动漫',
+            'pinyin' => 'dong-man',
+        ],
+
+        5 =>
+        [
+            'name' => '纪录片',
+            'pinyin' => 'ji-lu-pian',
+        ],
+
+        6 =>
+        [
+            'name' => '动作片',
+            'pinyin' => 'dong-zuo-pian',
+        ],
+
+        7 =>
+        [
+            'name' => '爱情片',
+            'pinyin' => 'ai-qing-pian',
+        ],
+
+        8 =>
+        [
+            'name' => '喜剧片',
+            'pinyin' => 'xi-ju-pian',
+        ],
+
+        9 =>
+        [
+            'name' => '科幻片',
+            'pinyin' => 'ke-huan-pian',
+        ],
+
+        10 =>
+        [
+            'name' => '恐怖片',
+            'pinyin' => 'kong-bu-pian',
+        ],
+
+        11 =>
+        [
+            'name' => '剧情片',
+            'pinyin' => 'ju-qing-pian',
+        ],
+
+        12 =>
+        [
+            'name' => '战争片',
+            'pinyin' => 'zhan-zheng-pian',
+        ],
+
+        13 =>
+        [
+            'name' => '国产剧',
+            'pinyin' => 'guo-chan-ju',
+        ],
+
+        14 =>
+        [
+            'name' => '香港剧',
+            'pinyin' => 'xiang-gang-ju',
+        ],
+
+        15 =>
+        [
+            'name' => '欧美剧',
+            'pinyin' => 'ou-mei-ju',
+        ],
+
+        16 =>
+        [
+            'name' => '韩剧',
+            'pinyin' => 'han-ju',
+        ],
+
+        20 =>
+        [
+            'name' => '动漫电影',
+            'pinyin' => 'dong-man-dian-ying',
+        ],
+
+        21 =>
+        [
+            'name' => '泰国剧',
+            'pinyin' => 'tai-guo-ju',
+        ],
+
+        22 =>
+        [
+            'name' => '日本剧',
+            'pinyin' => 'ri-ben-ju',
+        ],
+
+        23 =>
+        [
+            'name' => '台湾剧',
+            'pinyin' => 'tai-wan-ju',
+        ],
+
+        24 =>
+        [
+            'name' => '海外剧',
+            'pinyin' => 'hai-wai-ju',
+        ],
+
+        25 =>
+        [
+            'name' => '大陆综艺',
+            'pinyin' => 'da-lu-zong-yi',
+        ],
+
+        26 =>
+        [
+            'name' => '日韩综艺',
+            'pinyin' => 'ri-han-zong-yi',
+        ],
+
+        27 =>
+        [
+            'name' => '港台综艺',
+            'pinyin' => 'gang-tai-zong-yi',
+        ],
+
+        28 =>
+        [
+            'name' => '欧美综艺',
+            'pinyin' => 'ou-mei-zong-yi',
+        ],
+
+        29 =>
+        [
+            'name' => '演唱会',
+            'pinyin' => 'yan-chang-hui',
+        ],
+
+        30 =>
+        [
+            'name' => '国产动漫',
+            'pinyin' => 'guo-chan-dong-man',
+        ],
+
+        31 =>
+        [
+            'name' => '日本动漫',
+            'pinyin' => 'ri-ben-dong-man',
+        ],
+
+        32 =>
+        [
+            'name' => '欧美动漫',
+            'pinyin' => 'ou-mei-dong-man',
+        ],
+
+        33 =>
+        [
+            'name' => '海外动漫',
+            'pinyin' => 'hai-wai-dong-man',
+        ],
+
+        34 =>
+        [
+            'name' => '伦理片',
+            'pinyin' => 'lun-li-pian',
+        ],
+
+        35 =>
+        [
+            'name' => '新闻资讯',
+            'pinyin' => 'xin-wen-zi-xun',
+        ],
+
+        36 =>
+        [
+            'name' => '体育赛事',
+            'pinyin' => 'ti-yu-sai-shi',
+        ],
+
+        37 =>
+        [
+            'name' => '短剧大全',
+            'pinyin' => 'duan-ju-da-quan',
+        ],
+
+        38 =>
+        [
+            'name' => '篮球',
+            'pinyin' => 'lan-qiu',
+        ],
+
+        39 =>
+        [
+            'name' => '足球',
+            'pinyin' => 'zu-qiu',
+        ],
+
+        40 =>
+        [
+            'name' => '网球',
+            'pinyin' => 'wang-qiu',
+        ],
+
+        41 =>
+        [
+            'name' => '斯诺克',
+            'pinyin' => 'si-nuo-ke',
+        ],
+
+        42 =>
+        [
+            'name' => 'LPL',
+            'pinyin' => 'LPL',
+        ],
+
+        43 =>
+        [
+            'name' => '重生民国',
+            'pinyin' => 'zhong-sheng-min-guo',
+        ],
+
+        44 =>
+        [
+            'name' => '穿越现代',
+            'pinyin' => 'chuan-yue-xian-dai',
+        ],
+
+        45 =>
+        [
+            'name' => '反转爽剧',
+            'pinyin' => 'fan-zhuan-shuang-ju',
+        ],
+
+        46 =>
+        [
+            'name' => '言情总裁',
+            'pinyin' => 'yan-qing-zong-cai',
+        ],
+
+        47 =>
+        [
+            'name' => '现代都市',
+            'pinyin' => 'xian-dai-dou-shi',
+        ],
+
+        48 =>
+        [
+            'name' => '古装仙侠',
+            'pinyin' => 'gu-zhuang-xian-xia',
+        ],
+
+        49 =>
+        [
+            'name' => '悬疑烧脑',
+            'pinyin' => 'xuan-yi-shao-nao',
+        ],
+
+        50 =>
+        [
+            'name' => '惊悚片',
+            'pinyin' => 'jing-song-pian',
+        ],
+
+        51 =>
+        [
+            'name' => '电影资讯',
+            'pinyin' => 'dian-ying-zi-xun',
+        ],
+
+        52 =>
+        [
+            'name' => '娱乐新闻',
+            'pinyin' => 'yu-le-xin-wen',
+        ],
+    ];
+}

code/app/task/crawler/douban/page/VideoInfoPage.php

@@ -0,0 +1,260 @@
+<?php
+
+declare(strict_types=1);
+
+namespace app\task\crawler\douban\page;
+
+use app\model\CountersModel;
+use app\model\VideoModel;
+
+class VideoInfoPage extends BasePage
+{
+    /**
+     * Undocumented variable
+     *
+     * @var string
+     */
+    protected $strVideoContent = '';
+
+    /**
+     * Undocumented variable
+     *
+     * @var VideoModel
+     */
+    protected $VideoModel;
+
+    /**
+     * Undocumented variable
+     *
+     * @var array
+     */
+    protected $arrVideo;
+
+    /**
+     * Undocumented variable
+     *
+     * @var array
+     */
+    public $arrOptions = [
+        'typeMap' => [
+            'root' => 'array',
+            'document' => 'array',
+            'array' => 'array',
+        ],
+    ];
+
+    /**
+     * Undocumented function
+     *
+     * @return VideoModel
+     */
+    public function getVideoModel(): VideoModel
+    {
+        if ($this->VideoModel == NULL) {
+            $this->VideoModel = VideoModel::getInstance();
+        }
+        return  $this->VideoModel;
+    }
+
+
+    /**
+     * Undocumented function
+     *
+     * @return array
+     */
+    public function getVideo(): array
+    {
+        if (empty($this->arrVideo)) {
+
+            $arrVideo = json_decode($this->getContent(), true);
+            $arrVideo = $arrVideo['list'][0];
+            // print_r($arrVideo);
+
+            $strPlayUrl = $arrVideo['vod_play_url'];
+            unset($arrVideo['vod_play_url']);
+
+            $arrPlayUrl = str_replace('#', '$', $strPlayUrl);
+            $arrPlayUrl = explode('$', $arrPlayUrl);
+
+            $arrPlayUrl = array_filter($arrPlayUrl);
+
+            $arrPlayUrl = array_chunk($arrPlayUrl, 2);
+            foreach ($arrPlayUrl  as $intKey => $arrVal) {
+                $arrPlayUrl[$intKey] = [
+                    'name' => $arrVal[0],
+                    'url' => $arrVal[1] ?? $arrVal[0],
+                ];
+            }
+
+            $arrVideo['vod_actor']  = trim($arrVideo['vod_actor']);
+            $arrVideo['vod_director']  = trim($arrVideo['vod_director']);
+            $arrVideo['vod_class']  = trim($arrVideo['vod_class']);
+            $arrVideo['vod_lang']  = trim($arrVideo['vod_lang']);
+            $arrVideo['vod_area']  = trim($arrVideo['vod_area']);
+
+            $arrVideo['vod_play_url'][$this->getsite()->getSiteCode()] = $arrPlayUrl;
+
+            $arrVideo['vod_actor'] = $arrVideo['vod_actor'] === '' ? [] : explode(',', $arrVideo['vod_actor']);
+            $arrVideo['vod_director'] = $arrVideo['vod_director'] === '' ? [] : explode(',', $arrVideo['vod_director']);
+            $arrVideo['vod_class'] = $arrVideo['vod_class'] === '' ? [] :  explode(',', $arrVideo['vod_class']);
+            $arrVideo['vod_lang'] = $arrVideo['vod_lang'] === '' ? [] : explode(',', $arrVideo['vod_lang']);
+            $arrVideo['vod_area'] = $arrVideo['vod_area'] === '' ? [] :  explode(',', $arrVideo['vod_area']);
+
+            $arrVideo['vod_lang_en'] =     $arrVideo['vod_area_en'] = [];
+
+            foreach ($arrVideo['vod_lang'] as $strVal) {
+                $arrVideo['vod_lang_en'][] = zhToPinYin($strVal);
+            }
+
+            foreach ($arrVideo['vod_area'] as $strVal) {
+                $arrVideo['vod_area_en'][] = zhToPinYin($strVal);
+            }
+
+
+            $arrVideo['vod_content'] = strip_tags($arrVideo['vod_content']);
+            $arrVideo['vod_content'] = trim($arrVideo['vod_content']);
+            $arrVideo['vod_content'] = mb_rtrim($arrVideo['vod_content'], '\　');
+            $arrVideo['vod_content'] = ensureUtf8($arrVideo['vod_content']);
+
+            $arrVideo['v_category'] = VideoCategory::$arrCategory[$arrVideo['type_id']]['name'];
+            $arrVideo['v_category_en'] = VideoCategory::$arrCategory[$arrVideo['type_id']]['pinyin'];
+            $arrVideo['v_parent_category'] = VideoCategory::$arrCategory[$arrVideo['type_id_1']]['name'] ?? VideoCategory::$arrCategory[$arrVideo['type_id']]['name'];
+            $arrVideo['v_parent_category_en'] = VideoCategory::$arrCategory[$arrVideo['type_id_1']]['pinyin'] ?? VideoCategory::$arrCategory[$arrVideo['type_id']]['pinyin'];
+
+            foreach ($arrVideo as $strKey => $mixedVal) {
+                $strKey = str_replace('vod', 'v', $strKey);
+
+                if ($strKey == 'type_id') {
+                    $strKey = 'v_category_id';
+                }
+                if ($strKey == 'type_name') {
+                    $strKey = 'v_category';
+                }
+                if ($strKey == 'v_content') {
+                    $strKey = 'v_description';
+                }
+                if ($strKey == 'v_enname') {
+                    $strKey = 'v_name_en';
+                }
+                if ($strKey == 'v_time') {
+                    $strKey = 'v_publish_date';
+                }
+
+                $this->arrVideo[$strKey] = $mixedVal;
+            }
+        }
+
+        return  $this->arrVideo;
+    }
+
+
+    /**
+     * Undocumented function
+     *
+     * @return boolean
+     */
+    public function saveVideo(): bool
+    {
+        $arrPageVideo = $this->getVideo();
+
+        if (
+            empty($arrPageVideo) ||
+            empty($arrPageVideo['v_name']) ||
+            empty($arrPageVideo['v_category']) ||
+            empty($arrPageVideo['v_play_url']) ||
+            empty($arrPageVideo['v_play_url']['douban'])
+        ) {
+            return false;
+        }
+
+        $arrExistsVideo = $this->getVideoModel()->findOne(['v_name' => $arrPageVideo['v_name']], $this->arrOptions);
+
+        if ($arrExistsVideo) {
+
+            $arrExistsVideo = (array) $arrExistsVideo;
+
+            $arrUpdate = [
+                'updated_at' => new \MongoDB\BSON\UTCDateTime(),
+                'v_play_url.douban' => $arrPageVideo['v_play_url']['douban'],
+            ];
+
+            $this->getVideoModel()->updateOne(
+                ['v_name' =>  $arrPageVideo['v_name']],
+                ['$set' =>   $arrUpdate],
+            );
+            $this->strPicLocalPath = $arrExistsVideo['v_pic'];
+        } else {
+
+            $arrInsertVideo = [
+                'v_id' =>   CountersModel::getInstance()->getNextSequence('video'),
+                'v_name' => $arrPageVideo['v_name'],
+                'v_name_en' => zhToPinYin($arrPageVideo['v_name']),
+                'v_parent_category' => $arrPageVideo['v_parent_category'],
+                'v_parent_category_en' => $arrPageVideo['v_parent_category_en'],
+                'v_category' => $arrPageVideo['v_category'],
+                'v_category_en' => $arrPageVideo['v_category_en'],
+                'v_actor' => $arrPageVideo['v_actor'],
+                'v_director' => $arrPageVideo['v_director'],
+                'v_class' => $arrPageVideo['v_class'],
+                'v_letter' => $arrPageVideo['v_letter'],
+                'v_pic' => '',
+                'v_lang' => $arrPageVideo['v_lang'],
+                'v_lang_en' => $arrPageVideo['v_lang_en'],
+                'v_area' => $arrPageVideo['v_area'],
+                'v_area_en' =>  $arrPageVideo['v_area_en'],
+                'v_year' => $arrPageVideo['v_year'],
+                'v_remarks' => $arrPageVideo['v_remarks'],
+                'v_isend' => $arrPageVideo['v_isend'],
+                'v_score' => $arrPageVideo['v_score'],
+                'v_description' => $arrPageVideo['v_description'],
+                'v_play_url' => $arrPageVideo['v_play_url'],
+                'v_publish_date' => $arrPageVideo['v_publish_date'],
+                'created_at' => new \MongoDB\BSON\UTCDateTime(),
+            ];
+            $arrInsertVideo['v_pic'] = $this->generateVideoPath($arrInsertVideo['v_name_en']);
+
+            $this->strPicLocalPath =  $arrInsertVideo['v_pic'];
+
+            try {
+                $this->getVideoModel()->insert($arrInsertVideo);
+            } catch (\Exception $e) {
+                if ($e->getCode() === 11000) {
+                    return $this->saveVideo();
+                } else {
+                    throw $e;
+                }
+            }
+        }
+
+        return true;
+    }
+
+    private $strPicLocalPath = '';
+
+    /**
+     * Undocumented function
+     *
+     * @param string $strName
+     * @return string
+     */
+    public function generateVideoPath($strName): string
+    {
+        $intTime = time() - mt_rand(1, 2 * 24 * 3600 * 360);
+
+        return sprintf('/img/video/%s/%s.webp', date('Y/m/d/H', $intTime), $strName);
+    }
+
+
+    /**
+     * Undocumented function
+     *
+     * @return array
+     */
+    public function getDownPicInfo(): array
+    {
+        return [
+            'uri' => $this->getVideo()['v_pic'],
+            'local_path' => $this->strPicLocalPath,
+        ];
+    }
+}

code/app/task/crawler/douban/page/VideoPagerPage.php

@@ -0,0 +1,43 @@
+<?php
+
+declare(strict_types=1);
+
+namespace app\task\crawler\douban\page;
+
+use app\model\CategoryModel;
+use app\model\CountersModel;
+use app\model\NovelModel;
+use storage\StorageCore;
+
+class VideoPagerPage extends BasePage
+{
+    /**
+     * Undocumented variable
+     *
+     * @var array
+     */
+    protected $arrVideo = [];
+
+    public $arrOptions = [
+        'typeMap' => [
+            'root' => 'array',
+            'document' => 'array',
+            'array' => 'array',
+        ],
+    ];
+
+    /**
+     * Undocumented function
+     *
+     * @return array
+     */
+    public function getVideoPager(): array
+    {
+        if (empty($this->arrVideo)) {
+
+            $this->arrVideo = json_decode($this->getContent(), true);
+        }
+
+        return  $this->arrVideo;
+    }
+}

code/app/task/module/PlanTask.php

@@ -15,6 +15,7 @@ use app\task\crawler\zw74\Scheduler as Zw74Scheduler;
 use app\task\crawler\xbiqugu\Scheduler as XBiQuGeScheduler;
 use app\task\crawler\youzhi\Scheduler as YouZhiScheduler;
 use app\task\crawler\fengchao\Scheduler as FengChaoScheduler;
+use app\task\crawler\douban\Scheduler as DouBanScheduler;
 use app\task\crawler\wuxian\Scheduler as WuXianScheduler;
 
 use app\task\crawler\custom\Scheduler as CustomScheduler;
@@ -66,7 +67,6 @@ class PlanTask
                 case 'PULL_XBIQUGE_NOVEL_LATEST':
                     self::pullXBiQuGeNovelLatest();
                     break;
-                      
                 case 'PULL_YOU_ZHI_SHI_PIN':
                     self::pullYouZhiShiPin();
                     break;
@@ -76,6 +76,9 @@ class PlanTask
                 case 'PULL_FENG_CHAO_SHI_PIN':
                     self::pullFengChaoShiPin();
                     break;
+                case 'PULL_DOU_BAN_SHI_PIN':
+                    self::pullDouBanShiPin();
+                    break;
                 case 'CUSTOM_CRAWLER':
                     self::customCrawler();
                     break;
@@ -117,6 +120,22 @@ class PlanTask
         $QueueManage->set($arrTask);
     }
 
+    /**
+     * Undocumented function
+     *
+     * @return void
+     */
+    public static function pullDouBanShiPin()
+    {
+        $arrTask = [
+            'callback' => [DouBanScheduler::class, 'crawlFull'],
+            'data' => []
+        ];
+
+        $QueueManage = QueueManage::getInstance();
+        $QueueManage->set($arrTask);
+    }
+
     /**
      * Undocumented function
      *

code/database/seeders/PlanTaskSeeder.php

@@ -59,7 +59,7 @@ class PlanTaskSeeder
             //     'pt_limit' => 1 * 24 * 3600,
             // ],
 
-            
+
             [
                 'pt_name' => '优质资源库全量采集',
                 'pt_code' => 'PULL_YOU_ZHI_SHI_PIN',
@@ -105,6 +105,12 @@ class PlanTaskSeeder
                 'pt_enable' => 0,
                 'pt_limit' => 7 * 24 * 3600,
             ],
+            [
+                'pt_name' => '豆瓣资源库全量采集',
+                'pt_code' => 'PULL_DOU_BAN_SHI_PIN',
+                'pt_enable' => 0,
+                'pt_limit' => 1 * 24 * 3600,
+            ],
         ];
 
         foreach ($arrDataNovel as $arrPlanTask) {