diff --git a/code/app/api/controller/Statis.php b/code/app/api/controller/Statis.php index 1ca45b9..3a35685 100644 --- a/code/app/api/controller/Statis.php +++ b/code/app/api/controller/Statis.php @@ -70,6 +70,26 @@ class Statis extends BaseController public function test0909() { + + + $a = '[{"type_id":1,"type_pid":0,"type_name":"电影"},{"type_id":2,"type_pid":0,"type_name":"连续剧"},{"type_id":3,"type_pid":0,"type_name":"综艺片"},{"type_id":4,"type_pid":0,"type_name":"动漫"},{"type_id":5,"type_pid":1,"type_name":"纪录片"},{"type_id":6,"type_pid":1,"type_name":"动作片"},{"type_id":7,"type_pid":1,"type_name":"爱情片"},{"type_id":8,"type_pid":1,"type_name":"喜剧片"},{"type_id":9,"type_pid":1,"type_name":"科幻片"},{"type_id":10,"type_pid":1,"type_name":"恐怖片"},{"type_id":11,"type_pid":1,"type_name":"剧情片"},{"type_id":12,"type_pid":1,"type_name":"战争片"},{"type_id":13,"type_pid":2,"type_name":"国产剧"},{"type_id":14,"type_pid":2,"type_name":"香港剧"},{"type_id":15,"type_pid":2,"type_name":"欧美剧"},{"type_id":16,"type_pid":2,"type_name":"韩剧"},{"type_id":20,"type_pid":1,"type_name":"动漫电影"},{"type_id":21,"type_pid":2,"type_name":"泰国剧"},{"type_id":22,"type_pid":2,"type_name":"日本剧"},{"type_id":23,"type_pid":2,"type_name":"台湾剧"},{"type_id":24,"type_pid":2,"type_name":"海外剧"},{"type_id":25,"type_pid":3,"type_name":"大陆综艺"},{"type_id":26,"type_pid":3,"type_name":"日韩综艺"},{"type_id":27,"type_pid":3,"type_name":"港台综艺"},{"type_id":28,"type_pid":3,"type_name":"欧美综艺"},{"type_id":29,"type_pid":3,"type_name":"演唱会"},{"type_id":30,"type_pid":4,"type_name":"国产动漫"},{"type_id":31,"type_pid":4,"type_name":"日本动漫"},{"type_id":32,"type_pid":4,"type_name":"欧美动漫"},{"type_id":33,"type_pid":4,"type_name":"海外动漫"},{"type_id":34,"type_pid":0,"type_name":"伦理片"},{"type_id":35,"type_pid":0,"type_name":"新闻资讯"},{"type_id":36,"type_pid":0,"type_name":"体育赛事"},{"type_id":37,"type_pid":0,"type_name":"短剧大全"},{"type_id":38,"type_pid":36,"type_name":"篮球"},{"type_id":39,"type_pid":36,"type_name":"足球"},{"type_id":40,"type_pid":36,"type_name":"网球"},{"type_id":41,"type_pid":36,"type_name":"斯诺克"},{"type_id":42,"type_pid":36,"type_name":"LPL"},{"type_id":43,"type_pid":37,"type_name":"重生民国"},{"type_id":44,"type_pid":37,"type_name":"穿越现代"},{"type_id":45,"type_pid":37,"type_name":"反转爽剧"},{"type_id":46,"type_pid":37,"type_name":"言情总裁"},{"type_id":47,"type_pid":37,"type_name":"现代都市"},{"type_id":48,"type_pid":37,"type_name":"古装仙侠"},{"type_id":49,"type_pid":37,"type_name":"悬疑烧脑"},{"type_id":50,"type_pid":1,"type_name":"惊悚片"},{"type_id":51,"type_pid":35,"type_name":"电影资讯"},{"type_id":52,"type_pid":35,"type_name":"娱乐新闻"}]'; + $b = json_decode($a, true); + + $c = []; + foreach ($b as $val) { + $c[$val['type_id']] = [ + 'name' => $val['type_name'], + 'pinyin' => zhToPinYin($val['type_name']), + ]; + if (empty($c[$val['type_id']]['pinyin'])) { + $c[$val['type_id']]['pinyin'] = $c[$val['type_id']]['name']; + } + } + echo '
';
+ print_r($c);
+ exit;
+
+
$a = VideoCategory::$arrCategory;
$b = PageVideoCategory::$arrCategory;
$c = YouzhiPageVideoCategory::$arrCategory;
diff --git a/code/app/task/crawler/douban/Scheduler.php b/code/app/task/crawler/douban/Scheduler.php
new file mode 100644
index 0000000..1213cc3
--- /dev/null
+++ b/code/app/task/crawler/douban/Scheduler.php
@@ -0,0 +1,206 @@
+QueueManage = new QueueManage();
+ $this->init();
+ }
+
+ /**
+ * Undocumented function
+ *
+ * @return void
+ */
+ public function init()
+ {
+ $this->intSlice = 8;
+ }
+
+ /**
+ * get site
+ *
+ * @return Site
+ */
+ public function getSite()
+ {
+ if ($this->Site == NULL) {
+ $this->Site = new Site;
+ }
+ return $this->Site;
+ }
+
+
+ /**
+ * create a site-wide craw task
+ *
+ * @return void
+ */
+ public function crawlFull()
+ {
+ // $arrVSourceId = [['v_source_id' => 14256]];
+ // $arrTask = [
+ // 'callback' => [static::class, 'fetchVideo'],
+ // 'data' => $arrVSourceId,
+ // ];
+ // $this->QueueManage->set($arrTask);
+
+ // $this->fetchVideo($arrVSourceId);
+
+ // return;
+
+ $Site = $this->getSite();
+
+ $arrAllCategory = $Site->getCategory();
+
+ print_r($arrAllCategory);
+
+ return;
+
+ $intLimit = 5;
+
+ $arrTask = [
+ 'callback' => [static::class, 'fetchVSourceId'],
+ 'data' => [],
+ ];
+
+ foreach ($arrAllCategory as $arrCategory) {
+
+ if ($arrCategory['total'] <= 0) {
+ continue;
+ }
+
+ if (in_array($arrCategory['v_category_name'], CrawlerConfig::$arrExcludeVideoCategory)) {
+ continue;
+ }
+
+
+ $intTotalPage = $arrCategory['total_page'];
+ $arrFilter = [];
+ for ($intStartPage = 1; $intStartPage <= $intTotalPage; $intStartPage++) {
+
+ $arrFilter[] = [
+ 'v_category_id' => $arrCategory['v_category_id'],
+ 'page' => $intStartPage,
+ ];
+ if (($intStartPage % $intLimit) == 0) {
+ $arrTask['data'] = $arrFilter;
+ $this->QueueManage->set($arrTask);
+ $arrFilter = [];
+ }
+ }
+ if (!empty($arrFilter)) {
+ $arrTask['data'] = $arrFilter;
+ $this->QueueManage->set($arrTask);
+ }
+ }
+ }
+
+ /**
+ * Undocumented function
+ *
+ * @param array $arrFilter
+ * @return void
+ */
+ public function fetchVSourceId(array $arrFilter)
+ {
+
+ $Site = $this->getSite();
+
+ $intLimit = 5;
+
+ $arrTask = [
+ 'callback' => [static::class, 'fetchVideo'],
+ 'data' => [],
+ ];
+
+ $arrAllVideoPagerPage = $Site->getVideoPagerPageList($arrFilter);
+
+ $arrVSourceId = [];
+
+ foreach ($arrAllVideoPagerPage as $VideoPagerPage) {
+ $arrVideoPager = $VideoPagerPage->getVideoPager();
+
+ foreach ($arrVideoPager['list'] as $arrVideoPager) {
+ $arrVSourceId[] = ['v_source_id' => (int)$arrVideoPager['vod_id']];
+
+ if ((count($arrVSourceId) % $intLimit) == 0) {
+ $arrTask['data'] = $arrVSourceId;
+ $this->QueueManage->set($arrTask);
+ $arrVSourceId = [];
+ }
+ }
+ }
+
+ if (!empty($arrVSourceId)) {
+ $arrTask['data'] = $arrVSourceId;
+ $this->QueueManage->set($arrTask);
+ }
+ }
+
+ /**
+ * Undocumented function
+ *
+ * @param array $arrVSourceId
+ * @return void
+ */
+ public function fetchVideo(array $arrVSourceId)
+ {
+ $Site = $this->getSite();
+
+ $arrAllVideoInfoPage = $Site->getVideoInfoPageList($arrVSourceId);
+
+ $arrPic = [];
+ foreach ($arrAllVideoInfoPage as $VideoInfoPage) {
+
+ $VideoInfoPage->saveVideo();
+
+ $arrPicItem = $VideoInfoPage->getDownPicInfo();
+
+ if (!StorageCore::getInstance()->has($arrPicItem['local_path'])) {
+ $arrPic[] = $arrPicItem;
+ }
+ }
+
+ if ($arrPic) {
+ $Site->downRemoteImgToLocal($arrPic);
+ }
+ }
+}
diff --git a/code/app/task/crawler/douban/page/BasePage.php b/code/app/task/crawler/douban/page/BasePage.php
new file mode 100644
index 0000000..ace5c3b
--- /dev/null
+++ b/code/app/task/crawler/douban/page/BasePage.php
@@ -0,0 +1,249 @@
+setSite($Site);
+
+ $this->strUri = $strUri;
+ }
+
+ /**
+ * Undocumented function
+ *
+ * @return string
+ */
+ public function getUri(): string
+ {
+ return $this->strUri;
+ }
+
+ /**
+ * Undocumented function
+ *
+ * @param array $arrData
+ * @return void
+ */
+ public function setData(array $arrData)
+ {
+ $this->arrData = $arrData;
+ }
+
+ /**
+ * Undocumented function
+ *
+ * @return array
+ */
+ public function getData(): array
+ {
+ return $this->arrData;
+ }
+
+ /**
+ * Undocumented function
+ *
+ * @return QueueManage
+ */
+ public function getQueueManage(): QueueManage
+ {
+ if ($this->QueueManage == NULL) {
+ $this->QueueManage = QueueManage::getInstance();
+ }
+
+ return $this->QueueManage;
+ }
+
+ /**
+ * Undocumented function
+ *
+ * @return MongoBase
+ */
+ public function getMongoBase(): MongoBase
+ {
+ if ($this->MongoBase == NULL) {
+ $this->MongoBase = MongoBase::getInstance();
+ }
+
+ return $this->MongoBase;
+ }
+
+ public function getQueryList(): QueryList
+ {
+ if ($this->QueryList == NULL) {
+ $this->QueryList = QueryList::html($this->getContent());
+ }
+
+ return $this->QueryList;
+ }
+
+
+ /**
+ * Undocumented function
+ *
+ * @return string
+ */
+ public function getContent(): string
+ {
+ if ($this->strContent == NULL) {
+ $this->Response = $this->getClient()->get($this->strUri);
+ $this->strContent = $this->Response->getBody()->getContents();
+ }
+
+ return $this->strContent;
+ }
+
+ /**
+ * force set page content
+ *
+ * @param string $strContent
+ * @return void
+ */
+ public function setContent($strContent)
+ {
+ $this->strContent = $strContent;
+ }
+
+
+ /**
+ * Undocumented function
+ *
+ * @return boolean
+ */
+ public function isMobile($strMark = 'content="mobile"'): bool
+ {
+ if ($this->boolIsMobile === NULL) {
+ $this->boolIsMobile = is_numeric(strpos($this->getContent(), $strMark));
+ }
+ return $this->boolIsMobile;
+ }
+
+
+ /**
+ * Undocumented function
+ *
+ * @return Client
+ */
+ protected function getClient(): Client
+ {
+ if ($this->Client == NULL) {
+ $this->Client = $this->getSite()->getClient();
+ }
+ return $this->Client;
+ }
+
+
+ /**
+ * Undocumented variable
+ *
+ * @var Site
+ */
+ protected $Site;
+
+ public function setSite(Site $Site)
+ {
+ $this->Site = $Site;
+ }
+
+ public function getSite()
+ {
+ return $this->Site;
+ }
+}
diff --git a/code/app/task/crawler/douban/page/Site.php b/code/app/task/crawler/douban/page/Site.php
new file mode 100644
index 0000000..3d7fb83
--- /dev/null
+++ b/code/app/task/crawler/douban/page/Site.php
@@ -0,0 +1,389 @@
+initClient();
+ }
+
+ /**
+ * Undocumented function
+ */
+ public function initClient()
+ {
+ $arrConfig = [
+ 'http_version' => '1.1', // 强制用 HTTP/1.1,已有,保持
+ 'base_uri' => $this->strDomain, // 基础域名,已有,保持
+ 'http_errors' => false, // 不抛 HTTP 错误,已有,保持
+ 'timeout' => 60, // 从 160 秒改为 60 秒
+ 'connect_timeout' => 10, // 从 30 秒改为 10 秒
+ 'curl' => [
+ CURLOPT_TCP_KEEPALIVE => 1, // 启用 TCP Keep-Alive,已有,保持
+ CURLOPT_TCP_KEEPIDLE => 120, // Keep-Alive 空闲时间
+ CURLOPT_TCP_KEEPINTVL => 60, // Keep-Alive 探测间隔
+ // CURLOPT_SSL_VERIFYPEER => true, // 验证 SSL,已有,保持
+ ],
+ 'headers' => [
+ 'User-Agent' => 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36',
+ 'Referer' => $this->strDomain,
+ 'Connection' => 'keep-alive',
+ ],
+ 'pool_size' => 20,
+ 'verify' => false,
+ ];
+
+ $arrProxy = config('proxy');
+
+
+ if ($this->boolProxy && $arrProxy['proxy_status']) {
+
+ $strCode = $arrProxy['proxy_code'];
+ $strProxyHost = $arrProxy['proxy_host'];
+ $strKey = $arrProxy['proxy_key'];
+ $strSecret = $arrProxy['proxy_secret'];
+
+
+ switch ($strCode) {
+ case 'XIONGMAO':
+ $intTime = time();
+ $strTxt = "orderno=" . $strKey . ",secret=" . $strSecret . ",timestamp=" . $intTime;
+ $strSign = strtoupper(md5($strTxt));
+ $strAuth = 'sign=' . $strSign . '&orderno=' . $strKey . '×tamp=' . $intTime;
+ $arrConfig['proxy'] = $strProxyHost;
+ $arrConfig['headers']['Proxy-Authorization'] = $strAuth;
+ break;
+ case 'KUAI':
+ $arrConfig['proxy'] = $strProxyHost;
+ $arrConfig['curl'][CURLOPT_PROXYAUTH] = CURLAUTH_BASIC;
+ $arrConfig['curl'][CURLOPT_PROXYUSERPWD] = "$strKey:$strSecret";
+ break;
+ case 'QING':
+ $arrConfig['proxy'] = $strProxyHost;
+ $strProxyAuth = base64_encode($strKey . ":" . $strSecret);
+ $arrConfig['headers']['Proxy-Authorization'] = "Basic " . $strProxyAuth;
+ break;
+ case 'JULIANG':
+ // 隧道代理信息
+ // $strProxy = 'http://username:password@XXX.juliangip.com:23456';
+ $strProxy = "http://" . $strKey . ":" . $strSecret . "@" . $strProxyHost;
+ $arrConfig['proxy'] = $strProxy;
+ break;
+ }
+ }
+
+
+ $this->Client = new Client($arrConfig);
+ }
+
+ /**
+ * Undocumented function
+ *
+ * @return string
+ */
+ public function getSiteCode(): string
+ {
+ return $this->strSiteCode;
+ }
+
+ /**
+ * Undocumented function
+ *
+ * @param integer|string $strSign
+ * @return string
+ */
+ public function getSiteUUId(int|string $strSign): string
+ {
+ return $this->getSiteCode() . '-' . $strSign;
+ }
+
+ /**
+ * Undocumented function
+ *
+ * @return Client
+ */
+ public function getClient(): Client
+ {
+ return $this->Client;
+ }
+
+ /**
+ * retrieve multiple novel page model
+ *
+ * @param string $strUri
+ * @return array
+ */
+ public function getCategory(): array
+ {
+ $strUri = '/api.php/provide/vod/at/josn?ac=list';
+
+ $Response = $this->getClient()->get($strUri);
+ $strContent = $Response->getBody()->getContents();
+
+ $arrContent = json_decode($strContent, true);
+ $arrFilter = $arrContent['class'];
+
+ // print_r($arrFilter);
+
+ // $arrA = [];
+ // foreach ($arrFilter as $b) {
+ // $arrA[$b['type_id']] = [
+ // 'name' => $b['type_name'],
+ // 'pinyin' => zhToPinYin($b['type_name']),
+ // ];
+ // }
+ // print_r($arrA);
+
+
+ $arrCategory = [];
+
+ foreach ($arrFilter as $intKey => $arrItem) {
+
+ $arrFilter[$intKey]['uri'] = $this->getVideoPageUri((int)$arrItem['type_id'], 1);
+
+ $arrPromises[$intKey] = $this->getClient()->getAsync($arrFilter[$intKey]['uri']);
+ }
+
+ $arrResults = Promise\Utils::settle($arrPromises)->wait();
+
+ $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
+ return $arrResult['state'] === 'fulfilled';
+ });
+
+ foreach ($arrSuccessfulResults as $intKey => $arrResult) {
+ $Response = $arrResult['value'];
+
+ $strContent = $Response->getBody()->getContents();
+
+ $arrTemp = json_decode($strContent, true);
+
+ $arrCategory[] = [
+ 'v_category_id' => (int)$arrFilter[$intKey]['type_id'],
+ 'v_category_name' => $arrFilter[$intKey]['type_name'],
+ 'v_category_name_en' => zhToPinYin($arrFilter[$intKey]['type_name']),
+ 'total' => $arrTemp['total'],
+ 'total_page' => $arrTemp['pagecount'],
+ ];
+ }
+
+ return $arrCategory;
+ }
+
+ /**
+ * generate video list uri
+ *
+ * @param integer $intVCategoryId
+ * @param integer $intPage
+ * @return string
+ */
+ public function getVideoPageUri(int $intVCategoryId, int $intPage): string
+ {
+ return sprintf('/api.php/provide/vod/at/josn/?ac=list&t=%s&pg=%s', $intVCategoryId, $intPage);
+ }
+
+ /**
+ * retrieve multiple novel page model
+ *
+ * @param string $strUri
+ * @return VideoPagerPage[]
+ */
+ public function getVideoPagerPageList(array $arrFilter): array
+ {
+
+ /** @var VideoPagerPage[] */
+ $arrVideoPagerPage = [];
+
+ foreach ($arrFilter as $intKey => $arrItem) {
+
+ $arrFilter[$intKey]['uri'] = $this->getVideoPageUri($arrItem['v_category_id'], $arrItem['page']);
+
+ $arrPromises[$intKey] = $this->getClient()->getAsync($arrFilter[$intKey]['uri']);
+ }
+
+ $arrResults = Promise\Utils::settle($arrPromises)->wait();
+
+
+ $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
+ return $arrResult['state'] === 'fulfilled';
+ });
+
+ foreach ($arrSuccessfulResults as $intKey => $arrResult) {
+ $Response = $arrResult['value'];
+
+ $strContent = $Response->getBody()->getContents();
+
+ if (is_numeric(strpos($strContent, 'System Error'))) {
+ continue;
+ }
+
+ $VideoPagerPage = $this->getVideoPagerPage($this, $arrFilter[$intKey]['uri']);
+
+ $VideoPagerPage->setContent($strContent);
+ $VideoPagerPage->setData($arrFilter[$intKey]);
+
+ $arrVideoPagerPage[] = $VideoPagerPage;
+ }
+
+ return $arrVideoPagerPage;
+ }
+
+ /**
+ * generate video list uri
+ *
+ * @param integer $intVCategoryId
+ * @param integer $intPage
+ * @return string
+ */
+ public function getVideoUri(int $intVSourceId): string
+ {
+ return sprintf('/api.php/provide/vod/at/josn?ac=detail&ids=%s', $intVSourceId);
+ }
+
+ /**
+ * retrieve multiple chapter page model
+ *
+ * @param string $strUri
+ * @return VideoInfoPage[]
+ */
+ public function getVideoInfoPageList(array $arrFilter): array
+ {
+
+ /** @var VideoInfoPage[] */
+ $arrVideoInfoPage = [];
+
+ $arrPromises = [];
+ foreach ($arrFilter as $intKey => $arrItem) {
+ $arrFilter[$intKey]['uri'] = $this->getVideoUri($arrItem['v_source_id']);
+
+ $arrPromises[$intKey] = $this->getClient()->getAsync($arrFilter[$intKey]['uri']);
+ }
+
+ $arrResults = Promise\Utils::settle($arrPromises)->wait();
+
+ $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
+ return $arrResult['state'] === 'fulfilled';
+ });
+
+ foreach ($arrSuccessfulResults as $intKey => $arrResult) {
+ $Response = $arrResult['value'];
+
+ $strContent = $Response->getBody()->getContents();;
+
+ if (is_numeric(strpos($strContent, 'System Error'))) {
+ continue;
+ }
+
+ $VideoInfoPage = $this->getVideoInfoPage($this, $arrFilter[$intKey]['uri']);
+
+ $VideoInfoPage->setContent($strContent);
+ $VideoInfoPage->setData($arrFilter[$intKey]);
+
+ $arrVideoInfoPage[] = $VideoInfoPage;
+ }
+
+ return $arrVideoInfoPage;
+ }
+
+ /**
+ * download the image to the local system
+ *
+ * @param array $arrFilter
+ * @return void
+ */
+ public function downRemoteImgToLocal(array $arrFilter)
+ {
+ $StorageCore = StorageCore::getInstance();
+
+ $Client = new Client();
+
+ foreach ($arrFilter as $arrItem) {
+ if (!$StorageCore->has($arrItem['local_path'])) {
+ try {
+ $Response = $Client->get($arrItem['uri']);
+ if ($Response->getStatusCode() === 200) {
+ if (str_ends_with(strtolower($arrItem['local_path']), '.webp')) {
+ $strTmpFile = str_replace('.webp', '.jpg', $arrItem['local_path']);
+ $StorageCore->writeStream($strTmpFile, $Response->getBody()->detach());
+ convertToWebP(
+ $StorageCore->getRealPath($strTmpFile),
+ $StorageCore->getRealPath($arrItem['local_path']),
+ 40
+ );
+ @unlink($StorageCore->getRealPath($strTmpFile));
+ } else {
+ $StorageCore->writeStream($arrItem['local_path'], $Response->getBody()->detach());
+ }
+ }
+ } catch (\Exception $e) {
+ // echo $e->getMessage();
+ }
+ }
+ }
+ }
+
+ /**
+ * Undocumented function
+ *
+ * @param object $Site
+ * @param string $strUri
+ * @return VideoPagerPage
+ */
+ public function getVideoPagerPage($Site, $strUri)
+ {
+ return new VideoPagerPage($Site, $strUri);
+ }
+
+ /**
+ * Undocumented function
+ *
+ * @param object $Site
+ * @param string $strUri
+ * @return VideoInfoPage
+ */
+ public function getVideoInfoPage($Site, $strUri)
+ {
+ return new VideoInfoPage($Site, $strUri);
+ }
+}
diff --git a/code/app/task/crawler/douban/page/VideoCategory.php b/code/app/task/crawler/douban/page/VideoCategory.php
new file mode 100644
index 0000000..c67480d
--- /dev/null
+++ b/code/app/task/crawler/douban/page/VideoCategory.php
@@ -0,0 +1,304 @@
+
+ [
+ 'name' => '电影',
+ 'pinyin' => 'dian-ying',
+ ],
+
+ 2 =>
+ [
+ 'name' => '连续剧',
+ 'pinyin' => 'lian-xu-ju',
+ ],
+
+ 3 =>
+ [
+ 'name' => '综艺片',
+ 'pinyin' => 'zong-yi-pian',
+ ],
+
+ 4 =>
+ [
+ 'name' => '动漫',
+ 'pinyin' => 'dong-man',
+ ],
+
+ 5 =>
+ [
+ 'name' => '纪录片',
+ 'pinyin' => 'ji-lu-pian',
+ ],
+
+ 6 =>
+ [
+ 'name' => '动作片',
+ 'pinyin' => 'dong-zuo-pian',
+ ],
+
+ 7 =>
+ [
+ 'name' => '爱情片',
+ 'pinyin' => 'ai-qing-pian',
+ ],
+
+ 8 =>
+ [
+ 'name' => '喜剧片',
+ 'pinyin' => 'xi-ju-pian',
+ ],
+
+ 9 =>
+ [
+ 'name' => '科幻片',
+ 'pinyin' => 'ke-huan-pian',
+ ],
+
+ 10 =>
+ [
+ 'name' => '恐怖片',
+ 'pinyin' => 'kong-bu-pian',
+ ],
+
+ 11 =>
+ [
+ 'name' => '剧情片',
+ 'pinyin' => 'ju-qing-pian',
+ ],
+
+ 12 =>
+ [
+ 'name' => '战争片',
+ 'pinyin' => 'zhan-zheng-pian',
+ ],
+
+ 13 =>
+ [
+ 'name' => '国产剧',
+ 'pinyin' => 'guo-chan-ju',
+ ],
+
+ 14 =>
+ [
+ 'name' => '香港剧',
+ 'pinyin' => 'xiang-gang-ju',
+ ],
+
+ 15 =>
+ [
+ 'name' => '欧美剧',
+ 'pinyin' => 'ou-mei-ju',
+ ],
+
+ 16 =>
+ [
+ 'name' => '韩剧',
+ 'pinyin' => 'han-ju',
+ ],
+
+ 20 =>
+ [
+ 'name' => '动漫电影',
+ 'pinyin' => 'dong-man-dian-ying',
+ ],
+
+ 21 =>
+ [
+ 'name' => '泰国剧',
+ 'pinyin' => 'tai-guo-ju',
+ ],
+
+ 22 =>
+ [
+ 'name' => '日本剧',
+ 'pinyin' => 'ri-ben-ju',
+ ],
+
+ 23 =>
+ [
+ 'name' => '台湾剧',
+ 'pinyin' => 'tai-wan-ju',
+ ],
+
+ 24 =>
+ [
+ 'name' => '海外剧',
+ 'pinyin' => 'hai-wai-ju',
+ ],
+
+ 25 =>
+ [
+ 'name' => '大陆综艺',
+ 'pinyin' => 'da-lu-zong-yi',
+ ],
+
+ 26 =>
+ [
+ 'name' => '日韩综艺',
+ 'pinyin' => 'ri-han-zong-yi',
+ ],
+
+ 27 =>
+ [
+ 'name' => '港台综艺',
+ 'pinyin' => 'gang-tai-zong-yi',
+ ],
+
+ 28 =>
+ [
+ 'name' => '欧美综艺',
+ 'pinyin' => 'ou-mei-zong-yi',
+ ],
+
+ 29 =>
+ [
+ 'name' => '演唱会',
+ 'pinyin' => 'yan-chang-hui',
+ ],
+
+ 30 =>
+ [
+ 'name' => '国产动漫',
+ 'pinyin' => 'guo-chan-dong-man',
+ ],
+
+ 31 =>
+ [
+ 'name' => '日本动漫',
+ 'pinyin' => 'ri-ben-dong-man',
+ ],
+
+ 32 =>
+ [
+ 'name' => '欧美动漫',
+ 'pinyin' => 'ou-mei-dong-man',
+ ],
+
+ 33 =>
+ [
+ 'name' => '海外动漫',
+ 'pinyin' => 'hai-wai-dong-man',
+ ],
+
+ 34 =>
+ [
+ 'name' => '伦理片',
+ 'pinyin' => 'lun-li-pian',
+ ],
+
+ 35 =>
+ [
+ 'name' => '新闻资讯',
+ 'pinyin' => 'xin-wen-zi-xun',
+ ],
+
+ 36 =>
+ [
+ 'name' => '体育赛事',
+ 'pinyin' => 'ti-yu-sai-shi',
+ ],
+
+ 37 =>
+ [
+ 'name' => '短剧大全',
+ 'pinyin' => 'duan-ju-da-quan',
+ ],
+
+ 38 =>
+ [
+ 'name' => '篮球',
+ 'pinyin' => 'lan-qiu',
+ ],
+
+ 39 =>
+ [
+ 'name' => '足球',
+ 'pinyin' => 'zu-qiu',
+ ],
+
+ 40 =>
+ [
+ 'name' => '网球',
+ 'pinyin' => 'wang-qiu',
+ ],
+
+ 41 =>
+ [
+ 'name' => '斯诺克',
+ 'pinyin' => 'si-nuo-ke',
+ ],
+
+ 42 =>
+ [
+ 'name' => 'LPL',
+ 'pinyin' => 'LPL',
+ ],
+
+ 43 =>
+ [
+ 'name' => '重生民国',
+ 'pinyin' => 'zhong-sheng-min-guo',
+ ],
+
+ 44 =>
+ [
+ 'name' => '穿越现代',
+ 'pinyin' => 'chuan-yue-xian-dai',
+ ],
+
+ 45 =>
+ [
+ 'name' => '反转爽剧',
+ 'pinyin' => 'fan-zhuan-shuang-ju',
+ ],
+
+ 46 =>
+ [
+ 'name' => '言情总裁',
+ 'pinyin' => 'yan-qing-zong-cai',
+ ],
+
+ 47 =>
+ [
+ 'name' => '现代都市',
+ 'pinyin' => 'xian-dai-dou-shi',
+ ],
+
+ 48 =>
+ [
+ 'name' => '古装仙侠',
+ 'pinyin' => 'gu-zhuang-xian-xia',
+ ],
+
+ 49 =>
+ [
+ 'name' => '悬疑烧脑',
+ 'pinyin' => 'xuan-yi-shao-nao',
+ ],
+
+ 50 =>
+ [
+ 'name' => '惊悚片',
+ 'pinyin' => 'jing-song-pian',
+ ],
+
+ 51 =>
+ [
+ 'name' => '电影资讯',
+ 'pinyin' => 'dian-ying-zi-xun',
+ ],
+
+ 52 =>
+ [
+ 'name' => '娱乐新闻',
+ 'pinyin' => 'yu-le-xin-wen',
+ ],
+ ];
+}
diff --git a/code/app/task/crawler/douban/page/VideoInfoPage.php b/code/app/task/crawler/douban/page/VideoInfoPage.php
new file mode 100644
index 0000000..f40397c
--- /dev/null
+++ b/code/app/task/crawler/douban/page/VideoInfoPage.php
@@ -0,0 +1,260 @@
+ [
+ 'root' => 'array',
+ 'document' => 'array',
+ 'array' => 'array',
+ ],
+ ];
+
+ /**
+ * Undocumented function
+ *
+ * @return VideoModel
+ */
+ public function getVideoModel(): VideoModel
+ {
+ if ($this->VideoModel == NULL) {
+ $this->VideoModel = VideoModel::getInstance();
+ }
+ return $this->VideoModel;
+ }
+
+
+ /**
+ * Undocumented function
+ *
+ * @return array
+ */
+ public function getVideo(): array
+ {
+ if (empty($this->arrVideo)) {
+
+ $arrVideo = json_decode($this->getContent(), true);
+ $arrVideo = $arrVideo['list'][0];
+ // print_r($arrVideo);
+
+ $strPlayUrl = $arrVideo['vod_play_url'];
+ unset($arrVideo['vod_play_url']);
+
+ $arrPlayUrl = str_replace('#', '$', $strPlayUrl);
+ $arrPlayUrl = explode('$', $arrPlayUrl);
+
+ $arrPlayUrl = array_filter($arrPlayUrl);
+
+ $arrPlayUrl = array_chunk($arrPlayUrl, 2);
+ foreach ($arrPlayUrl as $intKey => $arrVal) {
+ $arrPlayUrl[$intKey] = [
+ 'name' => $arrVal[0],
+ 'url' => $arrVal[1] ?? $arrVal[0],
+ ];
+ }
+
+ $arrVideo['vod_actor'] = trim($arrVideo['vod_actor']);
+ $arrVideo['vod_director'] = trim($arrVideo['vod_director']);
+ $arrVideo['vod_class'] = trim($arrVideo['vod_class']);
+ $arrVideo['vod_lang'] = trim($arrVideo['vod_lang']);
+ $arrVideo['vod_area'] = trim($arrVideo['vod_area']);
+
+ $arrVideo['vod_play_url'][$this->getsite()->getSiteCode()] = $arrPlayUrl;
+
+ $arrVideo['vod_actor'] = $arrVideo['vod_actor'] === '' ? [] : explode(',', $arrVideo['vod_actor']);
+ $arrVideo['vod_director'] = $arrVideo['vod_director'] === '' ? [] : explode(',', $arrVideo['vod_director']);
+ $arrVideo['vod_class'] = $arrVideo['vod_class'] === '' ? [] : explode(',', $arrVideo['vod_class']);
+ $arrVideo['vod_lang'] = $arrVideo['vod_lang'] === '' ? [] : explode(',', $arrVideo['vod_lang']);
+ $arrVideo['vod_area'] = $arrVideo['vod_area'] === '' ? [] : explode(',', $arrVideo['vod_area']);
+
+ $arrVideo['vod_lang_en'] = $arrVideo['vod_area_en'] = [];
+
+ foreach ($arrVideo['vod_lang'] as $strVal) {
+ $arrVideo['vod_lang_en'][] = zhToPinYin($strVal);
+ }
+
+ foreach ($arrVideo['vod_area'] as $strVal) {
+ $arrVideo['vod_area_en'][] = zhToPinYin($strVal);
+ }
+
+
+ $arrVideo['vod_content'] = strip_tags($arrVideo['vod_content']);
+ $arrVideo['vod_content'] = trim($arrVideo['vod_content']);
+ $arrVideo['vod_content'] = mb_rtrim($arrVideo['vod_content'], '\ ');
+ $arrVideo['vod_content'] = ensureUtf8($arrVideo['vod_content']);
+
+ $arrVideo['v_category'] = VideoCategory::$arrCategory[$arrVideo['type_id']]['name'];
+ $arrVideo['v_category_en'] = VideoCategory::$arrCategory[$arrVideo['type_id']]['pinyin'];
+ $arrVideo['v_parent_category'] = VideoCategory::$arrCategory[$arrVideo['type_id_1']]['name'] ?? VideoCategory::$arrCategory[$arrVideo['type_id']]['name'];
+ $arrVideo['v_parent_category_en'] = VideoCategory::$arrCategory[$arrVideo['type_id_1']]['pinyin'] ?? VideoCategory::$arrCategory[$arrVideo['type_id']]['pinyin'];
+
+ foreach ($arrVideo as $strKey => $mixedVal) {
+ $strKey = str_replace('vod', 'v', $strKey);
+
+ if ($strKey == 'type_id') {
+ $strKey = 'v_category_id';
+ }
+ if ($strKey == 'type_name') {
+ $strKey = 'v_category';
+ }
+ if ($strKey == 'v_content') {
+ $strKey = 'v_description';
+ }
+ if ($strKey == 'v_enname') {
+ $strKey = 'v_name_en';
+ }
+ if ($strKey == 'v_time') {
+ $strKey = 'v_publish_date';
+ }
+
+ $this->arrVideo[$strKey] = $mixedVal;
+ }
+ }
+
+ return $this->arrVideo;
+ }
+
+
+ /**
+ * Undocumented function
+ *
+ * @return boolean
+ */
+ public function saveVideo(): bool
+ {
+ $arrPageVideo = $this->getVideo();
+
+ if (
+ empty($arrPageVideo) ||
+ empty($arrPageVideo['v_name']) ||
+ empty($arrPageVideo['v_category']) ||
+ empty($arrPageVideo['v_play_url']) ||
+ empty($arrPageVideo['v_play_url']['douban'])
+ ) {
+ return false;
+ }
+
+ $arrExistsVideo = $this->getVideoModel()->findOne(['v_name' => $arrPageVideo['v_name']], $this->arrOptions);
+
+ if ($arrExistsVideo) {
+
+ $arrExistsVideo = (array) $arrExistsVideo;
+
+ $arrUpdate = [
+ 'updated_at' => new \MongoDB\BSON\UTCDateTime(),
+ 'v_play_url.douban' => $arrPageVideo['v_play_url']['douban'],
+ ];
+
+ $this->getVideoModel()->updateOne(
+ ['v_name' => $arrPageVideo['v_name']],
+ ['$set' => $arrUpdate],
+ );
+ $this->strPicLocalPath = $arrExistsVideo['v_pic'];
+ } else {
+
+ $arrInsertVideo = [
+ 'v_id' => CountersModel::getInstance()->getNextSequence('video'),
+ 'v_name' => $arrPageVideo['v_name'],
+ 'v_name_en' => zhToPinYin($arrPageVideo['v_name']),
+ 'v_parent_category' => $arrPageVideo['v_parent_category'],
+ 'v_parent_category_en' => $arrPageVideo['v_parent_category_en'],
+ 'v_category' => $arrPageVideo['v_category'],
+ 'v_category_en' => $arrPageVideo['v_category_en'],
+ 'v_actor' => $arrPageVideo['v_actor'],
+ 'v_director' => $arrPageVideo['v_director'],
+ 'v_class' => $arrPageVideo['v_class'],
+ 'v_letter' => $arrPageVideo['v_letter'],
+ 'v_pic' => '',
+ 'v_lang' => $arrPageVideo['v_lang'],
+ 'v_lang_en' => $arrPageVideo['v_lang_en'],
+ 'v_area' => $arrPageVideo['v_area'],
+ 'v_area_en' => $arrPageVideo['v_area_en'],
+ 'v_year' => $arrPageVideo['v_year'],
+ 'v_remarks' => $arrPageVideo['v_remarks'],
+ 'v_isend' => $arrPageVideo['v_isend'],
+ 'v_score' => $arrPageVideo['v_score'],
+ 'v_description' => $arrPageVideo['v_description'],
+ 'v_play_url' => $arrPageVideo['v_play_url'],
+ 'v_publish_date' => $arrPageVideo['v_publish_date'],
+ 'created_at' => new \MongoDB\BSON\UTCDateTime(),
+ ];
+ $arrInsertVideo['v_pic'] = $this->generateVideoPath($arrInsertVideo['v_name_en']);
+
+ $this->strPicLocalPath = $arrInsertVideo['v_pic'];
+
+ try {
+ $this->getVideoModel()->insert($arrInsertVideo);
+ } catch (\Exception $e) {
+ if ($e->getCode() === 11000) {
+ return $this->saveVideo();
+ } else {
+ throw $e;
+ }
+ }
+ }
+
+ return true;
+ }
+
+ private $strPicLocalPath = '';
+
+ /**
+ * Undocumented function
+ *
+ * @param string $strName
+ * @return string
+ */
+ public function generateVideoPath($strName): string
+ {
+ $intTime = time() - mt_rand(1, 2 * 24 * 3600 * 360);
+
+ return sprintf('/img/video/%s/%s.webp', date('Y/m/d/H', $intTime), $strName);
+ }
+
+
+ /**
+ * Undocumented function
+ *
+ * @return array
+ */
+ public function getDownPicInfo(): array
+ {
+ return [
+ 'uri' => $this->getVideo()['v_pic'],
+ 'local_path' => $this->strPicLocalPath,
+ ];
+ }
+}
diff --git a/code/app/task/crawler/douban/page/VideoPagerPage.php b/code/app/task/crawler/douban/page/VideoPagerPage.php
new file mode 100644
index 0000000..fd8c4c9
--- /dev/null
+++ b/code/app/task/crawler/douban/page/VideoPagerPage.php
@@ -0,0 +1,43 @@
+ [
+ 'root' => 'array',
+ 'document' => 'array',
+ 'array' => 'array',
+ ],
+ ];
+
+ /**
+ * Undocumented function
+ *
+ * @return array
+ */
+ public function getVideoPager(): array
+ {
+ if (empty($this->arrVideo)) {
+
+ $this->arrVideo = json_decode($this->getContent(), true);
+ }
+
+ return $this->arrVideo;
+ }
+}
diff --git a/code/app/task/module/PlanTask.php b/code/app/task/module/PlanTask.php
index daf0ce9..0969ae1 100644
--- a/code/app/task/module/PlanTask.php
+++ b/code/app/task/module/PlanTask.php
@@ -15,6 +15,7 @@ use app\task\crawler\zw74\Scheduler as Zw74Scheduler;
use app\task\crawler\xbiqugu\Scheduler as XBiQuGeScheduler;
use app\task\crawler\youzhi\Scheduler as YouZhiScheduler;
use app\task\crawler\fengchao\Scheduler as FengChaoScheduler;
+use app\task\crawler\douban\Scheduler as DouBanScheduler;
use app\task\crawler\wuxian\Scheduler as WuXianScheduler;
use app\task\crawler\custom\Scheduler as CustomScheduler;
@@ -66,7 +67,6 @@ class PlanTask
case 'PULL_XBIQUGE_NOVEL_LATEST':
self::pullXBiQuGeNovelLatest();
break;
-
case 'PULL_YOU_ZHI_SHI_PIN':
self::pullYouZhiShiPin();
break;
@@ -76,6 +76,9 @@ class PlanTask
case 'PULL_FENG_CHAO_SHI_PIN':
self::pullFengChaoShiPin();
break;
+ case 'PULL_DOU_BAN_SHI_PIN':
+ self::pullDouBanShiPin();
+ break;
case 'CUSTOM_CRAWLER':
self::customCrawler();
break;
@@ -117,6 +120,22 @@ class PlanTask
$QueueManage->set($arrTask);
}
+ /**
+ * Undocumented function
+ *
+ * @return void
+ */
+ public static function pullDouBanShiPin()
+ {
+ $arrTask = [
+ 'callback' => [DouBanScheduler::class, 'crawlFull'],
+ 'data' => []
+ ];
+
+ $QueueManage = QueueManage::getInstance();
+ $QueueManage->set($arrTask);
+ }
+
/**
* Undocumented function
*
diff --git a/code/database/seeders/PlanTaskSeeder.php b/code/database/seeders/PlanTaskSeeder.php
index 7012825..89be9bb 100644
--- a/code/database/seeders/PlanTaskSeeder.php
+++ b/code/database/seeders/PlanTaskSeeder.php
@@ -59,7 +59,7 @@ class PlanTaskSeeder
// 'pt_limit' => 1 * 24 * 3600,
// ],
-
+
[
'pt_name' => '优质资源库全量采集',
'pt_code' => 'PULL_YOU_ZHI_SHI_PIN',
@@ -105,6 +105,12 @@ class PlanTaskSeeder
'pt_enable' => 0,
'pt_limit' => 7 * 24 * 3600,
],
+ [
+ 'pt_name' => '豆瓣资源库全量采集',
+ 'pt_code' => 'PULL_DOU_BAN_SHI_PIN',
+ 'pt_enable' => 0,
+ 'pt_limit' => 1 * 24 * 3600,
+ ],
];
foreach ($arrDataNovel as $arrPlanTask) {
diff --git a/code/extend/ddl/ext/1.ddl b/code/extend/ddl/ext/1.ddl
index f85ec3b..c7034bd 100644
Binary files a/code/extend/ddl/ext/1.ddl and b/code/extend/ddl/ext/1.ddl differ
diff --git a/code/extend/tools/pinyin-tool b/code/extend/tools/pinyin-tool
index fc3f78e..20e93c8 100644
Binary files a/code/extend/tools/pinyin-tool and b/code/extend/tools/pinyin-tool differ