From 172c0f8c1928ec2f04d7af67d8b3f4caa1b1cc0a Mon Sep 17 00:00:00 2001 From: make Date: Tue, 21 Oct 2025 21:39:04 +0800 Subject: [PATCH] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E8=8C=85=E5=8F=B0=E8=B5=84?= =?UTF-8?q?=E6=BA=90=E9=87=87=E9=9B=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- code/app/task/crawler/maotai/Scheduler.php | 206 ++++++++++ .../app/task/crawler/maotai/page/BasePage.php | 249 +++++++++++ code/app/task/crawler/maotai/page/Site.php | 389 ++++++++++++++++++ .../crawler/maotai/page/VideoCategory.php | 304 ++++++++++++++ .../crawler/maotai/page/VideoInfoPage.php | 260 ++++++++++++ .../crawler/maotai/page/VideoPagerPage.php | 43 ++ code/app/task/module/PlanTask.php | 20 + code/database/seeders/PlanTaskSeeder.php | 6 + 8 files changed, 1477 insertions(+) create mode 100644 code/app/task/crawler/maotai/Scheduler.php create mode 100644 code/app/task/crawler/maotai/page/BasePage.php create mode 100644 code/app/task/crawler/maotai/page/Site.php create mode 100644 code/app/task/crawler/maotai/page/VideoCategory.php create mode 100644 code/app/task/crawler/maotai/page/VideoInfoPage.php create mode 100644 code/app/task/crawler/maotai/page/VideoPagerPage.php diff --git a/code/app/task/crawler/maotai/Scheduler.php b/code/app/task/crawler/maotai/Scheduler.php new file mode 100644 index 0000000..4d3c846 --- /dev/null +++ b/code/app/task/crawler/maotai/Scheduler.php @@ -0,0 +1,206 @@ +QueueManage = new QueueManage(); + $this->init(); + } + + /** + * Undocumented function + * + * @return void + */ + public function init() + { + $this->intSlice = 8; + } + + /** + * get site + * + * @return Site + */ + public function getSite() + { + if ($this->Site == NULL) { + $this->Site = new Site; + } + return $this->Site; + } + + + /** + * create a site-wide craw task + * + * @return void + */ + public function crawlFull() + { + // $arrVSourceId = [['v_source_id' => 14256]]; + // $arrTask = [ + // 'callback' => [static::class, 'fetchVideo'], + // 'data' => $arrVSourceId, + // ]; + // $this->QueueManage->set($arrTask); + + // $this->fetchVideo($arrVSourceId); + + // return; + + $Site = $this->getSite(); + + $arrAllCategory = $Site->getCategory(); + + print_r($arrAllCategory); + + // return; + + $intLimit = 5; + + $arrTask = [ + 'callback' => [static::class, 'fetchVSourceId'], + 'data' => [], + ]; + + foreach ($arrAllCategory as $arrCategory) { + + if ($arrCategory['total'] <= 0) { + continue; + } + + if (in_array($arrCategory['v_category_name'], CrawlerConfig::$arrExcludeVideoCategory)) { + continue; + } + + + $intTotalPage = $arrCategory['total_page']; + $arrFilter = []; + for ($intStartPage = 1; $intStartPage <= $intTotalPage; $intStartPage++) { + + $arrFilter[] = [ + 'v_category_id' => $arrCategory['v_category_id'], + 'page' => $intStartPage, + ]; + if (($intStartPage % $intLimit) == 0) { + $arrTask['data'] = $arrFilter; + $this->QueueManage->set($arrTask); + $arrFilter = []; + } + } + if (!empty($arrFilter)) { + $arrTask['data'] = $arrFilter; + $this->QueueManage->set($arrTask); + } + } + } + + /** + * Undocumented function + * + * @param array $arrFilter + * @return void + */ + public function fetchVSourceId(array $arrFilter) + { + + $Site = $this->getSite(); + + $intLimit = 5; + + $arrTask = [ + 'callback' => [static::class, 'fetchVideo'], + 'data' => [], + ]; + + $arrAllVideoPagerPage = $Site->getVideoPagerPageList($arrFilter); + + $arrVSourceId = []; + + foreach ($arrAllVideoPagerPage as $VideoPagerPage) { + $arrVideoPager = $VideoPagerPage->getVideoPager(); + + foreach ($arrVideoPager['list'] as $arrVideoPager) { + $arrVSourceId[] = ['v_source_id' => (int)$arrVideoPager['vod_id']]; + + if ((count($arrVSourceId) % $intLimit) == 0) { + $arrTask['data'] = $arrVSourceId; + $this->QueueManage->set($arrTask); + $arrVSourceId = []; + } + } + } + + if (!empty($arrVSourceId)) { + $arrTask['data'] = $arrVSourceId; + $this->QueueManage->set($arrTask); + } + } + + /** + * Undocumented function + * + * @param array $arrVSourceId + * @return void + */ + public function fetchVideo(array $arrVSourceId) + { + $Site = $this->getSite(); + + $arrAllVideoInfoPage = $Site->getVideoInfoPageList($arrVSourceId); + + $arrPic = []; + foreach ($arrAllVideoInfoPage as $VideoInfoPage) { + + $VideoInfoPage->saveVideo(); + + $arrPicItem = $VideoInfoPage->getDownPicInfo(); + + if (!StorageCore::getInstance()->has($arrPicItem['local_path'])) { + $arrPic[] = $arrPicItem; + } + } + + if ($arrPic) { + $Site->downRemoteImgToLocal($arrPic); + } + } +} diff --git a/code/app/task/crawler/maotai/page/BasePage.php b/code/app/task/crawler/maotai/page/BasePage.php new file mode 100644 index 0000000..ace5c3b --- /dev/null +++ b/code/app/task/crawler/maotai/page/BasePage.php @@ -0,0 +1,249 @@ +setSite($Site); + + $this->strUri = $strUri; + } + + /** + * Undocumented function + * + * @return string + */ + public function getUri(): string + { + return $this->strUri; + } + + /** + * Undocumented function + * + * @param array $arrData + * @return void + */ + public function setData(array $arrData) + { + $this->arrData = $arrData; + } + + /** + * Undocumented function + * + * @return array + */ + public function getData(): array + { + return $this->arrData; + } + + /** + * Undocumented function + * + * @return QueueManage + */ + public function getQueueManage(): QueueManage + { + if ($this->QueueManage == NULL) { + $this->QueueManage = QueueManage::getInstance(); + } + + return $this->QueueManage; + } + + /** + * Undocumented function + * + * @return MongoBase + */ + public function getMongoBase(): MongoBase + { + if ($this->MongoBase == NULL) { + $this->MongoBase = MongoBase::getInstance(); + } + + return $this->MongoBase; + } + + public function getQueryList(): QueryList + { + if ($this->QueryList == NULL) { + $this->QueryList = QueryList::html($this->getContent()); + } + + return $this->QueryList; + } + + + /** + * Undocumented function + * + * @return string + */ + public function getContent(): string + { + if ($this->strContent == NULL) { + $this->Response = $this->getClient()->get($this->strUri); + $this->strContent = $this->Response->getBody()->getContents(); + } + + return $this->strContent; + } + + /** + * force set page content + * + * @param string $strContent + * @return void + */ + public function setContent($strContent) + { + $this->strContent = $strContent; + } + + + /** + * Undocumented function + * + * @return boolean + */ + public function isMobile($strMark = 'content="mobile"'): bool + { + if ($this->boolIsMobile === NULL) { + $this->boolIsMobile = is_numeric(strpos($this->getContent(), $strMark)); + } + return $this->boolIsMobile; + } + + + /** + * Undocumented function + * + * @return Client + */ + protected function getClient(): Client + { + if ($this->Client == NULL) { + $this->Client = $this->getSite()->getClient(); + } + return $this->Client; + } + + + /** + * Undocumented variable + * + * @var Site + */ + protected $Site; + + public function setSite(Site $Site) + { + $this->Site = $Site; + } + + public function getSite() + { + return $this->Site; + } +} diff --git a/code/app/task/crawler/maotai/page/Site.php b/code/app/task/crawler/maotai/page/Site.php new file mode 100644 index 0000000..d8ed468 --- /dev/null +++ b/code/app/task/crawler/maotai/page/Site.php @@ -0,0 +1,389 @@ +initClient(); + } + + /** + * Undocumented function + */ + public function initClient() + { + $arrConfig = [ + 'http_version' => '1.1', // 强制用 HTTP/1.1，已有，保持 + 'base_uri' => $this->strDomain, // 基础域名，已有，保持 + 'http_errors' => false, // 不抛 HTTP 错误，已有，保持 + 'timeout' => 60, // 从 160 秒改为 60 秒 + 'connect_timeout' => 10, // 从 30 秒改为 10 秒 + 'curl' => [ + CURLOPT_TCP_KEEPALIVE => 1, // 启用 TCP Keep-Alive，已有，保持 + CURLOPT_TCP_KEEPIDLE => 120, // Keep-Alive 空闲时间 + CURLOPT_TCP_KEEPINTVL => 60, // Keep-Alive 探测间隔 + // CURLOPT_SSL_VERIFYPEER => true, // 验证 SSL，已有，保持 + ], + 'headers' => [ + 'User-Agent' => 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36', + 'Referer' => $this->strDomain, + 'Connection' => 'keep-alive', + ], + 'pool_size' => 20, + 'verify' => false, + ]; + + $arrProxy = config('proxy'); + + + if ($this->boolProxy && $arrProxy['proxy_status']) { + + $strCode = $arrProxy['proxy_code']; + $strProxyHost = $arrProxy['proxy_host']; + $strKey = $arrProxy['proxy_key']; + $strSecret = $arrProxy['proxy_secret']; + + + switch ($strCode) { + case 'XIONGMAO': + $intTime = time(); + $strTxt = "orderno=" . $strKey . ",secret=" . $strSecret . ",timestamp=" . $intTime; + $strSign = strtoupper(md5($strTxt)); + $strAuth = 'sign=' . $strSign . '&orderno=' . $strKey . '×tamp=' . $intTime; + $arrConfig['proxy'] = $strProxyHost; + $arrConfig['headers']['Proxy-Authorization'] = $strAuth; + break; + case 'KUAI': + $arrConfig['proxy'] = $strProxyHost; + $arrConfig['curl'][CURLOPT_PROXYAUTH] = CURLAUTH_BASIC; + $arrConfig['curl'][CURLOPT_PROXYUSERPWD] = "$strKey:$strSecret"; + break; + case 'QING': + $arrConfig['proxy'] = $strProxyHost; + $strProxyAuth = base64_encode($strKey . ":" . $strSecret); + $arrConfig['headers']['Proxy-Authorization'] = "Basic " . $strProxyAuth; + break; + case 'JULIANG': + // 隧道代理信息 + // $strProxy = 'http://username:password@XXX.juliangip.com:23456'; + $strProxy = "http://" . $strKey . ":" . $strSecret . "@" . $strProxyHost; + $arrConfig['proxy'] = $strProxy; + break; + } + } + + + $this->Client = new Client($arrConfig); + } + + /** + * Undocumented function + * + * @return string + */ + public function getSiteCode(): string + { + return $this->strSiteCode; + } + + /** + * Undocumented function + * + * @param integer|string $strSign + * @return string + */ + public function getSiteUUId(int|string $strSign): string + { + return $this->getSiteCode() . '-' . $strSign; + } + + /** + * Undocumented function + * + * @return Client + */ + public function getClient(): Client + { + return $this->Client; + } + + /** + * retrieve multiple novel page model + * + * @param string $strUri + * @return array + */ + public function getCategory(): array + { + $strUri = '/api.php/provide/vod/at/josn?ac=list'; + + $Response = $this->getClient()->get($strUri); + $strContent = $Response->getBody()->getContents(); + + $arrContent = json_decode($strContent, true); + $arrFilter = $arrContent['class']; + + // print_r($arrFilter); + + // $arrA = []; + // foreach ($arrFilter as $b) { + // $arrA[$b['type_id']] = [ + // 'name' => $b['type_name'], + // 'pinyin' => zhToPinYin($b['type_name']), + // ]; + // } + // print_r($arrA); + + + $arrCategory = []; + + foreach ($arrFilter as $intKey => $arrItem) { + + $arrFilter[$intKey]['uri'] = $this->getVideoPageUri((int)$arrItem['type_id'], 1); + + $arrPromises[$intKey] = $this->getClient()->getAsync($arrFilter[$intKey]['uri']); + } + + $arrResults = Promise\Utils::settle($arrPromises)->wait(); + + $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) { + return $arrResult['state'] === 'fulfilled'; + }); + + foreach ($arrSuccessfulResults as $intKey => $arrResult) { + $Response = $arrResult['value']; + + $strContent = $Response->getBody()->getContents(); + + $arrTemp = json_decode($strContent, true); + + $arrCategory[] = [ + 'v_category_id' => (int)$arrFilter[$intKey]['type_id'], + 'v_category_name' => $arrFilter[$intKey]['type_name'], + 'v_category_name_en' => zhToPinYin($arrFilter[$intKey]['type_name']), + 'total' => $arrTemp['total'], + 'total_page' => $arrTemp['pagecount'], + ]; + } + + return $arrCategory; + } + + /** + * generate video list uri + * + * @param integer $intVCategoryId + * @param integer $intPage + * @return string + */ + public function getVideoPageUri(int $intVCategoryId, int $intPage): string + { + return sprintf('/api.php/provide/vod/at/josn/?ac=list&t=%s&pg=%s', $intVCategoryId, $intPage); + } + + /** + * retrieve multiple novel page model + * + * @param string $strUri + * @return VideoPagerPage[] + */ + public function getVideoPagerPageList(array $arrFilter): array + { + + /** @var VideoPagerPage[] */ + $arrVideoPagerPage = []; + + foreach ($arrFilter as $intKey => $arrItem) { + + $arrFilter[$intKey]['uri'] = $this->getVideoPageUri($arrItem['v_category_id'], $arrItem['page']); + + $arrPromises[$intKey] = $this->getClient()->getAsync($arrFilter[$intKey]['uri']); + } + + $arrResults = Promise\Utils::settle($arrPromises)->wait(); + + + $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) { + return $arrResult['state'] === 'fulfilled'; + }); + + foreach ($arrSuccessfulResults as $intKey => $arrResult) { + $Response = $arrResult['value']; + + $strContent = $Response->getBody()->getContents(); + + if (is_numeric(strpos($strContent, 'System Error'))) { + continue; + } + + $VideoPagerPage = $this->getVideoPagerPage($this, $arrFilter[$intKey]['uri']); + + $VideoPagerPage->setContent($strContent); + $VideoPagerPage->setData($arrFilter[$intKey]); + + $arrVideoPagerPage[] = $VideoPagerPage; + } + + return $arrVideoPagerPage; + } + + /** + * generate video list uri + * + * @param integer $intVCategoryId + * @param integer $intPage + * @return string + */ + public function getVideoUri(int $intVSourceId): string + { + return sprintf('/api.php/provide/vod/at/josn?ac=detail&ids=%s', $intVSourceId); + } + + /** + * retrieve multiple chapter page model + * + * @param string $strUri + * @return VideoInfoPage[] + */ + public function getVideoInfoPageList(array $arrFilter): array + { + + /** @var VideoInfoPage[] */ + $arrVideoInfoPage = []; + + $arrPromises = []; + foreach ($arrFilter as $intKey => $arrItem) { + $arrFilter[$intKey]['uri'] = $this->getVideoUri($arrItem['v_source_id']); + + $arrPromises[$intKey] = $this->getClient()->getAsync($arrFilter[$intKey]['uri']); + } + + $arrResults = Promise\Utils::settle($arrPromises)->wait(); + + $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) { + return $arrResult['state'] === 'fulfilled'; + }); + + foreach ($arrSuccessfulResults as $intKey => $arrResult) { + $Response = $arrResult['value']; + + $strContent = $Response->getBody()->getContents();; + + if (is_numeric(strpos($strContent, 'System Error'))) { + continue; + } + + $VideoInfoPage = $this->getVideoInfoPage($this, $arrFilter[$intKey]['uri']); + + $VideoInfoPage->setContent($strContent); + $VideoInfoPage->setData($arrFilter[$intKey]); + + $arrVideoInfoPage[] = $VideoInfoPage; + } + + return $arrVideoInfoPage; + } + + /** + * download the image to the local system + * + * @param array $arrFilter + * @return void + */ + public function downRemoteImgToLocal(array $arrFilter) + { + $StorageCore = StorageCore::getInstance(); + + $Client = new Client(); + + foreach ($arrFilter as $arrItem) { + if (!$StorageCore->has($arrItem['local_path'])) { + try { + $Response = $Client->get($arrItem['uri']); + if ($Response->getStatusCode() === 200) { + if (str_ends_with(strtolower($arrItem['local_path']), '.webp')) { + $strTmpFile = str_replace('.webp', '.jpg', $arrItem['local_path']); + $StorageCore->writeStream($strTmpFile, $Response->getBody()->detach()); + convertToWebP( + $StorageCore->getRealPath($strTmpFile), + $StorageCore->getRealPath($arrItem['local_path']), + 40 + ); + @unlink($StorageCore->getRealPath($strTmpFile)); + } else { + $StorageCore->writeStream($arrItem['local_path'], $Response->getBody()->detach()); + } + } + } catch (\Exception $e) { + // echo $e->getMessage(); + } + } + } + } + + /** + * Undocumented function + * + * @param object $Site + * @param string $strUri + * @return VideoPagerPage + */ + public function getVideoPagerPage($Site, $strUri) + { + return new VideoPagerPage($Site, $strUri); + } + + /** + * Undocumented function + * + * @param object $Site + * @param string $strUri + * @return VideoInfoPage + */ + public function getVideoInfoPage($Site, $strUri) + { + return new VideoInfoPage($Site, $strUri); + } +} diff --git a/code/app/task/crawler/maotai/page/VideoCategory.php b/code/app/task/crawler/maotai/page/VideoCategory.php new file mode 100644 index 0000000..c67480d --- /dev/null +++ b/code/app/task/crawler/maotai/page/VideoCategory.php @@ -0,0 +1,304 @@ + + [ + 'name' => '电影', + 'pinyin' => 'dian-ying', + ], + + 2 => + [ + 'name' => '连续剧', + 'pinyin' => 'lian-xu-ju', + ], + + 3 => + [ + 'name' => '综艺片', + 'pinyin' => 'zong-yi-pian', + ], + + 4 => + [ + 'name' => '动漫', + 'pinyin' => 'dong-man', + ], + + 5 => + [ + 'name' => '纪录片', + 'pinyin' => 'ji-lu-pian', + ], + + 6 => + [ + 'name' => '动作片', + 'pinyin' => 'dong-zuo-pian', + ], + + 7 => + [ + 'name' => '爱情片', + 'pinyin' => 'ai-qing-pian', + ], + + 8 => + [ + 'name' => '喜剧片', + 'pinyin' => 'xi-ju-pian', + ], + + 9 => + [ + 'name' => '科幻片', + 'pinyin' => 'ke-huan-pian', + ], + + 10 => + [ + 'name' => '恐怖片', + 'pinyin' => 'kong-bu-pian', + ], + + 11 => + [ + 'name' => '剧情片', + 'pinyin' => 'ju-qing-pian', + ], + + 12 => + [ + 'name' => '战争片', + 'pinyin' => 'zhan-zheng-pian', + ], + + 13 => + [ + 'name' => '国产剧', + 'pinyin' => 'guo-chan-ju', + ], + + 14 => + [ + 'name' => '香港剧', + 'pinyin' => 'xiang-gang-ju', + ], + + 15 => + [ + 'name' => '欧美剧', + 'pinyin' => 'ou-mei-ju', + ], + + 16 => + [ + 'name' => '韩剧', + 'pinyin' => 'han-ju', + ], + + 20 => + [ + 'name' => '动漫电影', + 'pinyin' => 'dong-man-dian-ying', + ], + + 21 => + [ + 'name' => '泰国剧', + 'pinyin' => 'tai-guo-ju', + ], + + 22 => + [ + 'name' => '日本剧', + 'pinyin' => 'ri-ben-ju', + ], + + 23 => + [ + 'name' => '台湾剧', + 'pinyin' => 'tai-wan-ju', + ], + + 24 => + [ + 'name' => '海外剧', + 'pinyin' => 'hai-wai-ju', + ], + + 25 => + [ + 'name' => '大陆综艺', + 'pinyin' => 'da-lu-zong-yi', + ], + + 26 => + [ + 'name' => '日韩综艺', + 'pinyin' => 'ri-han-zong-yi', + ], + + 27 => + [ + 'name' => '港台综艺', + 'pinyin' => 'gang-tai-zong-yi', + ], + + 28 => + [ + 'name' => '欧美综艺', + 'pinyin' => 'ou-mei-zong-yi', + ], + + 29 => + [ + 'name' => '演唱会', + 'pinyin' => 'yan-chang-hui', + ], + + 30 => + [ + 'name' => '国产动漫', + 'pinyin' => 'guo-chan-dong-man', + ], + + 31 => + [ + 'name' => '日本动漫', + 'pinyin' => 'ri-ben-dong-man', + ], + + 32 => + [ + 'name' => '欧美动漫', + 'pinyin' => 'ou-mei-dong-man', + ], + + 33 => + [ + 'name' => '海外动漫', + 'pinyin' => 'hai-wai-dong-man', + ], + + 34 => + [ + 'name' => '伦理片', + 'pinyin' => 'lun-li-pian', + ], + + 35 => + [ + 'name' => '新闻资讯', + 'pinyin' => 'xin-wen-zi-xun', + ], + + 36 => + [ + 'name' => '体育赛事', + 'pinyin' => 'ti-yu-sai-shi', + ], + + 37 => + [ + 'name' => '短剧大全', + 'pinyin' => 'duan-ju-da-quan', + ], + + 38 => + [ + 'name' => '篮球', + 'pinyin' => 'lan-qiu', + ], + + 39 => + [ + 'name' => '足球', + 'pinyin' => 'zu-qiu', + ], + + 40 => + [ + 'name' => '网球', + 'pinyin' => 'wang-qiu', + ], + + 41 => + [ + 'name' => '斯诺克', + 'pinyin' => 'si-nuo-ke', + ], + + 42 => + [ + 'name' => 'LPL', + 'pinyin' => 'LPL', + ], + + 43 => + [ + 'name' => '重生民国', + 'pinyin' => 'zhong-sheng-min-guo', + ], + + 44 => + [ + 'name' => '穿越现代', + 'pinyin' => 'chuan-yue-xian-dai', + ], + + 45 => + [ + 'name' => '反转爽剧', + 'pinyin' => 'fan-zhuan-shuang-ju', + ], + + 46 => + [ + 'name' => '言情总裁', + 'pinyin' => 'yan-qing-zong-cai', + ], + + 47 => + [ + 'name' => '现代都市', + 'pinyin' => 'xian-dai-dou-shi', + ], + + 48 => + [ + 'name' => '古装仙侠', + 'pinyin' => 'gu-zhuang-xian-xia', + ], + + 49 => + [ + 'name' => '悬疑烧脑', + 'pinyin' => 'xuan-yi-shao-nao', + ], + + 50 => + [ + 'name' => '惊悚片', + 'pinyin' => 'jing-song-pian', + ], + + 51 => + [ + 'name' => '电影资讯', + 'pinyin' => 'dian-ying-zi-xun', + ], + + 52 => + [ + 'name' => '娱乐新闻', + 'pinyin' => 'yu-le-xin-wen', + ], + ]; +} diff --git a/code/app/task/crawler/maotai/page/VideoInfoPage.php b/code/app/task/crawler/maotai/page/VideoInfoPage.php new file mode 100644 index 0000000..f40397c --- /dev/null +++ b/code/app/task/crawler/maotai/page/VideoInfoPage.php @@ -0,0 +1,260 @@ + [ + 'root' => 'array', + 'document' => 'array', + 'array' => 'array', + ], + ]; + + /** + * Undocumented function + * + * @return VideoModel + */ + public function getVideoModel(): VideoModel + { + if ($this->VideoModel == NULL) { + $this->VideoModel = VideoModel::getInstance(); + } + return $this->VideoModel; + } + + + /** + * Undocumented function + * + * @return array + */ + public function getVideo(): array + { + if (empty($this->arrVideo)) { + + $arrVideo = json_decode($this->getContent(), true); + $arrVideo = $arrVideo['list'][0]; + // print_r($arrVideo); + + $strPlayUrl = $arrVideo['vod_play_url']; + unset($arrVideo['vod_play_url']); + + $arrPlayUrl = str_replace('#', '$', $strPlayUrl); + $arrPlayUrl = explode('$', $arrPlayUrl); + + $arrPlayUrl = array_filter($arrPlayUrl); + + $arrPlayUrl = array_chunk($arrPlayUrl, 2); + foreach ($arrPlayUrl as $intKey => $arrVal) { + $arrPlayUrl[$intKey] = [ + 'name' => $arrVal[0], + 'url' => $arrVal[1] ?? $arrVal[0], + ]; + } + + $arrVideo['vod_actor'] = trim($arrVideo['vod_actor']); + $arrVideo['vod_director'] = trim($arrVideo['vod_director']); + $arrVideo['vod_class'] = trim($arrVideo['vod_class']); + $arrVideo['vod_lang'] = trim($arrVideo['vod_lang']); + $arrVideo['vod_area'] = trim($arrVideo['vod_area']); + + $arrVideo['vod_play_url'][$this->getsite()->getSiteCode()] = $arrPlayUrl; + + $arrVideo['vod_actor'] = $arrVideo['vod_actor'] === '' ? [] : explode(',', $arrVideo['vod_actor']); + $arrVideo['vod_director'] = $arrVideo['vod_director'] === '' ? [] : explode(',', $arrVideo['vod_director']); + $arrVideo['vod_class'] = $arrVideo['vod_class'] === '' ? [] : explode(',', $arrVideo['vod_class']); + $arrVideo['vod_lang'] = $arrVideo['vod_lang'] === '' ? [] : explode(',', $arrVideo['vod_lang']); + $arrVideo['vod_area'] = $arrVideo['vod_area'] === '' ? [] : explode(',', $arrVideo['vod_area']); + + $arrVideo['vod_lang_en'] = $arrVideo['vod_area_en'] = []; + + foreach ($arrVideo['vod_lang'] as $strVal) { + $arrVideo['vod_lang_en'][] = zhToPinYin($strVal); + } + + foreach ($arrVideo['vod_area'] as $strVal) { + $arrVideo['vod_area_en'][] = zhToPinYin($strVal); + } + + + $arrVideo['vod_content'] = strip_tags($arrVideo['vod_content']); + $arrVideo['vod_content'] = trim($arrVideo['vod_content']); + $arrVideo['vod_content'] = mb_rtrim($arrVideo['vod_content'], '\　'); + $arrVideo['vod_content'] = ensureUtf8($arrVideo['vod_content']); + + $arrVideo['v_category'] = VideoCategory::$arrCategory[$arrVideo['type_id']]['name']; + $arrVideo['v_category_en'] = VideoCategory::$arrCategory[$arrVideo['type_id']]['pinyin']; + $arrVideo['v_parent_category'] = VideoCategory::$arrCategory[$arrVideo['type_id_1']]['name'] ?? VideoCategory::$arrCategory[$arrVideo['type_id']]['name']; + $arrVideo['v_parent_category_en'] = VideoCategory::$arrCategory[$arrVideo['type_id_1']]['pinyin'] ?? VideoCategory::$arrCategory[$arrVideo['type_id']]['pinyin']; + + foreach ($arrVideo as $strKey => $mixedVal) { + $strKey = str_replace('vod', 'v', $strKey); + + if ($strKey == 'type_id') { + $strKey = 'v_category_id'; + } + if ($strKey == 'type_name') { + $strKey = 'v_category'; + } + if ($strKey == 'v_content') { + $strKey = 'v_description'; + } + if ($strKey == 'v_enname') { + $strKey = 'v_name_en'; + } + if ($strKey == 'v_time') { + $strKey = 'v_publish_date'; + } + + $this->arrVideo[$strKey] = $mixedVal; + } + } + + return $this->arrVideo; + } + + + /** + * Undocumented function + * + * @return boolean + */ + public function saveVideo(): bool + { + $arrPageVideo = $this->getVideo(); + + if ( + empty($arrPageVideo) || + empty($arrPageVideo['v_name']) || + empty($arrPageVideo['v_category']) || + empty($arrPageVideo['v_play_url']) || + empty($arrPageVideo['v_play_url']['douban']) + ) { + return false; + } + + $arrExistsVideo = $this->getVideoModel()->findOne(['v_name' => $arrPageVideo['v_name']], $this->arrOptions); + + if ($arrExistsVideo) { + + $arrExistsVideo = (array) $arrExistsVideo; + + $arrUpdate = [ + 'updated_at' => new \MongoDB\BSON\UTCDateTime(), + 'v_play_url.douban' => $arrPageVideo['v_play_url']['douban'], + ]; + + $this->getVideoModel()->updateOne( + ['v_name' => $arrPageVideo['v_name']], + ['$set' => $arrUpdate], + ); + $this->strPicLocalPath = $arrExistsVideo['v_pic']; + } else { + + $arrInsertVideo = [ + 'v_id' => CountersModel::getInstance()->getNextSequence('video'), + 'v_name' => $arrPageVideo['v_name'], + 'v_name_en' => zhToPinYin($arrPageVideo['v_name']), + 'v_parent_category' => $arrPageVideo['v_parent_category'], + 'v_parent_category_en' => $arrPageVideo['v_parent_category_en'], + 'v_category' => $arrPageVideo['v_category'], + 'v_category_en' => $arrPageVideo['v_category_en'], + 'v_actor' => $arrPageVideo['v_actor'], + 'v_director' => $arrPageVideo['v_director'], + 'v_class' => $arrPageVideo['v_class'], + 'v_letter' => $arrPageVideo['v_letter'], + 'v_pic' => '', + 'v_lang' => $arrPageVideo['v_lang'], + 'v_lang_en' => $arrPageVideo['v_lang_en'], + 'v_area' => $arrPageVideo['v_area'], + 'v_area_en' => $arrPageVideo['v_area_en'], + 'v_year' => $arrPageVideo['v_year'], + 'v_remarks' => $arrPageVideo['v_remarks'], + 'v_isend' => $arrPageVideo['v_isend'], + 'v_score' => $arrPageVideo['v_score'], + 'v_description' => $arrPageVideo['v_description'], + 'v_play_url' => $arrPageVideo['v_play_url'], + 'v_publish_date' => $arrPageVideo['v_publish_date'], + 'created_at' => new \MongoDB\BSON\UTCDateTime(), + ]; + $arrInsertVideo['v_pic'] = $this->generateVideoPath($arrInsertVideo['v_name_en']); + + $this->strPicLocalPath = $arrInsertVideo['v_pic']; + + try { + $this->getVideoModel()->insert($arrInsertVideo); + } catch (\Exception $e) { + if ($e->getCode() === 11000) { + return $this->saveVideo(); + } else { + throw $e; + } + } + } + + return true; + } + + private $strPicLocalPath = ''; + + /** + * Undocumented function + * + * @param string $strName + * @return string + */ + public function generateVideoPath($strName): string + { + $intTime = time() - mt_rand(1, 2 * 24 * 3600 * 360); + + return sprintf('/img/video/%s/%s.webp', date('Y/m/d/H', $intTime), $strName); + } + + + /** + * Undocumented function + * + * @return array + */ + public function getDownPicInfo(): array + { + return [ + 'uri' => $this->getVideo()['v_pic'], + 'local_path' => $this->strPicLocalPath, + ]; + } +} diff --git a/code/app/task/crawler/maotai/page/VideoPagerPage.php b/code/app/task/crawler/maotai/page/VideoPagerPage.php new file mode 100644 index 0000000..fd8c4c9 --- /dev/null +++ b/code/app/task/crawler/maotai/page/VideoPagerPage.php @@ -0,0 +1,43 @@ + [ + 'root' => 'array', + 'document' => 'array', + 'array' => 'array', + ], + ]; + + /** + * Undocumented function + * + * @return array + */ + public function getVideoPager(): array + { + if (empty($this->arrVideo)) { + + $this->arrVideo = json_decode($this->getContent(), true); + } + + return $this->arrVideo; + } +} diff --git a/code/app/task/module/PlanTask.php b/code/app/task/module/PlanTask.php index 0969ae1..9473999 100644 --- a/code/app/task/module/PlanTask.php +++ b/code/app/task/module/PlanTask.php @@ -16,6 +16,7 @@ use app\task\crawler\xbiqugu\Scheduler as XBiQuGeScheduler; use app\task\crawler\youzhi\Scheduler as YouZhiScheduler; use app\task\crawler\fengchao\Scheduler as FengChaoScheduler; use app\task\crawler\douban\Scheduler as DouBanScheduler; +use app\task\crawler\maotai\Scheduler as MaoTaiScheduler; use app\task\crawler\wuxian\Scheduler as WuXianScheduler; use app\task\crawler\custom\Scheduler as CustomScheduler; @@ -79,6 +80,10 @@ class PlanTask case 'PULL_DOU_BAN_SHI_PIN': self::pullDouBanShiPin(); break; + case 'PULL_MAO_TAI_SHI_PIN': + self::pullMaoTaiShiPin(); + break; + case 'CUSTOM_CRAWLER': self::customCrawler(); break; @@ -119,7 +124,22 @@ class PlanTask $QueueManage = QueueManage::getInstance(); $QueueManage->set($arrTask); } + + /** + * Undocumented function + * + * @return void + */ + public static function pullMaoTaiShiPin() + { + $arrTask = [ + 'callback' => [MaoTaiScheduler::class, 'crawlFull'], + 'data' => [] + ]; + $QueueManage = QueueManage::getInstance(); + $QueueManage->set($arrTask); + } /** * Undocumented function * diff --git a/code/database/seeders/PlanTaskSeeder.php b/code/database/seeders/PlanTaskSeeder.php index 89be9bb..cdc9ab7 100644 --- a/code/database/seeders/PlanTaskSeeder.php +++ b/code/database/seeders/PlanTaskSeeder.php @@ -110,7 +110,13 @@ class PlanTaskSeeder 'pt_code' => 'PULL_DOU_BAN_SHI_PIN', 'pt_enable' => 0, 'pt_limit' => 1 * 24 * 3600, + ], [ + 'pt_name' => '茅台资源库全量采集', + 'pt_code' => 'PULL_MAO_TAI_SHI_PIN', + 'pt_enable' => 0, + 'pt_limit' => 1 * 24 * 3600, ], + ]; foreach ($arrDataNovel as $arrPlanTask) {