From 5fdd741c30a2472f8d13ecce2a2e614563286194 Mon Sep 17 00:00:00 2001 From: Default Date: Thu, 1 May 2025 19:46:29 +0800 Subject: [PATCH] feature : crawler fengchao video --- code/app/task/crawler/fengchao/Scheduler.php | 196 ++++++++++ .../task/crawler/fengchao/page/BasePage.php | 251 ++++++++++++ code/app/task/crawler/fengchao/page/Site.php | 359 ++++++++++++++++++ .../crawler/fengchao/page/VideoInfoPage.php | 230 +++++++++++ .../crawler/fengchao/page/VideoPagerPage.php | 43 +++ .../crawler/youzhi/page/VideoInfoPage.php | 2 +- code/app/task/module/PlanTask.php | 38 +- code/database/seeders/PlanTaskSeeder.php | 7 + code/extend/storage/StorageCore.php | 30 +- 9 files changed, 1138 insertions(+), 18 deletions(-) create mode 100644 code/app/task/crawler/fengchao/Scheduler.php create mode 100644 code/app/task/crawler/fengchao/page/BasePage.php create mode 100644 code/app/task/crawler/fengchao/page/Site.php create mode 100644 code/app/task/crawler/fengchao/page/VideoInfoPage.php create mode 100644 code/app/task/crawler/fengchao/page/VideoPagerPage.php diff --git a/code/app/task/crawler/fengchao/Scheduler.php b/code/app/task/crawler/fengchao/Scheduler.php new file mode 100644 index 0000000..b93c794 --- /dev/null +++ b/code/app/task/crawler/fengchao/Scheduler.php @@ -0,0 +1,196 @@ +QueueManage = new QueueManage(); + $this->init(); + } + + /** + * Undocumented function + * + * @return void + */ + public function init() + { + $this->intSlice = 8; + } + + /** + * get site + * + * @return Site + */ + public function getSite() + { + if ($this->Site == NULL) { + $this->Site = new Site; + } + return $this->Site; + } + + + /** + * create a site-wide craw task + * + * @return void + */ + public function crawlFull() + { + // $arrVSourceId = [['v_source_id' => 14256]]; + // $arrTask = [ + // 'callback' => [static::class, 'fetchVideo'], + // 'data' => $arrVSourceId, + // ]; + // $this->QueueManage->set($arrTask); + // return; + + $Site = $this->getSite(); + + $arrAllCategory = $Site->getCategory(); + + // print_r($arrAllCategory); + + // return; + + $intLimit = 5; + + $arrTask = [ + 'callback' => [static::class, 'fetchVSourceId'], + 'data' => [], + ]; + + foreach ($arrAllCategory as $arrCategory) { + + if ($arrCategory['total'] <= 0) { + continue; + } + + $intTotalPage = $arrCategory['total_page']; + $arrFilter = []; + for ($intStartPage = 1; $intStartPage <= $intTotalPage; $intStartPage++) { + + $arrFilter[] = [ + 'v_category_id' => $arrCategory['v_category_id'], + 'page' => $intStartPage, + ]; + if (($intStartPage % $intLimit) == 0) { + $arrTask['data'] = $arrFilter; + $this->QueueManage->set($arrTask); + $arrFilter = []; + } + } + if (!empty($arrFilter)) { + $arrTask['data'] = $arrFilter; + $this->QueueManage->set($arrTask); + } + } + } + + /** + * Undocumented function + * + * @param array $arrFilter + * @return void + */ + public function fetchVSourceId(array $arrFilter) + { + + $Site = $this->getSite(); + + $intLimit = 5; + + $arrTask = [ + 'callback' => [static::class, 'fetchVideo'], + 'data' => [], + ]; + + $arrAllVideoPagerPage = $Site->getVideoPagerPageList($arrFilter); + + $arrVSourceId = []; + + foreach ($arrAllVideoPagerPage as $VideoPagerPage) { + $arrVideoPager = $VideoPagerPage->getVideoPager(); + + foreach ($arrVideoPager['list'] as $arrVideoPager) { + $arrVSourceId[] = ['v_source_id' => (int)$arrVideoPager['vod_id']]; + + if ((count($arrVSourceId) % $intLimit) == 0) { + $arrTask['data'] = $arrVSourceId; + $this->QueueManage->set($arrTask); + $arrVSourceId = []; + } + } + } + + if (!empty($arrVSourceId)) { + $arrTask['data'] = $arrVSourceId; + $this->QueueManage->set($arrTask); + } + } + + /** + * Undocumented function + * + * @param array $arrVSourceId + * @return void + */ + public function fetchVideo(array $arrVSourceId) + { + $Site = $this->getSite(); + + $arrAllVideoInfoPage = $Site->getVideoInfoPageList($arrVSourceId); + + $arrPic = []; + foreach ($arrAllVideoInfoPage as $VideoInfoPage) { + $VideoInfoPage->saveVideo(); + + $arrPicItem = $VideoInfoPage->getDownPicInfo(); + + if (!StorageCore::getInstance()->has($arrPicItem['local_path'])) { + $arrPic[] = $arrPicItem; + } + } + + if ($arrPic) { + $Site->downRemoteImgToLocal($arrPic); + } + } +} diff --git a/code/app/task/crawler/fengchao/page/BasePage.php b/code/app/task/crawler/fengchao/page/BasePage.php new file mode 100644 index 0000000..9d7fa1f --- /dev/null +++ b/code/app/task/crawler/fengchao/page/BasePage.php @@ -0,0 +1,251 @@ +setSite($Site); + + $this->strUri = $strUri; + } + + /** + * Undocumented function + * + * @return string + */ + public function getUri(): string + { + return $this->strUri; + } + + /** + * Undocumented function + * + * @param array $arrData + * @return void + */ + public function setData(array $arrData) + { + $this->arrData = $arrData; + } + + /** + * Undocumented function + * + * @return array + */ + public function getData(): array + { + return $this->arrData; + } + + /** + * Undocumented function + * + * @return QueueManage + */ + public function getQueueManage(): QueueManage + { + if ($this->QueueManage == NULL) { + $this->QueueManage = QueueManage::getInstance(); + } + + return $this->QueueManage; + } + + /** + * Undocumented function + * + * @return MongoBase + */ + public function getMongoBase(): MongoBase + { + if ($this->MongoBase == NULL) { + DDLManage::load('MongoBase'); + $this->MongoBase = MongoBase::getInstance(); + } + + return $this->MongoBase; + } + + public function getQueryList(): QueryList + { + if ($this->QueryList == NULL) { + $this->QueryList = QueryList::html($this->getContent()); + } + + return $this->QueryList; + } + + + /** + * Undocumented function + * + * @return string + */ + public function getContent(): string + { + if ($this->strContent == NULL) { + $this->Response = $this->getClient()->get($this->strUri); + $this->strContent = $this->Response->getBody()->getContents(); + } + + return $this->strContent; + } + + /** + * force set page content + * + * @param string $strContent + * @return void + */ + public function setContent($strContent) + { + $this->strContent = $strContent; + } + + + /** + * Undocumented function + * + * @return boolean + */ + public function isMobile($strMark = 'content="mobile"'): bool + { + if ($this->boolIsMobile === NULL) { + $this->boolIsMobile = is_numeric(strpos($this->getContent(), $strMark)); + } + return $this->boolIsMobile; + } + + + /** + * Undocumented function + * + * @return Client + */ + protected function getClient(): Client + { + if ($this->Client == NULL) { + $this->Client = $this->getSite()->getClient(); + } + return $this->Client; + } + + + /** + * Undocumented variable + * + * @var Site + */ + protected $Site; + + public function setSite(Site $Site) + { + $this->Site = $Site; + } + + public function getSite() + { + return $this->Site; + } +} diff --git a/code/app/task/crawler/fengchao/page/Site.php b/code/app/task/crawler/fengchao/page/Site.php new file mode 100644 index 0000000..1c969a4 --- /dev/null +++ b/code/app/task/crawler/fengchao/page/Site.php @@ -0,0 +1,359 @@ +initClient(); + } + + /** + * Undocumented function + */ + public function initClient() + { + $arrConfig = [ + 'http_version' => '1.1', // 强制用 HTTP/1.1，已有，保持 + 'base_uri' => $this->strDomain, // 基础域名，已有，保持 + 'http_errors' => false, // 不抛 HTTP 错误，已有，保持 + 'timeout' => 60, // 从 160 秒改为 60 秒 + 'connect_timeout' => 10, // 从 30 秒改为 10 秒 + 'curl' => [ + CURLOPT_TCP_KEEPALIVE => 1, // 启用 TCP Keep-Alive，已有，保持 + CURLOPT_TCP_KEEPIDLE => 120, // Keep-Alive 空闲时间 + CURLOPT_TCP_KEEPINTVL => 60, // Keep-Alive 探测间隔 + // CURLOPT_SSL_VERIFYPEER => true, // 验证 SSL，已有，保持 + ], + 'headers' => [ + 'User-Agent' => 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36', + 'Referer' => $this->strDomain, + 'Connection' => 'keep-alive', + ], + 'pool_size' => 20, + 'verify' => false, + ]; + + $arrProxy = config('proxy'); + + + if ($this->boolProxy && $arrProxy['proxy_status']) { + + $strCode = $arrProxy['proxy_code']; + $strProxyHost = $arrProxy['proxy_host']; + $strKey = $arrProxy['proxy_key']; + $strSecret = $arrProxy['proxy_secret']; + + + switch ($strCode) { + case 'XIONGMAO': + $intTime = time(); + $strTxt = "orderno=" . $strKey . ",secret=" . $strSecret . ",timestamp=" . $intTime; + $strSign = strtoupper(md5($strTxt)); + $strAuth = 'sign=' . $strSign . '&orderno=' . $strKey . '×tamp=' . $intTime; + $arrConfig['proxy'] = $strProxyHost; + $arrConfig['headers']['Proxy-Authorization'] = $strAuth; + break; + case 'KUAI': + $arrConfig['proxy'] = $strProxyHost; + $arrConfig['curl'][CURLOPT_PROXYAUTH] = CURLAUTH_BASIC; + $arrConfig['curl'][CURLOPT_PROXYUSERPWD] = "$strKey:$strSecret"; + break; + } + } + + + $this->Client = new Client($arrConfig); + } + + /** + * Undocumented function + * + * @return string + */ + public function getSiteCode(): string + { + return $this->strSiteCode; + } + + /** + * Undocumented function + * + * @param integer|string $strSign + * @return string + */ + public function getSiteUUId(int|string $strSign): string + { + return $this->getSiteCode() . '-' . $strSign; + } + + /** + * Undocumented function + * + * @return Client + */ + public function getClient(): Client + { + return $this->Client; + } + + /** + * retrieve multiple novel page model + * + * @param string $strUri + * @return array + */ + public function getCategory(): array + { + $strUri = '/api.php/provide/vod/at/json?ac=list'; + + $Response = $this->getClient()->get($strUri); + $strContent = $Response->getBody()->getContents(); + + $arrContent = json_decode($strContent, true); + $arrFilter = $arrContent['class']; + + $arrCategory = []; + + foreach ($arrFilter as $intKey => $arrItem) { + + $arrFilter[$intKey]['uri'] = $this->getVideoPageUri((int)$arrItem['type_id'], 1); + + $arrPromises[$intKey] = $this->getClient()->getAsync($arrFilter[$intKey]['uri']); + } + + $arrResults = Promise\Utils::settle($arrPromises)->wait(); + + $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) { + return $arrResult['state'] === 'fulfilled'; + }); + + foreach ($arrSuccessfulResults as $intKey => $arrResult) { + $Response = $arrResult['value']; + + $strContent = $Response->getBody()->getContents(); + + $arrTemp = json_decode($strContent, true); + + $arrCategory[] = [ + 'v_category_id' => (int)$arrFilter[$intKey]['type_id'], + 'v_category_name' => $arrFilter[$intKey]['type_name'], + 'v_category_name_en' => zhToPinYin($arrFilter[$intKey]['type_name']), + 'total' => $arrTemp['total'], + 'total_page' => $arrTemp['pagecount'], + ]; + } + + return $arrCategory; + } + + /** + * generate video list uri + * + * @param integer $intVCategoryId + * @param integer $intPage + * @return string + */ + public function getVideoPageUri(int $intVCategoryId, int $intPage): string + { + return sprintf('/api.php/provide/vod/at/json?ac=list&t=%s&pg=%s', $intVCategoryId, $intPage); + } + + /** + * retrieve multiple novel page model + * + * @param string $strUri + * @return VideoPagerPage[] + */ + public function getVideoPagerPageList(array $arrFilter): array + { + + /** @var VideoPagerPage[] */ + $arrVideoPagerPage = []; + + foreach ($arrFilter as $intKey => $arrItem) { + + $arrFilter[$intKey]['uri'] = $this->getVideoPageUri($arrItem['v_category_id'], $arrItem['page']); + + $arrPromises[$intKey] = $this->getClient()->getAsync($arrFilter[$intKey]['uri']); + } + + $arrResults = Promise\Utils::settle($arrPromises)->wait(); + + + $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) { + return $arrResult['state'] === 'fulfilled'; + }); + + foreach ($arrSuccessfulResults as $intKey => $arrResult) { + $Response = $arrResult['value']; + + $strContent = $Response->getBody()->getContents(); + + + $VideoPagerPage = $this->getVideoPagerPage($this, $arrFilter[$intKey]['uri']); + + $VideoPagerPage->setContent($strContent); + $VideoPagerPage->setData($arrFilter[$intKey]); + + $arrVideoPagerPage[] = $VideoPagerPage; + } + + return $arrVideoPagerPage; + } + + /** + * generate video list uri + * + * @param integer $intVCategoryId + * @param integer $intPage + * @return string + */ + public function getVideoUri(int $intVSourceId): string + { + return sprintf('/api.php/provide/vod/at/json?ac=detail&ids=%s', $intVSourceId); + } + + /** + * retrieve multiple chapter page model + * + * @param string $strUri + * @return VideoInfoPage[] + */ + public function getVideoInfoPageList(array $arrFilter): array + { + + /** @var VideoInfoPage[] */ + $arrVideoInfoPage = []; + + $arrPromises = []; + foreach ($arrFilter as $intKey => $arrItem) { + $arrFilter[$intKey]['uri'] = $this->getVideoUri($arrItem['v_source_id']); + + $arrPromises[$intKey] = $this->getClient()->getAsync($arrFilter[$intKey]['uri']); + } + + $arrResults = Promise\Utils::settle($arrPromises)->wait(); + + $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) { + return $arrResult['state'] === 'fulfilled'; + }); + + foreach ($arrSuccessfulResults as $intKey => $arrResult) { + $Response = $arrResult['value']; + + $strContent = $Response->getBody()->getContents();; + + $VideoInfoPage = $this->getVideoInfoPage($this, $arrFilter[$intKey]['uri']); + + $VideoInfoPage->setContent($strContent); + $VideoInfoPage->setData($arrFilter[$intKey]); + + $arrVideoInfoPage[] = $VideoInfoPage; + } + + return $arrVideoInfoPage; + } + + /** + * download the image to the local system + * + * @param array $arrFilter + * @return void + */ + public function downRemoteImgToLocal(array $arrFilter) + { + $StorageCore = StorageCore::getInstance(); + + $Client = new Client(); + + foreach ($arrFilter as $arrItem) { + if (!$StorageCore->has($arrItem['local_path'])) { + try { + $Response = $Client->get($arrItem['uri']); + if ($Response->getStatusCode() === 200) { + if (str_ends_with(strtolower($arrItem['local_path']), '.webp')) { + $strTmpFile = str_replace('.webp', '.jpg', $arrItem['local_path']); + $StorageCore->writeStream($strTmpFile, $Response->getBody()->detach()); + convertToWebP( + $StorageCore->getRealPath($strTmpFile), + $StorageCore->getRealPath($arrItem['local_path']), + 40 + ); + @unlink($StorageCore->getRealPath($strTmpFile)); + } else { + $StorageCore->writeStream($arrItem['local_path'], $Response->getBody()->detach()); + } + } + } catch (\Exception $e) { + // echo $e->getMessage(); + } + } + } + } + + /** + * Undocumented function + * + * @param object $Site + * @param string $strUri + * @return VideoPagerPage + */ + public function getVideoPagerPage($Site, $strUri) + { + return new VideoPagerPage($Site, $strUri); + } + + /** + * Undocumented function + * + * @param object $Site + * @param string $strUri + * @return VideoInfoPage + */ + public function getVideoInfoPage($Site, $strUri) + { + return new VideoInfoPage($Site, $strUri); + } +} diff --git a/code/app/task/crawler/fengchao/page/VideoInfoPage.php b/code/app/task/crawler/fengchao/page/VideoInfoPage.php new file mode 100644 index 0000000..9593852 --- /dev/null +++ b/code/app/task/crawler/fengchao/page/VideoInfoPage.php @@ -0,0 +1,230 @@ + [ + 'root' => 'array', + 'document' => 'array', + 'array' => 'array', + ], + ]; + + /** + * Undocumented function + * + * @return VideoModel + */ + public function getVideoModel(): VideoModel + { + if ($this->VideoModel == NULL) { + $this->VideoModel = VideoModel::getInstance(); + } + return $this->VideoModel; + } + + + /** + * Undocumented function + * + * @return array + */ + public function getVideo(): array + { + if (empty($this->arrVideo)) { + + $arrVideo = json_decode($this->getContent(), true); + $arrVideo = $arrVideo['list'][0]; + // print_r($arrVideo); + + $strPlayUrl = $arrVideo['vod_play_url']; + unset($arrVideo['vod_play_url']); + + $arrPlayUrl = str_replace('#', '$', $strPlayUrl); + $arrPlayUrl = explode('$', $arrPlayUrl); + $arrPlayUrl = array_chunk($arrPlayUrl, 2); + foreach ($arrPlayUrl as $intKey => $arrVal) { + $arrPlayUrl[$intKey] = [ + 'name' => $arrVal[0], + 'url' => $arrVal[1] ?? $arrVal[0], + ]; + } + + $arrVideo['vod_class'] = trim($arrVideo['vod_class']); + + $arrVideo['vod_play_url'][$this->getsite()->getSiteCode()] = $arrPlayUrl; + $arrVideo['vod_actor'] = explode(',', $arrVideo['vod_actor']); + $arrVideo['vod_director'] = explode(',', $arrVideo['vod_director']); + $arrVideo['vod_class'] = explode(',', $arrVideo['vod_class']); + $arrVideo['vod_content'] = trim($arrVideo['vod_content']); + $arrVideo['vod_content'] = mb_rtrim($arrVideo['vod_content'], '\　'); + + $arrVideo['vod_content'] = ensureUtf8($arrVideo['vod_content']); + + foreach ($arrVideo as $strKey => $mixedVal) { + $strKey = str_replace('vod', 'v', $strKey); + + if ($strKey == 'type_id') { + $strKey = 'v_category_id'; + } + if ($strKey == 'type_name') { + $strKey = 'v_category'; + } + if ($strKey == 'v_content') { + $strKey = 'v_description'; + } + if ($strKey == 'v_enname') { + $strKey = 'v_name_en'; + } + if ($strKey == 'v_time') { + $strKey = 'v_publish_date'; + } + + $this->arrVideo[$strKey] = $mixedVal; + } + } + + return $this->arrVideo; + } + + + /** + * Undocumented function + * + * @return boolean + */ + public function saveVideo(): bool + { + $arrPageVideo = $this->getVideo(); + + if ( + empty($arrPageVideo) || + empty($arrPageVideo['v_name']) || + empty($arrPageVideo['v_category']) || + empty($arrPageVideo['v_play_url']) || + empty($arrPageVideo['v_play_url']['fengchao']) + ) { + return false; + } + + $arrExistsVideo = $this->getVideoModel()->findOne(['v_name' => $arrPageVideo['v_name']], $this->arrOptions); + + if ($arrExistsVideo) { + + $arrExistsVideo = (array) $arrExistsVideo; + + $arrUpdate = [ + 'updated_at' => new \MongoDB\BSON\UTCDateTime(), + 'v_play_url.fengchao' => $arrPageVideo['v_play_url']['fengchao'], + ]; + + $this->getVideoModel()->updateOne( + ['v_name' => $arrPageVideo['v_name']], + ['$set' => $arrUpdate], + ); + $this->strPicLocalPath = $arrExistsVideo['v_pic']; + } else { + + $arrInsertVideo = [ + 'v_id' => CountersModel::getInstance()->getNextSequence('video'), + 'v_name' => $arrPageVideo['v_name'], + 'v_name_en' => zhToPinYin($arrPageVideo['v_name']), + 'v_category' => $arrPageVideo['v_category'], + 'v_category_en' => zhToPinYin($arrPageVideo['v_category']), + 'v_actor' => $arrPageVideo['v_actor'], + 'v_director' => $arrPageVideo['v_director'], + 'v_class' => $arrPageVideo['v_class'], + 'v_letter' => $arrPageVideo['v_letter'], + 'v_pic' => '', + 'v_lang' => $arrPageVideo['v_lang'], + 'v_area' => $arrPageVideo['v_area'], + 'v_year' => $arrPageVideo['v_year'], + 'v_remarks' => $arrPageVideo['v_remarks'], + 'v_isend' => $arrPageVideo['v_isend'], + 'v_score' => $arrPageVideo['v_score'], + 'v_description' => $arrPageVideo['v_description'], + 'v_play_url' => $arrPageVideo['v_play_url'], + 'v_publish_date' => $arrPageVideo['v_publish_date'], + 'created_at' => new \MongoDB\BSON\UTCDateTime(), + ]; + $arrInsertVideo['v_pic'] = $this->generateVideoPath($arrInsertVideo['v_name_en']); + + $this->strPicLocalPath = $arrInsertVideo['v_pic']; + + try { + $this->getVideoModel()->insert($arrInsertVideo); + } catch (\Exception $e) { + if ($e->getCode() === 11000) { + return $this->saveVideo(); + } else { + throw $e; + } + } + } + + return true; + } + + private $strPicLocalPath = ''; + + /** + * Undocumented function + * + * @param string $strName + * @return string + */ + public function generateVideoPath($strName): string + { + $intTime = time() - mt_rand(1, 2 * 24 * 3600 * 360); + + return sprintf('/img/video/%s/%s.webp', date('Y/m/d/H', $intTime), $strName); + } + + + /** + * Undocumented function + * + * @return array + */ + public function getDownPicInfo(): array + { + return [ + 'uri' => $this->getVideo()['v_pic'], + 'local_path' => $this->strPicLocalPath, + ]; + } +} diff --git a/code/app/task/crawler/fengchao/page/VideoPagerPage.php b/code/app/task/crawler/fengchao/page/VideoPagerPage.php new file mode 100644 index 0000000..43f6af5 --- /dev/null +++ b/code/app/task/crawler/fengchao/page/VideoPagerPage.php @@ -0,0 +1,43 @@ + [ + 'root' => 'array', + 'document' => 'array', + 'array' => 'array', + ], + ]; + + /** + * Undocumented function + * + * @return array + */ + public function getVideoPager(): array + { + if (empty($this->arrVideo)) { + + $this->arrVideo = json_decode($this->getContent(), true); + } + + return $this->arrVideo; + } +} diff --git a/code/app/task/crawler/youzhi/page/VideoInfoPage.php b/code/app/task/crawler/youzhi/page/VideoInfoPage.php index ab1dd05..e561bd0 100644 --- a/code/app/task/crawler/youzhi/page/VideoInfoPage.php +++ b/code/app/task/crawler/youzhi/page/VideoInfoPage.php @@ -156,7 +156,7 @@ class VideoInfoPage extends BasePage ['v_name' => $arrPageVideo['v_name']], ['$set' => $arrUpdate], ); - $this->strPicLocalPath = $arrExistsVideo['pic']; + $this->strPicLocalPath = $arrExistsVideo['v_pic']; } else { $arrInsertVideo = [ diff --git a/code/app/task/module/PlanTask.php b/code/app/task/module/PlanTask.php index bad83f7..d7b9ae3 100644 --- a/code/app/task/module/PlanTask.php +++ b/code/app/task/module/PlanTask.php @@ -5,13 +5,18 @@ declare(strict_types=1); namespace app\task\module; use app\model\PlanTaskModel; -use app\task\crawler\zw630\Scheduler as Zw630Scheduler; -use app\task\crawler\zw74\Scheduler as Zw74Scheduler; -use app\task\crawler\youzhi\Scheduler as YouZhiScheduler; use app\task\logic\SiteMapLogic; use microserver\QueueManage; use microserver\ProcessSanitizer; +use app\task\crawler\zw630\Scheduler as Zw630Scheduler; +use app\task\crawler\zw74\Scheduler as Zw74Scheduler; +use app\task\crawler\youzhi\Scheduler as YouZhiScheduler; +use app\task\crawler\fengchao\Scheduler as FengChaoScheduler; + + + + class PlanTask { @@ -49,6 +54,10 @@ class PlanTask self::pullYouZhiShiPin(); break; + case 'PULL_FENG_CHAO_SHI_PIN': + self::pullFengChaoShiPin(); + break; + case 'GENERATE_SITE_MAP': self::generateSiteMap(); break; @@ -61,18 +70,17 @@ class PlanTask * * @return void */ - public static function generateSiteMap() + public static function pullFengChaoShiPin() { $arrTask = [ - 'callback' => [SiteMapLogic::class, 'generateSiteMap'], + 'callback' => [FengChaoScheduler::class, 'crawlFull'], 'data' => [] ]; - $QueueManage = new QueueManage(2); + $QueueManage = new QueueManage(); $QueueManage->set($arrTask); } - /** * Undocumented function * @@ -154,4 +162,20 @@ class PlanTask $QueueManage = new QueueManage(); $QueueManage->set($arrTask); } + + /** + * Undocumented function + * + * @return void + */ + public static function generateSiteMap() + { + $arrTask = [ + 'callback' => [SiteMapLogic::class, 'generateSiteMap'], + 'data' => [] + ]; + + $QueueManage = new QueueManage(2); + $QueueManage->set($arrTask); + } } diff --git a/code/database/seeders/PlanTaskSeeder.php b/code/database/seeders/PlanTaskSeeder.php index 47895cc..307341c 100644 --- a/code/database/seeders/PlanTaskSeeder.php +++ b/code/database/seeders/PlanTaskSeeder.php @@ -53,6 +53,13 @@ class PlanTaskSeeder 'pt_limit' => 1 * 24 * 3600, ], + [ + 'pt_name' => '蜂巢资源库全量采集', + 'pt_code' => 'PULL_FENG_CHAO_SHI_PIN', + 'pt_enable' => 0, + 'pt_limit' => 1 * 24 * 3600, + ], + [ 'pt_name' => '生成网站地图', 'pt_code' => 'GENERATE_SITE_MAP', diff --git a/code/extend/storage/StorageCore.php b/code/extend/storage/StorageCore.php index 9c77695..d11ca23 100644 --- a/code/extend/storage/StorageCore.php +++ b/code/extend/storage/StorageCore.php @@ -10,7 +10,12 @@ use League\Flysystem\Local\LocalFilesystemAdapter; class StorageCore { - private static $instance = null; + /** + * Undocumented variable + * + * @var StorageCore + */ + private static $StorageCore = null; /** * Undocumented variable @@ -19,7 +24,6 @@ class StorageCore */ private $Filesystem; - /** * Undocumented variable * @@ -27,15 +31,20 @@ class StorageCore */ private $Adapter; + /** + * Undocumented variable + * + * @var string + */ + private $strLocationPath; + private function __construct() { $arrConfig = config('storage'); - // $this->Adapter = new Local( - // $arrConfig['local']['dir'] - // ); + $this->strLocationPath = rtrim($arrConfig['local']['dir'], '/'); - $this->Adapter = new LocalFilesystemAdapter($arrConfig['local']['dir']); + $this->Adapter = new LocalFilesystemAdapter($this->strLocationPath); $this->Filesystem = new Filesystem($this->Adapter); } @@ -47,10 +56,10 @@ class StorageCore */ public static function getInstance(): self { - if (self::$instance == null) { - self::$instance = new self(); + if (self::$StorageCore == null) { + self::$StorageCore = new self(); } - return self::$instance; + return self::$StorageCore; } public function set($strKey, $strContent) @@ -101,6 +110,7 @@ class StorageCore public function getRealPath($strKey): string { - return $this->Adapter->applyPathPrefix($strKey); + return $this->strLocationPath . $strKey; + // return $this->Adapter->applyPathPrefix($strKey); } }