diff --git a/code/app/common.php b/code/app/common.php index 35f5a03..fd9e926 100644 --- a/code/app/common.php +++ b/code/app/common.php @@ -815,15 +815,51 @@ namespace { * @param integer $length * @return string */ - function randomString(int $length): string { + function randomString(int $length): string + { $chars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'; $max = 51; // 字符集长度减1 (0-51) $result = ''; - + for ($i = 0; $i < $length; $i++) { $result .= $chars[random_int(0, $max)]; } - + return $result; } + + /** + * 图像转换函数 + * + * @param string $strInputFile + * @param string $strOutputFile + * @param integer $intQuality + * @return boolean + */ + function convertToWebP(string $strInputFile, string $strOutputFile, int $intQuality = 80): bool + { + $strCwebpPath = root_path() . '/extend/tools/cwebp'; + + $strCMD = escapeshellcmd("$strCwebpPath -q $intQuality " . escapeshellarg($strInputFile) . " -o " . escapeshellarg($strOutputFile)); + + exec($strCMD . ' 2>&1', $strOutput, $intReturnCode); + + if ($intReturnCode === 0) { + return true; + } else { + echo "Conversion failed: " . implode("\n", $strOutput); + return false; + } + } + + function ensureUtf8(string $input): string + { + // 检查是否为有效的 UTF-8 + if (mb_check_encoding($input, 'UTF-8')) { + return $input; // 已经是有效的 UTF-8 + } + + // 尝试将非 UTF-8 转换为 UTF-8,忽略无效字符 + return iconv('UTF-8', 'UTF-8//IGNORE', $input); + } } diff --git a/code/app/model/ConverterMovel.php b/code/app/model/ConverterMovel.php index d4caace..bf654db 100644 --- a/code/app/model/ConverterMovel.php +++ b/code/app/model/ConverterMovel.php @@ -88,7 +88,7 @@ class ConverterMovel * Undocumented function * * @param string $strKey - * @return void + * @return string */ static public function getVal(string $strKey) { diff --git a/code/app/model/VideoClicksModel.php b/code/app/model/VideoClicksModel.php index 06984f4..49f18a3 100644 --- a/code/app/model/VideoClicksModel.php +++ b/code/app/model/VideoClicksModel.php @@ -70,7 +70,6 @@ class VideoClicksModel extends MongoModel ], [ '$inc' => ['vc_clicks' => 1], - '$set' => ['v_category_id' => $Novel->v_category_id] ], [ 'upsert' => true diff --git a/code/app/model/VideoModel.php b/code/app/model/VideoModel.php index 33bcd7d..f4c6bbc 100644 --- a/code/app/model/VideoModel.php +++ b/code/app/model/VideoModel.php @@ -43,9 +43,6 @@ class VideoModel extends MongoModel ], 'projection' => [ '_id' => 0, - 'n_site_code' => 0, - 'n_source_uuid' => 0, - 'n_source_id' => 0, ], ]; @@ -63,12 +60,12 @@ class VideoModel extends MongoModel try { $arrConditions = array_map(function ($strKeyword) { - return ['n_name' => ['$regex' => trim($strKeyword), '$options' => 'i']]; + return ['v_name' => ['$regex' => trim($strKeyword), '$options' => 'i']]; }, $arrKeywords); $UpdateResult = $this->getCol()->updateMany( ['$or' => $arrConditions], - ['$set' => ['n_level' => $intLevel]] + ['$set' => ['v_level' => $intLevel]] ); $UpdateResult->getModifiedCount(); @@ -99,7 +96,6 @@ class VideoModel extends MongoModel $arrOptions['sort'] = $arrSort; } - $arrFilter['n_have_chapter'] = 1; $Cursor = $this->getCol()->find($arrFilter, $arrOptions); @@ -179,7 +175,7 @@ class VideoModel extends MongoModel */ public function getVideoByNId(int $intNId): null|array { - $arrFilter = ['n_id' => $intNId, 'n_have_chapter' => 1]; + $arrFilter = ['v_id' => $intNId]; return $this->findOne($arrFilter); } @@ -202,7 +198,6 @@ class VideoModel extends MongoModel return $arrResult; } - $arrFilter['n_have_chapter'] = 1; $arrPipeline = [ ['$match' => $arrFilter], @@ -236,12 +231,11 @@ class VideoModel extends MongoModel array $arrFilter, int $intPage = 1, int $intLimit = 20, - array $arrSort = ['n_level' => -1], + array $arrSort = ['v_level' => -1], string $strKey = '', int $intLifeTime = 24 * 3600, ): array { try { - $arrFilter['n_have_chapter'] = 1; $arrResult = $this->checkCacheByKey($strKey); if ($arrResult !== NULL) { @@ -295,23 +289,23 @@ class VideoModel extends MongoModel int $intLifeTime = 24 * 3600, int $intType = 0, $strStatus = NULL, - array $arrSort = ['n_level' => -1] + array $arrSort = ['v_level' => -1] ): array { - $arrFilter = ['n_have_chapter' => 1,]; + $arrFilter = []; switch ($intType) { case 1: $arrCategory = array_values(CategoryModel::getManCategory()); - $arrFilter['n_category'] = ['$in' => $arrCategory]; + $arrFilter['v_category'] = ['$in' => $arrCategory]; break; case 2: $arrCategory = array_values(CategoryModel::getWomenCategory()); - $arrFilter['n_category'] = ['$in' => $arrCategory]; + $arrFilter['v_category'] = ['$in' => $arrCategory]; break; } if ($strStatus !== NULL) { - $arrFilter['n_status'] = $strStatus; + $arrFilter['v_status'] = $strStatus; } return $this->findPaginatedWithCache($arrFilter, $intPage, $intLimit, $arrSort, $strKey, $intLifeTime,); } @@ -340,7 +334,6 @@ class VideoModel extends MongoModel return $arrResult; } - $arrFilter['n_have_chapter'] = 1; $arrOptions = self::$arrOptions; @@ -385,17 +378,17 @@ class VideoModel extends MongoModel int $intCount = 10, int $intType = 0, $strStatus = NULL, - array $arrSort = ['n_level' => -1] + array $arrSort = ['v_level' => -1] ): array { try { $arrFilter = []; if (in_array($intType, [1, 2])) { - $arrFilter['n_category'] = ['$in' => CategoryModel::getCategoryByType($intType)]; + $arrFilter['v_category'] = ['$in' => CategoryModel::getCategoryByType($intType)]; } if ($strStatus !== NULL) { - $arrFilter['n_status'] = $strStatus; + $arrFilter['v_status'] = $strStatus; } return $this->findManyShuffledWithCache($arrFilter, $intCount, $arrSort, $strKey, $intLifeTime); @@ -409,7 +402,6 @@ class VideoModel extends MongoModel * * @param string $strSign * @param integer $intCount - * @param integer $intType * @param string strDateType * * daily 天 * * weekly 周 @@ -417,7 +409,7 @@ class VideoModel extends MongoModel * * total 总 * @return array */ - public function getRankVideoOnCache(string $strKey, int $intCount = 10, int $intType = 0, $strDateType = 'daily', $strStatus = NULL): array + public function getRankVideoOnCache(string $strKey, int $intCount = 10, $strDateType = 'daily', $strStatus = NULL): array { try { $intLifeTime = 24 * 3600; @@ -431,18 +423,15 @@ class VideoModel extends MongoModel ]; if ($strStatus !== NULL) { - $arrFilter['n_status'] = $strStatus; + $arrFilter['v_status'] = $strStatus; } - if (in_array($intType, [1, 2])) { - $arrFilter['n_category'] = ['$in' => CategoryModel::getCategoryByType($intType)]; - } $arrOptions = [ 'sort' => ['nc_clicks' => -1], 'limit' => $intCount * 3, 'projection' => [ - 'n_id' => 1, + 'v_id' => 1, ], ]; @@ -452,10 +441,10 @@ class VideoModel extends MongoModel $arrNId = []; foreach ($Cursor as $Doc) { - $arrNId[] = $Doc['n_id']; + $arrNId[] = $Doc['v_id']; } - $arrFilter = ['n_id' => ['$in' => $arrNId]]; + $arrFilter = ['v_id' => ['$in' => $arrNId]]; $arrOptions = self::$arrOptions; unset($arrOptions['typeMap']); diff --git a/code/app/task/crawler/youzhi/Scheduler.php b/code/app/task/crawler/youzhi/Scheduler.php index ea59121..f11875a 100644 --- a/code/app/task/crawler/youzhi/Scheduler.php +++ b/code/app/task/crawler/youzhi/Scheduler.php @@ -74,15 +74,12 @@ class Scheduler */ public function crawlFull() { - // $Site = $this->getSite(); - // $arrVSourceId = []; - // $arrVSourceId[] = ['v_source_id' => 14256]; - // $arrAllVideoInfoPage = $Site->getVideoInfoPageList($arrVSourceId); - // foreach ($arrAllVideoInfoPage as $VideoInfoPage) { - // $arrVideo = $VideoInfoPage->getVideo(); - // $VideoInfoPage->saveVideo(); - // print_r($arrVideo); - // } + // $arrVSourceId = [['v_source_id' => 14256]]; + // $arrTask = [ + // 'callback' => [static::class, 'fetchVideo'], + // 'data' => $arrVSourceId, + // ]; + // $this->QueueManage->set($arrTask); // return; $Site = $this->getSite(); @@ -141,13 +138,12 @@ class Scheduler 'data' => [], ]; - # 访问分页,获取 v_source_id 集合 - $arrAllVideoPageList = $Site->getVideoPageList($arrFilter); + $arrAllVideoPagerPage = $Site->getVideoPagerPageList($arrFilter); $arrVSourceId = []; - foreach ($arrAllVideoPageList as $VideoPageList) { - $arrVideoPager = $VideoPageList->getVideoPager(); + foreach ($arrAllVideoPagerPage as $VideoPagerPage) { + $arrVideoPager = $VideoPagerPage->getVideoPager(); foreach ($arrVideoPager['list'] as $arrVideoPager) { $arrVSourceId[] = ['v_source_id' => (int)$arrVideoPager['vod_id']]; @@ -177,8 +173,20 @@ class Scheduler $Site = $this->getSite(); $arrAllVideoInfoPage = $Site->getVideoInfoPageList($arrVSourceId); + + $arrPic = []; foreach ($arrAllVideoInfoPage as $VideoInfoPage) { $VideoInfoPage->saveVideo(); + + $arrPicItem = $VideoInfoPage->getDownPicInfo(); + + if (!StorageCore::getInstance()->has($arrPicItem['local_path'])) { + $arrPic[] = $arrPicItem; + } + } + + if ($arrPic) { + $Site->downRemoteImgToLocal($arrPic); } } } diff --git a/code/app/task/crawler/youzhi/page/Site.php b/code/app/task/crawler/youzhi/page/Site.php index 218e48f..f17c2c0 100644 --- a/code/app/task/crawler/youzhi/page/Site.php +++ b/code/app/task/crawler/youzhi/page/Site.php @@ -142,7 +142,7 @@ class Site * retrieve multiple novel page model * * @param string $strUri - * @return VideoPage[] + * @return array */ public function getCategory(): array { @@ -204,13 +204,13 @@ class Site * retrieve multiple novel page model * * @param string $strUri - * @return VideoPage[] + * @return VideoPagerPage[] */ - public function getVideoPageList(array $arrFilter): array + public function getVideoPagerPageList(array $arrFilter): array { - /** @var VideoPage[] */ - $arrVideoPage = []; + /** @var VideoPagerPage[] */ + $arrVideoPagerPage = []; foreach ($arrFilter as $intKey => $arrItem) { @@ -232,15 +232,15 @@ class Site $strContent = $Response->getBody()->getContents(); - $VideoPage = $this->getVideoPage($this, $arrFilter[$intKey]['uri']); + $VideoPagerPage = $this->getVideoPagerPage($this, $arrFilter[$intKey]['uri']); - $VideoPage->setContent($strContent); - $VideoPage->setData($arrFilter[$intKey]); + $VideoPagerPage->setContent($strContent); + $VideoPagerPage->setData($arrFilter[$intKey]); - $arrVideoPage[] = $VideoPage; + $arrVideoPagerPage[] = $VideoPagerPage; } - return $arrVideoPage; + return $arrVideoPagerPage; } /** @@ -306,25 +306,31 @@ class Site { $StorageCore = StorageCore::getInstance(); - $arrPromises = []; + $Client = new Client(); - foreach ($arrFilter as $intKey => $arrItem) { + foreach ($arrFilter as $arrItem) { if (!$StorageCore->has($arrItem['local_path'])) { - $arrPromises[$intKey] = $this->getClient()->getAsync($arrItem['uri']); + try { + $Response = $Client->get($arrItem['uri']); + if ($Response->getStatusCode() === 200) { + if (str_ends_with(strtolower($arrItem['local_path']), '.webp')) { + $strTmpFile = str_replace('.webp', '.jpg', $arrItem['local_path']); + $StorageCore->writeStream($strTmpFile, $Response->getBody()->detach()); + convertToWebP( + $StorageCore->getRealPath($strTmpFile), + $StorageCore->getRealPath($arrItem['local_path']), + 40 + ); + @unlink($StorageCore->getRealPath($strTmpFile)); + } else { + $StorageCore->writeStream($arrItem['local_path'], $Response->getBody()->detach()); + } + } + } catch (\Exception $e) { + // echo $e->getMessage(); + } } } - - $arrResults = Promise\Utils::settle($arrPromises)->wait(); - - $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) { - return $arrResult['state'] === 'fulfilled'; - }); - - foreach ($arrSuccessfulResults as $intKey => $arrResult) { - $Response = $arrResult['value']; - - $StorageCore->writeStream($arrFilter[$intKey]['local_path'], $Response->getBody()->detach());; - } } /** @@ -332,11 +338,11 @@ class Site * * @param object $Site * @param string $strUri - * @return VideoPage + * @return VideoPagerPage */ - public function getVideoPage($Site, $strUri) + public function getVideoPagerPage($Site, $strUri) { - return new VideoPage($Site, $strUri); + return new VideoPagerPage($Site, $strUri); } /** diff --git a/code/app/task/crawler/youzhi/page/VideoInfoPage.php b/code/app/task/crawler/youzhi/page/VideoInfoPage.php index 8892965..c6810d5 100644 --- a/code/app/task/crawler/youzhi/page/VideoInfoPage.php +++ b/code/app/task/crawler/youzhi/page/VideoInfoPage.php @@ -24,6 +24,26 @@ class VideoInfoPage extends BasePage */ protected $VideoModel; + /** + * Undocumented variable + * + * @var array + */ + protected $arrVideo; + + /** + * Undocumented variable + * + * @var array + */ + public $arrOptions = [ + 'typeMap' => [ + 'root' => 'array', + 'document' => 'array', + 'array' => 'array', + ], + ]; + /** * Undocumented function * @@ -37,13 +57,6 @@ class VideoInfoPage extends BasePage return $this->VideoModel; } - public $arrOptions = [ - 'typeMap' => [ - 'root' => 'array', - 'document' => 'array', - 'array' => 'array', - ], - ]; /** * Undocumented function @@ -67,7 +80,7 @@ class VideoInfoPage extends BasePage foreach ($arrPlayUrl as $intKey => $arrVal) { $arrPlayUrl[$intKey] = [ 'name' => $arrVal[0], - 'url' => $arrVal[1]?? $arrVal[0], + 'url' => $arrVal[1] ?? $arrVal[0], ]; } $arrVideo['vod_play_url'][$this->getsite()->getSiteCode()] = $arrPlayUrl; @@ -75,7 +88,9 @@ class VideoInfoPage extends BasePage $arrVideo['vod_director'] = explode(',', $arrVideo['vod_director']); $arrVideo['vod_class'] = explode(',', $arrVideo['vod_class']); $arrVideo['vod_content'] = trim($arrVideo['vod_content']); - + $arrVideo['vod_content'] = trim($arrVideo['vod_content'], '\ '); + $arrVideo['vod_content'] = ensureUtf8($arrVideo['vod_content']); + foreach ($arrVideo as $strKey => $mixedVal) { $strKey = str_replace('vod', 'v', $strKey); @@ -102,12 +117,6 @@ class VideoInfoPage extends BasePage return $this->arrVideo; } - /** - * Undocumented variable - * - * @var array - */ - protected $arrVideo; /** * Undocumented function @@ -143,6 +152,7 @@ class VideoInfoPage extends BasePage ['v_name' => $arrPageVideo['v_name']], ['$set' => $arrUpdate], ); + $this->strPicLocalPath = $arrExistsVideo['pic']; } else { $arrInsertVideo = [ @@ -155,7 +165,7 @@ class VideoInfoPage extends BasePage 'v_director' => $arrPageVideo['v_director'], 'v_class' => $arrPageVideo['v_class'], 'v_letter' => $arrPageVideo['v_letter'], - 'v_pic' => $arrPageVideo['v_pic'], + // 'v_pic' => $arrPageVideo['v_pic'], 'v_lang' => $arrPageVideo['v_lang'], 'v_area' => $arrPageVideo['v_area'], 'v_year' => $arrPageVideo['v_year'], @@ -167,6 +177,9 @@ class VideoInfoPage extends BasePage 'v_publish_date' => $arrPageVideo['v_publish_date'], 'created_at' => new \MongoDB\BSON\UTCDateTime(), ]; + $arrInsertVideo['v_pic'] = $this->generateVideoPath($arrInsertVideo['v_name_en']); + + $this->strPicLocalPath = $arrInsertVideo['v_pic']; try { $this->getVideoModel()->insert($arrInsertVideo); @@ -181,4 +194,33 @@ class VideoInfoPage extends BasePage return true; } + + private $strPicLocalPath = ''; + + /** + * Undocumented function + * + * @param string $strName + * @return string + */ + public function generateVideoPath($strName): string + { + $intTime = time() - mt_rand(1, 2 * 24 * 3600 * 360); + + return sprintf('/img/video/%s/%s.webp', date('Y/m/d/H', $intTime), $strName); + } + + + /** + * Undocumented function + * + * @return array + */ + public function getDownPicInfo(): array + { + return [ + 'uri' => $this->getVideo()['v_pic'], + 'local_path' => $this->strPicLocalPath, + ]; + } } diff --git a/code/app/task/crawler/youzhi/page/VideoPage.php b/code/app/task/crawler/youzhi/page/VideoPage.php deleted file mode 100644 index 2cb1c83..0000000 --- a/code/app/task/crawler/youzhi/page/VideoPage.php +++ /dev/null @@ -1,417 +0,0 @@ -NovelModel == NULL) { - $this->NovelModel = NovelModel::getInstance(); - } - return $this->NovelModel; - } - - public $arrOptions = [ - 'typeMap' => [ - 'root' => 'array', - 'document' => 'array', - 'array' => 'array', - ], - ]; - - /** - * 查询看的小说 - * - * @param integer $intNSourceId - * @param string $strName - * @return array - */ - protected function findOrCreateNovel(int $intNSourceId, string $strName): array - { - $strUUId = $this->getSite()->getSiteUUId($intNSourceId); - - $arrExistsNovel = $this->getNovelModel()->findOne(['n_source_uuid' => $strUUId], $this->arrOptions); - - if ($arrExistsNovel) { - $arrVideo = (array) $arrExistsNovel; - } else { - $arrVideo = [ - 'n_id' => CountersModel::getInstance()->getNextSequence('novel'), - 'n_name' => $strName, - 'n_name_pinyin' => zhToPinYin($strName), - 'n_site_code' => $this->getSite()->getSiteCode(), - 'n_source_uuid' => $strUUId, - 'n_source_id' => $intNSourceId, - 'n_level' => 1, - 'n_have_chapter' => 0, - 'created_at' => new \MongoDB\BSON\UTCDateTime(), - ]; - try { - $this->getNovelModel()->insert($arrVideo); - } catch (\Exception $e) { - if ($e->getCode() === 11000) { - return $this->findOrCreateNovel($intNSourceId, $strName); - } else { - throw $e; - } - } - } - return $arrVideo; - } - - - /** - * 分析关联数据 - * - * @return void - */ - public function getReletedNovel() {} - - - /** - * Undocumented function - * - * @return array - */ - public function getVideoPager(): array - { - if (empty($this->arrVideo)) { - - $this->arrVideo = json_decode($this->getContent(), true); - - $this->getReletedNovel(); - } - - return $this->arrVideo; - } - - /** - * Undocumented function - * - * @param string $strName - * @return string - */ - public function generateCoverPath($strName): string - { - $intTime = time() - mt_rand(1, 2 * 24 * 3600 * 360); - - return sprintf('/img/%s/%s.jpg', date('Y/m/d/H', $intTime), $strName); - } - - public function filterOtherFields(&$arrNovel) - { - $arrFields = [ - 'n_id', - 'n_name', - 'n_name_pinyin', - 'n_status', - 'n_author', - 'n_description', - 'n_category', - 'n_category_pinyin', - 'n_category_sex_en', - 'n_category_sex_zh', - 'n_cover_path', - 'n_latest_chapter_name', - 'n_update_time', - 'n_forge_novel', - 'n_related_novel', - 'n_site_code', - 'n_source_uuid', - 'n_source_id', - 'n_have_chapter', - 'created_at', - 'updated_at', - ]; - - foreach ($arrNovel as $strKey => $strVal) { - if (!in_array($strKey, $arrFields)) { - unset($arrNovel[$strKey]); - } - } - } - - /** - * Undocumented function - * - * @return bool - */ - public function saveVideo(): bool - { - $arrPageInfo = $this->getVideo(); - - print_r($arrPageInfo); - return false; - if ( - empty($arrPageInfo) || - empty($arrPageInfo['og_title']) || - empty($arrPageInfo['og_image']) || - empty($arrPageInfo['og_novel_category']) - ) { - return false; - } - - $arrVideo = [ - // 'n_id'=> $arrPageInfo['xxxxx'], - 'n_name' => $arrPageInfo['og_title'], - 'n_name_pinyin' => zhToPinYin($arrPageInfo['og_title']), - 'n_status' => $arrPageInfo['og_novel_status'], - 'n_author' => $arrPageInfo['og_novel_author'], - 'n_description' => $arrPageInfo['og_description'], - 'n_category' => $arrPageInfo['og_novel_category'], - // 'n_category_pinyin' => $arrPageInfo['xxxxx'], - // 'n_category_sex_en' => $arrPageInfo['xxxxx'], - // 'n_category_sex_zh' => $arrPageInfo['xxxxx'], - // 'n_cover_path' => $arrPageInfo['xxxxx'], - 'n_latest_chapter_name' => $arrPageInfo['og_novel_latest_chapter_name'], - 'n_update_time' => $arrPageInfo['og_novel_update_time'], - 'n_forge_novel' => $this->arrForgeNovel, - 'n_related_novel' => $this->arrRelatedNovel, - - 'n_site_code' => $this->getSite()->getSiteCode(), - 'n_source_id' => $this->arrData['n_source_id'], - 'n_source_uuid' => $this->getSite()->getSiteUUId($this->arrData['n_source_id']), - - // 'n_have_chapter' => $arrPageInfo['xxxxx'], - // 'created_at' => $arrPageInfo['xxxxx'], - // 'updated_at' => $arrPageInfo['xxxxx'], - - ]; - - $arrExistsNovel = $this->getNovelModel()->findOne(['n_source_uuid' => $arrVideo['n_source_uuid']], $this->arrOptions); - - if ($arrExistsNovel) { - - $arrExistsNovel = (array) $arrExistsNovel; - - $arrUpdate = [ - 'n_latest_chapter_name' => $arrVideo['n_latest_chapter_name'], - 'n_update_time' => $arrVideo['n_update_time'], - 'updated_at' => new \MongoDB\BSON\UTCDateTime(), - ]; - - # 如果没有分类,说明这条数据是最初从关联shu那里采集的,需要补全全部信息 - if (empty($arrExistsNovel['n_category'])) { - $arrUpdate = array_merge($arrUpdate, $arrVideo); - $arrUpdate['n_related_novel'] = $this->arrRelatedNovel; - $arrUpdate['n_forge_novel'] = $this->arrForgeNovel; - $arrUpdate['n_category_pinyin'] = CategoryModel::getPinYin($this->arrVideo['og_novel_category']); - $arrUpdate['n_category_sex_en'] = CategoryModel::getSexEn($this->arrVideo['og_novel_category']); - $arrUpdate['n_category_sex_zh'] = CategoryModel::getSexZh($this->arrVideo['og_novel_category']); - $arrUpdate['n_related_novel'] = $this->arrRelatedNovel; - $arrUpdate['n_forge_novel'] = $this->arrForgeNovel; - } - - # 图片不存在的时候,需要重新生成 - if ( - !isset($arrExistsNovel['n_cover_path']) || - empty($arrExistsNovel['n_cover_path']) || - !StorageCore::getInstance()->has($arrExistsNovel['n_cover_path']) || - (basename($arrExistsNovel['n_cover_path']) == '.jpg') - ) { - $arrUpdate['n_cover_path'] = $this->generateCoverPath($arrVideo['n_name_pinyin']); - $this->strCoverLocalPath = $arrUpdate['n_cover_path']; - } else { - $this->strCoverLocalPath = $arrExistsNovel['n_cover_path']; - } - - $this->getNovelModel()->updateOne( - ['n_source_uuid' => $arrVideo['n_source_uuid']], - ['$set' => $arrUpdate], - ); - $this->intNId = $arrExistsNovel['n_id']; - } else { - - $arrVideo['n_category_pinyin'] = CategoryModel::getPinYin($this->arrVideo['og_novel_category']); - $arrVideo['n_category_sex_en'] = CategoryModel::getSexEn($this->arrVideo['og_novel_category']); - $arrVideo['n_category_sex_zh'] = CategoryModel::getSexZh($this->arrVideo['og_novel_category']); - $arrVideo['n_id'] = CountersModel::getInstance()->getNextSequence('novel'); - $arrVideo['n_level'] = 1; - $arrVideo['n_have_chapter'] = 0; - $arrVideo['n_cover_path'] = $this->generateCoverPath($arrVideo['n_name_pinyin']); - $arrVideo['n_related_novel'] = $this->arrRelatedNovel; - $arrVideo['n_forge_novel'] = $this->arrForgeNovel; - $arrVideo['created_at'] = new \MongoDB\BSON\UTCDateTime(); - - try { - $this->getNovelModel()->insert($arrVideo); - $this->intNId = $arrVideo['n_id']; - $this->strCoverLocalPath = $arrVideo['n_cover_path']; - } catch (\Exception $e) { - if ($e->getCode() === 11000) { - return $this->saveVideo(); - } else { - throw $e; - } - } - } - - return true; - } - - - public $strCoverLocalPath = ''; - - public $intNId; - - /** - * Undocumented function - * - * @return array - */ - public function getDownCoverInfo(): array - { - return [ - 'uri' => $this->arrVideo['og_image'], - 'local_path' => $this->strCoverLocalPath, - ]; - } - - /** - * Undocumented variable - * - * @var array - */ - protected $arrLatestChapters = NULL; - - /** - * Undocumented variable - * - * @var array - */ - protected $arrChapters = NULL; - - public function getLatestChapterByNIdsList(): array - { - if ($this->arrLatestChapters === NULL) { - $this->arrLatestChapters = []; - - if ($this->isMobile()) { - $strClass = '.chapter-list'; - } else { - $strClass = '.section-list'; - } - - if ($this->arrData['page'] == 0) { - $this->getQueryList()->find($strClass)->eq(0)->find("a")->map(function ($A) { - $this->arrLatestChapters[] = [ - 'name' => $A->text(), - 'uri' => $A->attr('href'), - ]; - }); - } - } - return $this->arrLatestChapters; - } - - /** - * Undocumented function - * - * @return array - */ - public function getChapters(): array - { - if ($this->arrChapters === NULL) { - $this->arrChapters = []; - if ($this->isMobile()) { - $strClass = '.chapter-list'; - } else { - $strClass = '.section-list'; - } - - $intDivIndex = 0; - if ($this->arrData['page'] == 0) { - $intDivIndex = 1; - } - - $this->getQueryList()->find($strClass)->eq($intDivIndex)->find("a")->map(function ($A) { - $this->arrChapters[] = [ - 'name' => $A->text(), - 'uri' => $A->attr('href'), - ]; - }); - } - return $this->arrChapters; - } - - - protected $arrNextPageArgs = NULL; - - /** - * Undocumented function - * - * @return array - */ - public function getNextPageArgs(): array - { - $this->arrNextPageArgs = []; - if ($this->arrData['page'] == 0) { - $this->getLatestChapterByNIdsList(); - $this->getChapters(); - - $arrLatestChapterFirstBlock = count($this->arrLatestChapters) > 0 ? $this->arrLatestChapters[0] : null; - $arrLatestChapterSecondBlock = count($this->arrChapters) > 0 ? $this->arrChapters[count($this->arrChapters) - 1] : null; - - if ( - !empty($arrLatestChapterFirstBlock) && !empty($arrLatestChapterSecondBlock) - && ($arrLatestChapterFirstBlock['uri'] != $arrLatestChapterSecondBlock['uri']) - - ) { - $this->arrNextPageArgs = [ - 'n_source_id' => $this->arrData['n_source_id'], - 'page' => $this->arrData['page'] + 1, - ]; - } - } else { - $strLastUri = $this->getQueryList()->find('select:first option:last')->attr('value'); - if ($strLastUri != '' && $strLastUri != $this->strUri) { - $this->arrNextPageArgs = [ - 'n_source_id' => $this->arrData['n_source_id'], - 'page' => $this->arrData['page'] + 1, - ]; - } - } - return $this->arrNextPageArgs; - } -} diff --git a/code/app/task/crawler/youzhi/page/VideoPagerPage.php b/code/app/task/crawler/youzhi/page/VideoPagerPage.php new file mode 100644 index 0000000..8c64b8c --- /dev/null +++ b/code/app/task/crawler/youzhi/page/VideoPagerPage.php @@ -0,0 +1,43 @@ + [ + 'root' => 'array', + 'document' => 'array', + 'array' => 'array', + ], + ]; + + /** + * Undocumented function + * + * @return array + */ + public function getVideoPager(): array + { + if (empty($this->arrVideo)) { + + $this->arrVideo = json_decode($this->getContent(), true); + } + + return $this->arrVideo; + } +} diff --git a/code/database/mongo/migrations/video_clicks.php b/code/database/mongo/migrations/video_clicks.php new file mode 100644 index 0000000..1417362 --- /dev/null +++ b/code/database/mongo/migrations/video_clicks.php @@ -0,0 +1,10 @@ + 'video_clicks', + 'indexes' => [ + [ + 'key' => ['v_id' => 1, 'vc_date' => 1, 'vc_type' => 1], + 'options' => ['unique' => true] + ], + ] +]; diff --git a/code/extend/storage/StorageCore.php b/code/extend/storage/StorageCore.php index 78d2612..fa7b777 100644 --- a/code/extend/storage/StorageCore.php +++ b/code/extend/storage/StorageCore.php @@ -95,4 +95,9 @@ class StorageCore { return $this->Filesystem->has($strKey); } + + public function getRealPath($strKey): string + { + return $this->Adapter->applyPathPrefix($strKey); + } } diff --git a/code/extend/tools/cwebp b/code/extend/tools/cwebp new file mode 100644 index 0000000..8f7e598 Binary files /dev/null and b/code/extend/tools/cwebp differ