diff --git a/code/app/home/view/default/test/test01/index.html b/code/app/home/view/default/test/test01/index.html index 77c3344..5ba0b9d 100644 --- a/code/app/home/view/default/test/test01/index.html +++ b/code/app/home/view/default/test/test01/index.html @@ -1,7 +1,26 @@ -{novel:pagerexp page="1" limit="10" n_category="$Request.route.category" d_key="d_key" d_val="novel" p_val="page_data" + {site:forgelist count="3" d_key="d_key" d_val="novel" cache_life="1000"} +
  • + Index: {$d_key} + Title: {$novel.n_name} + Author: {$novel.n_category} +
  • +{/site:forgelist} + + + + + + + diff --git a/code/app/model/NovelModel.php b/code/app/model/NovelModel.php index 7b40318..efdf686 100644 --- a/code/app/model/NovelModel.php +++ b/code/app/model/NovelModel.php @@ -242,7 +242,7 @@ class NovelModel extends MongoModel ): array { try { $arrFilter['n_have_chapter'] = 1; - + $arrResult = $this->checkCacheByKey($strKey); if ($arrResult !== NULL) { return $arrResult; @@ -417,10 +417,16 @@ class NovelModel extends MongoModel * * total 总 * @return array */ - public function getRankNovelOnCache(string $strKey, int $intCount = 10, int $intType = 0, $strDateType = 'daily', $strStatus = NULL): array - { + public function getRankNovelOnCache( + string $strKey, + int $intCount = 10, + int $intType = 0, + $strDateType = 'daily', + $strStatus = NULL, + $intLifeTime = 24 * 3600, + ): array { try { - $intLifeTime = 24 * 3600; + $arrResult = $this->checkCacheByKey($strKey); if ($arrResult !== NULL) { @@ -477,8 +483,4 @@ class NovelModel extends MongoModel return []; } } - - - - } diff --git a/code/app/model/VideoClicksModel.php b/code/app/model/VideoClicksModel.php new file mode 100644 index 0000000..06984f4 --- /dev/null +++ b/code/app/model/VideoClicksModel.php @@ -0,0 +1,124 @@ +getCol()->findOne([ + "v_id" => $intVId, + ]); + + if (empty($Novel)) { + return false; + } + + $Now = new DateTime(); + + $strDailyDate = $Now->format('Y-m-d'); + $strWeeklyDate = $Now->modify('monday this week')->format('Y-m-d'); + $MonthlyDate = $Now->format('Y-m-01'); + + $arrClickTypes = [ + 'daily' => $strDailyDate, + 'weekly' => $strWeeklyDate, + 'monthly' => $MonthlyDate, + 'total' => null, + ]; + + foreach ($arrClickTypes as $strType => $strDate) { + $this->getCol()->updateOne( + [ + 'v_id' => $intVId, + 'vc_date' => $strDate, + 'vc_type' => $strType, + ], + [ + '$inc' => ['vc_clicks' => 1], + '$set' => ['v_category_id' => $Novel->v_category_id] + ], + [ + 'upsert' => true + ] + ); + } + return true; + } + + /** + * GetStats + * + * @param integer $intVId + * @return array + */ + public function getStats(int $intVId): array + { + $Now = new DateTime(); + $strDailyDate = $Now->format('Y-m-d'); + $strWeeklyDate = $Now->modify('monday this week')->format('Y-m-d'); + $strMonthlyDate = $Now->format('Y-m-01'); + + $query = [ + 'v_id' => $intVId, + '$or' => [ + ['vc_type' => 'daily', 'vc_date' => $strDailyDate], + ['vc_type' => 'weekly', 'vc_date' => $strWeeklyDate], + ['vc_type' => 'monthly', 'vc_date' => $strMonthlyDate], + ['vc_type' => 'total', 'vc_date' => null] + ] + ]; + $arrOptions = [ + 'projection' => ['vc_type' => 1, 'vc_clicks' => 1, '_id' => 0], + 'typeMap' => ['root' => 'array', 'document' => 'array', 'array' => 'array'] + ]; + + $arrResults = $this->getCol()->find($query, $arrOptions)->toArray(); + + $arrStats = array_column($arrResults, 'vc_clicks', 'vc_type'); + + $defaultStats = [ + 'daily' => 0, + 'weekly' => 0, + 'monthly' => 0, + 'total' => 0 + ]; + $arrStats = array_merge($defaultStats, $arrStats); + + return $arrStats; + } +} diff --git a/code/app/model/VideoModel.php b/code/app/model/VideoModel.php new file mode 100644 index 0000000..33bcd7d --- /dev/null +++ b/code/app/model/VideoModel.php @@ -0,0 +1,484 @@ + [ + 'root' => 'array', + 'document' => 'array', + 'array' => 'array', + ], + 'projection' => [ + '_id' => 0, + 'n_site_code' => 0, + 'n_source_uuid' => 0, + 'n_source_id' => 0, + ], + ]; + + /** + * 根据关键词数组更新小说推荐等级 + * @param array $arrKeywords 关键词数组,例如 ["斗罗", "遮天", "凡人"] + * @param int $intLevel 推荐等级1~9,例如 3 + * @return array 更新结果 + */ + public function updateByKeywords(array $arrKeywords, int $intLevel) + { + if (empty($arrKeywords)) { + return false; + } + + try { + $arrConditions = array_map(function ($strKeyword) { + return ['n_name' => ['$regex' => trim($strKeyword), '$options' => 'i']]; + }, $arrKeywords); + + $UpdateResult = $this->getCol()->updateMany( + ['$or' => $arrConditions], + ['$set' => ['n_level' => $intLevel]] + ); + + $UpdateResult->getModifiedCount(); + } catch (Exception $e) { + return false; + } + + return true; + } + + /** + * 直接读库查询大量小说 + * + * @param integer $intNId + * @return null|array + */ + public function findMany(array $arrFilter, int $intCount = 10, array $arrSort = [], array $arrOptions = []): null|array + { + if (empty($arrOptions)) { + $arrOptions = self::$arrOptions; + } + + if ($intCount > 0) { + $arrOptions['limit'] = $intCount; + } + + if (empty($arrSort)) { + $arrOptions['sort'] = $arrSort; + } + + $arrFilter['n_have_chapter'] = 1; + + $Cursor = $this->getCol()->find($arrFilter, $arrOptions); + + $arrResult = iterator_to_array($Cursor); + + foreach ($arrResult as &$arrItem) { + applyToKeys($arrItem, ['created_at', 'updated_at'], 'formatMongoDate'); + } + + return $arrResult; + } + + + /** + * 查询多条数据并缓存 + * + * @param array $arrFilter + * @param integer $intCount + * @param array $arrSort + * @param array $arrOptions + * @param string $strKey + * @param integer $intLifeTime + * @return null|array + */ + public function findManyWithCache( + array $arrFilter, + int $intCount = 0, + array $arrSort = [], + array $arrOptions = [], + string $strKey = '', + int $intLifeTime = 24 * 3600, + ): null|array { + try { + + $arrResult = $this->checkCacheByKey($strKey); + if ($arrResult !== NULL) { + return $arrResult; + } + + $arrResult = $this->findMany($arrFilter, $intCount, $arrSort, $arrOptions); + + $this->setCacheByKey($strKey, $arrResult, $intLifeTime); + + return $arrResult; + } catch (\Exception $e) { + return []; + } + } + + /** + * find one Video + * + * @param array $arrFilter + * @param array $arrOptions + * @return null|array + */ + public function findOne(array $arrFilter, array $arrOptions = []): null|array + { + if (empty($arrOptions)) { + $arrOptions = self::$arrOptions; + } + + $arrVideo = $this->getOne($arrFilter, $arrOptions); + + if (!empty($arrVideo)) { + applyToKeys($arrVideo, ['created_at', 'updated_at'], 'formatMongoDate'); + } + + return $arrVideo; + } + + /** + * 根据小说ID查询指定小说 + * + * @param integer $intNId + * @return array + */ + public function getVideoByNId(int $intNId): null|array + { + $arrFilter = ['n_id' => $intNId, 'n_have_chapter' => 1]; + return $this->findOne($arrFilter); + } + + + /** + * 查询符合条件的随机多条小说 + * + * @param string $strKey + * @param array $arrFilter + * @param integer $intCount + * @param integer $intLifeTime + * @return array + */ + public function getRandVideoWithCache(array $arrFilter = [], int $intCount = 10, string $strKey = '', int $intLifeTime = 24 * 3600,): array + { + try { + + $arrResult = $this->checkCacheByKey($strKey); + if ($arrResult !== NULL) { + return $arrResult; + } + + $arrFilter['n_have_chapter'] = 1; + + $arrPipeline = [ + ['$match' => $arrFilter], + ['$sample' => ['size' => $intCount]] + ]; + + $Video = $this->getCol()->aggregate($arrPipeline); + + $arrResult = iterator_to_array($Video); + + $this->setCacheByKey($strKey, $arrResult, $intLifeTime); + + return $arrResult; + } catch (\Exception $e) { + return []; + } + } + + /** + * 通用小说分页 + * + * @param array $arrFilter + * @param integer $intPage + * @param integer $intLimit + * @param array $arrSort + * @param string $strKey + * @param int $intLifeTime + * @return array + */ + public function findPaginatedWithCache( + array $arrFilter, + int $intPage = 1, + int $intLimit = 20, + array $arrSort = ['n_level' => -1], + string $strKey = '', + int $intLifeTime = 24 * 3600, + ): array { + try { + $arrFilter['n_have_chapter'] = 1; + + $arrResult = $this->checkCacheByKey($strKey); + if ($arrResult !== NULL) { + return $arrResult; + } + $arrOptions = self::$arrOptions; + $arrOptions['sort'] = $arrSort; + $arrOptions['skip'] = ($intPage - 1) * $intLimit; + $arrOptions['limit'] = $intLimit; + unset($arrOptions['typeMap']); + $Cursor = $this->getCol()->find($arrFilter, $arrOptions); + $intTotal = $this->getCol()->countDocuments($arrFilter); + + $arrResult = iterator_to_array($Cursor); + + shuffle($arrResult); + + $arrResult = [ + 'data' => $arrResult, + 'total' => $intTotal, + 'page' => $intPage, + 'limit' => $intLimit, + 'pages' => ceil($intTotal / $intLimit) + ]; + + $this->setCacheByKey($strKey, $arrResult, $intLifeTime); + + return $arrResult; + } catch (\Exception $e) { + return []; + } + } + + + /** + * 用于Home下的书库分页 + * + * @param string $strKey + * @param integer $intPage + * @param integer $intLimit + * @param int $intLifeTime + * @param integer $intType + * @param string $strStatus + * @param array $arrSort + * @return array + */ + public function getCustomPage01( + string $strKey, + int $intPage = 1, + int $intLimit = 20, + int $intLifeTime = 24 * 3600, + int $intType = 0, + $strStatus = NULL, + array $arrSort = ['n_level' => -1] + ): array { + $arrFilter = ['n_have_chapter' => 1,]; + + switch ($intType) { + case 1: + $arrCategory = array_values(CategoryModel::getManCategory()); + $arrFilter['n_category'] = ['$in' => $arrCategory]; + break; + case 2: + $arrCategory = array_values(CategoryModel::getWomenCategory()); + $arrFilter['n_category'] = ['$in' => $arrCategory]; + break; + } + + if ($strStatus !== NULL) { + $arrFilter['n_status'] = $strStatus; + } + return $this->findPaginatedWithCache($arrFilter, $intPage, $intLimit, $arrSort, $strKey, $intLifeTime,); + } + + /** + * Undocumented function + * + * @param array $arrFilter + * @param integer $intCount + * @param array $arrSort + * @param string $strKey + * @param integer $intLifeTime + * @return array + */ + public function findManyShuffledWithCache( + array $arrFilter, + int $intCount = 10, + array $arrSort = [], + string $strKey = '', + int $intLifeTime = 24 * 3600, + ): array { + try { + + $arrResult = $this->checkCacheByKey($strKey); + if ($arrResult !== NULL) { + return $arrResult; + } + + $arrFilter['n_have_chapter'] = 1; + + $arrOptions = self::$arrOptions; + + if (!empty($arrSort)) { + $arrOptions['sort'] = $arrSort; + } + + $arrOptions['limit'] = $intCount * 3; + + unset($arrOptions['typeMap']); + + $Cursor = $this->getCol()->find($arrFilter, $arrOptions); + + $arrResult = iterator_to_array($Cursor); + + shuffle($arrResult); + + $arrResult = array_slice($arrResult, 0, $intCount); + + $this->setCacheByKey($strKey, $arrResult, $intLifeTime); + + return $arrResult; + } catch (\Exception $e) { + return []; + } + } + + /** + * Undocumented function + * + * @param string $strKey + * @param int $intLifeTime + * @param integer $intCount + * @param integer $intType + * @param string $strStatus + * @param array $arrSort + * @return array + */ + public function getCommonVideoOnCache( + string $strKey, + int $intLifeTime = 24 * 3600, + int $intCount = 10, + int $intType = 0, + $strStatus = NULL, + array $arrSort = ['n_level' => -1] + ): array { + try { + $arrFilter = []; + + if (in_array($intType, [1, 2])) { + $arrFilter['n_category'] = ['$in' => CategoryModel::getCategoryByType($intType)]; + } + + if ($strStatus !== NULL) { + $arrFilter['n_status'] = $strStatus; + } + + return $this->findManyShuffledWithCache($arrFilter, $intCount, $arrSort, $strKey, $intLifeTime); + } catch (\Exception $e) { + return []; + } + } + + /** + * Undocumented function + * + * @param string $strSign + * @param integer $intCount + * @param integer $intType + * @param string strDateType + * * daily 天 + * * weekly 周 + * * monthly 月 + * * total 总 + * @return array + */ + public function getRankVideoOnCache(string $strKey, int $intCount = 10, int $intType = 0, $strDateType = 'daily', $strStatus = NULL): array + { + try { + $intLifeTime = 24 * 3600; + + $arrResult = $this->checkCacheByKey($strKey); + if ($arrResult !== NULL) { + return $arrResult; + } + $arrFilter = [ + 'nc_type' => $strDateType, + ]; + + if ($strStatus !== NULL) { + $arrFilter['n_status'] = $strStatus; + } + + if (in_array($intType, [1, 2])) { + $arrFilter['n_category'] = ['$in' => CategoryModel::getCategoryByType($intType)]; + } + + $arrOptions = [ + 'sort' => ['nc_clicks' => -1], + 'limit' => $intCount * 3, + 'projection' => [ + 'n_id' => 1, + ], + ]; + + $VideoClicksModel = VideoClicksModel::getInstance(); + + $Cursor = $VideoClicksModel->getCol()->find($arrFilter, $arrOptions); + + $arrNId = []; + foreach ($Cursor as $Doc) { + $arrNId[] = $Doc['n_id']; + } + + $arrFilter = ['n_id' => ['$in' => $arrNId]]; + $arrOptions = self::$arrOptions; + unset($arrOptions['typeMap']); + + $Cursor = $this->getCol()->find( + $arrFilter, + $arrOptions + ); + + $arrResult = iterator_to_array($Cursor); + + shuffle($arrResult); + + $arrResult = array_slice($arrResult, 0, $intCount); + + $this->setCacheByKey($strKey, $arrResult, $intLifeTime); + + return $arrResult; + } catch (\Exception $e) { + return []; + } + } + + + + +} diff --git a/code/app/services/NovelService.php b/code/app/services/NovelService.php index cfc5827..65f2596 100644 --- a/code/app/services/NovelService.php +++ b/code/app/services/NovelService.php @@ -216,6 +216,13 @@ class NovelService 'update' => '最新更新', ]; + public $arrRankSortType = [ + 'daily' => '天', + 'weekly' => '周', + 'monthly' => '月', + 'total' => '总', + ]; + public $arrStatus = [ 'all' => '所有', 'lianzai' => '连载中', @@ -234,18 +241,108 @@ class NovelService * * @return array */ - public function getCategoryFilter(string $n_sex=''): array + public function getCategoryFilter(string $strNSex = ''): array { - if($n_sex == 'man'){ + if ($strNSex == 'man') { return CategoryModel::$arrCategoryAll['man']; - }else if($n_sex == 'women'){ + } else if ($strNSex == 'women') { return CategoryModel::$arrCategoryAll['women']; - }else{ + } else { return array_combine(CategoryModel::$arrCategoryPinYin, CategoryModel::$arrCategory); } - } + /** + * 获取排行榜小说 + * + * @param array $arrParams + * @return array + */ + public function getRankList(array $arrParams): array + { + $intCount = (int)$arrParams['count'] ?? 10; + $strSortType = (string)$arrParams['sort_type'] ?? ''; + $strNStatus = (string) $arrParams['n_status'] ?? ''; + $strNSex = (string) $arrParams['n_sex'] ?? ''; + $strDiffKey = (string) $arrParams['diff_key'] ?? ''; + $intCacheLifeTime = (int)$arrParams['cache_life'] ?? 0; + + $arrFilter = []; + + # n_status filter + if (!empty($strNStatus)) { + switch ($strNStatus) { + case 'lianzai': + $arrFilter['n_status'] = $this->arrStatus[$strNStatus]; + break; + case 'wanjie': + $arrFilter['n_status'] = $this->arrStatus[$strNStatus]; + break; + } + } + + # n_category_sex_zh filter + if (!empty($strNSex)) { + switch ($strNSex) { + case 'man': + $arrFilter['n_category_sex_zh'] = 1; + break; + case 'women': + $arrFilter['n_category_sex_zh'] = 2; + break; + } + } + + $arrSort = []; + + # sort filter + if (!empty($strSortType)) { + switch ($strSortType) { + case 'daily': + $arrSort['n_sort_type'] = 'daily'; + break; + case 'weekly': + $arrSort['n_sort_type'] = 'weekly'; + break; + case 'monthly': + $arrSort['n_sort_type'] = 'monthly'; + break; + case 'total': + $arrSort['n_sort_type'] = 'total'; + break; + } + } + + $strCacheKey = ''; + $intLifeTime = 0; + + # cache filter + if ($intCacheLifeTime > 0) { + $strCacheKey = sprintf( + 'NovelService:getRankList:%s:%s:%s:%s:%s', + $this->SiteContext->DomainModel->d_domain, + md5(serialize($arrFilter)), + md5(serialize($arrSort)), + $intCount, + $strDiffKey, + ); + + $intLifeTime = $intLifeTime; + } + + $arrNovel = $this->NovelModel->getRankNovelOnCache( + $strCacheKey, + $intCount, + $arrFilter['n_category_sex_zh'] ?? 0, + $arrSort['n_sort_type'] ?? 'daily', + $arrFilter['n_status'] ?? NULL, + $intLifeTime, + ); + + return $arrNovel; + } + + /** * 获取小说数据 * @@ -319,7 +416,7 @@ class NovelService # cache filter if ($intCacheLifeTime > 0) { $strCacheKey = sprintf( - 'Service:getNovelList:%s:%s:%s:%s:%s', + 'NovelService:getNovelList:%s:%s:%s:%s:%s', $this->SiteContext->DomainModel->d_domain, md5(serialize($arrFilter)), md5(serialize($arrSort)), @@ -464,7 +561,7 @@ class NovelService # cache filter if ($intCacheLifeTime > 0) { $strCacheKey = sprintf( - 'Service:getNovelList:%s:%s:%s:%s:%s:%s', + 'NovelService:getNovelList:%s:%s:%s:%s:%s:%s', $this->SiteContext->DomainModel->d_domain, md5(serialize($arrFilter)), md5(serialize($arrSort)), @@ -619,4 +716,7 @@ class NovelService return $arrNovel; } + + + } diff --git a/code/app/services/SiteContext.php b/code/app/services/SiteContext.php index 2294e32..b719229 100644 --- a/code/app/services/SiteContext.php +++ b/code/app/services/SiteContext.php @@ -7,6 +7,9 @@ use app\model\DomainModel; use app\model\GanRaoMaModel; use app\model\TemplatesModel; use app\model\CategoryModel; +use app\model\ChapterModel; +use app\model\NovelClicksModel; +use app\model\NovelModel; use think\facade\Request; use think\facade\Config; use think\exception\HttpException; @@ -35,9 +38,32 @@ class SiteContext */ public $TemplatesModel; + /** + * NovelModel + * + * @var NovelModel + */ + public $NovelModel; + + /** + * ChapterModel + * + * @var ChapterModel + */ + public $ChapterModel; + + /** + * NovelClicksModel + * + * @var NovelClicksModel + */ + public $NovelClicksModel; public function __construct() { $this->Request = request(); + $this->NovelModel = NovelModel::getInstance(); + $this->ChapterModel = ChapterModel::getInstance(); + $this->NovelClicksModel = NovelClicksModel::getInstance(); } /** @@ -81,32 +107,32 @@ class SiteContext */ public function initCfg() { - // 全局使用的URL模板 - $strPcUrlTemp = $this->DomainModel->getFomartSubject('PC_URL_PATH','',false); - $strNanUrlTemp = $this->DomainModel->getFomartSubject('NAN_SHENG_URL','',false); - $strNvUrlTemp = $this->DomainModel->getFomartSubject('NV_SHENG_URL','',false); - $strCategoryIndexUrlTemp = $this->DomainModel->getFomartSubject('CATEGORY_INDEX_URL','',false); - $strRankUrlTemp = $this->DomainModel->getFomartSubject('RANK_INFO_URL','',false); - $strSearchIndexUrlTemp = $this->DomainModel->getFomartSubject('SEARCH_INDEX_URL','',false); - $strHistoryUrlTemp = $this->DomainModel->getFomartSubject('HISTORY_INFO_URL','',false); - $strBookShelfUrlTemp = $this->DomainModel->getFomartSubject('SHUJIA_INFO_URL','',false); - $strUserUrlTemp = $this->DomainModel->getFomartSubject('USER_INFO_URL','',false); + // 全局使用的URL模板 + $strPcUrlTemp = $this->DomainModel->getFomartSubject('PC_URL_PATH', '', false); + $strNanUrlTemp = $this->DomainModel->getFomartSubject('NAN_SHENG_URL', '', false); + $strNvUrlTemp = $this->DomainModel->getFomartSubject('NV_SHENG_URL', '', false); + $strCategoryIndexUrlTemp = $this->DomainModel->getFomartSubject('CATEGORY_INDEX_URL', '', false); + $strRankUrlTemp = $this->DomainModel->getFomartSubject('RANK_INFO_URL', '', false); + $strSearchIndexUrlTemp = $this->DomainModel->getFomartSubject('SEARCH_INDEX_URL', '', false); + $strHistoryUrlTemp = $this->DomainModel->getFomartSubject('HISTORY_INFO_URL', '', false); + $strBookShelfUrlTemp = $this->DomainModel->getFomartSubject('SHUJIA_INFO_URL', '', false); + $strUserUrlTemp = $this->DomainModel->getFomartSubject('USER_INFO_URL', '', false); $strNanUrl = str_replace("/", "", $strNanUrlTemp); $strNvUrl = str_replace("/", "", $strNvUrlTemp); - - View::assign("strPcPath", $strPcUrlTemp); - View::assign("strNanUrlTemp", $strNanUrlTemp); - View::assign("strNvUrlTemp", $strNvUrlTemp); - View::assign("strNanUrl", $strNanUrl); - View::assign("strNvUrl", $strNvUrl); - View::assign("strCategoryIndexUrlTemp", $strCategoryIndexUrlTemp); - View::assign("strRankUrlTemp", $strRankUrlTemp); - View::assign("strSearchIndexUrlTemp", $strSearchIndexUrlTemp); - View::assign("strHistoryUrlTemp", $strHistoryUrlTemp); - View::assign("strBookShelfUrlTemp", $strBookShelfUrlTemp); - View::assign("strUserUrlTemp", $strUserUrlTemp); - + + View::assign("strPcPath", $strPcUrlTemp); + View::assign("strNanUrlTemp", $strNanUrlTemp); + View::assign("strNvUrlTemp", $strNvUrlTemp); + View::assign("strNanUrl", $strNanUrl); + View::assign("strNvUrl", $strNvUrl); + View::assign("strCategoryIndexUrlTemp", $strCategoryIndexUrlTemp); + View::assign("strRankUrlTemp", $strRankUrlTemp); + View::assign("strSearchIndexUrlTemp", $strSearchIndexUrlTemp); + View::assign("strHistoryUrlTemp", $strHistoryUrlTemp); + View::assign("strBookShelfUrlTemp", $strBookShelfUrlTemp); + View::assign("strUserUrlTemp", $strUserUrlTemp); + View::assign('Request', $this->Request); View::assign('DomainModel', $this->DomainModel); View::assign('TemplatesModel', $this->TemplatesModel); @@ -166,14 +192,72 @@ class SiteContext } else { $strView = 'index/index.html'; } - }else{ + } else { if ($this->DomainModel->n_id > 0) { $strView = 'pc/getNovelInfo.html'; } else { $strView = 'pc/index.html'; } } - + return $strView; } + + /** + * 获取伪造小说列表 + * + * @param array $arrParams + * @return array + */ + public function getForgeList(array $arrParams): array + { + $intCount = (int)$arrParams['count'] ?? 10; + $strDiffKey = (string) $arrParams['diff_key'] ?? ''; + $intCacheLifeTime = (int)$arrParams['cache_life'] ?? 0; + + $arrFilter = [ + 'n_forge_novel' => [ + '$ne' => null, // 不为 null + '$ne' => '', // 不为空字符串 + '$ne' => [], // 不为空数组 + '$exists' => true // 确保字段存在 + ], + ]; + + $strCacheKey = ''; + $intLifeTime = 0; + + # cache filter + if ($intCacheLifeTime > 0) { + $strCacheKey = sprintf( + 'SiteContext:getForgeList:%s:%s:%s:%s', + $this->DomainModel->d_domain, + md5(serialize($arrFilter)), + $intCount, + $strDiffKey, + ); + + $intLifeTime = $intLifeTime; + } + + $arrAllNovel = $this->NovelModel->getRandNovelWithCache( + $arrFilter, + $intCount, + $strCacheKey, + $intLifeTime, + ); + + + $arrResult = []; + foreach ($arrAllNovel as $arrNovel) { + $arrResult[] = [ + 'n_id' => $arrNovel['n_id'], + 'n_name_pinyin' => $arrNovel['n_name_pinyin'], + 'n_forge_id' => $arrNovel['n_forge_novel'][0]->n_forge_id, + 'n_forge_title' => $arrNovel['n_forge_novel'][0]->n_forge_title, + ]; + } + + return $arrResult; + } } diff --git a/code/app/task/crawler/youzhi/Scheduler.php b/code/app/task/crawler/youzhi/Scheduler.php new file mode 100644 index 0000000..05aa1cf --- /dev/null +++ b/code/app/task/crawler/youzhi/Scheduler.php @@ -0,0 +1,122 @@ +QueueManage = new QueueManage(); + $this->init(); + } + + /** + * Undocumented function + * + * @return void + */ + public function init() + { + $this->intSlice = 8; + } + + /** + * get site + * + * @return Site + */ + public function getSite() + { + if ($this->Site == NULL) { + $this->Site = new Site; + } + return $this->Site; + } + + + /** + * create a site-wide craw task + * + * @return void + */ + public function crawlFull() + { + $intLimit = 10; + + $Site = $this->getSite(); + + $arrAllCategory = $Site->getCategory(); + + foreach ($arrAllCategory as $arrCategory) { + $intTotalPage = ceil($arrCategory['total'] / $intLimit); + for ($intStartPage = 1; $intStartPage <= $intTotalPage; $intStartPage++) { + $arrFilter = [ + [ + 'v_category_id' => $arrCategory['v_category_id'], + 'page' => $intStartPage, + ], + ]; + + $arrVideoPageList = $Site->getVideoPageList($arrFilter); + + foreach ($arrVideoPageList as $VideoPage) { + $arrAllVideoPager = $VideoPage->getVideoPager(); + + $arrVSourceId = []; + foreach ($arrAllVideoPager['list'] as $arrVideoPager) { + $arrVSourceId[] = ['v_source_id' => (int)$arrVideoPager['vod_id']]; + } + + $arrAllVideoInfoPage = $Site->getVideoInfoPageList($arrVSourceId); + foreach ($arrAllVideoInfoPage as $VideoInfoPage) { + $arrVideo = $VideoInfoPage->getVideo(); + print_r($arrVideo); + } + } + } + } + + + $arrCover = []; + + + if ($arrCover) { + // $Site->downRemoteImgToLocal($arrCover); + } + + return; + } +} diff --git a/code/app/task/crawler/youzhi/page/BasePage.php b/code/app/task/crawler/youzhi/page/BasePage.php new file mode 100644 index 0000000..67b5d48 --- /dev/null +++ b/code/app/task/crawler/youzhi/page/BasePage.php @@ -0,0 +1,251 @@ +setSite($Site); + + $this->strUri = $strUri; + } + + /** + * Undocumented function + * + * @return string + */ + public function getUri(): string + { + return $this->strUri; + } + + /** + * Undocumented function + * + * @param array $arrData + * @return void + */ + public function setData(array $arrData) + { + $this->arrData = $arrData; + } + + /** + * Undocumented function + * + * @return array + */ + public function getData(): array + { + return $this->arrData; + } + + /** + * Undocumented function + * + * @return QueueManage + */ + public function getQueueManage(): QueueManage + { + if ($this->QueueManage == NULL) { + $this->QueueManage = QueueManage::getInstance(); + } + + return $this->QueueManage; + } + + /** + * Undocumented function + * + * @return MongoBase + */ + public function getMongoBase(): MongoBase + { + if ($this->MongoBase == NULL) { + DDLManage::load('MongoBase'); + $this->MongoBase = MongoBase::getInstance(); + } + + return $this->MongoBase; + } + + public function getQueryList(): QueryList + { + if ($this->QueryList == NULL) { + $this->QueryList = QueryList::html($this->getContent()); + } + + return $this->QueryList; + } + + + /** + * Undocumented function + * + * @return string + */ + public function getContent(): string + { + if ($this->strContent == NULL) { + $this->Response = $this->getClient()->get($this->strUri); + $this->strContent = $this->Response->getBody()->getContents(); + } + + return $this->strContent; + } + + /** + * force set page content + * + * @param string $strContent + * @return void + */ + public function setContent($strContent) + { + $this->strContent = $strContent; + } + + + /** + * Undocumented function + * + * @return boolean + */ + public function isMobile($strMark = 'content="mobile"'): bool + { + if ($this->boolIsMobile === NULL) { + $this->boolIsMobile = is_numeric(strpos($this->getContent(), $strMark)); + } + return $this->boolIsMobile; + } + + + /** + * Undocumented function + * + * @return Client + */ + protected function getClient(): Client + { + if ($this->Client == NULL) { + $this->Client = $this->getSite()->getClient(); + } + return $this->Client; + } + + + /** + * Undocumented variable + * + * @var Site + */ + protected $Site; + + public function setSite(Site $Site) + { + $this->Site = $Site; + } + + public function getSite() + { + return $this->Site; + } +} diff --git a/code/app/task/crawler/youzhi/page/Site.php b/code/app/task/crawler/youzhi/page/Site.php new file mode 100644 index 0000000..d66ee60 --- /dev/null +++ b/code/app/task/crawler/youzhi/page/Site.php @@ -0,0 +1,369 @@ +initClient(); + } + + /** + * Undocumented function + */ + public function initClient() + { + $arrConfig = [ + 'http_version' => '1.1', // 强制用 HTTP/1.1,已有,保持 + 'base_uri' => $this->strDomain, // 基础域名,已有,保持 + 'http_errors' => false, // 不抛 HTTP 错误,已有,保持 + 'timeout' => 60, // 从 160 秒改为 60 秒 + 'connect_timeout' => 10, // 从 30 秒改为 10 秒 + 'curl' => [ + CURLOPT_TCP_KEEPALIVE => 1, // 启用 TCP Keep-Alive,已有,保持 + CURLOPT_TCP_KEEPIDLE => 120, // Keep-Alive 空闲时间 + CURLOPT_TCP_KEEPINTVL => 60, // Keep-Alive 探测间隔 + // CURLOPT_SSL_VERIFYPEER => true, // 验证 SSL,已有,保持 + ], + 'headers' => [ + 'User-Agent' => 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36', + 'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7', + 'Accept-Language' => 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6', + 'Referer' => $this->strDomain, + 'Connection' => 'keep-alive', + 'sec-ch-ua' => '"Not/A)Brand";v="8", "Chromium";v="130", "Google Chrome";v="130"', + 'sec-ch-ua-mobile' => '?1', + 'sec-ch-ua-platform' => '"Android"', + 'sec-fetch-dest' => 'document', + 'sec-fetch-mode' => 'navigate', + 'sec-fetch-site' => 'same-origin', + 'sec-fetch-user' => '?1', + 'upgrade-insecure-requests' => '1', + ], + 'pool_size' => 20, + 'verify' => false, + ]; + + $arrProxy = config('proxy'); + + + if ($this->boolProxy && $arrProxy['proxy_status']) { + + $strCode = $arrProxy['proxy_code']; + $strProxyHost = $arrProxy['proxy_host']; + $strKey = $arrProxy['proxy_key']; + $strSecret = $arrProxy['proxy_secret']; + + + switch ($strCode) { + case 'XIONGMAO': + $intTime = time(); + $strTxt = "orderno=" . $strKey . ",secret=" . $strSecret . ",timestamp=" . $intTime; + $strSign = strtoupper(md5($strTxt)); + $strAuth = 'sign=' . $strSign . '&orderno=' . $strKey . '×tamp=' . $intTime; + $arrConfig['proxy'] = $strProxyHost; + $arrConfig['headers']['Proxy-Authorization'] = $strAuth; + break; + case 'KUAI': + $arrConfig['proxy'] = $strProxyHost; + $arrConfig['curl'][CURLOPT_PROXYAUTH] = CURLAUTH_BASIC; + $arrConfig['curl'][CURLOPT_PROXYUSERPWD] = "$strKey:$strSecret"; + break; + } + } + + + $this->Client = new Client($arrConfig); + } + + /** + * Undocumented function + * + * @return string + */ + public function getSiteCode(): string + { + return $this->strSiteCode; + } + + /** + * Undocumented function + * + * @param integer|string $strSign + * @return string + */ + public function getSiteUUId(int|string $strSign): string + { + return $this->getSiteCode() . '-' . $strSign; + } + + /** + * Undocumented function + * + * @return Client + */ + public function getClient(): Client + { + return $this->Client; + } + + /** + * retrieve multiple novel page model + * + * @param string $strUri + * @return VideoPage[] + */ + public function getCategory(): array + { + $strUri = '/inc/api_mac10.php?ac=list&t=1&page=1'; + + $Response = $this->getClient()->get($strUri); + $strContent = $Response->getBody()->getContents(); + + $arrContent = json_decode($strContent, true); + $arrFilter = $arrContent['class']; + + $arrCategory = []; + + foreach ($arrFilter as $intKey => $arrItem) { + + $arrFilter[$intKey]['uri'] = $this->getVideoPageUri((int)$arrItem['type_id'], 1); + + $arrPromises[$intKey] = $this->getClient()->getAsync($arrFilter[$intKey]['uri']); + } + + $arrResults = Promise\Utils::settle($arrPromises)->wait(); + + $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) { + return $arrResult['state'] === 'fulfilled'; + }); + + foreach ($arrSuccessfulResults as $intKey => $arrResult) { + $Response = $arrResult['value']; + + $strContent = $Response->getBody()->getContents(); + + $arrTemp = json_decode($strContent, true); + + $arrCategory[] = [ + 'v_category_id' => (int)$arrFilter[$intKey]['type_id'], + 'v_category_name' => $arrFilter[$intKey]['type_name'], + 'v_category_name_en' => zhToPinYin($arrFilter[$intKey]['type_name']), + 'total' => $arrTemp['total'], + ]; + } + + return $arrCategory; + } + + /** + * generate video list uri + * + * @param integer $intVCategoryId + * @param integer $intPage + * @return string + */ + public function getVideoPageUri(int $intVCategoryId, int $intPage): string + { + return sprintf('/inc/api_mac10.php?ac=list&t=%s&page=%s', $intVCategoryId, $intPage); + } + + /** + * retrieve multiple novel page model + * + * @param string $strUri + * @return VideoPage[] + */ + public function getVideoPageList(array $arrFilter): array + { + + /** @var VideoPage[] */ + $arrVideoPage = []; + + foreach ($arrFilter as $intKey => $arrItem) { + + $arrFilter[$intKey]['uri'] = $this->getVideoPageUri($arrItem['v_category_id'], $arrItem['page']); + + $arrPromises[$intKey] = $this->getClient()->getAsync($arrFilter[$intKey]['uri']); + } + + $arrResults = Promise\Utils::settle($arrPromises)->wait(); + + + $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) { + return $arrResult['state'] === 'fulfilled'; + }); + + foreach ($arrSuccessfulResults as $intKey => $arrResult) { + $Response = $arrResult['value']; + + $strContent = $Response->getBody()->getContents(); + + + $VideoPage = $this->getVideoPage($this, $arrFilter[$intKey]['uri']); + + $VideoPage->setContent($strContent); + $VideoPage->setData($arrFilter[$intKey]); + + $arrVideoPage[] = $VideoPage; + } + + return $arrVideoPage; + } + + /** + * generate video list uri + * + * @param integer $intVCategoryId + * @param integer $intPage + * @return string + */ + public function getVideoUri(int $intVSourceId): string + { + return sprintf('/inc/api_mac10.php?ac=detail&tds=%s', $intVSourceId); + } + + /** + * retrieve multiple chapter page model + * + * @param string $strUri + * @return VideoInfoPage[] + */ + public function getVideoInfoPageList(array $arrFilter): array + { + + /** @var VideoInfoPage[] */ + $arrVideoInfoPage = []; + + $arrPromises = []; + foreach ($arrFilter as $intKey => $arrItem) { + $arrFilter[$intKey]['uri'] = $this->getVideoUri($arrItem['v_source_id']); + $arrPromises[$intKey] = $arrFilter[$intKey]['uri']; + } + + print_r($arrFilter); + + $arrResults = Promise\Utils::settle($arrPromises)->wait(); + + + var_dump($arrResults); + return []; + + $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) { + return $arrResult['state'] === 'fulfilled'; + }); + + foreach ($arrSuccessfulResults as $intKey => $arrResult) { + $Response = $arrResult['value']; + + var_dump($arrResult); + + $strContent = $Response->getBody()->getContents();; + + $VideoInfoPage = $this->getVideoInfoPage($this, $arrFilter[$intKey]['uri']); + + $VideoInfoPage->setContent($strContent); + $VideoInfoPage->setData($arrFilter[$intKey]); + + $arrVideoInfoPage[] = $VideoInfoPage; + } + + return $arrVideoInfoPage; + } + + /** + * download the image to the local system + * + * @param array $arrFilter + * @return void + */ + public function downRemoteImgToLocal(array $arrFilter) + { + $StorageCore = StorageCore::getInstance(); + + $arrPromises = []; + + foreach ($arrFilter as $intKey => $arrItem) { + if (!$StorageCore->has($arrItem['local_path'])) { + $arrPromises[$intKey] = $this->getClient()->getAsync($arrItem['uri']); + } + } + + $arrResults = Promise\Utils::settle($arrPromises)->wait(); + + $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) { + return $arrResult['state'] === 'fulfilled'; + }); + + foreach ($arrSuccessfulResults as $intKey => $arrResult) { + $Response = $arrResult['value']; + + $StorageCore->writeStream($arrFilter[$intKey]['local_path'], $Response->getBody()->detach());; + } + } + + /** + * Undocumented function + * + * @param object $Site + * @param string $strUri + * @return VideoPage + */ + public function getVideoPage($Site, $strUri) + { + return new VideoPage($Site, $strUri); + } + + /** + * Undocumented function + * + * @param object $Site + * @param string $strUri + * @return VideoInfoPage + */ + public function getVideoInfoPage($Site, $strUri) + { + return new VideoInfoPage($Site, $strUri); + } +} diff --git a/code/app/task/crawler/youzhi/page/VideoInfoPage.php b/code/app/task/crawler/youzhi/page/VideoInfoPage.php new file mode 100644 index 0000000..4a63308 --- /dev/null +++ b/code/app/task/crawler/youzhi/page/VideoInfoPage.php @@ -0,0 +1,194 @@ +VideoModel == NULL) { + $this->VideoModel = VideoModel::getInstance(); + } + return $this->VideoModel; + } + + public $arrOptions = [ + 'typeMap' => [ + 'root' => 'array', + 'document' => 'array', + 'array' => 'array', + ], + ]; + + /** + * Undocumented function + * + * @return array + */ + public function getVideo(): array + { + if (empty($this->arrVideo)) { + + $this->arrVideo = json_decode($this->getContent(), true); + } + + return $this->arrVideo; + } + + /** + * Undocumented variable + * + * @var array + */ + protected $arrVideo; + + + /** + * Undocumented function + * + * @param integer $intNId + * @return boolean + */ + public function saveChapter(int $intNId): bool + { + $arrPageChapter = $this->getVideo(); + + if ( + empty($arrPageChapter) || + empty($arrPageChapter['c_page_title']) || + empty($arrPageChapter['c_page_data']) || + count($arrPageChapter['c_page_data']) < 5 + ) { + return false; + } + + $strUUId = $this->getSite()->getSiteUUId('chapter-' . $this->arrData['c_source_id'] . '-' . $this->arrData['c_page']); + + $arrChapter = [ + 'n_id' => $intNId, + 'c_name' => $arrPageChapter['c_page_data'][3], + 'c_sort_num' => $this->arrData['c_sort_num'], + 'c_page' => $this->arrData['c_page'], + 'c_source_id' => $this->arrData['c_source_id'], + 'c_site_code' => $this->getSite()->getSiteCode(), + 'c_source_uuid' => $strUUId, + ]; + + $arrExistsChapter = $this->getVideoModel()->findOne(['c_source_uuid' => $strUUId], $this->arrOptions); + + if ($arrExistsChapter) { + + $arrExistsChapter = (array) $arrExistsChapter; + + $arrUpdate = [ + 'updated_at' => new \MongoDB\BSON\UTCDateTime(), + ]; + + $this->getVideoModel()->updateOne( + ['c_source_uuid' => $strUUId], + ['$set' => $arrUpdate], + ); + $this->strCContentPath = $arrExistsChapter['c_content_path']; + } else { + + $arrInsertNovel = $arrChapter; + + $arrInsertNovel['c_content_path'] = '/novel/' . ($intNId % 100) . '/' . $intNId . '/' . $arrChapter['c_sort_num'] . '_' . $arrChapter['c_page'] . '.txt'; + $arrInsertNovel['created_at'] = new \MongoDB\BSON\UTCDateTime(); + + try { + $this->getVideoModel()->insert($arrInsertNovel); + $this->updateNovelHaveChapter($intNId); + $this->strCContentPath = $arrInsertNovel['c_content_path']; + } catch (\Exception $e) { + if ($e->getCode() === 11000) { + return $this->saveChapter($intNId); + } else { + throw $e; + } + } + } + + StorageCore::getInstance()->put($this->strCContentPath, $this->strChapterContent); + + return true; + } + + public $strCContentPath; + /** + * Undocumented function + * + * @param integer $intNId + * @return void + */ + protected function updateNovelHaveChapter(int $intNId) + { + $arrFilter = [ + 'n_id' => $intNId, + ]; + + $VideoModel = VideoModel::getInstance(); + $arrLasteChapter = $VideoModel->getLatestChapterByNId($intNId, false); + + if (!empty($arrLasteChapter)) { + $arrUpdate['$set']['n_have_chapter'] = 1; + $arrUpdate['$set']['n_latest_chapter_name'] = $arrLasteChapter['c_name']; + $this->getMongoBase()->updateOne('novel', $arrFilter, $arrUpdate); + } + } + + protected $arrNextPage; + + /** + * Undocumented function + * + * @return array + */ + public function getNextPage(): array + { + if ($this->arrNextPage === NULL) { + $this->arrNextPage = []; + + if ($this->isMobile()) { + $strPattern = "/var\s+hhekgsv\s*=\s*['\"](.*?)['\"]/"; + preg_match_all($strPattern, $this->getContent(), $arrMatches); + $strNextPageUri = end($arrMatches[1]) ?: ''; + } else { + $strHref = $this->getQueryList()->find('.read_btn a:contains("下一章")')->attr('href'); + $strNextPageUri = $strHref ?: ''; + } + + if (is_numeric(strpos($strNextPageUri, '_'))) { + $this->arrNextPage = $this->arrData; + $this->arrNextPage['c_page'] += 1; + $this->arrNextPage['uri'] = $strNextPageUri; + } + } + + return $this->arrNextPage; + } +} diff --git a/code/app/task/crawler/youzhi/page/VideoPage.php b/code/app/task/crawler/youzhi/page/VideoPage.php new file mode 100644 index 0000000..2cb1c83 --- /dev/null +++ b/code/app/task/crawler/youzhi/page/VideoPage.php @@ -0,0 +1,417 @@ +NovelModel == NULL) { + $this->NovelModel = NovelModel::getInstance(); + } + return $this->NovelModel; + } + + public $arrOptions = [ + 'typeMap' => [ + 'root' => 'array', + 'document' => 'array', + 'array' => 'array', + ], + ]; + + /** + * 查询看的小说 + * + * @param integer $intNSourceId + * @param string $strName + * @return array + */ + protected function findOrCreateNovel(int $intNSourceId, string $strName): array + { + $strUUId = $this->getSite()->getSiteUUId($intNSourceId); + + $arrExistsNovel = $this->getNovelModel()->findOne(['n_source_uuid' => $strUUId], $this->arrOptions); + + if ($arrExistsNovel) { + $arrVideo = (array) $arrExistsNovel; + } else { + $arrVideo = [ + 'n_id' => CountersModel::getInstance()->getNextSequence('novel'), + 'n_name' => $strName, + 'n_name_pinyin' => zhToPinYin($strName), + 'n_site_code' => $this->getSite()->getSiteCode(), + 'n_source_uuid' => $strUUId, + 'n_source_id' => $intNSourceId, + 'n_level' => 1, + 'n_have_chapter' => 0, + 'created_at' => new \MongoDB\BSON\UTCDateTime(), + ]; + try { + $this->getNovelModel()->insert($arrVideo); + } catch (\Exception $e) { + if ($e->getCode() === 11000) { + return $this->findOrCreateNovel($intNSourceId, $strName); + } else { + throw $e; + } + } + } + return $arrVideo; + } + + + /** + * 分析关联数据 + * + * @return void + */ + public function getReletedNovel() {} + + + /** + * Undocumented function + * + * @return array + */ + public function getVideoPager(): array + { + if (empty($this->arrVideo)) { + + $this->arrVideo = json_decode($this->getContent(), true); + + $this->getReletedNovel(); + } + + return $this->arrVideo; + } + + /** + * Undocumented function + * + * @param string $strName + * @return string + */ + public function generateCoverPath($strName): string + { + $intTime = time() - mt_rand(1, 2 * 24 * 3600 * 360); + + return sprintf('/img/%s/%s.jpg', date('Y/m/d/H', $intTime), $strName); + } + + public function filterOtherFields(&$arrNovel) + { + $arrFields = [ + 'n_id', + 'n_name', + 'n_name_pinyin', + 'n_status', + 'n_author', + 'n_description', + 'n_category', + 'n_category_pinyin', + 'n_category_sex_en', + 'n_category_sex_zh', + 'n_cover_path', + 'n_latest_chapter_name', + 'n_update_time', + 'n_forge_novel', + 'n_related_novel', + 'n_site_code', + 'n_source_uuid', + 'n_source_id', + 'n_have_chapter', + 'created_at', + 'updated_at', + ]; + + foreach ($arrNovel as $strKey => $strVal) { + if (!in_array($strKey, $arrFields)) { + unset($arrNovel[$strKey]); + } + } + } + + /** + * Undocumented function + * + * @return bool + */ + public function saveVideo(): bool + { + $arrPageInfo = $this->getVideo(); + + print_r($arrPageInfo); + return false; + if ( + empty($arrPageInfo) || + empty($arrPageInfo['og_title']) || + empty($arrPageInfo['og_image']) || + empty($arrPageInfo['og_novel_category']) + ) { + return false; + } + + $arrVideo = [ + // 'n_id'=> $arrPageInfo['xxxxx'], + 'n_name' => $arrPageInfo['og_title'], + 'n_name_pinyin' => zhToPinYin($arrPageInfo['og_title']), + 'n_status' => $arrPageInfo['og_novel_status'], + 'n_author' => $arrPageInfo['og_novel_author'], + 'n_description' => $arrPageInfo['og_description'], + 'n_category' => $arrPageInfo['og_novel_category'], + // 'n_category_pinyin' => $arrPageInfo['xxxxx'], + // 'n_category_sex_en' => $arrPageInfo['xxxxx'], + // 'n_category_sex_zh' => $arrPageInfo['xxxxx'], + // 'n_cover_path' => $arrPageInfo['xxxxx'], + 'n_latest_chapter_name' => $arrPageInfo['og_novel_latest_chapter_name'], + 'n_update_time' => $arrPageInfo['og_novel_update_time'], + 'n_forge_novel' => $this->arrForgeNovel, + 'n_related_novel' => $this->arrRelatedNovel, + + 'n_site_code' => $this->getSite()->getSiteCode(), + 'n_source_id' => $this->arrData['n_source_id'], + 'n_source_uuid' => $this->getSite()->getSiteUUId($this->arrData['n_source_id']), + + // 'n_have_chapter' => $arrPageInfo['xxxxx'], + // 'created_at' => $arrPageInfo['xxxxx'], + // 'updated_at' => $arrPageInfo['xxxxx'], + + ]; + + $arrExistsNovel = $this->getNovelModel()->findOne(['n_source_uuid' => $arrVideo['n_source_uuid']], $this->arrOptions); + + if ($arrExistsNovel) { + + $arrExistsNovel = (array) $arrExistsNovel; + + $arrUpdate = [ + 'n_latest_chapter_name' => $arrVideo['n_latest_chapter_name'], + 'n_update_time' => $arrVideo['n_update_time'], + 'updated_at' => new \MongoDB\BSON\UTCDateTime(), + ]; + + # 如果没有分类,说明这条数据是最初从关联shu那里采集的,需要补全全部信息 + if (empty($arrExistsNovel['n_category'])) { + $arrUpdate = array_merge($arrUpdate, $arrVideo); + $arrUpdate['n_related_novel'] = $this->arrRelatedNovel; + $arrUpdate['n_forge_novel'] = $this->arrForgeNovel; + $arrUpdate['n_category_pinyin'] = CategoryModel::getPinYin($this->arrVideo['og_novel_category']); + $arrUpdate['n_category_sex_en'] = CategoryModel::getSexEn($this->arrVideo['og_novel_category']); + $arrUpdate['n_category_sex_zh'] = CategoryModel::getSexZh($this->arrVideo['og_novel_category']); + $arrUpdate['n_related_novel'] = $this->arrRelatedNovel; + $arrUpdate['n_forge_novel'] = $this->arrForgeNovel; + } + + # 图片不存在的时候,需要重新生成 + if ( + !isset($arrExistsNovel['n_cover_path']) || + empty($arrExistsNovel['n_cover_path']) || + !StorageCore::getInstance()->has($arrExistsNovel['n_cover_path']) || + (basename($arrExistsNovel['n_cover_path']) == '.jpg') + ) { + $arrUpdate['n_cover_path'] = $this->generateCoverPath($arrVideo['n_name_pinyin']); + $this->strCoverLocalPath = $arrUpdate['n_cover_path']; + } else { + $this->strCoverLocalPath = $arrExistsNovel['n_cover_path']; + } + + $this->getNovelModel()->updateOne( + ['n_source_uuid' => $arrVideo['n_source_uuid']], + ['$set' => $arrUpdate], + ); + $this->intNId = $arrExistsNovel['n_id']; + } else { + + $arrVideo['n_category_pinyin'] = CategoryModel::getPinYin($this->arrVideo['og_novel_category']); + $arrVideo['n_category_sex_en'] = CategoryModel::getSexEn($this->arrVideo['og_novel_category']); + $arrVideo['n_category_sex_zh'] = CategoryModel::getSexZh($this->arrVideo['og_novel_category']); + $arrVideo['n_id'] = CountersModel::getInstance()->getNextSequence('novel'); + $arrVideo['n_level'] = 1; + $arrVideo['n_have_chapter'] = 0; + $arrVideo['n_cover_path'] = $this->generateCoverPath($arrVideo['n_name_pinyin']); + $arrVideo['n_related_novel'] = $this->arrRelatedNovel; + $arrVideo['n_forge_novel'] = $this->arrForgeNovel; + $arrVideo['created_at'] = new \MongoDB\BSON\UTCDateTime(); + + try { + $this->getNovelModel()->insert($arrVideo); + $this->intNId = $arrVideo['n_id']; + $this->strCoverLocalPath = $arrVideo['n_cover_path']; + } catch (\Exception $e) { + if ($e->getCode() === 11000) { + return $this->saveVideo(); + } else { + throw $e; + } + } + } + + return true; + } + + + public $strCoverLocalPath = ''; + + public $intNId; + + /** + * Undocumented function + * + * @return array + */ + public function getDownCoverInfo(): array + { + return [ + 'uri' => $this->arrVideo['og_image'], + 'local_path' => $this->strCoverLocalPath, + ]; + } + + /** + * Undocumented variable + * + * @var array + */ + protected $arrLatestChapters = NULL; + + /** + * Undocumented variable + * + * @var array + */ + protected $arrChapters = NULL; + + public function getLatestChapterByNIdsList(): array + { + if ($this->arrLatestChapters === NULL) { + $this->arrLatestChapters = []; + + if ($this->isMobile()) { + $strClass = '.chapter-list'; + } else { + $strClass = '.section-list'; + } + + if ($this->arrData['page'] == 0) { + $this->getQueryList()->find($strClass)->eq(0)->find("a")->map(function ($A) { + $this->arrLatestChapters[] = [ + 'name' => $A->text(), + 'uri' => $A->attr('href'), + ]; + }); + } + } + return $this->arrLatestChapters; + } + + /** + * Undocumented function + * + * @return array + */ + public function getChapters(): array + { + if ($this->arrChapters === NULL) { + $this->arrChapters = []; + if ($this->isMobile()) { + $strClass = '.chapter-list'; + } else { + $strClass = '.section-list'; + } + + $intDivIndex = 0; + if ($this->arrData['page'] == 0) { + $intDivIndex = 1; + } + + $this->getQueryList()->find($strClass)->eq($intDivIndex)->find("a")->map(function ($A) { + $this->arrChapters[] = [ + 'name' => $A->text(), + 'uri' => $A->attr('href'), + ]; + }); + } + return $this->arrChapters; + } + + + protected $arrNextPageArgs = NULL; + + /** + * Undocumented function + * + * @return array + */ + public function getNextPageArgs(): array + { + $this->arrNextPageArgs = []; + if ($this->arrData['page'] == 0) { + $this->getLatestChapterByNIdsList(); + $this->getChapters(); + + $arrLatestChapterFirstBlock = count($this->arrLatestChapters) > 0 ? $this->arrLatestChapters[0] : null; + $arrLatestChapterSecondBlock = count($this->arrChapters) > 0 ? $this->arrChapters[count($this->arrChapters) - 1] : null; + + if ( + !empty($arrLatestChapterFirstBlock) && !empty($arrLatestChapterSecondBlock) + && ($arrLatestChapterFirstBlock['uri'] != $arrLatestChapterSecondBlock['uri']) + + ) { + $this->arrNextPageArgs = [ + 'n_source_id' => $this->arrData['n_source_id'], + 'page' => $this->arrData['page'] + 1, + ]; + } + } else { + $strLastUri = $this->getQueryList()->find('select:first option:last')->attr('value'); + if ($strLastUri != '' && $strLastUri != $this->strUri) { + $this->arrNextPageArgs = [ + 'n_source_id' => $this->arrData['n_source_id'], + 'page' => $this->arrData['page'] + 1, + ]; + } + } + return $this->arrNextPageArgs; + } +} diff --git a/code/app/task/module/PlanTask.php b/code/app/task/module/PlanTask.php index f0c873c..264f37c 100644 --- a/code/app/task/module/PlanTask.php +++ b/code/app/task/module/PlanTask.php @@ -7,6 +7,8 @@ namespace app\task\module; use app\model\PlanTaskModel; use app\task\crawler\zw630\Scheduler as Zw630Scheduler; use app\task\crawler\zw74\Scheduler as Zw74Scheduler; +use app\task\crawler\youzhi\Scheduler as YouZhiScheduler; + use microserver\QueueManage; use microserver\ProcessSanitizer; @@ -42,11 +44,30 @@ class PlanTask case 'PULL_74_ZW_NOVEL_LATEST': self::pull74ZwNovelLatest(); break; - default: + + case 'PULL_YOU_ZHI_SHI_PIN': + self::pullYouZhiShiPin(); break; + } } } + /** + * Undocumented function + * + * @return void + */ + public static function pullYouZhiShiPin() + { + $arrTask = [ + 'callback' => [YouZhiScheduler::class, 'crawlFull'], + 'data' => [] + ]; + + $QueueManage = new QueueManage(); + $QueueManage->set($arrTask); + } + /** * Undocumented function @@ -81,7 +102,7 @@ class PlanTask } - /** + /** * Undocumented function * * @return void diff --git a/code/database/seeders/PlanTaskSeeder.php b/code/database/seeders/PlanTaskSeeder.php index a8520a3..666f9d8 100644 --- a/code/database/seeders/PlanTaskSeeder.php +++ b/code/database/seeders/PlanTaskSeeder.php @@ -44,6 +44,14 @@ class PlanTaskSeeder 'pt_enable' => 0, 'pt_limit' => 1 * 24 * 3600, ], + + + [ + 'pt_name' => '优质资源库全量采集', + 'pt_code' => 'PULL_YOU_ZHI_SHI_PIN', + 'pt_enable' => 0, + 'pt_limit' => 1 * 24 * 3600, + ], ]; foreach ($arrDataNovel as $arrPlanTask) { diff --git a/code/extend/template/taglib/Chapter.php b/code/extend/template/taglib/Chapter.php index e92255d..c302390 100644 --- a/code/extend/template/taglib/Chapter.php +++ b/code/extend/template/taglib/Chapter.php @@ -7,7 +7,7 @@ use think\template\TagLib; class Chapter extends TagLib { protected $tags = [ - 'list' => ['attr' => 'limit,n_category,sort_type,n_status,n_sex,key,diff_key,cache_life,d_key,d_val', 'close' => 1], + 'list' => ['attr' => 'count,n_category,sort_type,n_status,n_sex,key,diff_key,cache_life,d_key,d_val', 'close' => 1], 'pager' => ['attr' => 'page,limit,n_category,sort_type,n_status,n_sex,key,diff_key,cache_life,d_key,d_val,p_val,export_name', 'close' => 1], 'pagerexp' => ['attr' => 'page,limit,n_category,sort_type,n_status,n_sex,key,diff_key,cache_life,d_key,d_val,p_val,func,export_name', 'close' => 0], 'info' => ['attr' => 'n_id,n_key', 'close' => 0], diff --git a/code/extend/template/taglib/Novel.php b/code/extend/template/taglib/Novel.php index b2b5668..872a2ab 100644 --- a/code/extend/template/taglib/Novel.php +++ b/code/extend/template/taglib/Novel.php @@ -7,7 +7,7 @@ use think\template\TagLib; class Novel extends TagLib { protected $tags = [ - 'list' => ['attr' => 'limit,n_category,sort_type,n_status,n_sex,key,diff_key,cache_life,d_key,d_val', 'close' => 1], + 'list' => ['attr' => 'count,n_category,sort_type,n_status,n_sex,diff_key,cache_life,d_key,d_val', 'close' => 1], 'pager' => ['attr' => 'page,limit,n_category,sort_type,n_status,n_sex,key,diff_key,cache_life,d_key,d_val,p_val,func,export_name', 'close' => 1], 'pagerexp' => ['attr' => 'page,limit,n_category,sort_type,n_status,n_sex,key,diff_key,cache_life,d_key,d_val,p_val,func,export_name', 'close' => 0], 'info' => ['attr' => 'n_id,n_key', 'close' => 0], @@ -15,6 +15,8 @@ class Novel extends TagLib 'status' => ['attr' => 'd_key,d_val'], 'sex' => ['attr' => 'd_key,d_val'], 'category' => ['attr' => 'd_key,d_val,n_sex'], + + 'ranklist' => ['attr' => 'count,sort_type,n_status,n_sex,diff_key,cache_life,d_key,d_val', 'close' => 1], ]; private function customBuildVar($mixedArgs) @@ -27,8 +29,67 @@ class Novel extends TagLib } + /** - * novel 自定义标签 + * tagRankList + * + * @param array $tag + * @param string $content + * @return void + */ + public function tagRankList($tag, $content) + { + $count = $tag['count'] ?? '10'; + $sort_type = $tag['sort_type'] ?? ""; + $n_status = $tag['n_status'] ?? "all"; + $n_sex = $tag['n_sex'] ?? "all"; + + $diff_key = $tag['diff_key'] ?? ""; + $cache_life = $tag['cache_life'] ?? '0'; + $d_key = $tag['d_key'] ?? 'd_key'; + $d_val = $tag['d_val'] ?? 'd_val'; + + $sort_type = $this->customBuildVar($sort_type); + $n_status = $this->customBuildVar($n_status); + $n_sex = $this->customBuildVar($n_sex); + $diff_key = $this->customBuildVar($diff_key); + $cache_life = $this->customBuildVar($cache_life); + + $arrDataName = randomString(5); + + $strParse = <<getRankList([ +'count' => {$count}, +'sort_type' => {$sort_type}, +'n_status' => {$n_status}, +'n_sex' => {$n_sex}, +'diff_key' => {$diff_key}, +'cache_life' => {$cache_life} +]); + +if (!empty(\${$arrDataName}) && is_array(\${$arrDataName})): + +?> +EOT; + $strParse .= << \${$d_val}): ?> +EOT; + $strParse .= $content; + $strParse .= << + +EOT; + return $strParse; + } + + + /** + * tagPagerExp + * + * @param array $tag + * @param string $content + * @return void */ public function tagPagerExp($tag, $content) { @@ -36,7 +97,11 @@ class Novel extends TagLib } /** - * novel 自定义标签 + * tagPager + * + * @param array $tag + * @param string $content + * @return void */ public function tagPager($tag, $content) { @@ -115,7 +180,11 @@ EOT; } /** - * novel 自定义标签 + * tagList + * + * @param array $tag + * @param string $content + * @return void */ public function tagList($tag, $content) { diff --git a/code/extend/template/taglib/Site.php b/code/extend/template/taglib/Site.php index 2f137c8..5d190e7 100644 --- a/code/extend/template/taglib/Site.php +++ b/code/extend/template/taglib/Site.php @@ -18,6 +18,8 @@ class Site extends TagLib 'nflurl' => ['attr' => 'gender,category,status,order,page', 'close' => 0], 'grm' => ['attr' => 'domain,code,line,diff,start,end,index,type', 'close' => 0], 'replace' => ['attr' => 'code', 'close' => 0], + + 'forgelist' => ['attr' => 'count,diff_key,cache_life,d_key,d_val', 'close' => 1], ]; private function customBuildVar($mixedArgs) @@ -29,6 +31,50 @@ class Site extends TagLib } } + + /** + * tagRankList + * + * @param array $tag + * @param string $content + * @return void + */ + public function tagForgeList($tag, $content) + { + $count = $tag['count'] ?? '10'; + $diff_key = $tag['diff_key'] ?? ""; + $cache_life = $tag['cache_life'] ?? '0'; + $d_key = $tag['d_key'] ?? 'd_key'; + $d_val = $tag['d_val'] ?? 'd_val'; + + $diff_key = $this->customBuildVar($diff_key); + $cache_life = $this->customBuildVar($cache_life); + + $arrDataName = randomString(5); + + $strParse = <<getForgeList([ +'count' => {$count}, +'diff_key' => {$diff_key}, +'cache_life' => {$cache_life} +]); + +if (!empty(\${$arrDataName}) && is_array(\${$arrDataName})): + +?> +EOT; + $strParse .= << \${$d_val}): ?> +EOT; + $strParse .= $content; + $strParse .= << + +EOT; + return $strParse; + } + /** * cfg 标签处理函数-系统配置 * diff --git a/doc/TemplateTag.md b/doc/TemplateTag.md index c41ea3a..2a4d1f4 100644 --- a/doc/TemplateTag.md +++ b/doc/TemplateTag.md @@ -83,8 +83,14 @@ $count ?? '10'; # 查询条数 $n_category ?? "all"; # 查询分类 $sort_type ?? "all"; # 排序方式 news tuijian update - $n_status ?? "all"; # 查询状态 - $n_sex ?? "all"; # 查询性别 man women + $n_status ?? "all"; # 查询状态: + # * * 'all' => '所有', + # * * 'lianzai' => '连载中', + # * * 'wanjie' => '已完结', + $n_sex ?? "all"; # 查询性别: + # * * 'all' => '所有', + # * * 'man' => '男生', + # * * 'women' => '女生', $diff_key ?? ""; # 缓存键值差异 $cache_life ?? '0'; # 缓存时间,如果需要开启缓存,这个字段必须传,而且必须大于0 $d_key ?? 'd_key'; # 查询出来的数据遍历的时候, 下标键变量的名称 @@ -106,8 +112,14 @@ $limit ?? '10'; # 查询条数 $n_category ?? "all"; # 查询分类 $sort_type ?? "all"; # 排序方式 - $n_status ?? "all"; # 查询状态 - $n_sex ?? "all"; # 查询性别 + $n_status ?? "all"; # 查询状态: + # * * 'all' => '所有', + # * * 'lianzai' => '连载中', + # * * 'wanjie' => '已完结', + $n_sex ?? "all"; # 查询性别: + # * * 'all' => '所有', + # * * 'man' => '男生', + # * * 'women' => '女生', $key ?? ""; # 查询 搜索 $button_num ?? "10"; # 分页按钮数量 $diff_key ?? ""; # 缓存键值差异 @@ -132,8 +144,14 @@ $limit ?? '10'; # 查询条数 $n_category ?? "all"; # 查询分类 $sort_type ?? "all"; # 排序方式 - $n_status ?? "all"; # 查询状态 - $n_sex ?? "all"; # 查询性别 + $n_status ?? "all"; # 查询状态: + # * * 'all' => '所有', + # * * 'lianzai' => '连载中', + # * * 'wanjie' => '已完结', + $n_sex ?? "all"; # 查询性别: + # * * 'all' => '所有', + # * * 'man' => '男生', + # * * 'women' => '女生', $key ?? ""; # 查询 搜索 $button_num ?? "10"; # 分页按钮数量 $diff_key ?? ""; # 缓存键值差异 @@ -307,4 +325,53 @@ resData = [ # 样例 {site:grm page_code="fdasfadsf" diff="fadsfdsa" num="10" g_key="arrGrm" /} {php}var_dump($arrGrm);{/php} + + +# 13、 获取指定类型小说排行榜并且遍历 +# 传参与默认值(传参用下面的变量名,名称不需要$符号, 值的话如果用第三方变量作为值,需要带$符号): + $count ?? '10'; # 查询条数 + $sort_type ?? "daily"; # 排序方式: + # * * 'daily' => '天', + # * * 'weekly' => '周', + # * * 'monthly' => '月', + # * * 'total' => '总', + $n_status ?? "all"; # 查询状态: + # * * 'all' => '所有', + # * * 'lianzai' => '连载中', + # * * 'wanjie' => '已完结', + $n_sex ?? "all"; # 查询性别: + # * * 'all' => '所有', + # * * 'man' => '男生', + # * * 'women' => '女生', + $diff_key ?? ""; # 缓存键值差异 + $cache_life ?? '0'; # 缓存时间,如果需要开启缓存,这个字段必须传,而且必须大于0 + $d_key ?? 'd_key'; # 查询出来的数据遍历的时候, 下标键变量的名称 + $d_val ?? 'd_val'; # 查询出来的数据遍历的时候, 值的名称 +# 样例 + {novel:ranklist count="10" sort_type="total" n_status="wanjie" n_sex="man" d_key="d_key" d_val="novel" cache_life="1000"} +
  • + Index: {$d_key} + Title: {$novel.n_name} + Author: {$novel.n_category} +
  • +{/novel:ranklist} + +# 14、 获取随机伪造小说列表并且遍历 +# 传参与默认值(传参用下面的变量名,名称不需要$符号, 值的话如果用第三方变量作为值,需要带$符号): + $count ?? '10'; # 查询条数 + $diff_key ?? ""; # 缓存键值差异 + $cache_life ?? '0'; # 缓存时间,如果需要开启缓存,这个字段必须传,而且必须大于0 + $d_key ?? 'd_key'; # 查询出来的数据遍历的时候, 下标键变量的名称 + $d_val ?? 'd_val'; # 查询出来的数据遍历的时候, 值的名称 +# 样例 + {site:forgelist count="10" d_key="d_key" d_val="novel" cache_life="1000"} +
  • + Index: {$d_key} + N ID: {$novel.n_id} + N NAME PINYIN: {$novel.n_name_pinyin} + N FORGE ID: {$novel.n_forge_id} + N FORGE TITLE: {$novel.n_forge_title} + +
  • +{/site:forgelist} ``` \ No newline at end of file