admin/SEONexus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feature: crawler wuxian video

Browse Source 
bug-fix : video author
This commit is contained in:
Default
2025-05-01 20:14:52 +08:00
parent 3e270591b2
commit 83adda543a
12 changed files with 1238 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
code/app/admin/config/router.php
View File

@@ -7,6 +7,7 @@ use app\admin\controller\Domain;
use app\admin\controller\Novel;
use app\admin\controller\Site;
use app\admin\controller\SystemConfig;
use app\admin\controller\Video;
use app\admin\middleware\AdminAuth;
use think\facade\Route;
@@ -38,7 +39,14 @@ Route::group("/system", function () {
Route::group("/novel", function () {
Route::get("/list", [Novel::class, "getNovelList"])->name("Domain@getNovelList");
Route::post("/level/set", [Novel::class, "updateNovelLevel"])->name("Novel@updateNovelLevel");
Route::get("/category/list", [Novel::class, "getCategory"])->name("Domain@getCategory");
Route::get("/category/list", [Novel::class, "getCategory"])->name("Novel@getCategory");
})->middleware(AdminAuth::class);
# 视频
Route::group("/video", function () {
Route::get("/list", [Video::class, "getVideoList"])->name("Domain@getVideoList");
Route::post("/level/set", [Video::class, "updateVideoLevel"])->name("Video@updateVideoLevel");
Route::get("/category/list", [Video::class, "getCategory"])->name("Video@getCategory");
})->middleware(AdminAuth::class);
# 站点

104
code/app/admin/controller/Video.php Normal file
View File

@@ -0,0 +1,104 @@
<?php
declare(strict_types=1);
namespace app\admin\controller;
use app\admin\BaseController;
use app\model\AdminUserModel;
use app\model\VideoModel;
use app\model\VideoCategoryModel;
use app\Request;
use think\Response;
class Video extends BaseController
{
/**
* 根据关键词修改小说评级
*
* @param Request $Request
* @param AdminUserModel|null $AdminUserModel
* @return Response
*/
public function updateVideoLevel(Request $Request, ?AdminUserModel $AdminUserModel)
{
$arrData = [
"keywords" => $Request->post('keywords'),
"v_level" => (int)$Request->post('v_level'),
];
$arrRule = [
'keywords' => 'require',
'v_level' => 'require',
];
$this->validate($arrData, $arrRule);
$arrKeywords = explode(',', $arrData['keywords']);
$VideoModel = VideoModel::getInstance();
$VideoModel->updateByKeywords($arrKeywords, $arrData['v_level']);
return $this->success();
}
/**
* 小说分页
*
* @param Request $Request
* @param AdminUserModel|null $AdminUserModel
* @return Response
*/
public function getVideoList(Request $Request, ?AdminUserModel $AdminUserModel)
{
$intPage = $Request->param('page', 1, 'intval');
$intLimit = $Request->param('limit', 10, 'intval');
try {
$VideoModel = VideoModel::getInstance();
$Col = $VideoModel->getCol();
// 查询条件（可选）
$arrFilter = [];
// 示例：如果需要筛选，例如 status=1
// $arrFilter['status'] = 1;
$arrOptions = [
'sort' => ['_id' => -1],
'skip' => ($intPage - 1) * $intLimit,
'limit' => $intLimit,
];
$Cursor = $Col->find($arrFilter, $arrOptions);
$arrData = iterator_to_array($Cursor);
$intTotal = $Col->countDocuments($arrFilter);
$arrResult = [
'data' => $arrData,
'total' => $intTotal,
'page' => $intPage,
'limit' => $intLimit,
'pages' => ceil($intTotal / $intLimit)
];
return $this->success($arrResult);
} catch (\Exception $e) {
return $this->error("9999", $e->getMessage());
}
}
/**
* get category data
*
* @param Request $Request
* @param AdminUserModel|null $AdminUserModel
* @return Response
*/
public function getCategory(Request $Request, ?AdminUserModel $AdminUserModel)
{
$arrResult = VideoCategoryModel::$arrCategory;
return $this->success($arrResult);
}
}

8
code/app/task/crawler/fengchao/page/VideoInfoPage.php
View File

@@ -87,9 +87,11 @@ class VideoInfoPage extends BasePage
$arrVideo['vod_class'] = trim($arrVideo['vod_class']);
$arrVideo['vod_play_url'][$this->getsite()->getSiteCode()] = $arrPlayUrl;
$arrVideo['vod_actor'] = explode(',', $arrVideo['vod_actor']);
$arrVideo['vod_director'] = explode(',', $arrVideo['vod_director']);
$arrVideo['vod_class'] = explode(',', $arrVideo['vod_class']);
$arrVideo['vod_actor'] = $arrVideo['vod_actor'] === '' ? [] : explode(',', $arrVideo['vod_actor']);
$arrVideo['vod_director'] = $arrVideo['vod_director'] === '' ? [] : explode(',', $arrVideo['vod_director']);
$arrVideo['vod_class'] = $arrVideo['vod_class'] === '' ? [] : explode(',', $arrVideo['vod_class']);
$arrVideo['vod_content'] = trim($arrVideo['vod_content']);
$arrVideo['vod_content'] = mb_rtrim($arrVideo['vod_content'], '\　');

196
code/app/task/crawler/wuxian/Scheduler.php Normal file
View File

@@ -0,0 +1,196 @@
<?php
declare(strict_types=1);
namespace app\task\crawler\wuxian;
use app\model\ChapterModel;
use app\task\crawler\wuxian\page\Site;
use microserver\QueueManage;
use storage\StorageCore;
/**
* @mixin think\Model
*/
class Scheduler
{
protected $intSlice = 8;
protected $intStartNSourceId = 1;
protected $intEndNSourceId = 500000;
/**
* Undocumented variable
*
* @var \microserver\QueueManage
*/
protected $QueueManage = NULL;
/**
* Undocumented variable
*
* @var Site
*/
public $Site;
/**
* Undocumented function
*/
public function __construct()
{
$this->QueueManage = new QueueManage();
$this->init();
}
/**
* Undocumented function
*
* @return void
*/
public function init()
{
$this->intSlice = 8;
}
/**
* get site
*
* @return Site
*/
public function getSite()
{
if ($this->Site == NULL) {
$this->Site = new Site;
}
return $this->Site;
}
/**
* create a site-wide craw task
*
* @return void
*/
public function crawlFull()
{
// $arrVSourceId = [['v_source_id' => 14256]];
// $arrTask = [
// 'callback' => [static::class, 'fetchVideo'],
// 'data' => $arrVSourceId,
// ];
// $this->QueueManage->set($arrTask);
// return;
$Site = $this->getSite();
$arrAllCategory = $Site->getCategory();
// print_r($arrAllCategory);
// return;
$intLimit = 5;
$arrTask = [
'callback' => [static::class, 'fetchVSourceId'],
'data' => [],
];
foreach ($arrAllCategory as $arrCategory) {
if ($arrCategory['total'] <= 0) {
continue;
}
$intTotalPage = $arrCategory['total_page'];
$arrFilter = [];
for ($intStartPage = 1; $intStartPage <= $intTotalPage; $intStartPage++) {
$arrFilter[] = [
'v_category_id' => $arrCategory['v_category_id'],
'page' => $intStartPage,
];
if (($intStartPage % $intLimit) == 0) {
$arrTask['data'] = $arrFilter;
$this->QueueManage->set($arrTask);
$arrFilter = [];
}
}
if (!empty($arrFilter)) {
$arrTask['data'] = $arrFilter;
$this->QueueManage->set($arrTask);
}
}
}
/**
* Undocumented function
*
* @param array $arrFilter
* @return void
*/
public function fetchVSourceId(array $arrFilter)
{
$Site = $this->getSite();
$intLimit = 5;
$arrTask = [
'callback' => [static::class, 'fetchVideo'],
'data' => [],
];
$arrAllVideoPagerPage = $Site->getVideoPagerPageList($arrFilter);
$arrVSourceId = [];
foreach ($arrAllVideoPagerPage as $VideoPagerPage) {
$arrVideoPager = $VideoPagerPage->getVideoPager();
foreach ($arrVideoPager['list'] as $arrVideoPager) {
$arrVSourceId[] = ['v_source_id' => (int)$arrVideoPager['vod_id']];
if ((count($arrVSourceId) % $intLimit) == 0) {
$arrTask['data'] = $arrVSourceId;
$this->QueueManage->set($arrTask);
$arrVSourceId = [];
}
}
}
if (!empty($arrVSourceId)) {
$arrTask['data'] = $arrVSourceId;
$this->QueueManage->set($arrTask);
}
}
/**
* Undocumented function
*
* @param array $arrVSourceId
* @return void
*/
public function fetchVideo(array $arrVSourceId)
{
$Site = $this->getSite();
$arrAllVideoInfoPage = $Site->getVideoInfoPageList($arrVSourceId);
$arrPic = [];
foreach ($arrAllVideoInfoPage as $VideoInfoPage) {
$VideoInfoPage->saveVideo();
$arrPicItem = $VideoInfoPage->getDownPicInfo();
if (!StorageCore::getInstance()->has($arrPicItem['local_path'])) {
$arrPic[] = $arrPicItem;
}
}
if ($arrPic) {
$Site->downRemoteImgToLocal($arrPic);
}
}
}

251
code/app/task/crawler/wuxian/page/BasePage.php Normal file
View File

@@ -0,0 +1,251 @@
<?php
declare(strict_types=1);
namespace app\task\crawler\wuxian\page;
use db\mongo\MongoBase;
use ddl\DDLManage;
use GuzzleHttp\Client;
use microserver\QueueManage;
use QL\QueryList;
use Psr\Http\Message\ResponseInterface;
/**
* @mixin think\Model
*/
abstract class BasePage
{
/**
* Undocumented variable
*
* @var string
*/
// protected $strDomain;
/**
* Undocumented variable
*
* @var string
*/
protected $strUri;
/**
* Undocumented variable
*
* @var MongoBase
*/
protected $MongoBase;
/**
* Undocumented variable
*
* @var \GuzzleHttp\Client
*/
protected $Client = NULL;
/**
* Undocumented variable
*
* @var \microserver\QueueManage
*/
protected $QueueManage = NULL;
/**
* Undocumented variable
*
* @var array
*/
protected $arrCaiJiPeiZhi = [];
/**
* Undocumented variable
*
* @var string
*/
protected $strContent;
/**
* Undocumented variable
*
* @var QueryList
*/
protected $QueryList;
/**
* Undocumented variable
*
* @var array
*/
protected $arrData = [];
/**
* Undocumented variable
*
* @var bool
*/
protected $boolIsMobile;
/**
* Undocumented variable
*
* @var ResponseInterface
*/
private $Response;
/**
* Undocumented function
*
* @param string $strUri
*/
public function __construct($Site, string $strUri)
{
$this->setSite($Site);
$this->strUri = $strUri;
}
/**
* Undocumented function
*
* @return string
*/
public function getUri(): string
{
return $this->strUri;
}
/**
* Undocumented function
*
* @param array $arrData
* @return void
*/
public function setData(array $arrData)
{
$this->arrData = $arrData;
}
/**
* Undocumented function
*
* @return array
*/
public function getData(): array
{
return $this->arrData;
}
/**
* Undocumented function
*
* @return QueueManage
*/
public function getQueueManage(): QueueManage
{
if ($this->QueueManage == NULL) {
$this->QueueManage = QueueManage::getInstance();
}
return $this->QueueManage;
}
/**
* Undocumented function
*
* @return MongoBase
*/
public function getMongoBase(): MongoBase
{
if ($this->MongoBase == NULL) {
DDLManage::load('MongoBase');
$this->MongoBase = MongoBase::getInstance();
}
return $this->MongoBase;
}
public function getQueryList(): QueryList
{
if ($this->QueryList == NULL) {
$this->QueryList = QueryList::html($this->getContent());
}
return $this->QueryList;
}
/**
* Undocumented function
*
* @return string
*/
public function getContent(): string
{
if ($this->strContent == NULL) {
$this->Response = $this->getClient()->get($this->strUri);
$this->strContent = $this->Response->getBody()->getContents();
}
return $this->strContent;
}
/**
* force set page content
*
* @param string $strContent
* @return void
*/
public function setContent($strContent)
{
$this->strContent = $strContent;
}
/**
* Undocumented function
*
* @return boolean
*/
public function isMobile($strMark = 'content="mobile"'): bool
{
if ($this->boolIsMobile === NULL) {
$this->boolIsMobile = is_numeric(strpos($this->getContent(), $strMark));
}
return $this->boolIsMobile;
}
/**
* Undocumented function
*
* @return Client
*/
protected function getClient(): Client
{
if ($this->Client == NULL) {
$this->Client = $this->getSite()->getClient();
}
return $this->Client;
}
/**
* Undocumented variable
*
* @var Site
*/
protected $Site;
public function setSite(Site $Site)
{
$this->Site = $Site;
}
public function getSite()
{
return $this->Site;
}
}

359
code/app/task/crawler/wuxian/page/Site.php Normal file
View File

@@ -0,0 +1,359 @@
<?php
declare(strict_types=1);
namespace app\task\crawler\wuxian\page;
use GuzzleHttp\Client;
use GuzzleHttp\Promise;
use storage\StorageCore;
/**
* @mixin think\Model
*/
class Site
{
/**
* Undocumented variable
*
* @var Client
*/
protected $Client;
/**
* Undocumented variable
*
* @var string
*/
protected $strDomain = 'https://api.wuxianzy.net';
/**
* Undocumented variable
*
* @var string
*/
protected $strSiteCode = 'wuxian';
/**
* use proxy
*
* @var boolean
*/
protected $boolProxy = false;
/**
* Undocumented function
*/
public function __construct()
{
$this->initClient();
}
/**
* Undocumented function
*/
public function initClient()
{
$arrConfig = [
'http_version' => '1.1', // 强制用 HTTP/1.1，已有，保持
'base_uri' => $this->strDomain, // 基础域名，已有，保持
'http_errors' => false, // 不抛 HTTP 错误，已有，保持
'timeout' => 60, // 从 160 秒改为 60 秒
'connect_timeout' => 10, // 从 30 秒改为 10 秒
'curl' => [
CURLOPT_TCP_KEEPALIVE => 1, // 启用 TCP Keep-Alive，已有，保持
CURLOPT_TCP_KEEPIDLE => 120, // Keep-Alive 空闲时间
CURLOPT_TCP_KEEPINTVL => 60, // Keep-Alive 探测间隔
// CURLOPT_SSL_VERIFYPEER => true, // 验证 SSL，已有，保持
],
'headers' => [
'User-Agent' => 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36',
'Referer' => $this->strDomain,
'Connection' => 'keep-alive',
],
'pool_size' => 20,
'verify' => false,
];
$arrProxy = config('proxy');
if ($this->boolProxy && $arrProxy['proxy_status']) {
$strCode = $arrProxy['proxy_code'];
$strProxyHost = $arrProxy['proxy_host'];
$strKey = $arrProxy['proxy_key'];
$strSecret = $arrProxy['proxy_secret'];
switch ($strCode) {
case 'XIONGMAO':
$intTime = time();
$strTxt = "orderno=" . $strKey . ",secret=" . $strSecret . ",timestamp=" . $intTime;
$strSign = strtoupper(md5($strTxt));
$strAuth = 'sign=' . $strSign . '&orderno=' . $strKey . '&timestamp=' . $intTime;
$arrConfig['proxy'] = $strProxyHost;
$arrConfig['headers']['Proxy-Authorization'] = $strAuth;
break;
case 'KUAI':
$arrConfig['proxy'] = $strProxyHost;
$arrConfig['curl'][CURLOPT_PROXYAUTH] = CURLAUTH_BASIC;
$arrConfig['curl'][CURLOPT_PROXYUSERPWD] = "$strKey:$strSecret";
break;
}
}
$this->Client = new Client($arrConfig);
}
/**
* Undocumented function
*
* @return string
*/
public function getSiteCode(): string
{
return $this->strSiteCode;
}
/**
* Undocumented function
*
* @param integer|string $strSign
* @return string
*/
public function getSiteUUId(int|string $strSign): string
{
return $this->getSiteCode() . '-' . $strSign;
}
/**
* Undocumented function
*
* @return Client
*/
public function getClient(): Client
{
return $this->Client;
}
/**
* retrieve multiple novel page model
*
* @param string $strUri
* @return array
*/
public function getCategory(): array
{
$strUri = '/api.php/provide/vod/?ac=list';
$Response = $this->getClient()->get($strUri);
$strContent = $Response->getBody()->getContents();
$arrContent = json_decode($strContent, true);
$arrFilter = $arrContent['class'];
$arrCategory = [];
foreach ($arrFilter as $intKey => $arrItem) {
$arrFilter[$intKey]['uri'] = $this->getVideoPageUri((int)$arrItem['type_id'], 1);
$arrPromises[$intKey] = $this->getClient()->getAsync($arrFilter[$intKey]['uri']);
}
$arrResults = Promise\Utils::settle($arrPromises)->wait();
$arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
return $arrResult['state'] === 'fulfilled';
});
foreach ($arrSuccessfulResults as $intKey => $arrResult) {
$Response = $arrResult['value'];
$strContent = $Response->getBody()->getContents();
$arrTemp = json_decode($strContent, true);
$arrCategory[] = [
'v_category_id' => (int)$arrFilter[$intKey]['type_id'],
'v_category_name' => $arrFilter[$intKey]['type_name'],
'v_category_name_en' => zhToPinYin($arrFilter[$intKey]['type_name']),
'total' => $arrTemp['total'],
'total_page' => $arrTemp['pagecount'],
];
}
return $arrCategory;
}
/**
* generate video list uri
*
* @param integer $intVCategoryId
* @param integer $intPage
* @return string
*/
public function getVideoPageUri(int $intVCategoryId, int $intPage): string
{
return sprintf('/api.php/provide/vod/?ac=list&t=%s&pg=%s', $intVCategoryId, $intPage);
}
/**
* retrieve multiple novel page model
*
* @param string $strUri
* @return VideoPagerPage[]
*/
public function getVideoPagerPageList(array $arrFilter): array
{
/** @var VideoPagerPage[] */
$arrVideoPagerPage = [];
foreach ($arrFilter as $intKey => $arrItem) {
$arrFilter[$intKey]['uri'] = $this->getVideoPageUri($arrItem['v_category_id'], $arrItem['page']);
$arrPromises[$intKey] = $this->getClient()->getAsync($arrFilter[$intKey]['uri']);
}
$arrResults = Promise\Utils::settle($arrPromises)->wait();
$arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
return $arrResult['state'] === 'fulfilled';
});
foreach ($arrSuccessfulResults as $intKey => $arrResult) {
$Response = $arrResult['value'];
$strContent = $Response->getBody()->getContents();
$VideoPagerPage = $this->getVideoPagerPage($this, $arrFilter[$intKey]['uri']);
$VideoPagerPage->setContent($strContent);
$VideoPagerPage->setData($arrFilter[$intKey]);
$arrVideoPagerPage[] = $VideoPagerPage;
}
return $arrVideoPagerPage;
}
/**
* generate video list uri
*
* @param integer $intVCategoryId
* @param integer $intPage
* @return string
*/
public function getVideoUri(int $intVSourceId): string
{
return sprintf('/api.php/provide/vod/at/json?ac=detail&ids=%s', $intVSourceId);
}
/**
* retrieve multiple chapter page model
*
* @param string $strUri
* @return VideoInfoPage[]
*/
public function getVideoInfoPageList(array $arrFilter): array
{
/** @var VideoInfoPage[] */
$arrVideoInfoPage = [];
$arrPromises = [];
foreach ($arrFilter as $intKey => $arrItem) {
$arrFilter[$intKey]['uri'] = $this->getVideoUri($arrItem['v_source_id']);
$arrPromises[$intKey] = $this->getClient()->getAsync($arrFilter[$intKey]['uri']);
}
$arrResults = Promise\Utils::settle($arrPromises)->wait();
$arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
return $arrResult['state'] === 'fulfilled';
});
foreach ($arrSuccessfulResults as $intKey => $arrResult) {
$Response = $arrResult['value'];
$strContent = $Response->getBody()->getContents();;
$VideoInfoPage = $this->getVideoInfoPage($this, $arrFilter[$intKey]['uri']);
$VideoInfoPage->setContent($strContent);
$VideoInfoPage->setData($arrFilter[$intKey]);
$arrVideoInfoPage[] = $VideoInfoPage;
}
return $arrVideoInfoPage;
}
/**
* download the image to the local system
*
* @param array $arrFilter
* @return void
*/
public function downRemoteImgToLocal(array $arrFilter)
{
$StorageCore = StorageCore::getInstance();
$Client = new Client();
foreach ($arrFilter as $arrItem) {
if (!$StorageCore->has($arrItem['local_path'])) {
try {
$Response = $Client->get($arrItem['uri']);
if ($Response->getStatusCode() === 200) {
if (str_ends_with(strtolower($arrItem['local_path']), '.webp')) {
$strTmpFile = str_replace('.webp', '.jpg', $arrItem['local_path']);
$StorageCore->writeStream($strTmpFile, $Response->getBody()->detach());
convertToWebP(
$StorageCore->getRealPath($strTmpFile),
$StorageCore->getRealPath($arrItem['local_path']),
40
);
@unlink($StorageCore->getRealPath($strTmpFile));
} else {
$StorageCore->writeStream($arrItem['local_path'], $Response->getBody()->detach());
}
}
} catch (\Exception $e) {
// echo $e->getMessage();
}
}
}
}
/**
* Undocumented function
*
* @param object $Site
* @param string $strUri
* @return VideoPagerPage
*/
public function getVideoPagerPage($Site, $strUri)
{
return new VideoPagerPage($Site, $strUri);
}
/**
* Undocumented function
*
* @param object $Site
* @param string $strUri
* @return VideoInfoPage
*/
public function getVideoInfoPage($Site, $strUri)
{
return new VideoInfoPage($Site, $strUri);
}
}

232
code/app/task/crawler/wuxian/page/VideoInfoPage.php Normal file
View File

@@ -0,0 +1,232 @@
<?php
declare(strict_types=1);
namespace app\task\crawler\wuxian\page;
use app\model\CountersModel;
use app\model\VideoModel;
use storage\StorageCore;
class VideoInfoPage extends BasePage
{
/**
* Undocumented variable
*
* @var string
*/
protected $strVideoContent = '';
/**
* Undocumented variable
*
* @var VideoModel
*/
protected $VideoModel;
/**
* Undocumented variable
*
* @var array
*/
protected $arrVideo;
/**
* Undocumented variable
*
* @var array
*/
public $arrOptions = [
'typeMap' => [
'root' => 'array',
'document' => 'array',
'array' => 'array',
],
];
/**
* Undocumented function
*
* @return VideoModel
*/
public function getVideoModel(): VideoModel
{
if ($this->VideoModel == NULL) {
$this->VideoModel = VideoModel::getInstance();
}
return $this->VideoModel;
}
/**
* Undocumented function
*
* @return array
*/
public function getVideo(): array
{
if (empty($this->arrVideo)) {
$arrVideo = json_decode($this->getContent(), true);
$arrVideo = $arrVideo['list'][0];
// print_r($arrVideo);
$strPlayUrl = $arrVideo['vod_play_url'];
unset($arrVideo['vod_play_url']);
$arrPlayUrl = str_replace('#', '$', $strPlayUrl);
$arrPlayUrl = explode('$', $arrPlayUrl);
$arrPlayUrl = array_chunk($arrPlayUrl, 2);
foreach ($arrPlayUrl as $intKey => $arrVal) {
$arrPlayUrl[$intKey] = [
'name' => $arrVal[0],
'url' => $arrVal[1] ?? $arrVal[0],
];
}
$arrVideo['vod_class'] = trim($arrVideo['vod_class']);
$arrVideo['vod_play_url'][$this->getsite()->getSiteCode()] = $arrPlayUrl;
$arrVideo['vod_actor'] = $arrVideo['vod_actor'] === '' ? [] : explode(',', $arrVideo['vod_actor']);
$arrVideo['vod_director'] = $arrVideo['vod_director'] === '' ? [] : explode(',', $arrVideo['vod_director']);
$arrVideo['vod_class'] = $arrVideo['vod_class'] === '' ? [] : explode(',', $arrVideo['vod_class']);
$arrVideo['vod_content'] = trim($arrVideo['vod_content']);
$arrVideo['vod_content'] = mb_rtrim($arrVideo['vod_content'], '\　');
$arrVideo['vod_content'] = ensureUtf8($arrVideo['vod_content']);
foreach ($arrVideo as $strKey => $mixedVal) {
$strKey = str_replace('vod', 'v', $strKey);
if ($strKey == 'type_id') {
$strKey = 'v_category_id';
}
if ($strKey == 'type_name') {
$strKey = 'v_category';
}
if ($strKey == 'v_content') {
$strKey = 'v_description';
}
if ($strKey == 'v_enname') {
$strKey = 'v_name_en';
}
if ($strKey == 'v_time') {
$strKey = 'v_publish_date';
}
$this->arrVideo[$strKey] = $mixedVal;
}
}
return $this->arrVideo;
}
/**
* Undocumented function
*
* @return boolean
*/
public function saveVideo(): bool
{
$arrPageVideo = $this->getVideo();
if (
empty($arrPageVideo) ||
empty($arrPageVideo['v_name']) ||
empty($arrPageVideo['v_category']) ||
empty($arrPageVideo['v_play_url']) ||
empty($arrPageVideo['v_play_url']['wuxian'])
) {
return false;
}
$arrExistsVideo = $this->getVideoModel()->findOne(['v_name' => $arrPageVideo['v_name']], $this->arrOptions);
if ($arrExistsVideo) {
$arrExistsVideo = (array) $arrExistsVideo;
$arrUpdate = [
'updated_at' => new \MongoDB\BSON\UTCDateTime(),
'v_play_url.wuxian' => $arrPageVideo['v_play_url']['wuxian'],
];
$this->getVideoModel()->updateOne(
['v_name' => $arrPageVideo['v_name']],
['$set' => $arrUpdate],
);
$this->strPicLocalPath = $arrExistsVideo['v_pic'];
} else {
$arrInsertVideo = [
'v_id' => CountersModel::getInstance()->getNextSequence('video'),
'v_name' => $arrPageVideo['v_name'],
'v_name_en' => zhToPinYin($arrPageVideo['v_name']),
'v_category' => $arrPageVideo['v_category'],
'v_category_en' => zhToPinYin($arrPageVideo['v_category']),
'v_actor' => $arrPageVideo['v_actor'],
'v_director' => $arrPageVideo['v_director'],
'v_class' => $arrPageVideo['v_class'],
'v_letter' => $arrPageVideo['v_letter'],
'v_pic' => '',
'v_lang' => $arrPageVideo['v_lang'],
'v_area' => $arrPageVideo['v_area'],
'v_year' => $arrPageVideo['v_year'],
'v_remarks' => $arrPageVideo['v_remarks'],
'v_isend' => $arrPageVideo['v_isend'],
'v_score' => $arrPageVideo['v_score'],
'v_description' => $arrPageVideo['v_description'],
'v_play_url' => $arrPageVideo['v_play_url'],
'v_publish_date' => $arrPageVideo['v_publish_date'],
'created_at' => new \MongoDB\BSON\UTCDateTime(),
];
$arrInsertVideo['v_pic'] = $this->generateVideoPath($arrInsertVideo['v_name_en']);
$this->strPicLocalPath = $arrInsertVideo['v_pic'];
try {
$this->getVideoModel()->insert($arrInsertVideo);
} catch (\Exception $e) {
if ($e->getCode() === 11000) {
return $this->saveVideo();
} else {
throw $e;
}
}
}
return true;
}
private $strPicLocalPath = '';
/**
* Undocumented function
*
* @param string $strName
* @return string
*/
public function generateVideoPath($strName): string
{
$intTime = time() - mt_rand(1, 2 * 24 * 3600 * 360);
return sprintf('/img/video/%s/%s.webp', date('Y/m/d/H', $intTime), $strName);
}
/**
* Undocumented function
*
* @return array
*/
public function getDownPicInfo(): array
{
return [
'uri' => $this->getVideo()['v_pic'],
'local_path' => $this->strPicLocalPath,
];
}
}

43
code/app/task/crawler/wuxian/page/VideoPagerPage.php Normal file
View File

@@ -0,0 +1,43 @@
<?php
declare(strict_types=1);
namespace app\task\crawler\wuxian\page;
use app\model\CategoryModel;
use app\model\CountersModel;
use app\model\NovelModel;
use storage\StorageCore;
class VideoPagerPage extends BasePage
{
/**
* Undocumented variable
*
* @var array
*/
protected $arrVideo = [];
public $arrOptions = [
'typeMap' => [
'root' => 'array',
'document' => 'array',
'array' => 'array',
],
];
/**
* Undocumented function
*
* @return array
*/
public function getVideoPager(): array
{
if (empty($this->arrVideo)) {
$this->arrVideo = json_decode($this->getContent(), true);
}
return $this->arrVideo;
}
}

8
code/app/task/crawler/youzhi/page/VideoInfoPage.php
View File

@@ -87,9 +87,11 @@ class VideoInfoPage extends BasePage
$arrVideo['vod_class'] = trim($arrVideo['vod_class']);
$arrVideo['vod_play_url'][$this->getsite()->getSiteCode()] = $arrPlayUrl;
$arrVideo['vod_actor'] = explode(',', $arrVideo['vod_actor']);
$arrVideo['vod_director'] = explode(',', $arrVideo['vod_director']);
$arrVideo['vod_class'] = explode(',', $arrVideo['vod_class']);
$arrVideo['vod_actor'] = $arrVideo['vod_actor'] === '' ? [] : explode(',', $arrVideo['vod_actor']);
$arrVideo['vod_director'] = $arrVideo['vod_director'] === '' ? [] : explode(',', $arrVideo['vod_director']);
$arrVideo['vod_class'] = $arrVideo['vod_class'] === '' ? [] : explode(',', $arrVideo['vod_class']);
$arrVideo['vod_content'] = trim($arrVideo['vod_content']);
$arrVideo['vod_content'] = mb_rtrim($arrVideo['vod_content'], '\　');

23
code/app/task/module/PlanTask.php
View File

@@ -13,6 +13,7 @@ use app\task\crawler\zw630\Scheduler as Zw630Scheduler;
use app\task\crawler\zw74\Scheduler as Zw74Scheduler;
use app\task\crawler\youzhi\Scheduler as YouZhiScheduler;
use app\task\crawler\fengchao\Scheduler as FengChaoScheduler;
use app\task\crawler\wuxian\Scheduler as WuXianScheduler;
@@ -54,6 +55,11 @@ class PlanTask
self::pullYouZhiShiPin();
break;
case 'PULL_WU_XIAN_SHI_PIN':
self::pullWuXianShiPin();
break;
case 'PULL_FENG_CHAO_SHI_PIN':
self::pullFengChaoShiPin();
break;
@@ -65,6 +71,23 @@ class PlanTask
}
}
/**
* Undocumented function
*
* @return void
*/
public static function pullWuXianShiPin()
{
$arrTask = [
'callback' => [WuXianScheduler::class, 'crawlFull'],
'data' => []
];
$QueueManage = new QueueManage();
$QueueManage->set($arrTask);
}
/**
* Undocumented function
*

4
code/database/mongo/migrations/video.php
View File

@@ -43,6 +43,10 @@ return [
'key' => ['v_director' => 1],
'options' => ['unique' => false]
],
[
'key' => ['v_level' => 1],
'options' => ['unique' => false]
],
]
];

7
code/database/seeders/PlanTaskSeeder.php
View File

@@ -60,6 +60,13 @@ class PlanTaskSeeder
'pt_limit' => 1 * 24 * 3600,
],
[
'pt_name' => '无限资源库全量采集',
'pt_code' => 'PULL_WU_XIAN_SHI_PIN',
'pt_enable' => 0,
'pt_limit' => 1 * 24 * 3600,
],
[
'pt_name' => '生成网站地图',
'pt_code' => 'GENERATE_SITE_MAP',