debug
This commit is contained in:
make
122
code/app/task/crawler/youzhi/Scheduler.php
Normal file
122
code/app/task/crawler/youzhi/Scheduler.php
Normal file
@@ -0,0 +1,122 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace app\task\crawler\youzhi;
|
||||
|
||||
use app\model\ChapterModel;
|
||||
use app\task\crawler\youzhi\page\Site;
|
||||
use microserver\QueueManage;
|
||||
use storage\StorageCore;
|
||||
|
||||
/**
|
||||
* @mixin think\Model
|
||||
*/
|
||||
class Scheduler
|
||||
{
|
||||
protected $intSlice = 8;
|
||||
|
||||
protected $intStartNSourceId = 1;
|
||||
|
||||
protected $intEndNSourceId = 500000;
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var \microserver\QueueManage
|
||||
*/
|
||||
protected $QueueManage = NULL;
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var Site
|
||||
*/
|
||||
public $Site;
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*/
|
||||
public function __construct()
|
||||
{
|
||||
$this->QueueManage = new QueueManage();
|
||||
$this->init();
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
public function init()
|
||||
{
|
||||
$this->intSlice = 8;
|
||||
}
|
||||
|
||||
/**
|
||||
* get site
|
||||
*
|
||||
* @return Site
|
||||
*/
|
||||
public function getSite()
|
||||
{
|
||||
if ($this->Site == NULL) {
|
||||
$this->Site = new Site;
|
||||
}
|
||||
return $this->Site;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* create a site-wide craw task
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
public function crawlFull()
|
||||
{
|
||||
$intLimit = 10;
|
||||
|
||||
$Site = $this->getSite();
|
||||
|
||||
$arrAllCategory = $Site->getCategory();
|
||||
|
||||
foreach ($arrAllCategory as $arrCategory) {
|
||||
$intTotalPage = ceil($arrCategory['total'] / $intLimit);
|
||||
for ($intStartPage = 1; $intStartPage <= $intTotalPage; $intStartPage++) {
|
||||
$arrFilter = [
|
||||
[
|
||||
'v_category_id' => $arrCategory['v_category_id'],
|
||||
'page' => $intStartPage,
|
||||
],
|
||||
];
|
||||
|
||||
$arrVideoPageList = $Site->getVideoPageList($arrFilter);
|
||||
|
||||
foreach ($arrVideoPageList as $VideoPage) {
|
||||
$arrAllVideoPager = $VideoPage->getVideoPager();
|
||||
|
||||
$arrVSourceId = [];
|
||||
foreach ($arrAllVideoPager['list'] as $arrVideoPager) {
|
||||
$arrVSourceId[] = ['v_source_id' => (int)$arrVideoPager['vod_id']];
|
||||
}
|
||||
|
||||
$arrAllVideoInfoPage = $Site->getVideoInfoPageList($arrVSourceId);
|
||||
foreach ($arrAllVideoInfoPage as $VideoInfoPage) {
|
||||
$arrVideo = $VideoInfoPage->getVideo();
|
||||
print_r($arrVideo);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
$arrCover = [];
|
||||
|
||||
|
||||
if ($arrCover) {
|
||||
// $Site->downRemoteImgToLocal($arrCover);
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
}
|
||||
251
code/app/task/crawler/youzhi/page/BasePage.php
Normal file
251
code/app/task/crawler/youzhi/page/BasePage.php
Normal file
@@ -0,0 +1,251 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace app\task\crawler\youzhi\page;
|
||||
|
||||
use db\mongo\MongoBase;
|
||||
use ddl\DDLManage;
|
||||
use GuzzleHttp\Client;
|
||||
use microserver\QueueManage;
|
||||
use QL\QueryList;
|
||||
use Psr\Http\Message\ResponseInterface;
|
||||
|
||||
/**
|
||||
* @mixin think\Model
|
||||
*/
|
||||
abstract class BasePage
|
||||
{
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
// protected $strDomain;
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $strUri;
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var MongoBase
|
||||
*/
|
||||
protected $MongoBase;
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var \GuzzleHttp\Client
|
||||
*/
|
||||
protected $Client = NULL;
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var \microserver\QueueManage
|
||||
*/
|
||||
protected $QueueManage = NULL;
|
||||
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var array
|
||||
*/
|
||||
protected $arrCaiJiPeiZhi = [];
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $strContent;
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var QueryList
|
||||
*/
|
||||
protected $QueryList;
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var array
|
||||
*/
|
||||
protected $arrData = [];
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var bool
|
||||
*/
|
||||
protected $boolIsMobile;
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var ResponseInterface
|
||||
*/
|
||||
private $Response;
|
||||
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @param string $strUri
|
||||
*/
|
||||
public function __construct($Site, string $strUri)
|
||||
{
|
||||
$this->setSite($Site);
|
||||
|
||||
$this->strUri = $strUri;
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return string
|
||||
*/
|
||||
public function getUri(): string
|
||||
{
|
||||
return $this->strUri;
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @param array $arrData
|
||||
* @return void
|
||||
*/
|
||||
public function setData(array $arrData)
|
||||
{
|
||||
$this->arrData = $arrData;
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
public function getData(): array
|
||||
{
|
||||
return $this->arrData;
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return QueueManage
|
||||
*/
|
||||
public function getQueueManage(): QueueManage
|
||||
{
|
||||
if ($this->QueueManage == NULL) {
|
||||
$this->QueueManage = QueueManage::getInstance();
|
||||
}
|
||||
|
||||
return $this->QueueManage;
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return MongoBase
|
||||
*/
|
||||
public function getMongoBase(): MongoBase
|
||||
{
|
||||
if ($this->MongoBase == NULL) {
|
||||
DDLManage::load('MongoBase');
|
||||
$this->MongoBase = MongoBase::getInstance();
|
||||
}
|
||||
|
||||
return $this->MongoBase;
|
||||
}
|
||||
|
||||
public function getQueryList(): QueryList
|
||||
{
|
||||
if ($this->QueryList == NULL) {
|
||||
$this->QueryList = QueryList::html($this->getContent());
|
||||
}
|
||||
|
||||
return $this->QueryList;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return string
|
||||
*/
|
||||
public function getContent(): string
|
||||
{
|
||||
if ($this->strContent == NULL) {
|
||||
$this->Response = $this->getClient()->get($this->strUri);
|
||||
$this->strContent = $this->Response->getBody()->getContents();
|
||||
}
|
||||
|
||||
return $this->strContent;
|
||||
}
|
||||
|
||||
/**
|
||||
* force set page content
|
||||
*
|
||||
* @param string $strContent
|
||||
* @return void
|
||||
*/
|
||||
public function setContent($strContent)
|
||||
{
|
||||
$this->strContent = $strContent;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return boolean
|
||||
*/
|
||||
public function isMobile($strMark = 'content="mobile"'): bool
|
||||
{
|
||||
if ($this->boolIsMobile === NULL) {
|
||||
$this->boolIsMobile = is_numeric(strpos($this->getContent(), $strMark));
|
||||
}
|
||||
return $this->boolIsMobile;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return Client
|
||||
*/
|
||||
protected function getClient(): Client
|
||||
{
|
||||
if ($this->Client == NULL) {
|
||||
$this->Client = $this->getSite()->getClient();
|
||||
}
|
||||
return $this->Client;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var Site
|
||||
*/
|
||||
protected $Site;
|
||||
|
||||
public function setSite(Site $Site)
|
||||
{
|
||||
$this->Site = $Site;
|
||||
}
|
||||
|
||||
public function getSite()
|
||||
{
|
||||
return $this->Site;
|
||||
}
|
||||
}
|
||||
369
code/app/task/crawler/youzhi/page/Site.php
Normal file
369
code/app/task/crawler/youzhi/page/Site.php
Normal file
@@ -0,0 +1,369 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace app\task\crawler\youzhi\page;
|
||||
|
||||
use GuzzleHttp\Client;
|
||||
use GuzzleHttp\Promise;
|
||||
use storage\StorageCore;
|
||||
|
||||
/**
|
||||
* @mixin think\Model
|
||||
*/
|
||||
class Site
|
||||
{
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var Client
|
||||
*/
|
||||
protected $Client;
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $strDomain = 'https://api.yzzy-api.com';
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $strSiteCode = 'youzhi';
|
||||
|
||||
/**
|
||||
* use proxy
|
||||
*
|
||||
* @var boolean
|
||||
*/
|
||||
protected $boolProxy = false;
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*/
|
||||
public function __construct()
|
||||
{
|
||||
$this->initClient();
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*/
|
||||
public function initClient()
|
||||
{
|
||||
$arrConfig = [
|
||||
'http_version' => '1.1', // 强制用 HTTP/1.1，已有，保持
|
||||
'base_uri' => $this->strDomain, // 基础域名，已有，保持
|
||||
'http_errors' => false, // 不抛 HTTP 错误，已有，保持
|
||||
'timeout' => 60, // 从 160 秒改为 60 秒
|
||||
'connect_timeout' => 10, // 从 30 秒改为 10 秒
|
||||
'curl' => [
|
||||
CURLOPT_TCP_KEEPALIVE => 1, // 启用 TCP Keep-Alive，已有，保持
|
||||
CURLOPT_TCP_KEEPIDLE => 120, // Keep-Alive 空闲时间
|
||||
CURLOPT_TCP_KEEPINTVL => 60, // Keep-Alive 探测间隔
|
||||
// CURLOPT_SSL_VERIFYPEER => true, // 验证 SSL，已有，保持
|
||||
],
|
||||
'headers' => [
|
||||
'User-Agent' => 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36',
|
||||
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
|
||||
'Accept-Language' => 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
|
||||
'Referer' => $this->strDomain,
|
||||
'Connection' => 'keep-alive',
|
||||
'sec-ch-ua' => '"Not/A)Brand";v="8", "Chromium";v="130", "Google Chrome";v="130"',
|
||||
'sec-ch-ua-mobile' => '?1',
|
||||
'sec-ch-ua-platform' => '"Android"',
|
||||
'sec-fetch-dest' => 'document',
|
||||
'sec-fetch-mode' => 'navigate',
|
||||
'sec-fetch-site' => 'same-origin',
|
||||
'sec-fetch-user' => '?1',
|
||||
'upgrade-insecure-requests' => '1',
|
||||
],
|
||||
'pool_size' => 20,
|
||||
'verify' => false,
|
||||
];
|
||||
|
||||
$arrProxy = config('proxy');
|
||||
|
||||
|
||||
if ($this->boolProxy && $arrProxy['proxy_status']) {
|
||||
|
||||
$strCode = $arrProxy['proxy_code'];
|
||||
$strProxyHost = $arrProxy['proxy_host'];
|
||||
$strKey = $arrProxy['proxy_key'];
|
||||
$strSecret = $arrProxy['proxy_secret'];
|
||||
|
||||
|
||||
switch ($strCode) {
|
||||
case 'XIONGMAO':
|
||||
$intTime = time();
|
||||
$strTxt = "orderno=" . $strKey . ",secret=" . $strSecret . ",timestamp=" . $intTime;
|
||||
$strSign = strtoupper(md5($strTxt));
|
||||
$strAuth = 'sign=' . $strSign . '&orderno=' . $strKey . '×tamp=' . $intTime;
|
||||
$arrConfig['proxy'] = $strProxyHost;
|
||||
$arrConfig['headers']['Proxy-Authorization'] = $strAuth;
|
||||
break;
|
||||
case 'KUAI':
|
||||
$arrConfig['proxy'] = $strProxyHost;
|
||||
$arrConfig['curl'][CURLOPT_PROXYAUTH] = CURLAUTH_BASIC;
|
||||
$arrConfig['curl'][CURLOPT_PROXYUSERPWD] = "$strKey:$strSecret";
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
$this->Client = new Client($arrConfig);
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return string
|
||||
*/
|
||||
public function getSiteCode(): string
|
||||
{
|
||||
return $this->strSiteCode;
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @param integer|string $strSign
|
||||
* @return string
|
||||
*/
|
||||
public function getSiteUUId(int|string $strSign): string
|
||||
{
|
||||
return $this->getSiteCode() . '-' . $strSign;
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return Client
|
||||
*/
|
||||
public function getClient(): Client
|
||||
{
|
||||
return $this->Client;
|
||||
}
|
||||
|
||||
/**
|
||||
* retrieve multiple novel page model
|
||||
*
|
||||
* @param string $strUri
|
||||
* @return VideoPage[]
|
||||
*/
|
||||
public function getCategory(): array
|
||||
{
|
||||
$strUri = '/inc/api_mac10.php?ac=list&t=1&page=1';
|
||||
|
||||
$Response = $this->getClient()->get($strUri);
|
||||
$strContent = $Response->getBody()->getContents();
|
||||
|
||||
$arrContent = json_decode($strContent, true);
|
||||
$arrFilter = $arrContent['class'];
|
||||
|
||||
$arrCategory = [];
|
||||
|
||||
foreach ($arrFilter as $intKey => $arrItem) {
|
||||
|
||||
$arrFilter[$intKey]['uri'] = $this->getVideoPageUri((int)$arrItem['type_id'], 1);
|
||||
|
||||
$arrPromises[$intKey] = $this->getClient()->getAsync($arrFilter[$intKey]['uri']);
|
||||
}
|
||||
|
||||
$arrResults = Promise\Utils::settle($arrPromises)->wait();
|
||||
|
||||
$arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
|
||||
return $arrResult['state'] === 'fulfilled';
|
||||
});
|
||||
|
||||
foreach ($arrSuccessfulResults as $intKey => $arrResult) {
|
||||
$Response = $arrResult['value'];
|
||||
|
||||
$strContent = $Response->getBody()->getContents();
|
||||
|
||||
$arrTemp = json_decode($strContent, true);
|
||||
|
||||
$arrCategory[] = [
|
||||
'v_category_id' => (int)$arrFilter[$intKey]['type_id'],
|
||||
'v_category_name' => $arrFilter[$intKey]['type_name'],
|
||||
'v_category_name_en' => zhToPinYin($arrFilter[$intKey]['type_name']),
|
||||
'total' => $arrTemp['total'],
|
||||
];
|
||||
}
|
||||
|
||||
return $arrCategory;
|
||||
}
|
||||
|
||||
/**
|
||||
* generate video list uri
|
||||
*
|
||||
* @param integer $intVCategoryId
|
||||
* @param integer $intPage
|
||||
* @return string
|
||||
*/
|
||||
public function getVideoPageUri(int $intVCategoryId, int $intPage): string
|
||||
{
|
||||
return sprintf('/inc/api_mac10.php?ac=list&t=%s&page=%s', $intVCategoryId, $intPage);
|
||||
}
|
||||
|
||||
/**
|
||||
* retrieve multiple novel page model
|
||||
*
|
||||
* @param string $strUri
|
||||
* @return VideoPage[]
|
||||
*/
|
||||
public function getVideoPageList(array $arrFilter): array
|
||||
{
|
||||
|
||||
/** @var VideoPage[] */
|
||||
$arrVideoPage = [];
|
||||
|
||||
foreach ($arrFilter as $intKey => $arrItem) {
|
||||
|
||||
$arrFilter[$intKey]['uri'] = $this->getVideoPageUri($arrItem['v_category_id'], $arrItem['page']);
|
||||
|
||||
$arrPromises[$intKey] = $this->getClient()->getAsync($arrFilter[$intKey]['uri']);
|
||||
}
|
||||
|
||||
$arrResults = Promise\Utils::settle($arrPromises)->wait();
|
||||
|
||||
|
||||
$arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
|
||||
return $arrResult['state'] === 'fulfilled';
|
||||
});
|
||||
|
||||
foreach ($arrSuccessfulResults as $intKey => $arrResult) {
|
||||
$Response = $arrResult['value'];
|
||||
|
||||
$strContent = $Response->getBody()->getContents();
|
||||
|
||||
|
||||
$VideoPage = $this->getVideoPage($this, $arrFilter[$intKey]['uri']);
|
||||
|
||||
$VideoPage->setContent($strContent);
|
||||
$VideoPage->setData($arrFilter[$intKey]);
|
||||
|
||||
$arrVideoPage[] = $VideoPage;
|
||||
}
|
||||
|
||||
return $arrVideoPage;
|
||||
}
|
||||
|
||||
/**
|
||||
* generate video list uri
|
||||
*
|
||||
* @param integer $intVCategoryId
|
||||
* @param integer $intPage
|
||||
* @return string
|
||||
*/
|
||||
public function getVideoUri(int $intVSourceId): string
|
||||
{
|
||||
return sprintf('/inc/api_mac10.php?ac=detail&tds=%s', $intVSourceId);
|
||||
}
|
||||
|
||||
/**
|
||||
* retrieve multiple chapter page model
|
||||
*
|
||||
* @param string $strUri
|
||||
* @return VideoInfoPage[]
|
||||
*/
|
||||
public function getVideoInfoPageList(array $arrFilter): array
|
||||
{
|
||||
|
||||
/** @var VideoInfoPage[] */
|
||||
$arrVideoInfoPage = [];
|
||||
|
||||
$arrPromises = [];
|
||||
foreach ($arrFilter as $intKey => $arrItem) {
|
||||
$arrFilter[$intKey]['uri'] = $this->getVideoUri($arrItem['v_source_id']);
|
||||
$arrPromises[$intKey] = $arrFilter[$intKey]['uri'];
|
||||
}
|
||||
|
||||
print_r($arrFilter);
|
||||
|
||||
$arrResults = Promise\Utils::settle($arrPromises)->wait();
|
||||
|
||||
|
||||
var_dump($arrResults);
|
||||
return [];
|
||||
|
||||
$arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
|
||||
return $arrResult['state'] === 'fulfilled';
|
||||
});
|
||||
|
||||
foreach ($arrSuccessfulResults as $intKey => $arrResult) {
|
||||
$Response = $arrResult['value'];
|
||||
|
||||
var_dump($arrResult);
|
||||
|
||||
$strContent = $Response->getBody()->getContents();;
|
||||
|
||||
$VideoInfoPage = $this->getVideoInfoPage($this, $arrFilter[$intKey]['uri']);
|
||||
|
||||
$VideoInfoPage->setContent($strContent);
|
||||
$VideoInfoPage->setData($arrFilter[$intKey]);
|
||||
|
||||
$arrVideoInfoPage[] = $VideoInfoPage;
|
||||
}
|
||||
|
||||
return $arrVideoInfoPage;
|
||||
}
|
||||
|
||||
/**
|
||||
* download the image to the local system
|
||||
*
|
||||
* @param array $arrFilter
|
||||
* @return void
|
||||
*/
|
||||
public function downRemoteImgToLocal(array $arrFilter)
|
||||
{
|
||||
$StorageCore = StorageCore::getInstance();
|
||||
|
||||
$arrPromises = [];
|
||||
|
||||
foreach ($arrFilter as $intKey => $arrItem) {
|
||||
if (!$StorageCore->has($arrItem['local_path'])) {
|
||||
$arrPromises[$intKey] = $this->getClient()->getAsync($arrItem['uri']);
|
||||
}
|
||||
}
|
||||
|
||||
$arrResults = Promise\Utils::settle($arrPromises)->wait();
|
||||
|
||||
$arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
|
||||
return $arrResult['state'] === 'fulfilled';
|
||||
});
|
||||
|
||||
foreach ($arrSuccessfulResults as $intKey => $arrResult) {
|
||||
$Response = $arrResult['value'];
|
||||
|
||||
$StorageCore->writeStream($arrFilter[$intKey]['local_path'], $Response->getBody()->detach());;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @param object $Site
|
||||
* @param string $strUri
|
||||
* @return VideoPage
|
||||
*/
|
||||
public function getVideoPage($Site, $strUri)
|
||||
{
|
||||
return new VideoPage($Site, $strUri);
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @param object $Site
|
||||
* @param string $strUri
|
||||
* @return VideoInfoPage
|
||||
*/
|
||||
public function getVideoInfoPage($Site, $strUri)
|
||||
{
|
||||
return new VideoInfoPage($Site, $strUri);
|
||||
}
|
||||
}
|
||||
194
code/app/task/crawler/youzhi/page/VideoInfoPage.php
Normal file
194
code/app/task/crawler/youzhi/page/VideoInfoPage.php
Normal file
@@ -0,0 +1,194 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace app\task\crawler\youzhi\page;
|
||||
|
||||
use app\model\VideoModel;
|
||||
use storage\StorageCore;
|
||||
|
||||
class VideoInfoPage extends BasePage
|
||||
{
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $strChapterContent = '';
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var VideoModel
|
||||
*/
|
||||
protected $VideoModel;
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return VideoModel
|
||||
*/
|
||||
public function getVideoModel(): VideoModel
|
||||
{
|
||||
if ($this->VideoModel == NULL) {
|
||||
$this->VideoModel = VideoModel::getInstance();
|
||||
}
|
||||
return $this->VideoModel;
|
||||
}
|
||||
|
||||
public $arrOptions = [
|
||||
'typeMap' => [
|
||||
'root' => 'array',
|
||||
'document' => 'array',
|
||||
'array' => 'array',
|
||||
],
|
||||
];
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
public function getVideo(): array
|
||||
{
|
||||
if (empty($this->arrVideo)) {
|
||||
|
||||
$this->arrVideo = json_decode($this->getContent(), true);
|
||||
}
|
||||
|
||||
return $this->arrVideo;
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var array
|
||||
*/
|
||||
protected $arrVideo;
|
||||
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @param integer $intNId
|
||||
* @return boolean
|
||||
*/
|
||||
public function saveChapter(int $intNId): bool
|
||||
{
|
||||
$arrPageChapter = $this->getVideo();
|
||||
|
||||
if (
|
||||
empty($arrPageChapter) ||
|
||||
empty($arrPageChapter['c_page_title']) ||
|
||||
empty($arrPageChapter['c_page_data']) ||
|
||||
count($arrPageChapter['c_page_data']) < 5
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$strUUId = $this->getSite()->getSiteUUId('chapter-' . $this->arrData['c_source_id'] . '-' . $this->arrData['c_page']);
|
||||
|
||||
$arrChapter = [
|
||||
'n_id' => $intNId,
|
||||
'c_name' => $arrPageChapter['c_page_data'][3],
|
||||
'c_sort_num' => $this->arrData['c_sort_num'],
|
||||
'c_page' => $this->arrData['c_page'],
|
||||
'c_source_id' => $this->arrData['c_source_id'],
|
||||
'c_site_code' => $this->getSite()->getSiteCode(),
|
||||
'c_source_uuid' => $strUUId,
|
||||
];
|
||||
|
||||
$arrExistsChapter = $this->getVideoModel()->findOne(['c_source_uuid' => $strUUId], $this->arrOptions);
|
||||
|
||||
if ($arrExistsChapter) {
|
||||
|
||||
$arrExistsChapter = (array) $arrExistsChapter;
|
||||
|
||||
$arrUpdate = [
|
||||
'updated_at' => new \MongoDB\BSON\UTCDateTime(),
|
||||
];
|
||||
|
||||
$this->getVideoModel()->updateOne(
|
||||
['c_source_uuid' => $strUUId],
|
||||
['$set' => $arrUpdate],
|
||||
);
|
||||
$this->strCContentPath = $arrExistsChapter['c_content_path'];
|
||||
} else {
|
||||
|
||||
$arrInsertNovel = $arrChapter;
|
||||
|
||||
$arrInsertNovel['c_content_path'] = '/novel/' . ($intNId % 100) . '/' . $intNId . '/' . $arrChapter['c_sort_num'] . '_' . $arrChapter['c_page'] . '.txt';
|
||||
$arrInsertNovel['created_at'] = new \MongoDB\BSON\UTCDateTime();
|
||||
|
||||
try {
|
||||
$this->getVideoModel()->insert($arrInsertNovel);
|
||||
$this->updateNovelHaveChapter($intNId);
|
||||
$this->strCContentPath = $arrInsertNovel['c_content_path'];
|
||||
} catch (\Exception $e) {
|
||||
if ($e->getCode() === 11000) {
|
||||
return $this->saveChapter($intNId);
|
||||
} else {
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
StorageCore::getInstance()->put($this->strCContentPath, $this->strChapterContent);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
public $strCContentPath;
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @param integer $intNId
|
||||
* @return void
|
||||
*/
|
||||
protected function updateNovelHaveChapter(int $intNId)
|
||||
{
|
||||
$arrFilter = [
|
||||
'n_id' => $intNId,
|
||||
];
|
||||
|
||||
$VideoModel = VideoModel::getInstance();
|
||||
$arrLasteChapter = $VideoModel->getLatestChapterByNId($intNId, false);
|
||||
|
||||
if (!empty($arrLasteChapter)) {
|
||||
$arrUpdate['$set']['n_have_chapter'] = 1;
|
||||
$arrUpdate['$set']['n_latest_chapter_name'] = $arrLasteChapter['c_name'];
|
||||
$this->getMongoBase()->updateOne('novel', $arrFilter, $arrUpdate);
|
||||
}
|
||||
}
|
||||
|
||||
protected $arrNextPage;
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
public function getNextPage(): array
|
||||
{
|
||||
if ($this->arrNextPage === NULL) {
|
||||
$this->arrNextPage = [];
|
||||
|
||||
if ($this->isMobile()) {
|
||||
$strPattern = "/var\s+hhekgsv\s*=\s*['\"](.*?)['\"]/";
|
||||
preg_match_all($strPattern, $this->getContent(), $arrMatches);
|
||||
$strNextPageUri = end($arrMatches[1]) ?: '';
|
||||
} else {
|
||||
$strHref = $this->getQueryList()->find('.read_btn a:contains("下一章")')->attr('href');
|
||||
$strNextPageUri = $strHref ?: '';
|
||||
}
|
||||
|
||||
if (is_numeric(strpos($strNextPageUri, '_'))) {
|
||||
$this->arrNextPage = $this->arrData;
|
||||
$this->arrNextPage['c_page'] += 1;
|
||||
$this->arrNextPage['uri'] = $strNextPageUri;
|
||||
}
|
||||
}
|
||||
|
||||
return $this->arrNextPage;
|
||||
}
|
||||
}
|
||||
417
code/app/task/crawler/youzhi/page/VideoPage.php
Normal file
417
code/app/task/crawler/youzhi/page/VideoPage.php
Normal file
@@ -0,0 +1,417 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace app\task\crawler\youzhi\page;
|
||||
|
||||
use app\model\CategoryModel;
|
||||
use app\model\CountersModel;
|
||||
use app\model\NovelModel;
|
||||
use storage\StorageCore;
|
||||
|
||||
class VideoPage extends BasePage
|
||||
{
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var array
|
||||
*/
|
||||
protected $arrVideo = [];
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var array
|
||||
*/
|
||||
protected $arrRelatedNovel = [];
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var array
|
||||
*/
|
||||
protected $arrForgeNovel = [];
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var NovelModel
|
||||
*/
|
||||
protected $NovelModel;
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return NovelModel
|
||||
*/
|
||||
public function getNovelModel(): NovelModel
|
||||
{
|
||||
if ($this->NovelModel == NULL) {
|
||||
$this->NovelModel = NovelModel::getInstance();
|
||||
}
|
||||
return $this->NovelModel;
|
||||
}
|
||||
|
||||
public $arrOptions = [
|
||||
'typeMap' => [
|
||||
'root' => 'array',
|
||||
'document' => 'array',
|
||||
'array' => 'array',
|
||||
],
|
||||
];
|
||||
|
||||
/**
|
||||
* 查询看的小说
|
||||
*
|
||||
* @param integer $intNSourceId
|
||||
* @param string $strName
|
||||
* @return array
|
||||
*/
|
||||
protected function findOrCreateNovel(int $intNSourceId, string $strName): array
|
||||
{
|
||||
$strUUId = $this->getSite()->getSiteUUId($intNSourceId);
|
||||
|
||||
$arrExistsNovel = $this->getNovelModel()->findOne(['n_source_uuid' => $strUUId], $this->arrOptions);
|
||||
|
||||
if ($arrExistsNovel) {
|
||||
$arrVideo = (array) $arrExistsNovel;
|
||||
} else {
|
||||
$arrVideo = [
|
||||
'n_id' => CountersModel::getInstance()->getNextSequence('novel'),
|
||||
'n_name' => $strName,
|
||||
'n_name_pinyin' => zhToPinYin($strName),
|
||||
'n_site_code' => $this->getSite()->getSiteCode(),
|
||||
'n_source_uuid' => $strUUId,
|
||||
'n_source_id' => $intNSourceId,
|
||||
'n_level' => 1,
|
||||
'n_have_chapter' => 0,
|
||||
'created_at' => new \MongoDB\BSON\UTCDateTime(),
|
||||
];
|
||||
try {
|
||||
$this->getNovelModel()->insert($arrVideo);
|
||||
} catch (\Exception $e) {
|
||||
if ($e->getCode() === 11000) {
|
||||
return $this->findOrCreateNovel($intNSourceId, $strName);
|
||||
} else {
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
}
|
||||
return $arrVideo;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 分析关联数据
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
public function getReletedNovel() {}
|
||||
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
public function getVideoPager(): array
|
||||
{
|
||||
if (empty($this->arrVideo)) {
|
||||
|
||||
$this->arrVideo = json_decode($this->getContent(), true);
|
||||
|
||||
$this->getReletedNovel();
|
||||
}
|
||||
|
||||
return $this->arrVideo;
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @param string $strName
|
||||
* @return string
|
||||
*/
|
||||
public function generateCoverPath($strName): string
|
||||
{
|
||||
$intTime = time() - mt_rand(1, 2 * 24 * 3600 * 360);
|
||||
|
||||
return sprintf('/img/%s/%s.jpg', date('Y/m/d/H', $intTime), $strName);
|
||||
}
|
||||
|
||||
public function filterOtherFields(&$arrNovel)
|
||||
{
|
||||
$arrFields = [
|
||||
'n_id',
|
||||
'n_name',
|
||||
'n_name_pinyin',
|
||||
'n_status',
|
||||
'n_author',
|
||||
'n_description',
|
||||
'n_category',
|
||||
'n_category_pinyin',
|
||||
'n_category_sex_en',
|
||||
'n_category_sex_zh',
|
||||
'n_cover_path',
|
||||
'n_latest_chapter_name',
|
||||
'n_update_time',
|
||||
'n_forge_novel',
|
||||
'n_related_novel',
|
||||
'n_site_code',
|
||||
'n_source_uuid',
|
||||
'n_source_id',
|
||||
'n_have_chapter',
|
||||
'created_at',
|
||||
'updated_at',
|
||||
];
|
||||
|
||||
foreach ($arrNovel as $strKey => $strVal) {
|
||||
if (!in_array($strKey, $arrFields)) {
|
||||
unset($arrNovel[$strKey]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return bool
|
||||
*/
|
||||
public function saveVideo(): bool
|
||||
{
|
||||
$arrPageInfo = $this->getVideo();
|
||||
|
||||
print_r($arrPageInfo);
|
||||
return false;
|
||||
if (
|
||||
empty($arrPageInfo) ||
|
||||
empty($arrPageInfo['og_title']) ||
|
||||
empty($arrPageInfo['og_image']) ||
|
||||
empty($arrPageInfo['og_novel_category'])
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$arrVideo = [
|
||||
// 'n_id'=> $arrPageInfo['xxxxx'],
|
||||
'n_name' => $arrPageInfo['og_title'],
|
||||
'n_name_pinyin' => zhToPinYin($arrPageInfo['og_title']),
|
||||
'n_status' => $arrPageInfo['og_novel_status'],
|
||||
'n_author' => $arrPageInfo['og_novel_author'],
|
||||
'n_description' => $arrPageInfo['og_description'],
|
||||
'n_category' => $arrPageInfo['og_novel_category'],
|
||||
// 'n_category_pinyin' => $arrPageInfo['xxxxx'],
|
||||
// 'n_category_sex_en' => $arrPageInfo['xxxxx'],
|
||||
// 'n_category_sex_zh' => $arrPageInfo['xxxxx'],
|
||||
// 'n_cover_path' => $arrPageInfo['xxxxx'],
|
||||
'n_latest_chapter_name' => $arrPageInfo['og_novel_latest_chapter_name'],
|
||||
'n_update_time' => $arrPageInfo['og_novel_update_time'],
|
||||
'n_forge_novel' => $this->arrForgeNovel,
|
||||
'n_related_novel' => $this->arrRelatedNovel,
|
||||
|
||||
'n_site_code' => $this->getSite()->getSiteCode(),
|
||||
'n_source_id' => $this->arrData['n_source_id'],
|
||||
'n_source_uuid' => $this->getSite()->getSiteUUId($this->arrData['n_source_id']),
|
||||
|
||||
// 'n_have_chapter' => $arrPageInfo['xxxxx'],
|
||||
// 'created_at' => $arrPageInfo['xxxxx'],
|
||||
// 'updated_at' => $arrPageInfo['xxxxx'],
|
||||
|
||||
];
|
||||
|
||||
$arrExistsNovel = $this->getNovelModel()->findOne(['n_source_uuid' => $arrVideo['n_source_uuid']], $this->arrOptions);
|
||||
|
||||
if ($arrExistsNovel) {
|
||||
|
||||
$arrExistsNovel = (array) $arrExistsNovel;
|
||||
|
||||
$arrUpdate = [
|
||||
'n_latest_chapter_name' => $arrVideo['n_latest_chapter_name'],
|
||||
'n_update_time' => $arrVideo['n_update_time'],
|
||||
'updated_at' => new \MongoDB\BSON\UTCDateTime(),
|
||||
];
|
||||
|
||||
# 如果没有分类，说明这条数据是最初从关联shu那里采集的，需要补全全部信息
|
||||
if (empty($arrExistsNovel['n_category'])) {
|
||||
$arrUpdate = array_merge($arrUpdate, $arrVideo);
|
||||
$arrUpdate['n_related_novel'] = $this->arrRelatedNovel;
|
||||
$arrUpdate['n_forge_novel'] = $this->arrForgeNovel;
|
||||
$arrUpdate['n_category_pinyin'] = CategoryModel::getPinYin($this->arrVideo['og_novel_category']);
|
||||
$arrUpdate['n_category_sex_en'] = CategoryModel::getSexEn($this->arrVideo['og_novel_category']);
|
||||
$arrUpdate['n_category_sex_zh'] = CategoryModel::getSexZh($this->arrVideo['og_novel_category']);
|
||||
$arrUpdate['n_related_novel'] = $this->arrRelatedNovel;
|
||||
$arrUpdate['n_forge_novel'] = $this->arrForgeNovel;
|
||||
}
|
||||
|
||||
# 图片不存在的时候，需要重新生成
|
||||
if (
|
||||
!isset($arrExistsNovel['n_cover_path']) ||
|
||||
empty($arrExistsNovel['n_cover_path']) ||
|
||||
!StorageCore::getInstance()->has($arrExistsNovel['n_cover_path']) ||
|
||||
(basename($arrExistsNovel['n_cover_path']) == '.jpg')
|
||||
) {
|
||||
$arrUpdate['n_cover_path'] = $this->generateCoverPath($arrVideo['n_name_pinyin']);
|
||||
$this->strCoverLocalPath = $arrUpdate['n_cover_path'];
|
||||
} else {
|
||||
$this->strCoverLocalPath = $arrExistsNovel['n_cover_path'];
|
||||
}
|
||||
|
||||
$this->getNovelModel()->updateOne(
|
||||
['n_source_uuid' => $arrVideo['n_source_uuid']],
|
||||
['$set' => $arrUpdate],
|
||||
);
|
||||
$this->intNId = $arrExistsNovel['n_id'];
|
||||
} else {
|
||||
|
||||
$arrVideo['n_category_pinyin'] = CategoryModel::getPinYin($this->arrVideo['og_novel_category']);
|
||||
$arrVideo['n_category_sex_en'] = CategoryModel::getSexEn($this->arrVideo['og_novel_category']);
|
||||
$arrVideo['n_category_sex_zh'] = CategoryModel::getSexZh($this->arrVideo['og_novel_category']);
|
||||
$arrVideo['n_id'] = CountersModel::getInstance()->getNextSequence('novel');
|
||||
$arrVideo['n_level'] = 1;
|
||||
$arrVideo['n_have_chapter'] = 0;
|
||||
$arrVideo['n_cover_path'] = $this->generateCoverPath($arrVideo['n_name_pinyin']);
|
||||
$arrVideo['n_related_novel'] = $this->arrRelatedNovel;
|
||||
$arrVideo['n_forge_novel'] = $this->arrForgeNovel;
|
||||
$arrVideo['created_at'] = new \MongoDB\BSON\UTCDateTime();
|
||||
|
||||
try {
|
||||
$this->getNovelModel()->insert($arrVideo);
|
||||
$this->intNId = $arrVideo['n_id'];
|
||||
$this->strCoverLocalPath = $arrVideo['n_cover_path'];
|
||||
} catch (\Exception $e) {
|
||||
if ($e->getCode() === 11000) {
|
||||
return $this->saveVideo();
|
||||
} else {
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
public $strCoverLocalPath = '';
|
||||
|
||||
public $intNId;
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
public function getDownCoverInfo(): array
|
||||
{
|
||||
return [
|
||||
'uri' => $this->arrVideo['og_image'],
|
||||
'local_path' => $this->strCoverLocalPath,
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var array
|
||||
*/
|
||||
protected $arrLatestChapters = NULL;
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var array
|
||||
*/
|
||||
protected $arrChapters = NULL;
|
||||
|
||||
public function getLatestChapterByNIdsList(): array
|
||||
{
|
||||
if ($this->arrLatestChapters === NULL) {
|
||||
$this->arrLatestChapters = [];
|
||||
|
||||
if ($this->isMobile()) {
|
||||
$strClass = '.chapter-list';
|
||||
} else {
|
||||
$strClass = '.section-list';
|
||||
}
|
||||
|
||||
if ($this->arrData['page'] == 0) {
|
||||
$this->getQueryList()->find($strClass)->eq(0)->find("a")->map(function ($A) {
|
||||
$this->arrLatestChapters[] = [
|
||||
'name' => $A->text(),
|
||||
'uri' => $A->attr('href'),
|
||||
];
|
||||
});
|
||||
}
|
||||
}
|
||||
return $this->arrLatestChapters;
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
public function getChapters(): array
|
||||
{
|
||||
if ($this->arrChapters === NULL) {
|
||||
$this->arrChapters = [];
|
||||
if ($this->isMobile()) {
|
||||
$strClass = '.chapter-list';
|
||||
} else {
|
||||
$strClass = '.section-list';
|
||||
}
|
||||
|
||||
$intDivIndex = 0;
|
||||
if ($this->arrData['page'] == 0) {
|
||||
$intDivIndex = 1;
|
||||
}
|
||||
|
||||
$this->getQueryList()->find($strClass)->eq($intDivIndex)->find("a")->map(function ($A) {
|
||||
$this->arrChapters[] = [
|
||||
'name' => $A->text(),
|
||||
'uri' => $A->attr('href'),
|
||||
];
|
||||
});
|
||||
}
|
||||
return $this->arrChapters;
|
||||
}
|
||||
|
||||
|
||||
protected $arrNextPageArgs = NULL;
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
public function getNextPageArgs(): array
|
||||
{
|
||||
$this->arrNextPageArgs = [];
|
||||
if ($this->arrData['page'] == 0) {
|
||||
$this->getLatestChapterByNIdsList();
|
||||
$this->getChapters();
|
||||
|
||||
$arrLatestChapterFirstBlock = count($this->arrLatestChapters) > 0 ? $this->arrLatestChapters[0] : null;
|
||||
$arrLatestChapterSecondBlock = count($this->arrChapters) > 0 ? $this->arrChapters[count($this->arrChapters) - 1] : null;
|
||||
|
||||
if (
|
||||
!empty($arrLatestChapterFirstBlock) && !empty($arrLatestChapterSecondBlock)
|
||||
&& ($arrLatestChapterFirstBlock['uri'] != $arrLatestChapterSecondBlock['uri'])
|
||||
|
||||
) {
|
||||
$this->arrNextPageArgs = [
|
||||
'n_source_id' => $this->arrData['n_source_id'],
|
||||
'page' => $this->arrData['page'] + 1,
|
||||
];
|
||||
}
|
||||
} else {
|
||||
$strLastUri = $this->getQueryList()->find('select:first option:last')->attr('value');
|
||||
if ($strLastUri != '' && $strLastUri != $this->strUri) {
|
||||
$this->arrNextPageArgs = [
|
||||
'n_source_id' => $this->arrData['n_source_id'],
|
||||
'page' => $this->arrData['page'] + 1,
|
||||
];
|
||||
}
|
||||
}
|
||||
return $this->arrNextPageArgs;
|
||||
}
|
||||
}
|
||||
319
code/app/task/crawler/zw/Scheduler.php
Normal file
319
code/app/task/crawler/zw/Scheduler.php
Normal file
@@ -0,0 +1,319 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace app\task\crawler\zw;
|
||||
|
||||
use app\model\ChapterModel;
|
||||
use app\task\crawler\zw\page\Site;
|
||||
use microserver\QueueManage;
|
||||
use storage\StorageCore;
|
||||
|
||||
/**
|
||||
* @mixin think\Model
|
||||
*/
|
||||
class Scheduler
|
||||
{
|
||||
protected $intSlice = 8;
|
||||
|
||||
protected $intStartNSourceId = 1;
|
||||
|
||||
protected $intEndNSourceId = 500000;
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var \microserver\QueueManage
|
||||
*/
|
||||
protected $QueueManage = NULL;
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*/
|
||||
public function __construct()
|
||||
{
|
||||
$this->QueueManage = new QueueManage();
|
||||
$this->init();
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
public function init()
|
||||
{
|
||||
$this->intSlice = 8;
|
||||
}
|
||||
|
||||
/**
|
||||
* create a site-wide craw task
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
public function crawlFull()
|
||||
{
|
||||
$intSliceSize = $this->intSlice;
|
||||
|
||||
$arrFilter = [];
|
||||
for ($intNId = $this->intStartNSourceId; $intNId <= $this->intEndNSourceId; $intNId++) {
|
||||
$arrFilter[] = [
|
||||
'n_source_id' => $intNId,
|
||||
'page' => 0,
|
||||
];
|
||||
|
||||
if ($intNId % $intSliceSize == 0) {
|
||||
$this->fetchNovel($arrFilter);
|
||||
$arrFilter = [];
|
||||
}
|
||||
}
|
||||
if (!empty($arrFilter)) {
|
||||
$this->fetchNovel($arrFilter);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @param array $arrFilter
|
||||
* @return void
|
||||
*/
|
||||
public function fetchNovel(array $arrFilter)
|
||||
{
|
||||
$arrTask = [
|
||||
'callback' => [static::class, 'getNovelInfo'],
|
||||
'data' => $arrFilter,
|
||||
];
|
||||
|
||||
$this->QueueManage->set($arrTask);
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @param array $arrFilter
|
||||
* @return void
|
||||
*/
|
||||
public function getNovelInfo(array $arrFilter)
|
||||
{
|
||||
$Site = $this->getSite();
|
||||
|
||||
$arrNovelPageList = $Site->getNovelPageList($arrFilter);
|
||||
|
||||
$arrCover = [];
|
||||
foreach ($arrNovelPageList as $NovelPage) {
|
||||
// $arrNovel = $NovelPage->getNovelInfo();
|
||||
$boolResult = $NovelPage->saveNovelInfo();
|
||||
// return;
|
||||
|
||||
if ($boolResult == false) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$arrNextPageData = $NovelPage->getNextPageArgs();
|
||||
if (!empty($arrNextPageData)) {
|
||||
$this->fetchNovel([
|
||||
[
|
||||
'n_source_id' => $arrNextPageData['n_source_id'],
|
||||
'page' => $arrNextPageData['page'],
|
||||
]
|
||||
]);
|
||||
}
|
||||
|
||||
$arrCoverItem = $NovelPage->getDownCoverInfo();
|
||||
|
||||
if (!StorageCore::getInstance()->has($arrCoverItem['local_path'])) {
|
||||
$arrCover[] = $arrCoverItem;
|
||||
}
|
||||
|
||||
$arrChapterList = $NovelPage->getChapters();
|
||||
$this->fetchChapter($NovelPage->intNId, $NovelPage->getData()['page'], $arrChapterList);
|
||||
}
|
||||
|
||||
if ($arrCover) {
|
||||
$Site->downRemoteImgToLocal($arrCover);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* create a task to craw the latest chapter
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
public function crawLatest()
|
||||
{
|
||||
$intSliceSize = $this->intSlice;
|
||||
$arrUri = [];
|
||||
$strUri = '/list/lastupdate_%s_0_0_%s.html';
|
||||
|
||||
$intEndCategory = $intEndPage = 10;
|
||||
|
||||
for ($intCategoryKey = 0; $intCategoryKey < $intEndCategory; $intCategoryKey++) {
|
||||
for ($intPage = 1; $intPage <= $intEndPage; $intPage++) {
|
||||
$arrUri[] = sprintf($strUri, $intCategoryKey, $intPage);
|
||||
}
|
||||
}
|
||||
|
||||
$arrUri = array_chunk($arrUri, $intSliceSize);
|
||||
|
||||
$arrTask = [
|
||||
'callback' => [static::class, 'getLatestChapter'],
|
||||
'data' => [],
|
||||
];
|
||||
|
||||
foreach ($arrUri as $arrUriSlice) {
|
||||
$arrTask['data'] = $arrUriSlice;
|
||||
$this->QueueManage->set($arrTask);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @param array $arrData
|
||||
* @return void
|
||||
*/
|
||||
public function getLatestChapter(array $arrFilter)
|
||||
{
|
||||
$Site = $this->getSite();
|
||||
|
||||
$arrLatestListPageList = $Site->getLatestListPageList($arrFilter);
|
||||
|
||||
$arrNSourceId = [];
|
||||
|
||||
foreach ($arrLatestListPageList as $LatestListPage) {
|
||||
$arrResult = $LatestListPage->getNovelFilter();
|
||||
$arrNSourceId = array_merge($arrNSourceId, $arrResult);
|
||||
}
|
||||
|
||||
$arrNSourceId = array_unique($arrNSourceId);
|
||||
$arrNSourceId = array_values($arrNSourceId);
|
||||
|
||||
$intSliceSize = $this->intSlice;
|
||||
$arrFilter = [];
|
||||
|
||||
foreach ($arrNSourceId as $intKey => $intNSourceId) {
|
||||
$arrFilter[] = [
|
||||
'n_source_id' => $intNSourceId,
|
||||
'page' => 0,
|
||||
];
|
||||
|
||||
if ($intKey % $intSliceSize == 0) {
|
||||
$this->fetchNovel($arrFilter);
|
||||
$arrFilter = [];
|
||||
}
|
||||
}
|
||||
|
||||
if (!empty($arrFilter)) {
|
||||
$this->fetchNovel($arrFilter);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @param integer $intNId
|
||||
* @param integer $intPage
|
||||
* @param array $arrChapterList
|
||||
* @return void
|
||||
*/
|
||||
public function fetchChapter(int $intNId, int $intPage, array $arrChapterList = [])
|
||||
{
|
||||
$intSliceSize = $this->intSlice;
|
||||
$intPageSize = 100;
|
||||
|
||||
$arrTask = [
|
||||
'callback' => [static::class, 'getChapterInfo'],
|
||||
'data' => [
|
||||
'n_id' => $intNId,
|
||||
]
|
||||
];
|
||||
$arrFilter = [];
|
||||
foreach ($arrChapterList as $intKey => $arrChapter) {
|
||||
$arrChapter['c_sort_num'] = $intPage * $intPageSize + $intKey + 1;
|
||||
$arrChapter['c_source_id'] = supperExtractFilename($arrChapter['uri']);
|
||||
$arrChapter['c_page'] = 0;
|
||||
|
||||
$arrFilter[] = $arrChapter;
|
||||
if ($intKey % $intSliceSize == 0) {
|
||||
$arrTask['data']['filter'] = $arrFilter;
|
||||
$this->QueueManage->set($arrTask);
|
||||
$arrFilter = [];
|
||||
}
|
||||
}
|
||||
|
||||
if (!empty($arrFilter)) {
|
||||
$arrTask['data']['filter'] = $arrFilter;
|
||||
$this->QueueManage->set($arrTask);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var Site
|
||||
*/
|
||||
public $Site;
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return Site
|
||||
*/
|
||||
public function getSite()
|
||||
{
|
||||
if ($this->Site == NULL) {
|
||||
$this->Site = new Site;
|
||||
}
|
||||
return $this->Site;
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @param array $arrData
|
||||
* @return void
|
||||
*/
|
||||
public function getChapterInfo(array $arrData)
|
||||
{
|
||||
$Site = $this->getSite();
|
||||
|
||||
$ChapterModel = ChapterModel::getInstance();
|
||||
|
||||
foreach ($arrData['filter'] as $intKey => $arrChapterFilter) {
|
||||
$strUUId = sprintf('%s-chapter-%s-%s', $Site->getSiteCode(), $arrChapterFilter['c_source_id'], $arrChapterFilter['c_page']);
|
||||
$arrChapter = $ChapterModel->findOne(['c_source_uuid' => $strUUId]);
|
||||
|
||||
if (!empty($arrChapter) && !empty($arrChapter['content'])) {
|
||||
unset($arrData['filter'][$intKey]);
|
||||
}
|
||||
}
|
||||
|
||||
$arrData['filter'] = array_values($arrData['filter']);
|
||||
|
||||
$arrChapterPageList = $Site->getChapterPageList($arrData['filter']);
|
||||
|
||||
$arrNextPageFilterList = [];
|
||||
foreach ($arrChapterPageList as $ChapterPage) {
|
||||
|
||||
$ChapterPage->saveChapter($arrData['n_id']);
|
||||
|
||||
$arrNextPageFilter = $ChapterPage->getNextPage();
|
||||
|
||||
if (!empty($arrNextPageFilter)) {
|
||||
$arrNextPageFilterList[] = $arrNextPageFilter;
|
||||
}
|
||||
}
|
||||
|
||||
if (!empty($arrNextPageFilterList)) {
|
||||
|
||||
$arrTask = $arrData;
|
||||
$arrTask['filter'] = $arrNextPageFilterList;
|
||||
$arrTask = [
|
||||
'callback' => [static::class, 'getChapterInfo'],
|
||||
'data' => $arrTask
|
||||
];
|
||||
$this->QueueManage->set($arrTask);
|
||||
}
|
||||
}
|
||||
}
|
||||
251
code/app/task/crawler/zw/page/BasePage.php
Normal file
251
code/app/task/crawler/zw/page/BasePage.php
Normal file
@@ -0,0 +1,251 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace app\task\crawler\zw\page;
|
||||
|
||||
use db\mongo\MongoBase;
|
||||
use ddl\DDLManage;
|
||||
use GuzzleHttp\Client;
|
||||
use microserver\QueueManage;
|
||||
use QL\QueryList;
|
||||
use Psr\Http\Message\ResponseInterface;
|
||||
|
||||
/**
|
||||
* @mixin think\Model
|
||||
*/
|
||||
abstract class BasePage
|
||||
{
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
// protected $strDomain;
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $strUri;
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var MongoBase
|
||||
*/
|
||||
protected $MongoBase;
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var \GuzzleHttp\Client
|
||||
*/
|
||||
protected $Client = NULL;
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var \microserver\QueueManage
|
||||
*/
|
||||
protected $QueueManage = NULL;
|
||||
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var array
|
||||
*/
|
||||
protected $arrCaiJiPeiZhi = [];
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $strContent;
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var QueryList
|
||||
*/
|
||||
protected $QueryList;
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var array
|
||||
*/
|
||||
protected $arrData = [];
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var bool
|
||||
*/
|
||||
protected $boolIsMobile;
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var ResponseInterface
|
||||
*/
|
||||
private $Response;
|
||||
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @param string $strUri
|
||||
*/
|
||||
public function __construct($Site, string $strUri)
|
||||
{
|
||||
$this->setSite($Site);
|
||||
|
||||
$this->strUri = $strUri;
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return string
|
||||
*/
|
||||
public function getUri(): string
|
||||
{
|
||||
return $this->strUri;
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @param array $arrData
|
||||
* @return void
|
||||
*/
|
||||
public function setData(array $arrData)
|
||||
{
|
||||
$this->arrData = $arrData;
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
public function getData(): array
|
||||
{
|
||||
return $this->arrData;
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return QueueManage
|
||||
*/
|
||||
public function getQueueManage(): QueueManage
|
||||
{
|
||||
if ($this->QueueManage == NULL) {
|
||||
$this->QueueManage = QueueManage::getInstance();
|
||||
}
|
||||
|
||||
return $this->QueueManage;
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return MongoBase
|
||||
*/
|
||||
public function getMongoBase(): MongoBase
|
||||
{
|
||||
if ($this->MongoBase == NULL) {
|
||||
DDLManage::load('MongoBase');
|
||||
$this->MongoBase = MongoBase::getInstance();
|
||||
}
|
||||
|
||||
return $this->MongoBase;
|
||||
}
|
||||
|
||||
public function getQueryList(): QueryList
|
||||
{
|
||||
if ($this->QueryList == NULL) {
|
||||
$this->QueryList = QueryList::html($this->getContent());
|
||||
}
|
||||
|
||||
return $this->QueryList;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return string
|
||||
*/
|
||||
public function getContent(): string
|
||||
{
|
||||
if ($this->strContent == NULL) {
|
||||
$this->Response = $this->getClient()->get($this->strUri);
|
||||
$this->strContent = $this->Response->getBody()->getContents();
|
||||
}
|
||||
|
||||
return $this->strContent;
|
||||
}
|
||||
|
||||
/**
|
||||
* force set page content
|
||||
*
|
||||
* @param string $strContent
|
||||
* @return void
|
||||
*/
|
||||
public function setContent($strContent)
|
||||
{
|
||||
$this->strContent = $strContent;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return boolean
|
||||
*/
|
||||
public function isMobile($strMark = 'content="mobile"'): bool
|
||||
{
|
||||
if ($this->boolIsMobile === NULL) {
|
||||
$this->boolIsMobile = is_numeric(strpos($this->getContent(), $strMark));
|
||||
}
|
||||
return $this->boolIsMobile;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return Client
|
||||
*/
|
||||
protected function getClient(): Client
|
||||
{
|
||||
if ($this->Client == NULL) {
|
||||
$this->Client = $this->getSite()->getClient();
|
||||
}
|
||||
return $this->Client;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var Site
|
||||
*/
|
||||
protected $Site;
|
||||
|
||||
public function setSite(Site $Site)
|
||||
{
|
||||
$this->Site = $Site;
|
||||
}
|
||||
|
||||
public function getSite()
|
||||
{
|
||||
return $this->Site;
|
||||
}
|
||||
}
|
||||
226
code/app/task/crawler/zw/page/ChapterPage.php
Normal file
226
code/app/task/crawler/zw/page/ChapterPage.php
Normal file
@@ -0,0 +1,226 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace app\task\crawler\zw\page;
|
||||
|
||||
use app\model\ChapterModel;
|
||||
use storage\StorageCore;
|
||||
|
||||
class ChapterPage extends BasePage
|
||||
{
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $strChapterContent = '';
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var ChapterModel
|
||||
*/
|
||||
protected $ChapterModel;
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return ChapterModel
|
||||
*/
|
||||
public function getChapterModel(): ChapterModel
|
||||
{
|
||||
if ($this->ChapterModel == NULL) {
|
||||
$this->ChapterModel = ChapterModel::getInstance();
|
||||
}
|
||||
return $this->ChapterModel;
|
||||
}
|
||||
|
||||
public $arrOptions = [
|
||||
'typeMap' => [
|
||||
'root' => 'array',
|
||||
'document' => 'array',
|
||||
'array' => 'array',
|
||||
],
|
||||
];
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
public function getChapterInfo(): array
|
||||
{
|
||||
if ($this->arrChapterInfo === NULL) {
|
||||
|
||||
$this->arrChapterInfo = [];
|
||||
|
||||
$strPattern = "/qsbs\.bb\('([^']*)'\)/";
|
||||
|
||||
preg_match_all($strPattern, $this->getContent(), $arrMatches);
|
||||
|
||||
$strContent = '';
|
||||
|
||||
foreach ($arrMatches[1] as $str) {
|
||||
$strContent .= base64_decode($str);
|
||||
}
|
||||
|
||||
$this->strChapterContent = $strContent;
|
||||
|
||||
$this->arrChapterInfo['c_page_title'] = $this->getQueryList()->find('title')->text();
|
||||
$this->arrChapterInfo['c_page_keywords'] = $this->getQueryList()->find('meta[name="keywords"]')->attr('content');
|
||||
$this->arrChapterInfo['c_page_description'] = $this->getQueryList()->find('meta[name="description"]')->attr('content');
|
||||
|
||||
$strPattern = "/lastread\.set\((.*?)\);/";
|
||||
|
||||
if (preg_match($strPattern, $this->getContent(), $arrMatches)) {
|
||||
$strMatch = $arrMatches[1]; // 提取括号内的内容
|
||||
|
||||
$arrData = array_map('trim', explode(",", $strMatch));
|
||||
|
||||
$arrData = array_map(function ($item) {
|
||||
return trim($item, "'");
|
||||
}, $arrData);
|
||||
|
||||
$this->arrChapterInfo['c_page_data'] = $arrData;
|
||||
} else {
|
||||
$this->arrChapterInfo['c_page_data'] = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
return $this->arrChapterInfo;
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var array
|
||||
*/
|
||||
protected $arrChapterInfo;
|
||||
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @param integer $intNId
|
||||
* @return boolean
|
||||
*/
|
||||
public function saveChapter(int $intNId): bool
|
||||
{
|
||||
$arrPageChapter = $this->getChapterInfo();
|
||||
|
||||
if (
|
||||
empty($arrPageChapter) ||
|
||||
empty($arrPageChapter['c_page_title']) ||
|
||||
empty($arrPageChapter['c_page_data']) ||
|
||||
count($arrPageChapter['c_page_data']) < 5
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$strUUId = $this->getSite()->getSiteUUId('chapter-' . $this->arrData['c_source_id'] . '-' . $this->arrData['c_page']);
|
||||
|
||||
$arrChapter = [
|
||||
'n_id' => $intNId,
|
||||
'c_name' => $arrPageChapter['c_page_data'][3],
|
||||
'c_sort_num' => $this->arrData['c_sort_num'],
|
||||
'c_page' => $this->arrData['c_page'],
|
||||
'c_source_id' => $this->arrData['c_source_id'],
|
||||
'c_site_code' => $this->getSite()->getSiteCode(),
|
||||
'c_source_uuid' => $strUUId,
|
||||
];
|
||||
|
||||
$arrExistsChapter = $this->getChapterModel()->findOne(['c_source_uuid' => $strUUId], $this->arrOptions);
|
||||
|
||||
if ($arrExistsChapter) {
|
||||
|
||||
$arrExistsChapter = (array) $arrExistsChapter;
|
||||
|
||||
$arrUpdate = [
|
||||
'updated_at' => new \MongoDB\BSON\UTCDateTime(),
|
||||
];
|
||||
|
||||
$this->getChapterModel()->updateOne(
|
||||
['c_source_uuid' => $strUUId],
|
||||
['$set' => $arrUpdate],
|
||||
);
|
||||
$this->strCContentPath = $arrExistsChapter['c_content_path'];
|
||||
} else {
|
||||
|
||||
$arrInsertNovel = $arrChapter;
|
||||
|
||||
$arrInsertNovel['c_content_path'] = '/novel/' . ($intNId % 100) . '/' . $intNId . '/' . $arrChapter['c_sort_num'] . '_' . $arrChapter['c_page'] . '.txt';
|
||||
$arrInsertNovel['created_at'] = new \MongoDB\BSON\UTCDateTime();
|
||||
|
||||
try {
|
||||
$this->getChapterModel()->insert($arrInsertNovel);
|
||||
$this->updateNovelHaveChapter($intNId);
|
||||
$this->strCContentPath = $arrInsertNovel['c_content_path'];
|
||||
} catch (\Exception $e) {
|
||||
if ($e->getCode() === 11000) {
|
||||
return $this->saveChapter($intNId);
|
||||
} else {
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
StorageCore::getInstance()->put($this->strCContentPath, $this->strChapterContent);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
public $strCContentPath;
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @param integer $intNId
|
||||
* @return void
|
||||
*/
|
||||
protected function updateNovelHaveChapter(int $intNId)
|
||||
{
|
||||
$arrFilter = [
|
||||
'n_id' => $intNId,
|
||||
];
|
||||
|
||||
$ChapterModel = ChapterModel::getInstance();
|
||||
$arrLasteChapter = $ChapterModel->getLatestChapterByNId($intNId, false);
|
||||
|
||||
if (!empty($arrLasteChapter)) {
|
||||
$arrUpdate['$set']['n_have_chapter'] = 1;
|
||||
$arrUpdate['$set']['n_latest_chapter_name'] = $arrLasteChapter['c_name'];
|
||||
$this->getMongoBase()->updateOne('novel', $arrFilter, $arrUpdate);
|
||||
}
|
||||
}
|
||||
|
||||
protected $arrNextPage;
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
public function getNextPage(): array
|
||||
{
|
||||
if ($this->arrNextPage === NULL) {
|
||||
$this->arrNextPage = [];
|
||||
|
||||
if ($this->isMobile()) {
|
||||
$strPattern = "/var\s+hhekgsv\s*=\s*['\"](.*?)['\"]/";
|
||||
preg_match_all($strPattern, $this->getContent(), $arrMatches);
|
||||
$strNextPageUri = end($arrMatches[1]) ?: '';
|
||||
} else {
|
||||
$strHref = $this->getQueryList()->find('.read_btn a:contains("下一章")')->attr('href');
|
||||
$strNextPageUri = $strHref ?: '';
|
||||
}
|
||||
|
||||
if (is_numeric(strpos($strNextPageUri, '_'))) {
|
||||
$this->arrNextPage = $this->arrData;
|
||||
$this->arrNextPage['c_page'] += 1;
|
||||
$this->arrNextPage['uri'] = $strNextPageUri;
|
||||
}
|
||||
}
|
||||
|
||||
return $this->arrNextPage;
|
||||
}
|
||||
}
|
||||
64
code/app/task/crawler/zw/page/LatestListPage.php
Normal file
64
code/app/task/crawler/zw/page/LatestListPage.php
Normal file
@@ -0,0 +1,64 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace app\task\crawler\zw\page;
|
||||
|
||||
use app\model\NovelModel;
|
||||
|
||||
|
||||
class LatestListPage extends BasePage
|
||||
{
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var NovelModel
|
||||
*/
|
||||
protected $NovelModel;
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return NovelModel
|
||||
*/
|
||||
public function getNovelModel(): NovelModel
|
||||
{
|
||||
if ($this->NovelModel == NULL) {
|
||||
$this->NovelModel = NovelModel::getInstance();
|
||||
}
|
||||
return $this->NovelModel;
|
||||
}
|
||||
|
||||
static public $arrOptions = [
|
||||
'typeMap' => [
|
||||
'root' => 'array',
|
||||
'document' => 'array',
|
||||
'array' => 'array',
|
||||
],
|
||||
];
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
public function getNovelFilter(): array
|
||||
{
|
||||
$arrFilter = [];
|
||||
if ($this->isMobile()) {
|
||||
$arrFilter = $this->getQueryList()->find('div.wrap-box')->eq(0)->find('ul.sort_list li')->map(function ($li) {
|
||||
$strUri = $li->find('a')->eq(0)->attr('href');
|
||||
return supperExtractFilename($strUri);
|
||||
})->filter()->all();
|
||||
} else {
|
||||
|
||||
$arrFilter = $this->getQueryList()->find('div.layout.layout2.layout-col2.fl ul.txt-list li')->map(function ($li) {
|
||||
$strUri = $li->find('a')->eq(0)->attr('href');
|
||||
return supperExtractFilename($strUri);
|
||||
})->filter()->all();
|
||||
}
|
||||
|
||||
return $arrFilter;
|
||||
}
|
||||
}
|
||||
418
code/app/task/crawler/zw/page/NovelPage.php
Normal file
418
code/app/task/crawler/zw/page/NovelPage.php
Normal file
@@ -0,0 +1,418 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace app\task\crawler\zw\page;
|
||||
|
||||
use app\model\CategoryModel;
|
||||
use app\model\CountersModel;
|
||||
use app\model\NovelModel;
|
||||
use storage\StorageCore;
|
||||
|
||||
class NovelPage extends BasePage
|
||||
{
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var array
|
||||
*/
|
||||
protected $arrNovelInfo = [];
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var array
|
||||
*/
|
||||
protected $arrRelatedNovel = [];
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var array
|
||||
*/
|
||||
protected $arrForgeNovel = [];
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var NovelModel
|
||||
*/
|
||||
protected $NovelModel;
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return NovelModel
|
||||
*/
|
||||
public function getNovelModel(): NovelModel
|
||||
{
|
||||
if ($this->NovelModel == NULL) {
|
||||
$this->NovelModel = NovelModel::getInstance();
|
||||
}
|
||||
return $this->NovelModel;
|
||||
}
|
||||
|
||||
public $arrOptions = [
|
||||
'typeMap' => [
|
||||
'root' => 'array',
|
||||
'document' => 'array',
|
||||
'array' => 'array',
|
||||
],
|
||||
];
|
||||
|
||||
/**
|
||||
* 查询看的小说
|
||||
*
|
||||
* @param integer $intNSourceId
|
||||
* @param string $strName
|
||||
* @return array
|
||||
*/
|
||||
protected function findOrCreateNovel(int $intNSourceId, string $strName): array
|
||||
{
|
||||
$strUUId = $this->getSite()->getSiteUUId($intNSourceId);
|
||||
|
||||
$arrExistsNovel = $this->getNovelModel()->findOne(['n_source_uuid' => $strUUId], $this->arrOptions);
|
||||
|
||||
if ($arrExistsNovel) {
|
||||
$arrNovelInfo = (array) $arrExistsNovel;
|
||||
} else {
|
||||
$arrNovelInfo = [
|
||||
'n_id' => CountersModel::getInstance()->getNextSequence('novel'),
|
||||
'n_name' => $strName,
|
||||
'n_name_pinyin' => zhToPinYin($strName),
|
||||
'n_site_code' => $this->getSite()->getSiteCode(),
|
||||
'n_source_uuid' => $strUUId,
|
||||
'n_source_id' => $intNSourceId,
|
||||
'n_level' => 1,
|
||||
'n_have_chapter' => 0,
|
||||
'created_at' => new \MongoDB\BSON\UTCDateTime(),
|
||||
];
|
||||
try {
|
||||
$this->getNovelModel()->insert($arrNovelInfo);
|
||||
} catch (\Exception $e) {
|
||||
if ($e->getCode() === 11000) {
|
||||
return $this->findOrCreateNovel($intNSourceId, $strName);
|
||||
} else {
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
}
|
||||
return $arrNovelInfo;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 分析关联数据
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
public function getReletedNovel() {}
|
||||
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
public function getNovelInfo(): array
|
||||
{
|
||||
if (empty($this->arrNovelInfo)) {
|
||||
|
||||
# 获取head字段
|
||||
$this->getQueryList()->find('meta[property]')->each(function ($Item) {
|
||||
$strKey = str_replace(':', '_', $Item->attr('property'));
|
||||
$this->arrNovelInfo[$strKey] = $Item->attr('content');
|
||||
});
|
||||
|
||||
$this->getReletedNovel();
|
||||
}
|
||||
|
||||
return $this->arrNovelInfo;
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @param string $strName
|
||||
* @return string
|
||||
*/
|
||||
public function generateCoverPath($strName): string
|
||||
{
|
||||
$intTime = time() - mt_rand(1, 2 * 24 * 3600 * 360);
|
||||
|
||||
return sprintf('/img/%s/%s.jpg', date('Y/m/d/H', $intTime), $strName);
|
||||
}
|
||||
|
||||
public function filterOtherFields(&$arrNovel)
|
||||
{
|
||||
$arrFields = [
|
||||
'n_id',
|
||||
'n_name',
|
||||
'n_name_pinyin',
|
||||
'n_status',
|
||||
'n_author',
|
||||
'n_description',
|
||||
'n_category',
|
||||
'n_category_pinyin',
|
||||
'n_category_sex_en',
|
||||
'n_category_sex_zh',
|
||||
'n_cover_path',
|
||||
'n_latest_chapter_name',
|
||||
'n_update_time',
|
||||
'n_forge_novel',
|
||||
'n_related_novel',
|
||||
'n_site_code',
|
||||
'n_source_uuid',
|
||||
'n_source_id',
|
||||
'n_have_chapter',
|
||||
'created_at',
|
||||
'updated_at',
|
||||
];
|
||||
|
||||
foreach ($arrNovel as $strKey => $strVal) {
|
||||
if (!in_array($strKey, $arrFields)) {
|
||||
unset($arrNovel[$strKey]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return bool
|
||||
*/
|
||||
public function saveNovelInfo(): bool
|
||||
{
|
||||
$arrPageInfo = $this->getNovelInfo();
|
||||
if (
|
||||
empty($arrPageInfo) ||
|
||||
empty($arrPageInfo['og_title']) ||
|
||||
empty($arrPageInfo['og_image']) ||
|
||||
empty($arrPageInfo['og_novel_category'])
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$arrNovelInfo = [
|
||||
// 'n_id'=> $arrPageInfo['xxxxx'],
|
||||
'n_name' => $arrPageInfo['og_title'],
|
||||
'n_name_pinyin' => zhToPinYin($arrPageInfo['og_title']),
|
||||
'n_status' => $arrPageInfo['og_novel_status'],
|
||||
'n_author' => $arrPageInfo['og_novel_author'],
|
||||
'n_description' => $arrPageInfo['og_description'],
|
||||
'n_category' => $arrPageInfo['og_novel_category'],
|
||||
// 'n_category_pinyin' => $arrPageInfo['xxxxx'],
|
||||
// 'n_category_sex_en' => $arrPageInfo['xxxxx'],
|
||||
// 'n_category_sex_zh' => $arrPageInfo['xxxxx'],
|
||||
// 'n_cover_path' => $arrPageInfo['xxxxx'],
|
||||
'n_latest_chapter_name' => $arrPageInfo['og_novel_latest_chapter_name'],
|
||||
'n_update_time' => $arrPageInfo['og_novel_update_time'],
|
||||
'n_forge_novel' => $this->arrForgeNovel,
|
||||
'n_related_novel' => $this->arrRelatedNovel,
|
||||
|
||||
'n_site_code' => $this->getSite()->getSiteCode(),
|
||||
'n_source_id' => $this->arrData['n_source_id'],
|
||||
'n_source_uuid' => $this->getSite()->getSiteUUId($this->arrData['n_source_id']),
|
||||
|
||||
// 'n_have_chapter' => $arrPageInfo['xxxxx'],
|
||||
// 'created_at' => $arrPageInfo['xxxxx'],
|
||||
// 'updated_at' => $arrPageInfo['xxxxx'],
|
||||
|
||||
];
|
||||
|
||||
$arrExistsNovel = $this->getNovelModel()->findOne(['n_source_uuid' => $arrNovelInfo['n_source_uuid']], $this->arrOptions);
|
||||
|
||||
if ($arrExistsNovel) {
|
||||
|
||||
$arrExistsNovel = (array) $arrExistsNovel;
|
||||
|
||||
$arrUpdate = [
|
||||
'n_latest_chapter_name' => $arrNovelInfo['n_latest_chapter_name'],
|
||||
'n_update_time' => $arrNovelInfo['n_update_time'],
|
||||
'updated_at' => new \MongoDB\BSON\UTCDateTime(),
|
||||
];
|
||||
|
||||
# 如果没有分类，说明这条数据是最初从关联shu那里采集的，需要补全全部信息
|
||||
if (empty($arrExistsNovel['n_category'])) {
|
||||
$arrUpdate = array_merge($arrUpdate, $arrNovelInfo);
|
||||
$arrUpdate['n_related_novel'] = $this->arrRelatedNovel;
|
||||
$arrUpdate['n_forge_novel'] = $this->arrForgeNovel;
|
||||
$arrUpdate['n_category_pinyin'] = CategoryModel::getPinYin($this->arrNovelInfo['og_novel_category']);
|
||||
$arrUpdate['n_category_sex_en'] = CategoryModel::getSexEn($this->arrNovelInfo['og_novel_category']);
|
||||
$arrUpdate['n_category_sex_zh'] = CategoryModel::getSexZh($this->arrNovelInfo['og_novel_category']);
|
||||
$arrUpdate['n_related_novel'] = $this->arrRelatedNovel;
|
||||
$arrUpdate['n_forge_novel'] = $this->arrForgeNovel;
|
||||
}
|
||||
|
||||
# 图片不存在的时候，需要重新生成
|
||||
if (
|
||||
!isset($arrExistsNovel['n_cover_path']) ||
|
||||
empty($arrExistsNovel['n_cover_path']) ||
|
||||
!StorageCore::getInstance()->has($arrExistsNovel['n_cover_path']) ||
|
||||
(basename($arrExistsNovel['n_cover_path']) == '.jpg')
|
||||
) {
|
||||
$arrUpdate['n_cover_path'] = $this->generateCoverPath($arrNovelInfo['n_name_pinyin']);
|
||||
$this->strCoverLocalPath = $arrUpdate['n_cover_path'];
|
||||
} else {
|
||||
$this->strCoverLocalPath = $arrExistsNovel['n_cover_path'];
|
||||
}
|
||||
|
||||
$this->getNovelModel()->updateOne(
|
||||
['n_source_uuid' => $arrNovelInfo['n_source_uuid']],
|
||||
['$set' => $arrUpdate],
|
||||
);
|
||||
$this->intNId = $arrExistsNovel['n_id'];
|
||||
} else {
|
||||
|
||||
$arrNovelInfo['n_category_pinyin'] = CategoryModel::getPinYin($this->arrNovelInfo['og_novel_category']);
|
||||
$arrNovelInfo['n_category_sex_en'] = CategoryModel::getSexEn($this->arrNovelInfo['og_novel_category']);
|
||||
$arrNovelInfo['n_category_sex_zh'] = CategoryModel::getSexZh($this->arrNovelInfo['og_novel_category']);
|
||||
$arrNovelInfo['n_id'] = CountersModel::getInstance()->getNextSequence('novel');
|
||||
$arrNovelInfo['n_level'] = 1;
|
||||
$arrNovelInfo['n_have_chapter'] = 0;
|
||||
$arrNovelInfo['n_cover_path'] = $this->generateCoverPath($arrNovelInfo['n_name_pinyin']);
|
||||
$arrNovelInfo['n_related_novel'] = $this->arrRelatedNovel;
|
||||
$arrNovelInfo['n_forge_novel'] = $this->arrForgeNovel;
|
||||
$arrNovelInfo['created_at'] = new \MongoDB\BSON\UTCDateTime();
|
||||
|
||||
try {
|
||||
$this->getNovelModel()->insert($arrNovelInfo);
|
||||
$this->intNId = $arrNovelInfo['n_id'];
|
||||
$this->strCoverLocalPath = $arrNovelInfo['n_cover_path'];
|
||||
} catch (\Exception $e) {
|
||||
if ($e->getCode() === 11000) {
|
||||
return $this->saveNovelInfo();
|
||||
} else {
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
public $strCoverLocalPath = '';
|
||||
|
||||
public $intNId;
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
public function getDownCoverInfo(): array
|
||||
{
|
||||
return [
|
||||
'uri' => $this->arrNovelInfo['og_image'],
|
||||
'local_path' => $this->strCoverLocalPath,
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var array
|
||||
*/
|
||||
protected $arrLatestChapters = NULL;
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var array
|
||||
*/
|
||||
protected $arrChapters = NULL;
|
||||
|
||||
public function getLatestChapterByNIdsList(): array
|
||||
{
|
||||
if ($this->arrLatestChapters === NULL) {
|
||||
$this->arrLatestChapters = [];
|
||||
|
||||
if ($this->isMobile()) {
|
||||
$strClass = '.chapter-list';
|
||||
} else {
|
||||
$strClass = '.section-list';
|
||||
}
|
||||
|
||||
if ($this->arrData['page'] == 0) {
|
||||
$this->getQueryList()->find($strClass)->eq(0)->find("a")->map(function ($A) {
|
||||
$this->arrLatestChapters[] = [
|
||||
'name' => $A->text(),
|
||||
'uri' => $A->attr('href'),
|
||||
];
|
||||
});
|
||||
}
|
||||
}
|
||||
return $this->arrLatestChapters;
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
public function getChapters(): array
|
||||
{
|
||||
if ($this->arrChapters === NULL) {
|
||||
$this->arrChapters = [];
|
||||
if ($this->isMobile()) {
|
||||
$strClass = '.chapter-list';
|
||||
} else {
|
||||
$strClass = '.section-list';
|
||||
}
|
||||
|
||||
$intDivIndex = 0;
|
||||
if ($this->arrData['page'] == 0) {
|
||||
$intDivIndex = 1;
|
||||
}
|
||||
|
||||
$this->getQueryList()->find($strClass)->eq($intDivIndex)->find("a")->map(function ($A) {
|
||||
$this->arrChapters[] = [
|
||||
'name' => $A->text(),
|
||||
'uri' => $A->attr('href'),
|
||||
];
|
||||
});
|
||||
}
|
||||
return $this->arrChapters;
|
||||
}
|
||||
|
||||
|
||||
protected $arrNextPageArgs = NULL;
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
public function getNextPageArgs(): array
|
||||
{
|
||||
$this->arrNextPageArgs = [];
|
||||
if ($this->arrData['page'] == 0) {
|
||||
$this->getLatestChapterByNIdsList();
|
||||
$this->getChapters();
|
||||
|
||||
$arrLatestChapterFirstBlock = count($this->arrLatestChapters) > 0 ? $this->arrLatestChapters[0] : null;
|
||||
$arrLatestChapterSecondBlock = count($this->arrChapters) > 0 ? $this->arrChapters[count($this->arrChapters) - 1] : null;
|
||||
|
||||
if (
|
||||
!empty($arrLatestChapterFirstBlock) && !empty($arrLatestChapterSecondBlock)
|
||||
&& ($arrLatestChapterFirstBlock['uri'] != $arrLatestChapterSecondBlock['uri'])
|
||||
|
||||
) {
|
||||
$this->arrNextPageArgs = [
|
||||
'n_source_id' => $this->arrData['n_source_id'],
|
||||
'page' => $this->arrData['page'] + 1,
|
||||
];
|
||||
}
|
||||
} else {
|
||||
$strLastUri = $this->getQueryList()->find('select:first option:last')->attr('value');
|
||||
if ($strLastUri != '' && $strLastUri != $this->strUri) {
|
||||
$this->arrNextPageArgs = [
|
||||
'n_source_id' => $this->arrData['n_source_id'],
|
||||
'page' => $this->arrData['page'] + 1,
|
||||
];
|
||||
}
|
||||
}
|
||||
return $this->arrNextPageArgs;
|
||||
}
|
||||
}
|
||||
347
code/app/task/crawler/zw/page/Site.php
Normal file
347
code/app/task/crawler/zw/page/Site.php
Normal file
@@ -0,0 +1,347 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace app\task\crawler\zw\page;
|
||||
|
||||
use GuzzleHttp\Client;
|
||||
use GuzzleHttp\Promise;
|
||||
use storage\StorageCore;
|
||||
|
||||
/**
|
||||
* @mixin think\Model
|
||||
*/
|
||||
class Site
|
||||
{
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var Client
|
||||
*/
|
||||
protected $Client;
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $strDomain = '';
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $strSiteCode = '';
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*/
|
||||
public function __construct()
|
||||
{
|
||||
$this->initClient();
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*/
|
||||
public function initClient()
|
||||
{
|
||||
$arrConfig = [
|
||||
'http_version' => '1.1', // 强制用 HTTP/1.1，已有，保持
|
||||
'base_uri' => $this->strDomain, // 基础域名，已有，保持
|
||||
'http_errors' => false, // 不抛 HTTP 错误，已有，保持
|
||||
'timeout' => 60, // 从 160 秒改为 60 秒
|
||||
'connect_timeout' => 10, // 从 30 秒改为 10 秒
|
||||
'curl' => [
|
||||
CURLOPT_TCP_KEEPALIVE => 1, // 启用 TCP Keep-Alive，已有，保持
|
||||
CURLOPT_TCP_KEEPIDLE => 120, // Keep-Alive 空闲时间
|
||||
CURLOPT_TCP_KEEPINTVL => 60, // Keep-Alive 探测间隔
|
||||
// CURLOPT_SSL_VERIFYPEER => true, // 验证 SSL，已有，保持
|
||||
],
|
||||
'headers' => [
|
||||
'User-Agent' => 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36',
|
||||
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
|
||||
'Accept-Language' => 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
|
||||
'Referer' => $this->strDomain,
|
||||
'Connection' => 'keep-alive',
|
||||
'sec-ch-ua' => '"Not/A)Brand";v="8", "Chromium";v="130", "Google Chrome";v="130"',
|
||||
'sec-ch-ua-mobile' => '?1',
|
||||
'sec-ch-ua-platform' => '"Android"',
|
||||
'sec-fetch-dest' => 'document',
|
||||
'sec-fetch-mode' => 'navigate',
|
||||
'sec-fetch-site' => 'same-origin',
|
||||
'sec-fetch-user' => '?1',
|
||||
'upgrade-insecure-requests' => '1',
|
||||
],
|
||||
'pool_size' => 10,
|
||||
'verify' => false,
|
||||
];
|
||||
|
||||
$arrProxy = config('proxy');
|
||||
if ($arrProxy['proxy_status']) {
|
||||
|
||||
$strCode = $arrProxy['proxy_code'];
|
||||
$strProxyHost = $arrProxy['proxy_host'];
|
||||
$strKey = $arrProxy['proxy_key'];
|
||||
$strSecret = $arrProxy['proxy_secret'];
|
||||
|
||||
|
||||
switch ($strCode) {
|
||||
case 'XIONGMAO':
|
||||
$intTime = time();
|
||||
$strTxt = "orderno=" . $strKey . ",secret=" . $strSecret . ",timestamp=" . $intTime;
|
||||
$strSign = strtoupper(md5($strTxt));
|
||||
$strAuth = 'sign=' . $strSign . '&orderno=' . $strKey . '×tamp=' . $intTime;
|
||||
$arrConfig['proxy'] = $strProxyHost;
|
||||
$arrConfig['headers']['Proxy-Authorization'] = $strAuth;
|
||||
break;
|
||||
case 'KUAI':
|
||||
$arrConfig['proxy'] = $strProxyHost;
|
||||
$arrConfig['curl'][CURLOPT_PROXYAUTH] = CURLAUTH_BASIC;
|
||||
$arrConfig['curl'][CURLOPT_PROXYUSERPWD] = "$strKey:$strSecret";
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
$this->Client = new Client($arrConfig);
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return string
|
||||
*/
|
||||
public function getSiteCode(): string
|
||||
{
|
||||
return $this->strSiteCode;
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @param integer|string $strSign
|
||||
* @return string
|
||||
*/
|
||||
public function getSiteUUId(int|string $strSign): string
|
||||
{
|
||||
return $this->getSiteCode() . '-' . $strSign;
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return Client
|
||||
*/
|
||||
public function getClient(): Client
|
||||
{
|
||||
return $this->Client;
|
||||
}
|
||||
|
||||
public function getNovelPageUri($intNSourceId, $intPage): string
|
||||
{
|
||||
$intUriFirst = intdiv($intNSourceId, 1000);
|
||||
|
||||
$strUri = '';
|
||||
|
||||
if ($intPage == 0) {
|
||||
$strUri = sprintf('/%s/%s/', $intUriFirst, $intNSourceId);
|
||||
} else {
|
||||
$strUri = sprintf('/book/%s/list%s.html', $intNSourceId, $intPage);
|
||||
}
|
||||
|
||||
return $strUri;
|
||||
}
|
||||
|
||||
/**
|
||||
* retrieve multiple novel page model
|
||||
*
|
||||
* @param string $strUri
|
||||
* @return NovelPage[]
|
||||
*/
|
||||
public function getNovelPageList(array $arrFilter): array
|
||||
{
|
||||
|
||||
/** @var NovelPage[] */
|
||||
$arrNovelPage = [];
|
||||
|
||||
foreach ($arrFilter as $intKey => $arrItem) {
|
||||
|
||||
$arrFilter[$intKey]['uri'] = $this->getNovelPageUri($arrItem['n_source_id'], $arrItem['page']);
|
||||
|
||||
$arrPromises[$intKey] = $this->getClient()->getAsync($arrFilter[$intKey]['uri']);
|
||||
}
|
||||
|
||||
$arrResults = Promise\Utils::settle($arrPromises)->wait();
|
||||
|
||||
|
||||
$arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
|
||||
return $arrResult['state'] === 'fulfilled';
|
||||
});
|
||||
|
||||
foreach ($arrSuccessfulResults as $intKey => $arrResult) {
|
||||
$Response = $arrResult['value'];
|
||||
|
||||
$strContent = $Response->getBody()->getContents();
|
||||
if ($strContent == '索引文件不存在！') {
|
||||
continue;
|
||||
}
|
||||
|
||||
$NovelPage = $this->getNovelPage($this, $arrFilter[$intKey]['uri']);
|
||||
|
||||
$NovelPage->setContent($strContent);
|
||||
$NovelPage->setData($arrFilter[$intKey]);
|
||||
|
||||
$arrNovelPage[] = $NovelPage;
|
||||
}
|
||||
|
||||
return $arrNovelPage;
|
||||
}
|
||||
|
||||
/**
|
||||
* retrieve multiple chapter page model
|
||||
*
|
||||
* @param string $strUri
|
||||
* @return ChapterPage[]
|
||||
*/
|
||||
public function getChapterPageList(array $arrFilter): array
|
||||
{
|
||||
|
||||
/** @var ChapterPage[] */
|
||||
$arrChapterPage = [];
|
||||
|
||||
$arrPromises = [];
|
||||
foreach ($arrFilter as $intKey => $arrItem) {
|
||||
$arrPromises[$intKey] = $this->getClient()->getAsync($arrItem['uri']);
|
||||
}
|
||||
|
||||
$arrResults = Promise\Utils::settle($arrPromises)->wait();
|
||||
|
||||
$arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
|
||||
return $arrResult['state'] === 'fulfilled';
|
||||
});
|
||||
|
||||
foreach ($arrSuccessfulResults as $intKey => $arrResult) {
|
||||
$Response = $arrResult['value'];
|
||||
|
||||
$strContent = $Response->getBody()->getContents();;
|
||||
|
||||
$ChapterPage = $this->getChapterPage($this, $arrFilter[$intKey]['uri']);
|
||||
|
||||
$ChapterPage->setContent($strContent);
|
||||
$ChapterPage->setData($arrFilter[$intKey]);
|
||||
|
||||
$arrChapterPage[] = $ChapterPage;
|
||||
}
|
||||
|
||||
return $arrChapterPage;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* retrieve multiple chapter page model
|
||||
*
|
||||
* @param string $strUri
|
||||
* @return LatestListPage[]
|
||||
*/
|
||||
public function getLatestListPageList(array $arrFilter): array
|
||||
{
|
||||
|
||||
/** @var LatestListPage[] */
|
||||
$arrLatestListPage = [];
|
||||
|
||||
$arrPromises = [];
|
||||
foreach ($arrFilter as $intKey => $strUri) {
|
||||
$arrPromises[$intKey] = $this->getClient()->getAsync($strUri);
|
||||
}
|
||||
|
||||
$arrResults = Promise\Utils::settle($arrPromises)->wait();
|
||||
|
||||
$arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
|
||||
return $arrResult['state'] === 'fulfilled';
|
||||
});
|
||||
|
||||
foreach ($arrSuccessfulResults as $intKey => $arrResult) {
|
||||
$Response = $arrResult['value'];
|
||||
|
||||
$strContent = $Response->getBody()->getContents();;
|
||||
|
||||
$LatestListPage = $this->getLatestListPage($this, $arrFilter[$intKey]);
|
||||
|
||||
$LatestListPage->setContent($strContent);
|
||||
|
||||
$arrLatestListPage[] = $LatestListPage;
|
||||
}
|
||||
|
||||
|
||||
return $arrLatestListPage;
|
||||
}
|
||||
|
||||
/**
|
||||
* download the image to the local system
|
||||
*
|
||||
* @param array $arrFilter
|
||||
* @return void
|
||||
*/
|
||||
public function downRemoteImgToLocal(array $arrFilter)
|
||||
{
|
||||
$StorageCore = StorageCore::getInstance();
|
||||
|
||||
$arrPromises = [];
|
||||
|
||||
foreach ($arrFilter as $intKey => $arrItem) {
|
||||
if (!$StorageCore->has($arrItem['local_path'])) {
|
||||
$arrPromises[$intKey] = $this->getClient()->getAsync($arrItem['uri']);
|
||||
}
|
||||
}
|
||||
|
||||
$arrResults = Promise\Utils::settle($arrPromises)->wait();
|
||||
|
||||
$arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
|
||||
return $arrResult['state'] === 'fulfilled';
|
||||
});
|
||||
|
||||
foreach ($arrSuccessfulResults as $intKey => $arrResult) {
|
||||
$Response = $arrResult['value'];
|
||||
|
||||
$StorageCore->writeStream($arrFilter[$intKey]['local_path'], $Response->getBody()->detach());;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @param object $Site
|
||||
* @param string $strUri
|
||||
* @return NovelPage
|
||||
*/
|
||||
public function getNovelPage($Site, $strUri)
|
||||
{
|
||||
return new NovelPage($Site, $strUri);
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @param object $Site
|
||||
* @param string $strUri
|
||||
* @return ChapterPage
|
||||
*/
|
||||
public function getChapterPage($Site, $strUri)
|
||||
{
|
||||
return new ChapterPage($Site, $strUri);
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @param object $Site
|
||||
* @param string $strUri
|
||||
* @return LatestListPage
|
||||
*/
|
||||
public function getLatestListPage($Site, $strUri)
|
||||
{
|
||||
return new LatestListPage($Site, $strUri);
|
||||
}
|
||||
}
|
||||
42
code/app/task/crawler/zw630/Scheduler.php
Normal file
42
code/app/task/crawler/zw630/Scheduler.php
Normal file
@@ -0,0 +1,42 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace app\task\crawler\zw630;
|
||||
|
||||
use app\task\crawler\zw630\page\Site;
|
||||
use app\task\crawler\zw\Scheduler as ZwScheduler;
|
||||
|
||||
/**
|
||||
* @mixin think\Model
|
||||
*/
|
||||
class Scheduler extends ZwScheduler
|
||||
{
|
||||
|
||||
protected $intStartNSourceId = 1;
|
||||
|
||||
protected $intEndNSourceId = 500000;
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
public function init()
|
||||
{
|
||||
$this->intSlice = 3;
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return Site
|
||||
*/
|
||||
public function getSite()
|
||||
{
|
||||
if ($this->Site == NULL) {
|
||||
$this->Site = new Site;
|
||||
}
|
||||
return $this->Site;
|
||||
}
|
||||
}
|
||||
9
code/app/task/crawler/zw630/page/ChapterPage.php
Normal file
9
code/app/task/crawler/zw630/page/ChapterPage.php
Normal file
@@ -0,0 +1,9 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace app\task\crawler\zw630\page;
|
||||
|
||||
use app\task\crawler\zw\page\ChapterPage as ZwChapterPage;
|
||||
|
||||
class ChapterPage extends ZwChapterPage {}
|
||||
11
code/app/task/crawler/zw630/page/LatestListPage.php
Normal file
11
code/app/task/crawler/zw630/page/LatestListPage.php
Normal file
@@ -0,0 +1,11 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace app\task\crawler\zw630\page;
|
||||
|
||||
|
||||
use app\model\NovelModel;
|
||||
use app\task\crawler\zw\page\LatestListPage as ZwLatestListPage;
|
||||
|
||||
class LatestListPage extends ZwLatestListPage {}
|
||||
43
code/app/task/crawler/zw630/page/NovelPage.php
Normal file
43
code/app/task/crawler/zw630/page/NovelPage.php
Normal file
@@ -0,0 +1,43 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace app\task\crawler\zw630\page;
|
||||
|
||||
|
||||
|
||||
use app\task\crawler\zw\page\NovelPage as ZwNovelPage;
|
||||
|
||||
class NovelPage extends ZwNovelPage
|
||||
{
|
||||
/**
|
||||
* 分析关联数据
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
public function getReletedNovel()
|
||||
{
|
||||
if (empty($this->arrRelatedNovel) || empty($this->arrForgeNovel)) {
|
||||
$this->getQueryList()->find('.first_txt')->eq(0)->find("a")->map(function ($A) {
|
||||
$strName = $A->text();
|
||||
$strUri = $A->attr('href');
|
||||
|
||||
if (is_numeric(strpos($strUri, 'shu'))) {
|
||||
$intNSourceId = (int)supperExtractFilename($strUri);
|
||||
$arrNovelInfo = $this->findOrCreateNovel($intNSourceId, $strName);
|
||||
$this->arrRelatedNovel[] = [
|
||||
"n_id" => $arrNovelInfo["n_id"],
|
||||
"n_source_id" => $arrNovelInfo["n_source_id"],
|
||||
"n_name" => $arrNovelInfo["n_name"],
|
||||
];
|
||||
} elseif (is_numeric(strpos($strUri, 'kan'))) {
|
||||
$intNForgeId = (int)supperExtractFilename($strUri);
|
||||
$this->arrForgeNovel[] = [
|
||||
"n_forge_id" => $intNForgeId,
|
||||
"n_forge_title" => $strName,
|
||||
];
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
148
code/app/task/crawler/zw630/page/Site.php
Normal file
148
code/app/task/crawler/zw630/page/Site.php
Normal file
@@ -0,0 +1,148 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace app\task\crawler\zw630\page;
|
||||
|
||||
use app\task\crawler\zw\page\Site as ZwSite;
|
||||
use GuzzleHttp\Promise;
|
||||
use GuzzleHttp\Client;
|
||||
|
||||
/**
|
||||
* @mixin think\Model
|
||||
*/
|
||||
class Site extends ZwSite
|
||||
{
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $strDomain = 'https://www.630zw.com';
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $strSiteCode = 'zw630';
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*/
|
||||
public function initClient()
|
||||
{
|
||||
$arrConfig = [
|
||||
'http_version' => '1.1', // 强制用 HTTP/1.1，已有，保持
|
||||
'base_uri' => $this->strDomain, // 基础域名，已有，保持
|
||||
'http_errors' => false, // 不抛 HTTP 错误，已有，保持
|
||||
'timeout' => 60, // 从 160 秒改为 60 秒
|
||||
'connect_timeout' => 10, // 从 30 秒改为 10 秒
|
||||
'curl' => [
|
||||
CURLOPT_TCP_KEEPALIVE => 1, // 启用 TCP Keep-Alive，已有，保持
|
||||
CURLOPT_TCP_KEEPIDLE => 120, // Keep-Alive 空闲时间
|
||||
CURLOPT_TCP_KEEPINTVL => 60, // Keep-Alive 探测间隔
|
||||
// CURLOPT_SSL_VERIFYPEER => true, // 验证 SSL，已有，保持
|
||||
],
|
||||
'headers' => [
|
||||
'User-Agent' => 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36',
|
||||
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
|
||||
'Accept-Language' => 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
|
||||
'Referer' => $this->strDomain,
|
||||
'Connection' => 'keep-alive',
|
||||
'sec-ch-ua' => '"Not/A)Brand";v="8", "Chromium";v="130", "Google Chrome";v="130"',
|
||||
'sec-ch-ua-mobile' => '?1',
|
||||
'sec-ch-ua-platform' => '"Android"',
|
||||
'sec-fetch-dest' => 'document',
|
||||
'sec-fetch-mode' => 'navigate',
|
||||
'sec-fetch-site' => 'same-origin',
|
||||
'sec-fetch-user' => '?1',
|
||||
'upgrade-insecure-requests' => '1',
|
||||
],
|
||||
'pool_size' => 10,
|
||||
'verify' => false,
|
||||
];
|
||||
|
||||
$arrProxy = config('proxy');
|
||||
if ($arrProxy['proxy_status']) {
|
||||
|
||||
$strCode = $arrProxy['proxy_code'];
|
||||
$strProxyHost = $arrProxy['proxy_host'];
|
||||
$strKey = $arrProxy['proxy_key'];
|
||||
$strSecret = $arrProxy['proxy_secret'];
|
||||
|
||||
|
||||
switch ($strCode) {
|
||||
case 'XIONGMAO':
|
||||
$intTime = time();
|
||||
$strTxt = "orderno=" . $strKey . ",secret=" . $strSecret . ",timestamp=" . $intTime;
|
||||
$strSign = strtoupper(md5($strTxt));
|
||||
$strAuth = 'sign=' . $strSign . '&orderno=' . $strKey . '×tamp=' . $intTime;
|
||||
$arrConfig['proxy'] = $strProxyHost;
|
||||
$arrConfig['headers']['Proxy-Authorization'] = $strAuth;
|
||||
break;
|
||||
case 'KUAI':
|
||||
$arrConfig['proxy'] = $strProxyHost;
|
||||
$arrConfig['curl'][CURLOPT_PROXYAUTH] = CURLAUTH_BASIC;
|
||||
$arrConfig['curl'][CURLOPT_PROXYUSERPWD] = "$strKey:$strSecret";
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
$this->Client = new Client($arrConfig);
|
||||
}
|
||||
|
||||
|
||||
public function getNovelPageUri($intNSourceId, $intPage): string
|
||||
{
|
||||
$strUri = '';
|
||||
|
||||
if ($intPage == 0) {
|
||||
|
||||
$strUri = sprintf('/shu/%s.html', $intNSourceId);
|
||||
} else {
|
||||
|
||||
$strUri = sprintf('/shu/%s_%s.html', $intNSourceId, $intPage);
|
||||
}
|
||||
|
||||
return $strUri;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @param object $Site
|
||||
* @param string $strUri
|
||||
* @return NovelPage
|
||||
*/
|
||||
public function getNovelPage($Site, $strUri)
|
||||
{
|
||||
return new NovelPage($Site, $strUri);
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @param object $Site
|
||||
* @param string $strUri
|
||||
* @return ChapterPage
|
||||
*/
|
||||
public function getChapterPage($Site, $strUri)
|
||||
{
|
||||
return new ChapterPage($Site, $strUri);
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @param object $Site
|
||||
* @param string $strUri
|
||||
* @return LatestListPage
|
||||
*/
|
||||
public function getLatestListPage($Site, $strUri)
|
||||
{
|
||||
return new LatestListPage($Site, $strUri);
|
||||
}
|
||||
}
|
||||
37
code/app/task/crawler/zw74/Scheduler.php
Normal file
37
code/app/task/crawler/zw74/Scheduler.php
Normal file
@@ -0,0 +1,37 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace app\task\crawler\zw74;
|
||||
|
||||
use app\task\crawler\zw74\page\Site;
|
||||
use app\task\crawler\zw\Scheduler as ZwScheduler;
|
||||
|
||||
/**
|
||||
* @mixin think\Model
|
||||
*/
|
||||
class Scheduler extends ZwScheduler
|
||||
{
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
public function init()
|
||||
{
|
||||
$this->intSlice = 8;
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return Site
|
||||
*/
|
||||
public function getSite()
|
||||
{
|
||||
if ($this->Site == NULL) {
|
||||
$this->Site = new Site;
|
||||
}
|
||||
return $this->Site;
|
||||
}
|
||||
}
|
||||
9
code/app/task/crawler/zw74/page/ChapterPage.php
Normal file
9
code/app/task/crawler/zw74/page/ChapterPage.php
Normal file
@@ -0,0 +1,9 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace app\task\crawler\zw74\page;
|
||||
|
||||
use app\task\crawler\zw\page\ChapterPage as ZwChapterPage;
|
||||
|
||||
class ChapterPage extends ZwChapterPage {}
|
||||
10
code/app/task/crawler/zw74/page/LatestListPage.php
Normal file
10
code/app/task/crawler/zw74/page/LatestListPage.php
Normal file
@@ -0,0 +1,10 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace app\task\crawler\zw74\page;
|
||||
|
||||
use app\model\NovelModel;
|
||||
use app\task\crawler\zw\page\LatestListPage as ZwLatestListPage;
|
||||
|
||||
class LatestListPage extends ZwLatestListPage {}
|
||||
48
code/app/task/crawler/zw74/page/NovelPage.php
Normal file
48
code/app/task/crawler/zw74/page/NovelPage.php
Normal file
@@ -0,0 +1,48 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace app\task\crawler\zw74\page;
|
||||
|
||||
|
||||
use app\task\crawler\zw\page\NovelPage as ZwNovelPage;
|
||||
|
||||
class NovelPage extends ZwNovelPage
|
||||
{
|
||||
/**
|
||||
* 分析关联数据
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
public function getReletedNovel()
|
||||
{
|
||||
if (empty($this->arrRelatedNovel) || empty($this->arrForgeNovel)) {
|
||||
$this->getQueryList()->find('.first_txt')->eq(0)->find("a")->map(function ($A) {
|
||||
$strName = $A->text();
|
||||
$strUri = $A->attr('href');
|
||||
|
||||
if (is_numeric(strpos($strUri, 'http'))) {
|
||||
return;
|
||||
}
|
||||
|
||||
$strPattern = '#^/\d+/(\d+)/$#';
|
||||
|
||||
if (is_numeric(strpos($strUri, 'info'))) {
|
||||
$intNForgeId = (int)supperExtractFilename($strUri);
|
||||
$this->arrForgeNovel[] = [
|
||||
"n_forge_id" => $intNForgeId,
|
||||
"n_forge_title" => $strName,
|
||||
];
|
||||
} elseif (preg_match($strPattern, $strUri, $arrMatches)) {
|
||||
$intNSourceId = (int)$arrMatches[1];
|
||||
$arrNovelInfo = $this->findOrCreateNovel($intNSourceId, $strName);
|
||||
$this->arrRelatedNovel[] = [
|
||||
"n_id" => $arrNovelInfo["n_id"],
|
||||
"n_source_id" => $arrNovelInfo["n_source_id"],
|
||||
"n_name" => $arrNovelInfo["n_name"],
|
||||
];
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
152
code/app/task/crawler/zw74/page/Site.php
Normal file
152
code/app/task/crawler/zw74/page/Site.php
Normal file
@@ -0,0 +1,152 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace app\task\crawler\zw74\page;
|
||||
|
||||
use app\task\crawler\zw\page\Site as ZwSite;
|
||||
use GuzzleHttp\Client;
|
||||
|
||||
/**
|
||||
* @mixin think\Model
|
||||
*/
|
||||
class Site extends ZwSite
|
||||
{
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $strDomain = 'https://www.74zw.cc';
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $strSiteCode = 'zw';
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*/
|
||||
public function initClient()
|
||||
{
|
||||
$arrConfig = [
|
||||
'http_version' => '1.1', // 强制用 HTTP/1.1，已有，保持
|
||||
'base_uri' => $this->strDomain, // 基础域名，已有，保持
|
||||
'http_errors' => false, // 不抛 HTTP 错误，已有，保持
|
||||
'timeout' => 60, // 从 160 秒改为 60 秒
|
||||
'connect_timeout' => 10, // 从 30 秒改为 10 秒
|
||||
'curl' => [
|
||||
CURLOPT_TCP_KEEPALIVE => 1, // 启用 TCP Keep-Alive，已有，保持
|
||||
CURLOPT_TCP_KEEPIDLE => 120, // Keep-Alive 空闲时间
|
||||
CURLOPT_TCP_KEEPINTVL => 60, // Keep-Alive 探测间隔
|
||||
// CURLOPT_SSL_VERIFYPEER => true, // 验证 SSL，已有，保持
|
||||
],
|
||||
'headers' => [
|
||||
'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36 Edg/135.0.0.0',
|
||||
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
|
||||
'Accept-Language' => 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
|
||||
'Referer' => $this->strDomain,
|
||||
'Connection' => 'keep-alive',
|
||||
'sec-ch-ua' => '"Microsoft Edge";v="135", "Not-A.Brand";v="8", "Chromium";v="135"',
|
||||
'sec-ch-ua-mobile' => '?0',
|
||||
'sec-ch-ua-platform' => '"Windows"',
|
||||
'sec-fetch-dest' => 'document',
|
||||
'sec-fetch-mode' => 'navigate',
|
||||
'sec-fetch-site' => 'same-origin',
|
||||
'sec-fetch-user' => '?1',
|
||||
'upgrade-insecure-requests' => '1',
|
||||
],
|
||||
'pool_size' => 10,
|
||||
'verify' => false,
|
||||
];
|
||||
|
||||
$arrProxy = config('proxy');
|
||||
if ($arrProxy['proxy_status']) {
|
||||
|
||||
$strCode = $arrProxy['proxy_code'];
|
||||
$strProxyHost = $arrProxy['proxy_host'];
|
||||
$strKey = $arrProxy['proxy_key'];
|
||||
$strSecret = $arrProxy['proxy_secret'];
|
||||
|
||||
|
||||
switch ($strCode) {
|
||||
case 'XIONGMAO':
|
||||
$intTime = time();
|
||||
$strTxt = "orderno=" . $strKey . ",secret=" . $strSecret . ",timestamp=" . $intTime;
|
||||
$strSign = strtoupper(md5($strTxt));
|
||||
$strAuth = 'sign=' . $strSign . '&orderno=' . $strKey . '×tamp=' . $intTime;
|
||||
$arrConfig['proxy'] = $strProxyHost;
|
||||
$arrConfig['headers']['Proxy-Authorization'] = $strAuth;
|
||||
break;
|
||||
case 'KUAI':
|
||||
$arrConfig['proxy'] = $strProxyHost;
|
||||
$arrConfig['curl'][CURLOPT_PROXYAUTH] = CURLAUTH_BASIC;
|
||||
$arrConfig['curl'][CURLOPT_PROXYUSERPWD] = "$strKey:$strSecret";
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
$this->Client = new Client($arrConfig);
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @param int $intNSourceId
|
||||
* @param int $intPage
|
||||
* @return string
|
||||
*/
|
||||
public function getNovelPageUri($intNSourceId, $intPage): string
|
||||
{
|
||||
$intUriFirst = intdiv($intNSourceId, 1000);
|
||||
|
||||
$strUri = '';
|
||||
|
||||
if ($intPage == 0) {
|
||||
$strUri = sprintf('/%s/%s/', $intUriFirst, $intNSourceId);
|
||||
} else {
|
||||
$strUri = sprintf('/book/%s/list%s.html', $intNSourceId, $intPage);
|
||||
}
|
||||
|
||||
return $strUri;
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @param object $Site
|
||||
* @param string $strUri
|
||||
* @return NovelPage
|
||||
*/
|
||||
public function getNovelPage($Site, $strUri)
|
||||
{
|
||||
return new NovelPage($Site, $strUri);
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @param object $Site
|
||||
* @param string $strUri
|
||||
* @return ChapterPage
|
||||
*/
|
||||
public function getChapterPage($Site, $strUri)
|
||||
{
|
||||
return new ChapterPage($Site, $strUri);
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @param object $Site
|
||||
* @param string $strUri
|
||||
* @return LatestListPage
|
||||
*/
|
||||
public function getLatestListPage($Site, $strUri)
|
||||
{
|
||||
return new LatestListPage($Site, $strUri);
|
||||
}
|
||||
}
|
||||
90
code/app/task/module/Listen.php
Normal file
90
code/app/task/module/Listen.php
Normal file
@@ -0,0 +1,90 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace app\task\module;
|
||||
|
||||
use microserver\ProcessSanitizer;
|
||||
use microserver\QueueManage;
|
||||
|
||||
class Listen
|
||||
{
|
||||
/**
|
||||
* 监听队列1代理
|
||||
*
|
||||
* @param \Swoole\Process $Process
|
||||
* @return void
|
||||
*/
|
||||
static public function innerConsume(\Swoole\Process $Process)
|
||||
{
|
||||
return self::innerConsumeCore(1);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 监听队列2代理
|
||||
*
|
||||
* @param \Swoole\Process $Process
|
||||
* @return void
|
||||
*/
|
||||
static public function innerConsume02(\Swoole\Process $Process)
|
||||
{
|
||||
return self::innerConsumeCore(2);
|
||||
}
|
||||
|
||||
/**
|
||||
* 监听队列
|
||||
*
|
||||
* @param \Swoole\Process $Process
|
||||
* @return void
|
||||
*/
|
||||
static private function innerConsumeCore(int $intIndex = 1)
|
||||
{
|
||||
|
||||
ProcessSanitizer::destructConnectSource();
|
||||
|
||||
$arrConfig = config('task.queue');
|
||||
|
||||
$intLimit = $arrConfig[$intIndex]['exec_num'];
|
||||
|
||||
$QueueManage = new QueueManage($intIndex);
|
||||
|
||||
return self::excuteTask($QueueManage, $intLimit);
|
||||
}
|
||||
|
||||
/**
|
||||
* 队列任务消费
|
||||
*
|
||||
* @param \Swoole\Process $Process
|
||||
* @return void
|
||||
*/
|
||||
static private function excuteTask(QueueManage $QueueManage, $intLimit)
|
||||
{
|
||||
while ($intLimit > 0) {
|
||||
$intLimit--;
|
||||
try {
|
||||
$strData = $QueueManage->get();
|
||||
if (empty($strData)) {
|
||||
sleep(1);
|
||||
continue;
|
||||
}
|
||||
$arrData = json_decode($strData, true);
|
||||
if (!is_array($arrData)) {
|
||||
echo '监听Redis队列读取到异常无法解析的数据[' . $strData . ']' . PHP_EOL;
|
||||
continue;
|
||||
}
|
||||
$Class = new $arrData['callback'][0];
|
||||
$Class->{$arrData['callback'][1]}($arrData['data']);
|
||||
unset($Class);
|
||||
} catch (\Throwable $t) {
|
||||
$strData = $strData ?? '';
|
||||
echo date('Y-m-d H:i:s') . ':出现致命错误需要处理' . PHP_EOL .
|
||||
' 队列数据:' . $strData . PHP_EOL .
|
||||
' 文件:' . $t->getFile() . PHP_EOL .
|
||||
' 行数:' . $t->getLine() . PHP_EOL .
|
||||
' 错误描述:' . $t->getMessage() . PHP_EOL .
|
||||
' 堆栈跟踪:' . $t->getTraceAsString() . PHP_EOL;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
136
code/app/task/module/PlanTask.php
Normal file
136
code/app/task/module/PlanTask.php
Normal file
@@ -0,0 +1,136 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace app\task\module;
|
||||
|
||||
use app\model\PlanTaskModel;
|
||||
use app\task\crawler\zw630\Scheduler as Zw630Scheduler;
|
||||
use app\task\crawler\zw74\Scheduler as Zw74Scheduler;
|
||||
use app\task\crawler\youzhi\Scheduler as YouZhiScheduler;
|
||||
|
||||
use microserver\QueueManage;
|
||||
use microserver\ProcessSanitizer;
|
||||
|
||||
|
||||
class PlanTask
|
||||
{
|
||||
public static function scanPlanTask($arrData)
|
||||
{
|
||||
ProcessSanitizer::destructConnectSource();
|
||||
|
||||
$PlanTaskAll = PlanTaskModel::where('pt_enable', 1)->select();
|
||||
|
||||
foreach ($PlanTaskAll as $PlanTask) {
|
||||
|
||||
if ($PlanTask->pt_last_exec != 0 && ($PlanTask->pt_last_exec + $PlanTask->pt_limit) > time()) continue;
|
||||
|
||||
$PlanTask->pt_last_exec = time();
|
||||
|
||||
$PlanTask->save();
|
||||
|
||||
switch ($PlanTask->pt_code) {
|
||||
|
||||
case 'PULL_630_ZW_NOVEL':
|
||||
self::pull630ZwNovel();
|
||||
break;
|
||||
case 'PULL_630_ZW_NOVEL_LATEST':
|
||||
self::pull630ZwNovelLatest();
|
||||
break;
|
||||
|
||||
case 'PULL_74_ZW_NOVEL':
|
||||
self::pull74ZwNovel();
|
||||
break;
|
||||
case 'PULL_74_ZW_NOVEL_LATEST':
|
||||
self::pull74ZwNovelLatest();
|
||||
break;
|
||||
|
||||
case 'PULL_YOU_ZHI_SHI_PIN':
|
||||
self::pullYouZhiShiPin();
|
||||
break;
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
public static function pullYouZhiShiPin()
|
||||
{
|
||||
$arrTask = [
|
||||
'callback' => [YouZhiScheduler::class, 'crawlFull'],
|
||||
'data' => []
|
||||
];
|
||||
|
||||
$QueueManage = new QueueManage();
|
||||
$QueueManage->set($arrTask);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
public static function pull630ZwNovel()
|
||||
{
|
||||
$arrTask = [
|
||||
'callback' => [Zw630Scheduler::class, 'crawlFull'],
|
||||
'data' => []
|
||||
];
|
||||
|
||||
$QueueManage = new QueueManage();
|
||||
$QueueManage->set($arrTask);
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
public static function pull630ZwNovelLatest()
|
||||
{
|
||||
$arrTask = [
|
||||
'callback' => [Zw630Scheduler::class, 'crawLatest'],
|
||||
'data' => []
|
||||
];
|
||||
|
||||
$QueueManage = new QueueManage();
|
||||
$QueueManage->set($arrTask);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
public static function pull74ZwNovel()
|
||||
{
|
||||
$arrTask = [
|
||||
'callback' => [Zw74Scheduler::class, 'crawlFull'],
|
||||
'data' => []
|
||||
];
|
||||
|
||||
$QueueManage = new QueueManage();
|
||||
$QueueManage->set($arrTask);
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
public static function pull74ZwNovelLatest()
|
||||
{
|
||||
$arrTask = [
|
||||
'callback' => [Zw74Scheduler::class, 'crawLatest'],
|
||||
'data' => []
|
||||
];
|
||||
|
||||
$QueueManage = new QueueManage();
|
||||
$QueueManage->set($arrTask);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user