123
This commit is contained in:
Default
@@ -37,8 +37,6 @@ class PullDataColBase
|
||||
$this->QueueManage = new QueueManage();
|
||||
}
|
||||
|
||||
|
||||
|
||||
public function getAilisiDomain()
|
||||
{
|
||||
if (!key_exists('AILISI_DOMAIN', $this->arrCaiJiPeiZhi)) {
|
||||
@@ -47,78 +45,4 @@ class PullDataColBase
|
||||
|
||||
return $this->arrCaiJiPeiZhi['AILISI_DOMAIN'];
|
||||
}
|
||||
|
||||
public function getAilisiCookie()
|
||||
{
|
||||
if (!key_exists('AILISI_COOKIE', $this->arrCaiJiPeiZhi)) {
|
||||
$strCookie = CaiJiPeiZhiModel::getValByCode('AILISI_COOKIE');
|
||||
$this->arrCaiJiPeiZhi['AILISI_COOKIE'] = parseCookies($strCookie);
|
||||
}
|
||||
|
||||
return $this->arrCaiJiPeiZhi['AILISI_COOKIE'];
|
||||
}
|
||||
|
||||
public function getTianLaiDomain()
|
||||
{
|
||||
if (!key_exists('TIANLAI_DOMAIN', $this->arrCaiJiPeiZhi)) {
|
||||
$this->arrCaiJiPeiZhi['TIANLAI_DOMAIN'] = CaiJiPeiZhiModel::getValByCode('TIANLAI_DOMAIN');
|
||||
}
|
||||
|
||||
return $this->arrCaiJiPeiZhi['TIANLAI_DOMAIN'];
|
||||
}
|
||||
|
||||
public function getBiquxsDomain()
|
||||
{
|
||||
if (!key_exists('BIQUXS_DOMAIN', $this->arrCaiJiPeiZhi)) {
|
||||
$this->arrCaiJiPeiZhi['BIQUXS_DOMAIN'] = CaiJiPeiZhiModel::getValByCode('BIQUXS_DOMAIN');
|
||||
}
|
||||
|
||||
return $this->arrCaiJiPeiZhi['BIQUXS_DOMAIN'];
|
||||
}
|
||||
|
||||
public function get5cccDomain()
|
||||
{
|
||||
if (!key_exists('5CCC_DOMAIN', $this->arrCaiJiPeiZhi)) {
|
||||
$this->arrCaiJiPeiZhi['5CCC_DOMAIN'] = CaiJiPeiZhiModel::getValByCode('5CCC_DOMAIN');
|
||||
}
|
||||
|
||||
return $this->arrCaiJiPeiZhi['5CCC_DOMAIN'];
|
||||
}
|
||||
|
||||
public function get630zwDomain()
|
||||
{
|
||||
if (!key_exists('630zw_DOMAIN', $this->arrCaiJiPeiZhi)) {
|
||||
$this->arrCaiJiPeiZhi['630zw_DOMAIN'] = CaiJiPeiZhiModel::getValByCode('630zw_DOMAIN');
|
||||
}
|
||||
|
||||
return $this->arrCaiJiPeiZhi['630zw_DOMAIN'];
|
||||
}
|
||||
|
||||
|
||||
public function getDadiDomain()
|
||||
{
|
||||
if (!key_exists('DADI_DOMAIN', $this->arrCaiJiPeiZhi)) {
|
||||
$this->arrCaiJiPeiZhi['DADI_DOMAIN'] = CaiJiPeiZhiModel::getValByCode('DADI_DOMAIN');
|
||||
}
|
||||
|
||||
return $this->arrCaiJiPeiZhi['DADI_DOMAIN'];
|
||||
}
|
||||
public function getHeiMuErDomain()
|
||||
{
|
||||
if (!key_exists('HEIMUER_DOMAIN', $this->arrCaiJiPeiZhi)) {
|
||||
$this->arrCaiJiPeiZhi['HEIMUER_DOMAIN'] = CaiJiPeiZhiModel::getValByCode('HEIMUER_DOMAIN');
|
||||
}
|
||||
|
||||
return $this->arrCaiJiPeiZhi['HEIMUER_DOMAIN'];
|
||||
}
|
||||
public function YouZhiZiYuanDomain()
|
||||
{
|
||||
if (!key_exists('YZZY_DOMAIN', $this->arrCaiJiPeiZhi)) {
|
||||
$this->arrCaiJiPeiZhi['YZZY_DOMAIN'] = CaiJiPeiZhiModel::getValByCode('YZZY_DOMAIN');
|
||||
}
|
||||
|
||||
return $this->arrCaiJiPeiZhi['YZZY_DOMAIN'];
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
@@ -7,37 +7,171 @@ namespace app\task\collection\zw630;
|
||||
|
||||
use app\model\XiaoShuoXiangQingModel;
|
||||
use app\task\collection\PullDataColBase;
|
||||
use db\mongo\MongoBase;
|
||||
use GuzzleHttp\Cookie\CookieJar;
|
||||
use processor\ImageProcessor;
|
||||
use think\facade\Cache;
|
||||
use QL\QueryList;
|
||||
|
||||
use minserver\QueueManage;
|
||||
use GuzzleHttp\Client;
|
||||
|
||||
/**
|
||||
* @mixin think\Model
|
||||
*/
|
||||
class Zw630Col extends PullDataColBase
|
||||
{
|
||||
private $strDomain = 'https://www.630zw.org';
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var MongoBase
|
||||
*/
|
||||
private $MongoBase;
|
||||
|
||||
public function __construct()
|
||||
{
|
||||
$this->MongoBase = MongoBase::getInstance();
|
||||
|
||||
$this->Client = new Client([
|
||||
'base_uri' => $this->strDomain,
|
||||
'http_errors' => false,
|
||||
'version' => 2.0, // 强制使用 HTTP/2
|
||||
'timeout' => 160,
|
||||
'connect_timeout' => 30, // 连接超时 10 秒
|
||||
'curl' => [
|
||||
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_2_0, // 强制 HTTP/2
|
||||
CURLOPT_TCP_KEEPALIVE => 1, // 启用 TCP Keep-Alive
|
||||
CURLOPT_SSL_VERIFYPEER => true, // 验证 SSL
|
||||
],
|
||||
'headers' => [
|
||||
'User-Agent' => 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Mobile Safari/537.36 Edg/134.0.0.0',
|
||||
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
|
||||
'Accept-Language' => 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
|
||||
'Referer' => $this->strDomain,
|
||||
'sec-ch-ua-mobile' => '?1',
|
||||
'sec-ch-ua-platform' => '"Android"',
|
||||
'sec-fetch-dest' => 'document',
|
||||
'sec-fetch-mode' => 'navigate',
|
||||
'sec-fetch-site' => 'same-origin',
|
||||
'sec-fetch-user' => '?1',
|
||||
'upgrade-insecure-requests' => '1',
|
||||
],
|
||||
]);
|
||||
|
||||
$this->QueueManage = new QueueManage();
|
||||
}
|
||||
|
||||
|
||||
public function pull630ZwNovel()
|
||||
{
|
||||
// 全局扫描任务
|
||||
$QueueManage = $this->QueueManage;
|
||||
for ($i = 1; $i <= 531208; $i++) {
|
||||
// for ($intNovelId = 1; $intNovelId <= 531208; $intNovelId++) {
|
||||
// for ($intNovelId = 1; $intNovelId <= 10000; $intNovelId++) {
|
||||
for ($intNovelId = 1; $intNovelId <= 1; $intNovelId++) {
|
||||
$arrTask = [
|
||||
'callback' => [self::class, 'getNovelInfo'],
|
||||
'data' => [
|
||||
'n_id' => $i,
|
||||
'n_id' => $intNovelId,
|
||||
'page' => 0,
|
||||
]
|
||||
];
|
||||
|
||||
print_r($arrTask);
|
||||
// $QueueManage->set($arrTask);
|
||||
// print_r($arrTask);
|
||||
$QueueManage->set($arrTask);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public function getNovelInfo($arrData)
|
||||
{
|
||||
// content="mobile" 有这个的是手机，否则是PC
|
||||
print_r($arrData);
|
||||
|
||||
if ($arrData['page'] == 0) {
|
||||
$strUri = sprintf('/shu/%s.html', $arrData['n_id']);
|
||||
} else {
|
||||
$strUri = sprintf('/shu/%s_%s.html', $arrData['n_id']);
|
||||
}
|
||||
|
||||
$ResponseBase = $this->Client->get($strUri);
|
||||
$strNovelContent = $ResponseBase->getBody()->getContents();
|
||||
|
||||
$NovelQuery = QueryList::html($strNovelContent);
|
||||
|
||||
$arrNovel = $this->getNovelHead($NovelQuery, $arrData['n_id'], $arrData['page']);
|
||||
|
||||
if (empty($arrNovel)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
print_r($arrNovel);
|
||||
// $strResult = (string)$Response->getBody();
|
||||
}
|
||||
|
||||
private function getNovelHead(QueryList $QueryList, $intSourceNovelId, $intPage = 0)
|
||||
{
|
||||
$arrResult = [];
|
||||
$QueryList->find('meta[property]')->each(function ($Item) use (&$arrResult) {
|
||||
$property = str_replace(':', '_', $Item->attr('property'));
|
||||
$arrResult[$property] = $Item->attr('content');
|
||||
});
|
||||
|
||||
if (empty($arrResult)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// print_r($arrResult);
|
||||
|
||||
$arrResult['novel_source_id'] = 'zw630-' . $intSourceNovelId;
|
||||
|
||||
try {
|
||||
$Col = $this->MongoBase->getDb()->selectCollection('novel');
|
||||
|
||||
$arrExisting = $Col->findOne(['novel_source_id' => $arrResult['novel_source_id']]);
|
||||
|
||||
// use novel
|
||||
// db.novel.createIndex({ "novel_source_id": 1 }, { unique: true })
|
||||
|
||||
if ($arrExisting) {
|
||||
|
||||
$arrExisting = (array) $arrExisting; // MongoDB\BSON\Document 转为数组
|
||||
$arrResult = array_merge($arrExisting, $arrResult); // 合并，保留原有字段
|
||||
|
||||
$arrResult['updated_at'] = new \MongoDB\BSON\UTCDateTime(); // 添加更新时间
|
||||
$Col->updateOne(
|
||||
['novel_source_id' => $arrResult['novel_source_id']],
|
||||
['$set' => $arrResult],
|
||||
// ['upsert' => false]
|
||||
);
|
||||
} else {
|
||||
$arrResult['novel_id'] = $this->MongoBase->getNextSequence('novel');
|
||||
$this->MongoBase->insertOne('novel', $arrResult);
|
||||
}
|
||||
return $arrResult;
|
||||
} catch (\MongoDB\Exception\Exception $e) {
|
||||
throw new \Exception("Upsert failed: " . $e->getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
public function getNovelInfo1($arrData)
|
||||
{
|
||||
//var_dump('getNovelFengMian');
|
||||
//var_dump($arrData);
|
||||
|
||||
Reference in New Issue
Block a user