debug
This commit is contained in:
make
319
code/app/task/crawler/zw/Scheduler.php
Normal file
319
code/app/task/crawler/zw/Scheduler.php
Normal file
@@ -0,0 +1,319 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace app\task\crawler\zw;
|
||||
|
||||
use app\model\ChapterModel;
|
||||
use app\task\crawler\zw\page\Site;
|
||||
use microserver\QueueManage;
|
||||
use storage\StorageCore;
|
||||
|
||||
/**
|
||||
* @mixin think\Model
|
||||
*/
|
||||
class Scheduler
|
||||
{
|
||||
protected $intSlice = 8;
|
||||
|
||||
protected $intStartNSourceId = 1;
|
||||
|
||||
protected $intEndNSourceId = 1000;
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var \microserver\QueueManage
|
||||
*/
|
||||
protected $QueueManage = NULL;
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*/
|
||||
public function __construct()
|
||||
{
|
||||
$this->QueueManage = new QueueManage();
|
||||
$this->init();
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
public function init()
|
||||
{
|
||||
$this->intSlice = 8;
|
||||
}
|
||||
|
||||
/**
|
||||
* create a site-wide craw task
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
public function crawlFull()
|
||||
{
|
||||
$intSliceSize = $this->intSlice;
|
||||
|
||||
$arrFilter = [];
|
||||
for ($intNId = $this->intStartNSourceId; $intNId <= $this->intEndNSourceId; $intNId++) {
|
||||
$arrFilter[] = [
|
||||
'n_source_id' => $intNId,
|
||||
'page' => 0,
|
||||
];
|
||||
|
||||
if ($intNId % $intSliceSize == 0) {
|
||||
$this->fetchNovel($arrFilter);
|
||||
$arrFilter = [];
|
||||
}
|
||||
}
|
||||
if (!empty($arrFilter)) {
|
||||
$this->fetchNovel($arrFilter);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @param array $arrFilter
|
||||
* @return void
|
||||
*/
|
||||
public function fetchNovel(array $arrFilter)
|
||||
{
|
||||
$arrTask = [
|
||||
'callback' => [static::class, 'getNovelInfo'],
|
||||
'data' => $arrFilter,
|
||||
];
|
||||
|
||||
$this->QueueManage->set($arrTask);
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @param array $arrFilter
|
||||
* @return void
|
||||
*/
|
||||
public function getNovelInfo(array $arrFilter)
|
||||
{
|
||||
$Site = $this->getSite();
|
||||
|
||||
$arrNovelPageList = $Site->getNovelPageList($arrFilter);
|
||||
|
||||
$arrCover = [];
|
||||
foreach ($arrNovelPageList as $NovelPage) {
|
||||
// $arrNovel = $NovelPage->getNovelInfo();
|
||||
$boolResult = $NovelPage->saveNovelInfo();
|
||||
// return;
|
||||
|
||||
if ($boolResult == false) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$arrNextPageData = $NovelPage->getNextPageArgs();
|
||||
if (!empty($arrNextPageData)) {
|
||||
$this->fetchNovel([
|
||||
[
|
||||
'n_source_id' => $arrNextPageData['n_source_id'],
|
||||
'page' => $arrNextPageData['page'],
|
||||
]
|
||||
]);
|
||||
}
|
||||
|
||||
$arrCoverItem = $NovelPage->getDownCoverInfo();
|
||||
|
||||
if (!StorageCore::getInstance()->has($arrCoverItem['local_path'])) {
|
||||
$arrCover[] = $arrCoverItem;
|
||||
}
|
||||
|
||||
$arrChapterList = $NovelPage->getChapters();
|
||||
$this->fetchChapter($NovelPage->intNId, $NovelPage->getData()['page'], $arrChapterList);
|
||||
}
|
||||
|
||||
if ($arrCover) {
|
||||
$Site->downRemoteImgToLocal($arrCover);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* create a task to craw the latest chapter
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
public function crawLatest()
|
||||
{
|
||||
$intSliceSize = $this->intSlice;
|
||||
$arrUri = [];
|
||||
$strUri = '/list/lastupdate_%s_0_0_%s.html';
|
||||
|
||||
$intEndCategory = $intEndPage = 10;
|
||||
|
||||
for ($intCategoryKey = 0; $intCategoryKey < $intEndCategory; $intCategoryKey++) {
|
||||
for ($intPage = 1; $intPage <= $intEndPage; $intPage++) {
|
||||
$arrUri[] = sprintf($strUri, $intCategoryKey, $intPage);
|
||||
}
|
||||
}
|
||||
|
||||
$arrUri = array_chunk($arrUri, $intSliceSize);
|
||||
|
||||
$arrTask = [
|
||||
'callback' => [static::class, 'getLatestChapter'],
|
||||
'data' => [],
|
||||
];
|
||||
|
||||
foreach ($arrUri as $arrUriSlice) {
|
||||
$arrTask['data'] = $arrUriSlice;
|
||||
$this->QueueManage->set($arrTask);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @param array $arrData
|
||||
* @return void
|
||||
*/
|
||||
public function getLatestChapter(array $arrFilter)
|
||||
{
|
||||
$Site = $this->getSite();
|
||||
|
||||
$arrLatestListPageList = $Site->getLatestListPageList($arrFilter);
|
||||
|
||||
$arrNSourceId = [];
|
||||
|
||||
foreach ($arrLatestListPageList as $LatestListPage) {
|
||||
$arrResult = $LatestListPage->getNovelFilter();
|
||||
$arrNSourceId = array_merge($arrNSourceId, $arrResult);
|
||||
}
|
||||
|
||||
$arrNSourceId = array_unique($arrNSourceId);
|
||||
$arrNSourceId = array_values($arrNSourceId);
|
||||
|
||||
$intSliceSize = $this->intSlice;
|
||||
$arrFilter = [];
|
||||
|
||||
foreach ($arrNSourceId as $intKey => $intNSourceId) {
|
||||
$arrFilter[] = [
|
||||
'n_source_id' => $intNSourceId,
|
||||
'page' => 0,
|
||||
];
|
||||
|
||||
if ($intKey % $intSliceSize == 0) {
|
||||
$this->fetchNovel($arrFilter);
|
||||
$arrFilter = [];
|
||||
}
|
||||
}
|
||||
|
||||
if (!empty($arrFilter)) {
|
||||
$this->fetchNovel($arrFilter);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @param integer $intNId
|
||||
* @param integer $intPage
|
||||
* @param array $arrChapterList
|
||||
* @return void
|
||||
*/
|
||||
public function fetchChapter(int $intNId, int $intPage, array $arrChapterList = [])
|
||||
{
|
||||
$intSliceSize = $this->intSlice;
|
||||
$intPageSize = 100;
|
||||
|
||||
$arrTask = [
|
||||
'callback' => [static::class, 'getChapterInfo'],
|
||||
'data' => [
|
||||
'n_id' => $intNId,
|
||||
]
|
||||
];
|
||||
$arrFilter = [];
|
||||
foreach ($arrChapterList as $intKey => $arrChapter) {
|
||||
$arrChapter['c_sort_num'] = $intPage * $intPageSize + $intKey + 1;
|
||||
$arrChapter['c_source_id'] = supperExtractFilename($arrChapter['uri']);
|
||||
$arrChapter['c_page'] = 0;
|
||||
|
||||
$arrFilter[] = $arrChapter;
|
||||
if ($intKey % $intSliceSize == 0) {
|
||||
$arrTask['data']['filter'] = $arrFilter;
|
||||
$this->QueueManage->set($arrTask);
|
||||
$arrFilter = [];
|
||||
}
|
||||
}
|
||||
|
||||
if (!empty($arrFilter)) {
|
||||
$arrTask['data']['filter'] = $arrFilter;
|
||||
$this->QueueManage->set($arrTask);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var Site
|
||||
*/
|
||||
public $Site;
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return Site
|
||||
*/
|
||||
public function getSite()
|
||||
{
|
||||
if ($this->Site == NULL) {
|
||||
$this->Site = new Site;
|
||||
}
|
||||
return $this->Site;
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @param array $arrData
|
||||
* @return void
|
||||
*/
|
||||
public function getChapterInfo(array $arrData)
|
||||
{
|
||||
$Site = $this->getSite();
|
||||
|
||||
$ChapterModel = ChapterModel::getInstance();
|
||||
|
||||
foreach ($arrData['filter'] as $intKey => $arrChapterFilter) {
|
||||
$strUUId = sprintf('%s-chapter-%s-%s', $Site->getSiteCode(), $arrChapterFilter['c_source_id'], $arrChapterFilter['c_page']);
|
||||
$arrChapter = $ChapterModel->findOne(['c_source_uuid' => $strUUId]);
|
||||
|
||||
if (!empty($arrChapter) && !empty($arrChapter['content'])) {
|
||||
unset($arrData['filter'][$intKey]);
|
||||
}
|
||||
}
|
||||
|
||||
$arrData['filter'] = array_values($arrData['filter']);
|
||||
|
||||
$arrChapterPageList = $Site->getChapterPageList($arrData['filter']);
|
||||
|
||||
$arrNextPageFilterList = [];
|
||||
foreach ($arrChapterPageList as $ChapterPage) {
|
||||
|
||||
$ChapterPage->saveChapter($arrData['n_id']);
|
||||
|
||||
$arrNextPageFilter = $ChapterPage->getNextPage();
|
||||
|
||||
if (!empty($arrNextPageFilter)) {
|
||||
$arrNextPageFilterList[] = $arrNextPageFilter;
|
||||
}
|
||||
}
|
||||
|
||||
if (!empty($arrNextPageFilterList)) {
|
||||
|
||||
$arrTask = $arrData;
|
||||
$arrTask['filter'] = $arrNextPageFilterList;
|
||||
$arrTask = [
|
||||
'callback' => [static::class, 'getChapterInfo'],
|
||||
'data' => $arrTask
|
||||
];
|
||||
$this->QueueManage->set($arrTask);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace app\task\crawler\zw630\page;
|
||||
namespace app\task\crawler\zw\page;
|
||||
|
||||
use db\mongo\MongoBase;
|
||||
use ddl\DDLManage;
|
||||
@@ -100,7 +100,7 @@ abstract class BasePage
|
||||
*
|
||||
* @param string $strUri
|
||||
*/
|
||||
public function __construct(Site $Site, string $strUri)
|
||||
public function __construct($Site, string $strUri)
|
||||
{
|
||||
$this->setSite($Site);
|
||||
|
||||
226
code/app/task/crawler/zw/page/ChapterPage.php
Normal file
226
code/app/task/crawler/zw/page/ChapterPage.php
Normal file
@@ -0,0 +1,226 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace app\task\crawler\zw\page;
|
||||
|
||||
use app\model\ChapterModel;
|
||||
use storage\StorageCore;
|
||||
|
||||
class ChapterPage extends BasePage
|
||||
{
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $strChapterContent = '';
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var ChapterModel
|
||||
*/
|
||||
protected $ChapterModel;
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return ChapterModel
|
||||
*/
|
||||
public function getChapterModel(): ChapterModel
|
||||
{
|
||||
if ($this->ChapterModel == NULL) {
|
||||
$this->ChapterModel = ChapterModel::getInstance();
|
||||
}
|
||||
return $this->ChapterModel;
|
||||
}
|
||||
|
||||
public $arrOptions = [
|
||||
'typeMap' => [
|
||||
'root' => 'array',
|
||||
'document' => 'array',
|
||||
'array' => 'array',
|
||||
],
|
||||
];
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
public function getChapterInfo(): array
|
||||
{
|
||||
if ($this->arrChapterInfo === NULL) {
|
||||
|
||||
$this->arrChapterInfo = [];
|
||||
|
||||
$strPattern = "/qsbs\.bb\('([^']*)'\)/";
|
||||
|
||||
preg_match_all($strPattern, $this->getContent(), $arrMatches);
|
||||
|
||||
$strContent = '';
|
||||
|
||||
foreach ($arrMatches[1] as $str) {
|
||||
$strContent .= base64_decode($str);
|
||||
}
|
||||
|
||||
$this->strChapterContent = $strContent;
|
||||
|
||||
$this->arrChapterInfo['c_page_title'] = $this->getQueryList()->find('title')->text();
|
||||
$this->arrChapterInfo['c_page_keywords'] = $this->getQueryList()->find('meta[name="keywords"]')->attr('content');
|
||||
$this->arrChapterInfo['c_page_description'] = $this->getQueryList()->find('meta[name="description"]')->attr('content');
|
||||
|
||||
$strPattern = "/lastread\.set\((.*?)\);/";
|
||||
|
||||
if (preg_match($strPattern, $this->getContent(), $arrMatches)) {
|
||||
$strMatch = $arrMatches[1]; // 提取括号内的内容
|
||||
|
||||
$arrData = array_map('trim', explode(",", $strMatch));
|
||||
|
||||
$arrData = array_map(function ($item) {
|
||||
return trim($item, "'");
|
||||
}, $arrData);
|
||||
|
||||
$this->arrChapterInfo['c_page_data'] = $arrData;
|
||||
} else {
|
||||
$this->arrChapterInfo['c_page_data'] = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
return $this->arrChapterInfo;
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var array
|
||||
*/
|
||||
protected $arrChapterInfo;
|
||||
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @param integer $intNId
|
||||
* @return boolean
|
||||
*/
|
||||
public function saveChapter(int $intNId): bool
|
||||
{
|
||||
$arrPageChapter = $this->getChapterInfo();
|
||||
|
||||
if (
|
||||
empty($arrPageChapter) ||
|
||||
empty($arrPageChapter['c_page_title']) ||
|
||||
empty($arrPageChapter['c_page_data']) ||
|
||||
count($arrPageChapter['c_page_data']) < 5
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$strUUId = $this->getSite()->getSiteUUId('chapter-' . $this->arrData['c_source_id'] . '-' . $this->arrData['c_page']);
|
||||
|
||||
$arrChapter = [
|
||||
'n_id' => $intNId,
|
||||
'c_name' => $arrPageChapter['c_page_data'][3],
|
||||
'c_sort_num' => $this->arrData['c_sort_num'],
|
||||
'c_page' => $this->arrData['c_page'],
|
||||
'c_source_id' => $this->arrData['c_source_id'],
|
||||
'c_site_code' => $this->getSite()->getSiteCode(),
|
||||
'c_source_uuid' => $strUUId,
|
||||
];
|
||||
|
||||
$arrExistsChapter = $this->getChapterModel()->findOne(['c_source_uuid' => $strUUId], $this->arrOptions);
|
||||
|
||||
if ($arrExistsChapter) {
|
||||
|
||||
$arrExistsChapter = (array) $arrExistsChapter;
|
||||
|
||||
$arrUpdate = [
|
||||
'updated_at' => new \MongoDB\BSON\UTCDateTime(),
|
||||
];
|
||||
|
||||
$this->getChapterModel()->updateOne(
|
||||
['c_source_uuid' => $strUUId],
|
||||
['$set' => $arrUpdate],
|
||||
);
|
||||
$this->strCContentPath = $arrExistsChapter['c_content_path'];
|
||||
} else {
|
||||
|
||||
$arrInsertNovel = $arrChapter;
|
||||
|
||||
$arrInsertNovel['c_content_path'] = '/novel/' . ($intNId % 100) . '/' . $intNId . '/' . $arrChapter['c_sort_num'] . '_' . $arrChapter['c_page'] . '.txt';
|
||||
$arrInsertNovel['created_at'] = new \MongoDB\BSON\UTCDateTime();
|
||||
|
||||
try {
|
||||
$this->getChapterModel()->insert($arrInsertNovel);
|
||||
$this->updateNovelHaveChapter($intNId);
|
||||
$this->strCContentPath = $arrInsertNovel['c_content_path'];
|
||||
} catch (\Exception $e) {
|
||||
if ($e->getCode() === 11000) {
|
||||
return $this->saveChapter($intNId);
|
||||
} else {
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
StorageCore::getInstance()->put($this->strCContentPath, $this->strChapterContent);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
public $strCContentPath;
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @param integer $intNId
|
||||
* @return void
|
||||
*/
|
||||
protected function updateNovelHaveChapter(int $intNId)
|
||||
{
|
||||
$arrFilter = [
|
||||
'n_id' => $intNId,
|
||||
];
|
||||
|
||||
$ChapterModel = ChapterModel::getInstance();
|
||||
$arrLasteChapter = $ChapterModel->getLatestChapterByNId($intNId, false);
|
||||
|
||||
if (!empty($arrLasteChapter)) {
|
||||
$arrUpdate['$set']['n_have_chapter'] = 1;
|
||||
$arrUpdate['$set']['n_latest_chapter_name'] = $arrLasteChapter['c_name'];
|
||||
$this->getMongoBase()->updateOne('novel', $arrFilter, $arrUpdate);
|
||||
}
|
||||
}
|
||||
|
||||
protected $arrNextPage;
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
public function getNextPage(): array
|
||||
{
|
||||
if ($this->arrNextPage === NULL) {
|
||||
$this->arrNextPage = [];
|
||||
|
||||
if ($this->isMobile()) {
|
||||
$strPattern = "/var\s+hhekgsv\s*=\s*['\"](.*?)['\"]/";
|
||||
preg_match_all($strPattern, $this->getContent(), $arrMatches);
|
||||
$strNextPageUri = end($arrMatches[1]) ?: '';
|
||||
} else {
|
||||
$strHref = $this->getQueryList()->find('.read_btn a:contains("下一章")')->attr('href');
|
||||
$strNextPageUri = $strHref ?: '';
|
||||
}
|
||||
|
||||
if (is_numeric(strpos($strNextPageUri, '_'))) {
|
||||
$this->arrNextPage = $this->arrData;
|
||||
$this->arrNextPage['c_page'] += 1;
|
||||
$this->arrNextPage['uri'] = $strNextPageUri;
|
||||
}
|
||||
}
|
||||
|
||||
return $this->arrNextPage;
|
||||
}
|
||||
}
|
||||
64
code/app/task/crawler/zw/page/LatestListPage.php
Normal file
64
code/app/task/crawler/zw/page/LatestListPage.php
Normal file
@@ -0,0 +1,64 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace app\task\crawler\zw\page;
|
||||
|
||||
use app\model\NovelModel;
|
||||
|
||||
|
||||
class LatestListPage extends BasePage
|
||||
{
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var NovelModel
|
||||
*/
|
||||
protected $NovelModel;
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return NovelModel
|
||||
*/
|
||||
public function getNovelModel(): NovelModel
|
||||
{
|
||||
if ($this->NovelModel == NULL) {
|
||||
$this->NovelModel = NovelModel::getInstance();
|
||||
}
|
||||
return $this->NovelModel;
|
||||
}
|
||||
|
||||
static public $arrOptions = [
|
||||
'typeMap' => [
|
||||
'root' => 'array',
|
||||
'document' => 'array',
|
||||
'array' => 'array',
|
||||
],
|
||||
];
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
public function getNovelFilter(): array
|
||||
{
|
||||
$arrFilter = [];
|
||||
if ($this->isMobile()) {
|
||||
$arrFilter = $this->getQueryList()->find('div.wrap-box')->eq(0)->find('ul.sort_list li')->map(function ($li) {
|
||||
$strUri = $li->find('a')->eq(0)->attr('href');
|
||||
return supperExtractFilename($strUri);
|
||||
})->filter()->all();
|
||||
} else {
|
||||
|
||||
$arrFilter = $this->getQueryList()->find('div.layout.layout2.layout-col2.fl ul.txt-list li')->map(function ($li) {
|
||||
$strUri = $li->find('a')->eq(0)->attr('href');
|
||||
return supperExtractFilename($strUri);
|
||||
})->filter()->all();
|
||||
}
|
||||
|
||||
return $arrFilter;
|
||||
}
|
||||
}
|
||||
418
code/app/task/crawler/zw/page/NovelPage.php
Normal file
418
code/app/task/crawler/zw/page/NovelPage.php
Normal file
@@ -0,0 +1,418 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace app\task\crawler\zw\page;
|
||||
|
||||
use app\model\CategoryModel;
|
||||
use app\model\CountersModel;
|
||||
use app\model\NovelModel;
|
||||
use storage\StorageCore;
|
||||
|
||||
class NovelPage extends BasePage
|
||||
{
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var array
|
||||
*/
|
||||
protected $arrNovelInfo = [];
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var array
|
||||
*/
|
||||
protected $arrRelatedNovel = [];
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var array
|
||||
*/
|
||||
protected $arrForgeNovel = [];
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var NovelModel
|
||||
*/
|
||||
protected $NovelModel;
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return NovelModel
|
||||
*/
|
||||
public function getNovelModel(): NovelModel
|
||||
{
|
||||
if ($this->NovelModel == NULL) {
|
||||
$this->NovelModel = NovelModel::getInstance();
|
||||
}
|
||||
return $this->NovelModel;
|
||||
}
|
||||
|
||||
public $arrOptions = [
|
||||
'typeMap' => [
|
||||
'root' => 'array',
|
||||
'document' => 'array',
|
||||
'array' => 'array',
|
||||
],
|
||||
];
|
||||
|
||||
/**
|
||||
* 查询看的小说
|
||||
*
|
||||
* @param integer $intNSourceId
|
||||
* @param string $strName
|
||||
* @return array
|
||||
*/
|
||||
protected function findOrCreateNovel(int $intNSourceId, string $strName): array
|
||||
{
|
||||
$strUUId = $this->getSite()->getSiteUUId($intNSourceId);
|
||||
|
||||
$arrExistsNovel = $this->getNovelModel()->findOne(['n_source_uuid' => $strUUId], $this->arrOptions);
|
||||
|
||||
if ($arrExistsNovel) {
|
||||
$arrNovelInfo = (array) $arrExistsNovel;
|
||||
} else {
|
||||
$arrNovelInfo = [
|
||||
'n_id' => CountersModel::getInstance()->getNextSequence('novel'),
|
||||
'n_name' => $strName,
|
||||
'n_name_pinyin' => zhToPinYin($strName),
|
||||
'n_site_code' => $this->getSite()->getSiteCode(),
|
||||
'n_source_uuid' => $strUUId,
|
||||
'n_source_id' => $intNSourceId,
|
||||
'n_level' => 1,
|
||||
'n_have_chapter' => 0,
|
||||
'created_at' => new \MongoDB\BSON\UTCDateTime(),
|
||||
];
|
||||
try {
|
||||
$this->getNovelModel()->insert($arrNovelInfo);
|
||||
} catch (\Exception $e) {
|
||||
if ($e->getCode() === 11000) {
|
||||
return $this->findOrCreateNovel($intNSourceId, $strName);
|
||||
} else {
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
}
|
||||
return $arrNovelInfo;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 分析关联数据
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
public function getReletedNovel() {}
|
||||
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
public function getNovelInfo(): array
|
||||
{
|
||||
if (empty($this->arrNovelInfo)) {
|
||||
|
||||
# 获取head字段
|
||||
$this->getQueryList()->find('meta[property]')->each(function ($Item) {
|
||||
$strKey = str_replace(':', '_', $Item->attr('property'));
|
||||
$this->arrNovelInfo[$strKey] = $Item->attr('content');
|
||||
});
|
||||
|
||||
$this->getReletedNovel();
|
||||
}
|
||||
|
||||
return $this->arrNovelInfo;
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @param string $strName
|
||||
* @return string
|
||||
*/
|
||||
public function generateCoverPath($strName): string
|
||||
{
|
||||
$intTime = time() - mt_rand(1, 2 * 24 * 3600 * 360);
|
||||
|
||||
return sprintf('/img/%s/%s.jpg', date('Y/m/d/H', $intTime), $strName);
|
||||
}
|
||||
|
||||
public function filterOtherFields(&$arrNovel)
|
||||
{
|
||||
$arrFields = [
|
||||
'n_id',
|
||||
'n_name',
|
||||
'n_name_pinyin',
|
||||
'n_status',
|
||||
'n_author',
|
||||
'n_description',
|
||||
'n_category',
|
||||
'n_category_pinyin',
|
||||
'n_category_sex_en',
|
||||
'n_category_sex_zh',
|
||||
'n_cover_path',
|
||||
'n_latest_chapter_name',
|
||||
'n_update_time',
|
||||
'n_forge_novel',
|
||||
'n_related_novel',
|
||||
'n_site_code',
|
||||
'n_source_uuid',
|
||||
'n_source_id',
|
||||
'n_have_chapter',
|
||||
'created_at',
|
||||
'updated_at',
|
||||
];
|
||||
|
||||
foreach ($arrNovel as $strKey => $strVal) {
|
||||
if (!in_array($strKey, $arrFields)) {
|
||||
unset($arrNovel[$strKey]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return bool
|
||||
*/
|
||||
public function saveNovelInfo(): bool
|
||||
{
|
||||
$arrPageInfo = $this->getNovelInfo();
|
||||
if (
|
||||
empty($arrPageInfo) ||
|
||||
empty($arrPageInfo['og_title']) ||
|
||||
empty($arrPageInfo['og_image']) ||
|
||||
empty($arrPageInfo['og_novel_category'])
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$arrNovelInfo = [
|
||||
// 'n_id'=> $arrPageInfo['xxxxx'],
|
||||
'n_name' => $arrPageInfo['og_title'],
|
||||
'n_name_pinyin' => zhToPinYin($arrPageInfo['og_title']),
|
||||
'n_status' => $arrPageInfo['og_novel_status'],
|
||||
'n_author' => $arrPageInfo['og_novel_author'],
|
||||
'n_description' => $arrPageInfo['og_description'],
|
||||
'n_category' => $arrPageInfo['og_novel_category'],
|
||||
// 'n_category_pinyin' => $arrPageInfo['xxxxx'],
|
||||
// 'n_category_sex_en' => $arrPageInfo['xxxxx'],
|
||||
// 'n_category_sex_zh' => $arrPageInfo['xxxxx'],
|
||||
// 'n_cover_path' => $arrPageInfo['xxxxx'],
|
||||
'n_latest_chapter_name' => $arrPageInfo['og_novel_latest_chapter_name'],
|
||||
'n_update_time' => $arrPageInfo['og_novel_update_time'],
|
||||
'n_forge_novel' => $this->arrForgeNovel,
|
||||
'n_related_novel' => $this->arrRelatedNovel,
|
||||
|
||||
'n_site_code' => $this->getSite()->getSiteCode(),
|
||||
'n_source_id' => $this->arrData['n_source_id'],
|
||||
'n_source_uuid' => $this->getSite()->getSiteUUId($this->arrData['n_source_id']),
|
||||
|
||||
// 'n_have_chapter' => $arrPageInfo['xxxxx'],
|
||||
// 'created_at' => $arrPageInfo['xxxxx'],
|
||||
// 'updated_at' => $arrPageInfo['xxxxx'],
|
||||
|
||||
];
|
||||
|
||||
$arrExistsNovel = $this->getNovelModel()->findOne(['n_source_uuid' => $arrNovelInfo['n_source_uuid']], $this->arrOptions);
|
||||
|
||||
if ($arrExistsNovel) {
|
||||
|
||||
$arrExistsNovel = (array) $arrExistsNovel;
|
||||
|
||||
$arrUpdate = [
|
||||
'n_latest_chapter_name' => $arrNovelInfo['n_latest_chapter_name'],
|
||||
'n_update_time' => $arrNovelInfo['n_update_time'],
|
||||
'updated_at' => new \MongoDB\BSON\UTCDateTime(),
|
||||
];
|
||||
|
||||
# 如果没有分类，说明这条数据是最初从关联shu那里采集的，需要补全全部信息
|
||||
if (empty($arrExistsNovel['n_category'])) {
|
||||
$arrUpdate = array_merge($arrUpdate, $arrNovelInfo);
|
||||
$arrUpdate['n_related_novel'] = $this->arrRelatedNovel;
|
||||
$arrUpdate['n_forge_novel'] = $this->arrForgeNovel;
|
||||
$arrUpdate['n_category_pinyin'] = CategoryModel::getPinYin($this->arrNovelInfo['og_novel_category']);
|
||||
$arrUpdate['n_category_sex_en'] = CategoryModel::getSexEn($this->arrNovelInfo['og_novel_category']);
|
||||
$arrUpdate['n_category_sex_zh'] = CategoryModel::getSexZh($this->arrNovelInfo['og_novel_category']);
|
||||
$arrUpdate['n_related_novel'] = $this->arrRelatedNovel;
|
||||
$arrUpdate['n_forge_novel'] = $this->arrForgeNovel;
|
||||
}
|
||||
|
||||
# 图片不存在的时候，需要重新生成
|
||||
if (
|
||||
!isset($arrExistsNovel['n_cover_path']) ||
|
||||
empty($arrExistsNovel['n_cover_path']) ||
|
||||
!StorageCore::getInstance()->has($arrExistsNovel['n_cover_path']) ||
|
||||
(basename($arrExistsNovel['n_cover_path']) == '.jpg')
|
||||
) {
|
||||
$arrUpdate['n_cover_path'] = $this->generateCoverPath($arrNovelInfo['n_name_pinyin']);
|
||||
$this->strCoverLocalPath = $arrUpdate['n_cover_path'];
|
||||
} else {
|
||||
$this->strCoverLocalPath = $arrExistsNovel['n_cover_path'];
|
||||
}
|
||||
|
||||
$this->getNovelModel()->updateOne(
|
||||
['n_source_uuid' => $arrNovelInfo['n_source_uuid']],
|
||||
['$set' => $arrUpdate],
|
||||
);
|
||||
$this->intNId = $arrExistsNovel['n_id'];
|
||||
} else {
|
||||
|
||||
$arrNovelInfo['n_category_pinyin'] = CategoryModel::getPinYin($this->arrNovelInfo['og_novel_category']);
|
||||
$arrNovelInfo['n_category_sex_en'] = CategoryModel::getSexEn($this->arrNovelInfo['og_novel_category']);
|
||||
$arrNovelInfo['n_category_sex_zh'] = CategoryModel::getSexZh($this->arrNovelInfo['og_novel_category']);
|
||||
$arrNovelInfo['n_id'] = CountersModel::getInstance()->getNextSequence('novel');
|
||||
$arrNovelInfo['n_level'] = 1;
|
||||
$arrNovelInfo['n_have_chapter'] = 0;
|
||||
$arrNovelInfo['n_cover_path'] = $this->generateCoverPath($arrNovelInfo['n_name_pinyin']);
|
||||
$arrNovelInfo['n_related_novel'] = $this->arrRelatedNovel;
|
||||
$arrNovelInfo['n_forge_novel'] = $this->arrForgeNovel;
|
||||
$arrNovelInfo['created_at'] = new \MongoDB\BSON\UTCDateTime();
|
||||
|
||||
try {
|
||||
$this->getNovelModel()->insert($arrNovelInfo);
|
||||
$this->intNId = $arrNovelInfo['n_id'];
|
||||
$this->strCoverLocalPath = $arrNovelInfo['n_cover_path'];
|
||||
} catch (\Exception $e) {
|
||||
if ($e->getCode() === 11000) {
|
||||
return $this->saveNovelInfo();
|
||||
} else {
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
public $strCoverLocalPath = '';
|
||||
|
||||
public $intNId;
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
public function getDownCoverInfo(): array
|
||||
{
|
||||
return [
|
||||
'uri' => $this->arrNovelInfo['og_image'],
|
||||
'local_path' => $this->strCoverLocalPath,
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var array
|
||||
*/
|
||||
protected $arrLatestChapters = NULL;
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var array
|
||||
*/
|
||||
protected $arrChapters = NULL;
|
||||
|
||||
public function getLatestChapterByNIdsList(): array
|
||||
{
|
||||
if ($this->arrLatestChapters === NULL) {
|
||||
$this->arrLatestChapters = [];
|
||||
|
||||
if ($this->isMobile()) {
|
||||
$strClass = '.chapter-list';
|
||||
} else {
|
||||
$strClass = '.section-list';
|
||||
}
|
||||
|
||||
if ($this->arrData['page'] == 0) {
|
||||
$this->getQueryList()->find($strClass)->eq(0)->find("a")->map(function ($A) {
|
||||
$this->arrLatestChapters[] = [
|
||||
'name' => $A->text(),
|
||||
'uri' => $A->attr('href'),
|
||||
];
|
||||
});
|
||||
}
|
||||
}
|
||||
return $this->arrLatestChapters;
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
public function getChapters(): array
|
||||
{
|
||||
if ($this->arrChapters === NULL) {
|
||||
$this->arrChapters = [];
|
||||
if ($this->isMobile()) {
|
||||
$strClass = '.chapter-list';
|
||||
} else {
|
||||
$strClass = '.section-list';
|
||||
}
|
||||
|
||||
$intDivIndex = 0;
|
||||
if ($this->arrData['page'] == 0) {
|
||||
$intDivIndex = 1;
|
||||
}
|
||||
|
||||
$this->getQueryList()->find($strClass)->eq($intDivIndex)->find("a")->map(function ($A) {
|
||||
$this->arrChapters[] = [
|
||||
'name' => $A->text(),
|
||||
'uri' => $A->attr('href'),
|
||||
];
|
||||
});
|
||||
}
|
||||
return $this->arrChapters;
|
||||
}
|
||||
|
||||
|
||||
protected $arrNextPageArgs = NULL;
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
public function getNextPageArgs(): array
|
||||
{
|
||||
$this->arrNextPageArgs = [];
|
||||
if ($this->arrData['page'] == 0) {
|
||||
$this->getLatestChapterByNIdsList();
|
||||
$this->getChapters();
|
||||
|
||||
$arrLatestChapterFirstBlock = count($this->arrLatestChapters) > 0 ? $this->arrLatestChapters[0] : null;
|
||||
$arrLatestChapterSecondBlock = count($this->arrChapters) > 0 ? $this->arrChapters[count($this->arrChapters) - 1] : null;
|
||||
|
||||
if (
|
||||
!empty($arrLatestChapterFirstBlock) && !empty($arrLatestChapterSecondBlock)
|
||||
&& ($arrLatestChapterFirstBlock['uri'] != $arrLatestChapterSecondBlock['uri'])
|
||||
|
||||
) {
|
||||
$this->arrNextPageArgs = [
|
||||
'n_source_id' => $this->arrData['n_source_id'],
|
||||
'page' => $this->arrData['page'] + 1,
|
||||
];
|
||||
}
|
||||
} else {
|
||||
$strLastUri = $this->getQueryList()->find('select:first option:last')->attr('value');
|
||||
if ($strLastUri != '' && $strLastUri != $this->strUri) {
|
||||
$this->arrNextPageArgs = [
|
||||
'n_source_id' => $this->arrData['n_source_id'],
|
||||
'page' => $this->arrData['page'] + 1,
|
||||
];
|
||||
}
|
||||
}
|
||||
return $this->arrNextPageArgs;
|
||||
}
|
||||
}
|
||||
347
code/app/task/crawler/zw/page/Site.php
Normal file
347
code/app/task/crawler/zw/page/Site.php
Normal file
@@ -0,0 +1,347 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace app\task\crawler\zw\page;
|
||||
|
||||
use GuzzleHttp\Client;
|
||||
use GuzzleHttp\Promise;
|
||||
use storage\StorageCore;
|
||||
|
||||
/**
|
||||
* @mixin think\Model
|
||||
*/
|
||||
class Site
|
||||
{
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var Client
|
||||
*/
|
||||
protected $Client;
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $strDomain = '';
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $strSiteCode = '';
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*/
|
||||
public function __construct()
|
||||
{
|
||||
$this->initClient();
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*/
|
||||
public function initClient()
|
||||
{
|
||||
$arrConfig = [
|
||||
'http_version' => '1.1', // 强制用 HTTP/1.1，已有，保持
|
||||
'base_uri' => $this->strDomain, // 基础域名，已有，保持
|
||||
'http_errors' => false, // 不抛 HTTP 错误，已有，保持
|
||||
'timeout' => 60, // 从 160 秒改为 60 秒
|
||||
'connect_timeout' => 10, // 从 30 秒改为 10 秒
|
||||
'curl' => [
|
||||
CURLOPT_TCP_KEEPALIVE => 1, // 启用 TCP Keep-Alive，已有，保持
|
||||
CURLOPT_TCP_KEEPIDLE => 120, // Keep-Alive 空闲时间
|
||||
CURLOPT_TCP_KEEPINTVL => 60, // Keep-Alive 探测间隔
|
||||
// CURLOPT_SSL_VERIFYPEER => true, // 验证 SSL，已有，保持
|
||||
],
|
||||
'headers' => [
|
||||
'User-Agent' => 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36',
|
||||
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
|
||||
'Accept-Language' => 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
|
||||
'Referer' => $this->strDomain,
|
||||
'Connection' => 'keep-alive',
|
||||
'sec-ch-ua' => '"Not/A)Brand";v="8", "Chromium";v="130", "Google Chrome";v="130"',
|
||||
'sec-ch-ua-mobile' => '?1',
|
||||
'sec-ch-ua-platform' => '"Android"',
|
||||
'sec-fetch-dest' => 'document',
|
||||
'sec-fetch-mode' => 'navigate',
|
||||
'sec-fetch-site' => 'same-origin',
|
||||
'sec-fetch-user' => '?1',
|
||||
'upgrade-insecure-requests' => '1',
|
||||
],
|
||||
'pool_size' => 10,
|
||||
'verify' => false,
|
||||
];
|
||||
|
||||
$arrProxy = config('proxy');
|
||||
if ($arrProxy['proxy_status']) {
|
||||
|
||||
$strCode = $arrProxy['proxy_code'];
|
||||
$strProxyHost = $arrProxy['proxy_host'];
|
||||
$strKey = $arrProxy['proxy_key'];
|
||||
$strSecret = $arrProxy['proxy_secret'];
|
||||
|
||||
|
||||
switch ($strCode) {
|
||||
case 'XIONGMAO':
|
||||
$intTime = time();
|
||||
$strTxt = "orderno=" . $strKey . ",secret=" . $strSecret . ",timestamp=" . $intTime;
|
||||
$strSign = strtoupper(md5($strTxt));
|
||||
$strAuth = 'sign=' . $strSign . '&orderno=' . $strKey . '×tamp=' . $intTime;
|
||||
$arrConfig['proxy'] = $strProxyHost;
|
||||
$arrConfig['headers']['Proxy-Authorization'] = $strAuth;
|
||||
break;
|
||||
case 'KUAI':
|
||||
$arrConfig['proxy'] = $strProxyHost;
|
||||
$arrConfig['curl'][CURLOPT_PROXYAUTH] = CURLAUTH_BASIC;
|
||||
$arrConfig['curl'][CURLOPT_PROXYUSERPWD] = "$strKey:$strSecret";
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
$this->Client = new Client($arrConfig);
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return string
|
||||
*/
|
||||
public function getSiteCode(): string
|
||||
{
|
||||
return $this->strSiteCode;
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @param integer|string $strSign
|
||||
* @return string
|
||||
*/
|
||||
public function getSiteUUId(int|string $strSign): string
|
||||
{
|
||||
return $this->getSiteCode() . '-' . $strSign;
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return Client
|
||||
*/
|
||||
public function getClient(): Client
|
||||
{
|
||||
return $this->Client;
|
||||
}
|
||||
|
||||
public function getNovelPageUri($intNSourceId, $intPage): string
|
||||
{
|
||||
$intUriFirst = intdiv($intNSourceId, 1000);
|
||||
|
||||
$strUri = '';
|
||||
|
||||
if ($intPage == 0) {
|
||||
$strUri = sprintf('/%s/%s/', $intUriFirst, $intNSourceId);
|
||||
} else {
|
||||
$strUri = sprintf('/book/%s/list%s.html', $intNSourceId, $intPage);
|
||||
}
|
||||
|
||||
return $strUri;
|
||||
}
|
||||
|
||||
/**
|
||||
* retrieve multiple novel page model
|
||||
*
|
||||
* @param string $strUri
|
||||
* @return NovelPage[]
|
||||
*/
|
||||
public function getNovelPageList(array $arrFilter): array
|
||||
{
|
||||
|
||||
/** @var NovelPage[] */
|
||||
$arrNovelPage = [];
|
||||
|
||||
foreach ($arrFilter as $intKey => $arrItem) {
|
||||
|
||||
$arrFilter[$intKey]['uri'] = $this->getNovelPageUri($arrItem['n_source_id'], $arrItem['page']);
|
||||
|
||||
$arrPromises[$intKey] = $this->getClient()->getAsync($arrFilter[$intKey]['uri']);
|
||||
}
|
||||
|
||||
$arrResults = Promise\Utils::settle($arrPromises)->wait();
|
||||
|
||||
|
||||
$arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
|
||||
return $arrResult['state'] === 'fulfilled';
|
||||
});
|
||||
|
||||
foreach ($arrSuccessfulResults as $intKey => $arrResult) {
|
||||
$Response = $arrResult['value'];
|
||||
|
||||
$strContent = $Response->getBody()->getContents();
|
||||
if ($strContent == '索引文件不存在！') {
|
||||
continue;
|
||||
}
|
||||
|
||||
$NovelPage = $this->getNovelPage($this, $arrFilter[$intKey]['uri']);
|
||||
|
||||
$NovelPage->setContent($strContent);
|
||||
$NovelPage->setData($arrFilter[$intKey]);
|
||||
|
||||
$arrNovelPage[] = $NovelPage;
|
||||
}
|
||||
|
||||
return $arrNovelPage;
|
||||
}
|
||||
|
||||
/**
|
||||
* retrieve multiple chapter page model
|
||||
*
|
||||
* @param string $strUri
|
||||
* @return ChapterPage[]
|
||||
*/
|
||||
public function getChapterPageList(array $arrFilter): array
|
||||
{
|
||||
|
||||
/** @var ChapterPage[] */
|
||||
$arrChapterPage = [];
|
||||
|
||||
$arrPromises = [];
|
||||
foreach ($arrFilter as $intKey => $arrItem) {
|
||||
$arrPromises[$intKey] = $this->getClient()->getAsync($arrItem['uri']);
|
||||
}
|
||||
|
||||
$arrResults = Promise\Utils::settle($arrPromises)->wait();
|
||||
|
||||
$arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
|
||||
return $arrResult['state'] === 'fulfilled';
|
||||
});
|
||||
|
||||
foreach ($arrSuccessfulResults as $intKey => $arrResult) {
|
||||
$Response = $arrResult['value'];
|
||||
|
||||
$strContent = $Response->getBody()->getContents();;
|
||||
|
||||
$ChapterPage = $this->getChapterPage($this, $arrFilter[$intKey]['uri']);
|
||||
|
||||
$ChapterPage->setContent($strContent);
|
||||
$ChapterPage->setData($arrFilter[$intKey]);
|
||||
|
||||
$arrChapterPage[] = $ChapterPage;
|
||||
}
|
||||
|
||||
return $arrChapterPage;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* retrieve multiple chapter page model
|
||||
*
|
||||
* @param string $strUri
|
||||
* @return LatestListPage[]
|
||||
*/
|
||||
public function getLatestListPageList(array $arrFilter): array
|
||||
{
|
||||
|
||||
/** @var LatestListPage[] */
|
||||
$arrLatestListPage = [];
|
||||
|
||||
$arrPromises = [];
|
||||
foreach ($arrFilter as $intKey => $strUri) {
|
||||
$arrPromises[$intKey] = $this->getClient()->getAsync($strUri);
|
||||
}
|
||||
|
||||
$arrResults = Promise\Utils::settle($arrPromises)->wait();
|
||||
|
||||
$arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
|
||||
return $arrResult['state'] === 'fulfilled';
|
||||
});
|
||||
|
||||
foreach ($arrSuccessfulResults as $intKey => $arrResult) {
|
||||
$Response = $arrResult['value'];
|
||||
|
||||
$strContent = $Response->getBody()->getContents();;
|
||||
|
||||
$LatestListPage = $this->getLatestListPage($this, $arrFilter[$intKey]);
|
||||
|
||||
$LatestListPage->setContent($strContent);
|
||||
|
||||
$arrLatestListPage[] = $LatestListPage;
|
||||
}
|
||||
|
||||
|
||||
return $arrLatestListPage;
|
||||
}
|
||||
|
||||
/**
|
||||
* download the image to the local system
|
||||
*
|
||||
* @param array $arrFilter
|
||||
* @return void
|
||||
*/
|
||||
public function downRemoteImgToLocal(array $arrFilter)
|
||||
{
|
||||
$StorageCore = StorageCore::getInstance();
|
||||
|
||||
$arrPromises = [];
|
||||
|
||||
foreach ($arrFilter as $intKey => $arrItem) {
|
||||
if (!$StorageCore->has($arrItem['local_path'])) {
|
||||
$arrPromises[$intKey] = $this->getClient()->getAsync($arrItem['uri']);
|
||||
}
|
||||
}
|
||||
|
||||
$arrResults = Promise\Utils::settle($arrPromises)->wait();
|
||||
|
||||
$arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
|
||||
return $arrResult['state'] === 'fulfilled';
|
||||
});
|
||||
|
||||
foreach ($arrSuccessfulResults as $intKey => $arrResult) {
|
||||
$Response = $arrResult['value'];
|
||||
|
||||
$StorageCore->writeStream($arrFilter[$intKey]['local_path'], $Response->getBody()->detach());;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @param object $Site
|
||||
* @param string $strUri
|
||||
* @return NovelPage
|
||||
*/
|
||||
public function getNovelPage($Site, $strUri)
|
||||
{
|
||||
return new NovelPage($Site, $strUri);
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @param object $Site
|
||||
* @param string $strUri
|
||||
* @return ChapterPage
|
||||
*/
|
||||
public function getChapterPage($Site, $strUri)
|
||||
{
|
||||
return new ChapterPage($Site, $strUri);
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @param object $Site
|
||||
* @param string $strUri
|
||||
* @return LatestListPage
|
||||
*/
|
||||
public function getLatestListPage($Site, $strUri)
|
||||
{
|
||||
return new LatestListPage($Site, $strUri);
|
||||
}
|
||||
}
|
||||
@@ -4,278 +4,34 @@ declare(strict_types=1);
|
||||
|
||||
namespace app\task\crawler\zw630;
|
||||
|
||||
use app\model\ChapterModel;
|
||||
use app\task\crawler\zw630\page\NovelPage;
|
||||
use app\task\crawler\zw630\page\Site;
|
||||
use microserver\QueueManage;
|
||||
use storage\StorageCore;
|
||||
use app\task\crawler\zw\Scheduler as ZwScheduler;
|
||||
|
||||
/**
|
||||
* @mixin think\Model
|
||||
*/
|
||||
class Scheduler
|
||||
class Scheduler extends ZwScheduler
|
||||
{
|
||||
|
||||
static $intSlice = 3;
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var \microserver\QueueManage
|
||||
*/
|
||||
protected $QueueManage = NULL;
|
||||
|
||||
public function __construct()
|
||||
{
|
||||
$this->QueueManage = new QueueManage();
|
||||
}
|
||||
|
||||
/**
|
||||
* create a site-wide craw task
|
||||
* Undocumented function
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
public function crawlFull()
|
||||
public function init()
|
||||
{
|
||||
$intSliceSize = self::$intSlice;
|
||||
|
||||
$arrFilter = [];
|
||||
for ($intNId = 1; $intNId <= 531208; $intNId++) {
|
||||
// for ($intNId = 1; $intNId <= 10000; $intNId++) {
|
||||
// for ($intNId = 1; $intNId <= 1; $intNId++) {
|
||||
$arrFilter[] = [
|
||||
'n_source_id' => $intNId,
|
||||
'page' => 0,
|
||||
];
|
||||
|
||||
if ($intNId % $intSliceSize == 0) {
|
||||
$this->fetchNovel($arrFilter);
|
||||
$arrFilter = [];
|
||||
}
|
||||
}
|
||||
if (!empty($arrFilter)) {
|
||||
$this->fetchNovel($arrFilter);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* create a task to craw the latest chapter
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
public function crawLatest()
|
||||
{
|
||||
$intSliceSize = self::$intSlice;
|
||||
$arrUri = [];
|
||||
$strUri = '/list/lastupdate_%s_0_0_%s.html';
|
||||
|
||||
$intEndCategory = $intEndPage = 10;
|
||||
|
||||
for ($intCategoryKey = 0; $intCategoryKey < $intEndCategory; $intCategoryKey++) {
|
||||
for ($intPage = 1; $intPage <= $intEndPage; $intPage++) {
|
||||
$arrUri[] = sprintf($strUri, $intCategoryKey, $intPage);
|
||||
}
|
||||
}
|
||||
|
||||
$arrUri = array_chunk($arrUri, $intSliceSize);
|
||||
|
||||
$arrTask = [
|
||||
'callback' => [self::class, 'getLatestChapter'],
|
||||
'data' => [],
|
||||
];
|
||||
|
||||
foreach ($arrUri as $arrUriSlice) {
|
||||
$arrTask['data'] = $arrUriSlice;
|
||||
$this->QueueManage->set($arrTask);
|
||||
}
|
||||
$this->intSlice = 8;
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @param array $arrData
|
||||
* @return void
|
||||
* @return Site
|
||||
*/
|
||||
public function getLatestChapter(array $arrFilter)
|
||||
public function getSite()
|
||||
{
|
||||
$Site = new Site;
|
||||
|
||||
$arrLatestListPageList = $Site->getLatestListPageList($arrFilter);
|
||||
|
||||
$arrNSourceId = [];
|
||||
|
||||
foreach ($arrLatestListPageList as $LatestListPage) {
|
||||
$arrResult = $LatestListPage->getNovelFilter();
|
||||
$arrNSourceId = array_merge($arrNSourceId, $arrResult);
|
||||
}
|
||||
|
||||
$arrNSourceId = array_unique($arrNSourceId);
|
||||
$arrNSourceId = array_values($arrNSourceId);
|
||||
|
||||
$intSliceSize = self::$intSlice;
|
||||
$arrFilter = [];
|
||||
|
||||
foreach ($arrNSourceId as $intKey => $intNSourceId) {
|
||||
$arrFilter[] = [
|
||||
'n_source_id' => $intNSourceId,
|
||||
'page' => 0,
|
||||
];
|
||||
|
||||
if ($intKey % $intSliceSize == 0) {
|
||||
$this->fetchNovel($arrFilter);
|
||||
$arrFilter = [];
|
||||
}
|
||||
}
|
||||
|
||||
if (!empty($arrFilter)) {
|
||||
$this->fetchNovel($arrFilter);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @param array $arrFilter
|
||||
* @return void
|
||||
*/
|
||||
public function fetchNovel(array $arrFilter)
|
||||
{
|
||||
$arrTask = [
|
||||
'callback' => [self::class, 'getNovelInfo'],
|
||||
'data' => $arrFilter,
|
||||
];
|
||||
$this->QueueManage->set($arrTask);
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @param integer $intNId
|
||||
* @param integer $intPage
|
||||
* @param array $arrChapterList
|
||||
* @return void
|
||||
*/
|
||||
public function fetchChapter(int $intNId, int $intPage, array $arrChapterList = [])
|
||||
{
|
||||
$intSliceSize = self::$intSlice;
|
||||
$intPageSize = 100;
|
||||
|
||||
$arrTask = [
|
||||
'callback' => [self::class, 'getChapterInfo'],
|
||||
'data' => [
|
||||
'n_id' => $intNId,
|
||||
]
|
||||
];
|
||||
$arrFilter = [];
|
||||
foreach ($arrChapterList as $intKey => $arrChapter) {
|
||||
$arrChapter['c_sort_num'] = $intPage * $intPageSize + $intKey + 1;
|
||||
$arrChapter['c_source_id'] = supperExtractFilename($arrChapter['uri']);
|
||||
$arrChapter['c_page'] = 0;
|
||||
|
||||
$arrFilter[] = $arrChapter;
|
||||
if ($intKey % $intSliceSize == 0) {
|
||||
$arrTask['data']['filter'] = $arrFilter;
|
||||
$this->QueueManage->set($arrTask);
|
||||
$arrFilter = [];
|
||||
}
|
||||
}
|
||||
|
||||
if (!empty($arrFilter)) {
|
||||
$arrTask['data']['filter'] = $arrFilter;
|
||||
$this->QueueManage->set($arrTask);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @param array $arrFilter
|
||||
* @return void
|
||||
*/
|
||||
public function getNovelInfo(array $arrFilter)
|
||||
{
|
||||
$Site = new Site;
|
||||
|
||||
$arrNovelPageList = $Site->getNovelPageList($arrFilter);
|
||||
|
||||
$arrCover = [];
|
||||
foreach ($arrNovelPageList as $NovelPage) {
|
||||
$arrNovel = $NovelPage->getNovelInfo();
|
||||
if (empty($arrNovel)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$arrNextPageData = $NovelPage->getNextPageArgs();
|
||||
if (!empty($arrNextPageData)) {
|
||||
$this->fetchNovel([
|
||||
[
|
||||
'n_source_id' => $arrNextPageData['n_source_id'],
|
||||
'page' => $arrNextPageData['page'],
|
||||
]
|
||||
]);
|
||||
}
|
||||
|
||||
$arrCoverItem = $NovelPage->getDownCoverInfo();
|
||||
|
||||
if (!StorageCore::getInstance()->has($arrCoverItem['local_path'])) {
|
||||
$arrCover[] = $arrCoverItem;
|
||||
}
|
||||
|
||||
$arrChapterList = $NovelPage->getChapters();
|
||||
$this->fetchChapter($arrNovel['n_id'], $NovelPage->getData()['page'], $arrChapterList);
|
||||
}
|
||||
|
||||
if ($arrCover) {
|
||||
$Site->downRemoteImgToLocal($arrCover);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @param array $arrData
|
||||
* @return void
|
||||
*/
|
||||
public function getChapterInfo(array $arrData)
|
||||
{
|
||||
$Site = new Site;
|
||||
|
||||
$ChapterModel = ChapterModel::getInstance();
|
||||
|
||||
foreach ($arrData['filter'] as $intKey => $arrChapterFilter) {
|
||||
$strUUId = sprintf('%s-chapter-%s-%s', $Site->getSiteCode(), $arrChapterFilter['c_source_id'], $arrChapterFilter['c_page']);
|
||||
$arrChapter = $ChapterModel->findOne(['c_site_uuid' => $strUUId]);
|
||||
|
||||
if (!empty($arrChapter) && !empty($arrChapter['content'])) {
|
||||
unset($arrData['filter'][$intKey]);
|
||||
}
|
||||
}
|
||||
|
||||
$arrData['filter'] = array_values($arrData['filter']);
|
||||
|
||||
$arrChapterPageList = $Site->getChapterPageList($arrData['filter']);
|
||||
|
||||
$arrNextPageFilterList = [];
|
||||
foreach ($arrChapterPageList as $ChapterPage) {
|
||||
|
||||
$ChapterPage->saveChapter($arrData['n_id']);
|
||||
|
||||
$arrNextPageFilter = $ChapterPage->getNextPage();
|
||||
|
||||
if (!empty($arrNextPageFilter)) {
|
||||
$arrNextPageFilterList[] = $arrNextPageFilter;
|
||||
}
|
||||
}
|
||||
|
||||
if (!empty($arrNextPageFilterList)) {
|
||||
|
||||
$arrTask = $arrData;
|
||||
$arrTask['filter'] = $arrNextPageFilterList;
|
||||
$arrTask = [
|
||||
'callback' => [self::class, 'getChapterInfo'],
|
||||
'data' => $arrTask
|
||||
];
|
||||
$this->QueueManage->set($arrTask);
|
||||
if ($this->Site == NULL) {
|
||||
$this->Site = new Site;
|
||||
}
|
||||
return $this->Site;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,214 +4,6 @@ declare(strict_types=1);
|
||||
|
||||
namespace app\task\crawler\zw630\page;
|
||||
|
||||
use app\model\ChapterModel;
|
||||
use storage\StorageCore;
|
||||
use app\task\crawler\zw\page\ChapterPage as ZwChapterPage;
|
||||
|
||||
class ChapterPage extends BasePage
|
||||
{
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $strChapterContent = '';
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var ChapterModel
|
||||
*/
|
||||
protected $ChapterModel;
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return ChapterModel
|
||||
*/
|
||||
public function getChapterModel(): ChapterModel
|
||||
{
|
||||
if ($this->ChapterModel == NULL) {
|
||||
$this->ChapterModel = ChapterModel::getInstance();
|
||||
}
|
||||
return $this->ChapterModel;
|
||||
}
|
||||
|
||||
static public $arrOptions = [
|
||||
'typeMap' => [
|
||||
'root' => 'array',
|
||||
'document' => 'array',
|
||||
'array' => 'array',
|
||||
],
|
||||
];
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
public function getChapterInfo(): array
|
||||
{
|
||||
if ($this->arrChapterInfo === NULL) {
|
||||
|
||||
$this->arrChapterInfo = [];
|
||||
|
||||
$strPattern = "/qsbs\.bb\('([^']*)'\)/";
|
||||
|
||||
preg_match_all($strPattern, $this->getContent(), $arrMatches);
|
||||
|
||||
$strContent = '';
|
||||
|
||||
foreach ($arrMatches[1] as $str) {
|
||||
$strContent .= base64_decode($str);
|
||||
}
|
||||
|
||||
$this->strChapterContent = $strContent;
|
||||
|
||||
$this->arrChapterInfo['c_page_title'] = $this->getQueryList()->find('title')->text();
|
||||
$this->arrChapterInfo['c_page_keywords'] = $this->getQueryList()->find('meta[name="keywords"]')->attr('content');
|
||||
$this->arrChapterInfo['c_page_description'] = $this->getQueryList()->find('meta[name="description"]')->attr('content');
|
||||
}
|
||||
|
||||
return $this->arrChapterInfo;
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var array
|
||||
*/
|
||||
protected $arrChapterInfo;
|
||||
|
||||
|
||||
public function filterOtherFields(&$arrChapter)
|
||||
{
|
||||
$arrFields = [
|
||||
'n_id',
|
||||
'name',
|
||||
'c_sort_num',
|
||||
'c_page',
|
||||
'c_source_id',
|
||||
'c_site_code',
|
||||
'c_site_uuid',
|
||||
'c_content_path',
|
||||
'created_at',
|
||||
'updated_at',
|
||||
|
||||
];
|
||||
|
||||
foreach ($arrChapter as $strKey => $strVal) {
|
||||
if (!in_array($strKey, $arrFields)) {
|
||||
unset($arrChapter[$strKey]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public function saveChapter(int $intNId): array
|
||||
{
|
||||
|
||||
try {
|
||||
$this->getChapterInfo();
|
||||
|
||||
$strUUId = $this->getSite()->getSiteUUId('chapter-' . $this->arrData['c_source_id'] . '-' . $this->arrData['c_page']);
|
||||
$arrExisting = $this->getChapterModel()->findOne(['c_site_uuid' => $strUUId], self::$arrOptions);
|
||||
|
||||
if ($arrExisting) {
|
||||
|
||||
$arrExisting = (array) $arrExisting;
|
||||
$this->arrChapterInfo = array_merge($arrExisting, $this->getChapterInfo());
|
||||
|
||||
$this->arrChapterInfo['updated_at'] = new \MongoDB\BSON\UTCDateTime();
|
||||
|
||||
$arrUpdate = $this->arrChapterInfo;
|
||||
|
||||
$this->filterOtherFields($arrUpdate);
|
||||
|
||||
$this->getChapterModel()->updateOne(
|
||||
['c_site_uuid' => $strUUId],
|
||||
['$set' => $arrUpdate],
|
||||
);
|
||||
} else {
|
||||
|
||||
$this->arrChapterInfo['n_id'] = $intNId;
|
||||
$this->arrChapterInfo['name'] = $this->arrData['name'];
|
||||
$this->arrChapterInfo['c_sort_num'] = $this->arrData['c_sort_num'];
|
||||
$this->arrChapterInfo['c_page'] = $this->arrData['c_page'];
|
||||
$this->arrChapterInfo['c_source_id'] = $this->arrData['c_source_id'];
|
||||
$this->arrChapterInfo['c_site_code'] = $this->getSite()->getSiteCode();
|
||||
$this->arrChapterInfo['c_site_uuid'] = $strUUId;
|
||||
$this->arrChapterInfo['c_content_path'] = '/novel/' . ($intNId % 100) . '/' . $intNId . '/' . $this->arrChapterInfo['c_sort_num'] . '_' . $this->arrChapterInfo['c_page'] . '.txt';
|
||||
$this->arrChapterInfo['created_at'] = new \MongoDB\BSON\UTCDateTime();
|
||||
$arrInsert = $this->arrChapterInfo;
|
||||
|
||||
$this->filterOtherFields($arrInsert);
|
||||
|
||||
$this->getChapterModel()->insert($arrInsert);
|
||||
|
||||
$this->updateNovelHaveChapter($intNId);
|
||||
}
|
||||
|
||||
if (mb_strlen($this->strChapterContent) > 20) {
|
||||
if (!StorageCore::getInstance()->has($this->arrChapterInfo['c_content_path'])) {
|
||||
StorageCore::getInstance()->set($this->arrChapterInfo['c_content_path'], $this->strChapterContent);
|
||||
}
|
||||
}
|
||||
return $this->arrChapterInfo;
|
||||
} catch (\Exception $e) {
|
||||
throw new \Exception("Upsert failed: " . $e->getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @param integer $intNId
|
||||
* @return void
|
||||
*/
|
||||
protected function updateNovelHaveChapter(int $intNId)
|
||||
{
|
||||
$arrFilter = [
|
||||
'n_id' => $intNId,
|
||||
];
|
||||
|
||||
$ChapterModel = ChapterModel::getInstance();
|
||||
$arrLasteChapter = $ChapterModel->getLatestChapterByNId($intNId, false);
|
||||
|
||||
if (!empty($arrLasteChapter)) {
|
||||
$arrUpdate['$set']['n_have_chapter'] = 1;
|
||||
$arrUpdate['$set']['og_novel_latest_chapter_name'] = $arrLasteChapter['name'];
|
||||
$this->getMongoBase()->updateOne('novel', $arrFilter, $arrUpdate);
|
||||
}
|
||||
}
|
||||
|
||||
protected $arrNextPage;
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
public function getNextPage(): array
|
||||
{
|
||||
if ($this->arrNextPage === NULL) {
|
||||
$this->arrNextPage = [];
|
||||
|
||||
if ($this->isMobile()) {
|
||||
$strPattern = "/var\s+hhekgsv\s*=\s*['\"](.*?)['\"]/";
|
||||
preg_match_all($strPattern, $this->getContent(), $arrMatches);
|
||||
$strNextPageUri = end($arrMatches[1]) ?: '';
|
||||
} else {
|
||||
$strHref = $this->getQueryList()->find('.read_btn a:contains("下一章")')->attr('href');
|
||||
$strNextPageUri = $strHref ?: '';
|
||||
}
|
||||
|
||||
if (is_numeric(strpos($strNextPageUri, '_'))) {
|
||||
$this->arrNextPage = $this->arrData;
|
||||
$this->arrNextPage['c_page'] += 1;
|
||||
$this->arrNextPage['uri'] = $strNextPageUri;
|
||||
}
|
||||
}
|
||||
|
||||
return $this->arrNextPage;
|
||||
}
|
||||
}
|
||||
class ChapterPage extends ZwChapterPage {}
|
||||
|
||||
@@ -4,63 +4,8 @@ declare(strict_types=1);
|
||||
|
||||
namespace app\task\crawler\zw630\page;
|
||||
|
||||
use app\model\CategoryModel;
|
||||
use app\model\CountersModel;
|
||||
|
||||
use app\model\NovelModel;
|
||||
use app\task\crawler\zw\page\LatestListPage as ZwLatestListPage;
|
||||
|
||||
|
||||
class LatestListPage extends BasePage
|
||||
{
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var NovelModel
|
||||
*/
|
||||
protected $NovelModel;
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return NovelModel
|
||||
*/
|
||||
public function getNovelModel(): NovelModel
|
||||
{
|
||||
if ($this->NovelModel == NULL) {
|
||||
$this->NovelModel = NovelModel::getInstance();
|
||||
}
|
||||
return $this->NovelModel;
|
||||
}
|
||||
|
||||
static public $arrOptions = [
|
||||
'typeMap' => [
|
||||
'root' => 'array',
|
||||
'document' => 'array',
|
||||
'array' => 'array',
|
||||
],
|
||||
];
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
public function getNovelFilter(): array
|
||||
{
|
||||
$arrFilter = [];
|
||||
if ($this->isMobile()) {
|
||||
$arrFilter = $this->getQueryList()->find('div.wrap-box')->eq(0)->find('ul.sort_list li')->map(function ($li) {
|
||||
$strUri = $li->find('a')->eq(0)->attr('href');
|
||||
return supperExtractFilename($strUri);
|
||||
})->filter()->all();
|
||||
} else {
|
||||
|
||||
$arrFilter = $this->getQueryList()->find('div.layout.layout2.layout-col2.fl ul.txt-list li')->map(function ($li) {
|
||||
$strUri = $li->find('a')->eq(0)->attr('href');
|
||||
return supperExtractFilename($strUri);
|
||||
})->filter()->all();
|
||||
}
|
||||
|
||||
return $arrFilter;
|
||||
}
|
||||
}
|
||||
class LatestListPage extends ZwLatestListPage {}
|
||||
|
||||
@@ -4,120 +4,31 @@ declare(strict_types=1);
|
||||
|
||||
namespace app\task\crawler\zw630\page;
|
||||
|
||||
use app\model\CategoryModel;
|
||||
use app\model\CountersModel;
|
||||
use app\model\NovelModel;
|
||||
|
||||
|
||||
class NovelPage extends BasePage
|
||||
use app\task\crawler\zw\page\NovelPage as ZwNovelPage;
|
||||
|
||||
class NovelPage extends ZwNovelPage
|
||||
{
|
||||
/**
|
||||
* Undocumented variable
|
||||
* 分析关联数据
|
||||
*
|
||||
* @var array
|
||||
* @return void
|
||||
*/
|
||||
protected $arrNovelInfo = [];
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var array
|
||||
*/
|
||||
protected $arrRelatedNovel = [];
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var array
|
||||
*/
|
||||
protected $arrForgeNovel = [];
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var NovelModel
|
||||
*/
|
||||
protected $NovelModel;
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return NovelModel
|
||||
*/
|
||||
public function getNovelModel(): NovelModel
|
||||
public function getReletedNovel()
|
||||
{
|
||||
if ($this->NovelModel == NULL) {
|
||||
$this->NovelModel = NovelModel::getInstance();
|
||||
}
|
||||
return $this->NovelModel;
|
||||
}
|
||||
|
||||
static public $arrOptions = [
|
||||
'typeMap' => [
|
||||
'root' => 'array',
|
||||
'document' => 'array',
|
||||
'array' => 'array',
|
||||
],
|
||||
];
|
||||
|
||||
/**
|
||||
* 查询看的小说
|
||||
*
|
||||
* @param integer $intNSourceId
|
||||
* @param string $strName
|
||||
* @return array
|
||||
*/
|
||||
private function findOrCreateNovel(int $intNSourceId, string $strName): array
|
||||
{
|
||||
$strUUId = $this->getSite()->getSiteUUId($intNSourceId);
|
||||
|
||||
$arrExisting = $this->getNovelModel()->findOne(['n_site_uuid' => $strUUId], self::$arrOptions);
|
||||
|
||||
if ($arrExisting) {
|
||||
$arrNovelInfo = (array) $arrExisting;
|
||||
} else {
|
||||
$arrNovelInfo = [
|
||||
'og_novel_book_name' => $strName,
|
||||
'og_novel_pin_yin' => zhToPinYin($strName),
|
||||
'n_site_code' => $this->getSite()->getSiteCode(),
|
||||
'n_site_uuid' => $strUUId,
|
||||
'n_source_id' => $intNSourceId,
|
||||
'n_level' => 1,
|
||||
'n_have_chapter' => 0,
|
||||
'n_id' => CountersModel::getInstance()->getNextSequence('novel'),
|
||||
];
|
||||
$this->getNovelModel()->insert($arrNovelInfo);
|
||||
}
|
||||
return $arrNovelInfo;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
public function getNovelInfo($boolSave = true): array
|
||||
{
|
||||
if (empty($this->arrNovelInfo)) {
|
||||
|
||||
# 获取head字段
|
||||
$this->getQueryList()->find('meta[property]')->each(function ($Item) {
|
||||
$strKey = str_replace(':', '_', $Item->attr('property'));
|
||||
$this->arrNovelInfo[$strKey] = $Item->attr('content');
|
||||
});
|
||||
|
||||
# 获取 kan 和 shu 的关联数据
|
||||
if (empty($this->arrRelatedNovel) || empty($this->arrForgeNovel)) {
|
||||
$this->getQueryList()->find('.first_txt')->eq(0)->find("a")->map(function ($A) {
|
||||
$strName = $A->text();
|
||||
$strUri = $A->attr('href');
|
||||
|
||||
if (is_numeric(strpos($strUri, 'shu'))) {
|
||||
$intNSourceId = (int)supperExtractFilename($strUri);
|
||||
$arrNovelInfo = $this->findOrCreateNovel($intNSourceId, $strName);
|
||||
$this->arrRelatedNovel[] = [
|
||||
"n_id" => $arrNovelInfo["n_id"],
|
||||
"n_source_id" => $arrNovelInfo["n_source_id"],
|
||||
"og_novel_book_name" => $arrNovelInfo["og_novel_book_name"],
|
||||
"n_name" => $arrNovelInfo["n_name"],
|
||||
];
|
||||
} elseif (is_numeric(strpos($strUri, 'kan'))) {
|
||||
$intNForgeId = (int)supperExtractFilename($strUri);
|
||||
@@ -127,244 +38,6 @@ class NovelPage extends BasePage
|
||||
];
|
||||
}
|
||||
});
|
||||
|
||||
$this->arrNovelInfo['n_related_novel'] = $this->arrRelatedNovel;
|
||||
$this->arrNovelInfo['n_forge_novel'] = $this->arrForgeNovel;
|
||||
|
||||
$this->arrNovelInfo['og_novel_pin_yin'] = zhToPinYin($this->arrNovelInfo['og_novel_book_name'] ?? '');
|
||||
|
||||
if (!empty($this->arrNovelInfo['og_novel_category'])) {
|
||||
$this->arrNovelInfo['category_pinyin'] = CategoryModel::getPinYinByZh($this->arrNovelInfo['og_novel_category']);
|
||||
$this->arrNovelInfo['category_sex'] = CategoryModel::checkSex($this->arrNovelInfo['og_novel_category']);
|
||||
$this->arrNovelInfo['category_sex_zh'] = CategoryModel::checkSexZh($this->arrNovelInfo['og_novel_category']);
|
||||
}
|
||||
|
||||
$this->arrNovelInfo['n_site_code'] = $this->getSite()->getSiteCode();
|
||||
$this->arrNovelInfo['n_source_id'] = $this->arrData['n_source_id'];
|
||||
}
|
||||
|
||||
if ($boolSave && !empty($this->arrNovelInfo)) {
|
||||
$this->saveNovelInfo();
|
||||
}
|
||||
|
||||
return $this->arrNovelInfo;
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @param string $strName
|
||||
* @return string
|
||||
*/
|
||||
public function generateCoverPath($strName): string
|
||||
{
|
||||
return sprintf('/img/%s/%s.jpg', date('Y/m/d', time() - 24 * 3600 * 14), $strName);
|
||||
}
|
||||
|
||||
public function filterOtherFields(&$arrNovel)
|
||||
{
|
||||
$arrFields = [
|
||||
'n_id',
|
||||
'og_novel_book_name',
|
||||
'og_novel_pin_yin',
|
||||
'og_novel_status',
|
||||
'og_novel_author',
|
||||
'og_description',
|
||||
'og_novel_category',
|
||||
'category_pinyin',
|
||||
'category_sex',
|
||||
'category_sex_zh',
|
||||
'n_cover_path',
|
||||
'og_novel_latest_chapter_name',
|
||||
'og_novel_update_time',
|
||||
'n_forge_novel',
|
||||
'n_related_novel',
|
||||
'og_site_code',
|
||||
'n_site_uuid',
|
||||
'n_source_id',
|
||||
'n_have_chapter',
|
||||
'created_at',
|
||||
'updated_at',
|
||||
];
|
||||
|
||||
foreach ($arrNovel as $strKey => $strVal) {
|
||||
if (!in_array($strKey, $arrFields)) {
|
||||
unset($arrNovel[$strKey]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
public function saveNovelInfo()
|
||||
{
|
||||
try {
|
||||
$strUUId = $this->getSite()->getSiteUUId($this->arrData['n_source_id']);
|
||||
|
||||
$arrExisting = $this->getNovelModel()->findOne(['n_site_uuid' => $strUUId], self::$arrOptions);
|
||||
|
||||
if ($arrExisting) {
|
||||
|
||||
$arrExisting = (array) $arrExisting;
|
||||
$this->arrNovelInfo = array_merge($arrExisting, $this->arrNovelInfo);
|
||||
|
||||
if (
|
||||
!isset($this->arrNovelInfo['n_cover_path']) ||
|
||||
empty($this->arrNovelInfo['n_cover_path']) ||
|
||||
!file_exists($this->arrNovelInfo['n_cover_path']) ||
|
||||
|
||||
(basename($this->arrNovelInfo['n_cover_path']) == '.jpg')
|
||||
) {
|
||||
$this->arrNovelInfo['n_cover_path'] = $this->generateCoverPath($this->arrNovelInfo['og_novel_pin_yin']);
|
||||
}
|
||||
|
||||
$this->arrNovelInfo['updated_at'] = new \MongoDB\BSON\UTCDateTime();
|
||||
|
||||
$arrUpdate = $this->arrNovelInfo;
|
||||
$this->filterOtherFields($arrUpdate);
|
||||
|
||||
$this->getNovelModel()->updateOne(
|
||||
['n_site_uuid' => $strUUId],
|
||||
['$set' => $arrUpdate],
|
||||
);
|
||||
} else {
|
||||
$this->arrNovelInfo['n_site_uuid'] = $strUUId;
|
||||
$this->arrNovelInfo['n_id'] = CountersModel::getInstance()->getNextSequence('novel');
|
||||
$this->arrNovelInfo['n_level'] = 1;
|
||||
$this->arrNovelInfo['n_have_chapter'] = 0;
|
||||
$this->arrNovelInfo['n_cover_path'] = $this->generateCoverPath($this->arrNovelInfo['og_novel_pin_yin']);
|
||||
|
||||
$arrInsert = $this->arrNovelInfo;
|
||||
$this->filterOtherFields($arrInsert);
|
||||
|
||||
$this->getNovelModel()->insert($arrInsert);
|
||||
}
|
||||
return $this->arrNovelInfo;
|
||||
} catch (\Exception $e) {
|
||||
throw new \Exception("Upsert failed: " . $e->getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
public function getDownCoverInfo(): array
|
||||
{
|
||||
return [
|
||||
'uri' => $this->arrNovelInfo['og_image'],
|
||||
'local_path' => $this->arrNovelInfo['n_cover_path'],
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var array
|
||||
*/
|
||||
protected $arrLatestChapters = NULL;
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var array
|
||||
*/
|
||||
protected $arrChapters = NULL;
|
||||
|
||||
public function getLatestChapterByNIdsList(): array
|
||||
{
|
||||
if ($this->arrLatestChapters === NULL) {
|
||||
$this->arrLatestChapters = [];
|
||||
|
||||
if ($this->isMobile()) {
|
||||
$strClass = '.chapter-list';
|
||||
} else {
|
||||
$strClass = '.section-list';
|
||||
}
|
||||
|
||||
if ($this->arrData['page'] == 0) {
|
||||
$this->getQueryList()->find($strClass)->eq(0)->find("a")->map(function ($A) {
|
||||
$this->arrLatestChapters[] = [
|
||||
'name' => $A->text(),
|
||||
'uri' => $A->attr('href'),
|
||||
];
|
||||
});
|
||||
}
|
||||
}
|
||||
return $this->arrLatestChapters;
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
public function getChapters(): array
|
||||
{
|
||||
if ($this->arrChapters === NULL) {
|
||||
$this->arrChapters = [];
|
||||
if ($this->isMobile()) {
|
||||
$strClass = '.chapter-list';
|
||||
} else {
|
||||
$strClass = '.section-list';
|
||||
}
|
||||
|
||||
$intDivIndex = 0;
|
||||
if ($this->arrData['page'] == 0) {
|
||||
$intDivIndex = 1;
|
||||
}
|
||||
|
||||
$this->getQueryList()->find($strClass)->eq($intDivIndex)->find("a")->map(function ($A) {
|
||||
$this->arrChapters[] = [
|
||||
'name' => $A->text(),
|
||||
'uri' => $A->attr('href'),
|
||||
];
|
||||
});
|
||||
}
|
||||
return $this->arrChapters;
|
||||
}
|
||||
|
||||
|
||||
protected $arrNextPageArgs = NULL;
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return array
|
||||
*/
|
||||
public function getNextPageArgs(): array
|
||||
{
|
||||
$this->arrNextPageArgs = [];
|
||||
if ($this->arrData['page'] == 0) {
|
||||
$this->getLatestChapterByNIdsList();
|
||||
$this->getChapters();
|
||||
|
||||
$arrLatestChapterFirstBlock = count($this->arrLatestChapters) > 0 ? $this->arrLatestChapters[0] : null;
|
||||
$arrLatestChapterSecondBlock = count($this->arrChapters) > 0 ? $this->arrChapters[count($this->arrChapters) - 1] : null;
|
||||
|
||||
if (
|
||||
!empty($arrLatestChapterFirstBlock) && !empty($arrLatestChapterSecondBlock)
|
||||
&& ($arrLatestChapterFirstBlock['uri'] != $arrLatestChapterSecondBlock['uri'])
|
||||
|
||||
) {
|
||||
$this->arrNextPageArgs = [
|
||||
'n_source_id' => $this->arrData['n_source_id'],
|
||||
'page' => $this->arrData['page'] + 1,
|
||||
];
|
||||
}
|
||||
} else {
|
||||
$strLastUri = $this->getQueryList()->find('select:first option:last')->attr('value');
|
||||
if ($strLastUri != '' && $strLastUri != $this->strUri) {
|
||||
$this->arrNextPageArgs = [
|
||||
'n_source_id' => $this->arrData['n_source_id'],
|
||||
'page' => $this->arrData['page'] + 1,
|
||||
];
|
||||
}
|
||||
}
|
||||
return $this->arrNextPageArgs;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,22 +4,14 @@ declare(strict_types=1);
|
||||
|
||||
namespace app\task\crawler\zw630\page;
|
||||
|
||||
use GuzzleHttp\Client;
|
||||
use app\task\crawler\zw\page\Site as ZwSite;
|
||||
use GuzzleHttp\Promise;
|
||||
use storage\StorageCore;
|
||||
|
||||
use GuzzleHttp\Client;
|
||||
/**
|
||||
* @mixin think\Model
|
||||
*/
|
||||
class Site
|
||||
class Site extends ZwSite
|
||||
{
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var Client
|
||||
*/
|
||||
private $Client;
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
@@ -27,33 +19,17 @@ class Site
|
||||
*/
|
||||
protected $strDomain = 'https://www.630zw.com';
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $strSiteCode = 'zw630';
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return string
|
||||
*/
|
||||
public function getSiteCode(): string
|
||||
{
|
||||
return $this->strSiteCode;
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @param integer|string $strSign
|
||||
* @return string
|
||||
*/
|
||||
public function getSiteUUId(int|string $strSign): string
|
||||
{
|
||||
return $this->getSiteCode() . '-' . $strSign;
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*/
|
||||
public function __construct()
|
||||
public function initClient()
|
||||
{
|
||||
$arrConfig = [
|
||||
'http_version' => '1.1', // 强制用 HTTP/1.1，已有，保持
|
||||
@@ -82,208 +58,90 @@ class Site
|
||||
'sec-fetch-user' => '?1',
|
||||
'upgrade-insecure-requests' => '1',
|
||||
],
|
||||
'pool_size' => 5,
|
||||
'verify' => false,
|
||||
'pool_size' => 10,
|
||||
'verify' => false,
|
||||
];
|
||||
|
||||
$arrProxy = config('proxy');
|
||||
if ($arrProxy['proxy_status']) {
|
||||
$intTime = time();
|
||||
$strOrderNo = $arrProxy['proxy_order'];
|
||||
|
||||
$strCode = $arrProxy['proxy_code'];
|
||||
$strProxyHost = $arrProxy['proxy_host'];
|
||||
$strKey = $arrProxy['proxy_key'];
|
||||
$strSecret = $arrProxy['proxy_secret'];
|
||||
|
||||
$strTxt = "orderno=" . $strOrderNo . ",secret=" . $strSecret . ",timestamp=" . $intTime;
|
||||
$strSign = strtoupper(md5($strTxt));
|
||||
$strAuth = 'sign=' . $strSign . '&orderno=' . $strOrderNo . '×tamp=' . $intTime;
|
||||
|
||||
$arrConfig['proxy'] = 'http://dtbf.xiongmaodaili.com:8089';
|
||||
$arrConfig['headers']['Proxy-Authorization'] = $strAuth;
|
||||
switch ($strCode) {
|
||||
case 'XIONGMAO':
|
||||
$intTime = time();
|
||||
$strTxt = "orderno=" . $strKey . ",secret=" . $strSecret . ",timestamp=" . $intTime;
|
||||
$strSign = strtoupper(md5($strTxt));
|
||||
$strAuth = 'sign=' . $strSign . '&orderno=' . $strKey . '×tamp=' . $intTime;
|
||||
$arrConfig['proxy'] = $strProxyHost;
|
||||
$arrConfig['headers']['Proxy-Authorization'] = $strAuth;
|
||||
break;
|
||||
case 'KUAI':
|
||||
$arrConfig['proxy'] = $strProxyHost;
|
||||
$arrConfig['curl'][CURLOPT_PROXYAUTH] = CURLAUTH_BASIC;
|
||||
$arrConfig['curl'][CURLOPT_PROXYUSERPWD] = "$strKey:$strSecret";
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
$this->Client = new Client($arrConfig);
|
||||
}
|
||||
|
||||
|
||||
public function getNovelPageUri($intNSourceId, $intPage): string
|
||||
{
|
||||
$strUri = '';
|
||||
|
||||
if ($intPage == 0) {
|
||||
|
||||
$arrItem['uri'] = sprintf('/shu/%s.html', $intNSourceId);
|
||||
} else {
|
||||
|
||||
$arrItem['uri'] = sprintf('/shu/%s_%s.html', $intNSourceId, $intPage);
|
||||
}
|
||||
|
||||
return $strUri;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return Client
|
||||
*/
|
||||
public function getClient(): Client
|
||||
{
|
||||
return $this->Client;
|
||||
}
|
||||
|
||||
/**
|
||||
* retrieve novel page model
|
||||
*
|
||||
* @param object $Site
|
||||
* @param string $strUri
|
||||
* @return NovelPage
|
||||
*/
|
||||
public function getNovelPage(string $strUri): NovelPage
|
||||
public function getNovelPage($Site, $strUri)
|
||||
{
|
||||
return new NovelPage($this, $strUri);
|
||||
return new NovelPage($Site, $strUri);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* retrieve multiple novel page model
|
||||
* Undocumented function
|
||||
*
|
||||
* @param object $Site
|
||||
* @param string $strUri
|
||||
* @return NovelPage[]
|
||||
* @return ChapterPage
|
||||
*/
|
||||
public function getNovelPageList(array $arrFilter): array
|
||||
public function getChapterPage($Site, $strUri)
|
||||
{
|
||||
|
||||
/** @var NovelPage[] */
|
||||
$arrNovelPage = [];
|
||||
|
||||
foreach ($arrFilter as $intKey => $arrItem) {
|
||||
|
||||
if ($arrItem['page'] == 0) {
|
||||
$arrItem['uri'] = sprintf('/shu/%s.html', $arrItem['n_source_id']);
|
||||
} else {
|
||||
$arrItem['uri'] = sprintf('/shu/%s_%s.html', $arrItem['n_source_id'], $arrItem['page']);
|
||||
}
|
||||
$arrFilter[$intKey]['uri'] = $arrItem['uri'];
|
||||
|
||||
$arrPromises[$intKey] = $this->getClient()->getAsync($arrItem['uri']);
|
||||
}
|
||||
|
||||
$arrResults = Promise\Utils::settle($arrPromises)->wait();
|
||||
|
||||
$arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
|
||||
return $arrResult['state'] === 'fulfilled';
|
||||
});
|
||||
|
||||
foreach ($arrSuccessfulResults as $intKey => $arrResult) {
|
||||
$Response = $arrResult['value'];
|
||||
|
||||
$strContent = $Response->getBody()->getContents();
|
||||
if ($strContent == '索引文件不存在！') {
|
||||
continue;
|
||||
}
|
||||
|
||||
$NovelPage = new NovelPage($this, $arrFilter[$intKey]['uri']);
|
||||
|
||||
$NovelPage->setContent($strContent);
|
||||
$NovelPage->setData($arrFilter[$intKey]);
|
||||
|
||||
$arrNovelPage[] = $NovelPage;
|
||||
}
|
||||
|
||||
|
||||
return $arrNovelPage;
|
||||
return new ChapterPage($Site, $strUri);
|
||||
}
|
||||
|
||||
/**
|
||||
* retrieve multiple chapter page model
|
||||
* Undocumented function
|
||||
*
|
||||
* @param object $Site
|
||||
* @param string $strUri
|
||||
* @return ChapterPage[]
|
||||
* @return LatestListPage
|
||||
*/
|
||||
public function getChapterPageList(array $arrFilter): array
|
||||
public function getLatestListPage($Site, $strUri)
|
||||
{
|
||||
|
||||
/** @var ChapterPage[] */
|
||||
$arrChapterPage = [];
|
||||
|
||||
$arrPromises = [];
|
||||
foreach ($arrFilter as $intKey => $arrItem) {
|
||||
$arrPromises[$intKey] = $this->getClient()->getAsync($arrItem['uri']);
|
||||
}
|
||||
|
||||
$arrResults = Promise\Utils::settle($arrPromises)->wait();
|
||||
|
||||
$arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
|
||||
return $arrResult['state'] === 'fulfilled';
|
||||
});
|
||||
|
||||
foreach ($arrSuccessfulResults as $intKey => $arrResult) {
|
||||
$Response = $arrResult['value'];
|
||||
|
||||
$strContent = $Response->getBody()->getContents();;
|
||||
|
||||
$ChapterPage = new ChapterPage($this, $arrFilter[$intKey]['uri']);
|
||||
|
||||
$ChapterPage->setContent($strContent);
|
||||
$ChapterPage->setData($arrFilter[$intKey]);
|
||||
|
||||
$arrChapterPage[] = $ChapterPage;
|
||||
}
|
||||
|
||||
|
||||
return $arrChapterPage;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* retrieve multiple chapter page model
|
||||
*
|
||||
* @param string $strUri
|
||||
* @return LatestListPage[]
|
||||
*/
|
||||
public function getLatestListPageList(array $arrFilter): array
|
||||
{
|
||||
|
||||
/** @var LatestListPage[] */
|
||||
$arrLatestListPage = [];
|
||||
|
||||
$arrPromises = [];
|
||||
foreach ($arrFilter as $intKey => $strUri) {
|
||||
$arrPromises[$intKey] = $this->getClient()->getAsync($strUri);
|
||||
}
|
||||
|
||||
$arrResults = Promise\Utils::settle($arrPromises)->wait();
|
||||
|
||||
$arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
|
||||
return $arrResult['state'] === 'fulfilled';
|
||||
});
|
||||
|
||||
foreach ($arrSuccessfulResults as $intKey => $arrResult) {
|
||||
$Response = $arrResult['value'];
|
||||
|
||||
$strContent = $Response->getBody()->getContents();;
|
||||
|
||||
$LatestListPage = new LatestListPage($this, $arrFilter[$intKey]);
|
||||
|
||||
$LatestListPage->setContent($strContent);
|
||||
|
||||
$arrLatestListPage[] = $LatestListPage;
|
||||
}
|
||||
|
||||
|
||||
return $arrLatestListPage;
|
||||
}
|
||||
|
||||
/**
|
||||
* download the image to the local system
|
||||
*
|
||||
* @param array $arrFilter
|
||||
* @return void
|
||||
*/
|
||||
public function downRemoteImgToLocal(array $arrFilter)
|
||||
{
|
||||
$StorageCore = StorageCore::getInstance();
|
||||
|
||||
$arrPromises = [];
|
||||
|
||||
foreach ($arrFilter as $intKey => $arrItem) {
|
||||
if (!$StorageCore->has($arrItem['local_path'])) {
|
||||
$arrPromises[$intKey] = $this->getClient()->getAsync($arrItem['uri']);
|
||||
}
|
||||
}
|
||||
|
||||
$arrResults = Promise\Utils::settle($arrPromises)->wait();
|
||||
|
||||
$arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
|
||||
return $arrResult['state'] === 'fulfilled';
|
||||
});
|
||||
|
||||
foreach ($arrSuccessfulResults as $intKey => $arrResult) {
|
||||
$Response = $arrResult['value'];
|
||||
|
||||
$StorageCore->writeStream($arrFilter[$intKey]['local_path'], $Response->getBody()->detach());;
|
||||
}
|
||||
return new LatestListPage($Site, $strUri);
|
||||
}
|
||||
}
|
||||
|
||||
37
code/app/task/crawler/zw74/Scheduler.php
Normal file
37
code/app/task/crawler/zw74/Scheduler.php
Normal file
@@ -0,0 +1,37 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace app\task\crawler\zw74;
|
||||
|
||||
use app\task\crawler\zw74\page\Site;
|
||||
use app\task\crawler\zw\Scheduler as ZwScheduler;
|
||||
|
||||
/**
|
||||
* @mixin think\Model
|
||||
*/
|
||||
class Scheduler extends ZwScheduler
|
||||
{
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
public function init()
|
||||
{
|
||||
$this->intSlice = 8;
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return Site
|
||||
*/
|
||||
public function getSite()
|
||||
{
|
||||
if ($this->Site == NULL) {
|
||||
$this->Site = new Site;
|
||||
}
|
||||
return $this->Site;
|
||||
}
|
||||
}
|
||||
9
code/app/task/crawler/zw74/page/ChapterPage.php
Normal file
9
code/app/task/crawler/zw74/page/ChapterPage.php
Normal file
@@ -0,0 +1,9 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace app\task\crawler\zw74\page;
|
||||
|
||||
use app\task\crawler\zw\page\ChapterPage as ZwChapterPage;
|
||||
|
||||
class ChapterPage extends ZwChapterPage {}
|
||||
10
code/app/task/crawler/zw74/page/LatestListPage.php
Normal file
10
code/app/task/crawler/zw74/page/LatestListPage.php
Normal file
@@ -0,0 +1,10 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace app\task\crawler\zw74\page;
|
||||
|
||||
use app\model\NovelModel;
|
||||
use app\task\crawler\zw\page\LatestListPage as ZwLatestListPage;
|
||||
|
||||
class LatestListPage extends ZwLatestListPage {}
|
||||
48
code/app/task/crawler/zw74/page/NovelPage.php
Normal file
48
code/app/task/crawler/zw74/page/NovelPage.php
Normal file
@@ -0,0 +1,48 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace app\task\crawler\zw74\page;
|
||||
|
||||
|
||||
use app\task\crawler\zw\page\NovelPage as ZwNovelPage;
|
||||
|
||||
class NovelPage extends ZwNovelPage
|
||||
{
|
||||
/**
|
||||
* 分析关联数据
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
public function getReletedNovel()
|
||||
{
|
||||
if (empty($this->arrRelatedNovel) || empty($this->arrForgeNovel)) {
|
||||
$this->getQueryList()->find('.first_txt')->eq(0)->find("a")->map(function ($A) {
|
||||
$strName = $A->text();
|
||||
$strUri = $A->attr('href');
|
||||
|
||||
if (is_numeric(strpos($strUri, 'http'))) {
|
||||
return;
|
||||
}
|
||||
|
||||
$strPattern = '#^/\d+/(\d+)/$#';
|
||||
|
||||
if (is_numeric(strpos($strUri, 'info'))) {
|
||||
$intNForgeId = (int)supperExtractFilename($strUri);
|
||||
$this->arrForgeNovel[] = [
|
||||
"n_forge_id" => $intNForgeId,
|
||||
"n_forge_title" => $strName,
|
||||
];
|
||||
} elseif (preg_match($strPattern, $strUri, $arrMatches)) {
|
||||
$intNSourceId = (int)$arrMatches[1];
|
||||
$arrNovelInfo = $this->findOrCreateNovel($intNSourceId, $strName);
|
||||
$this->arrRelatedNovel[] = [
|
||||
"n_id" => $arrNovelInfo["n_id"],
|
||||
"n_source_id" => $arrNovelInfo["n_source_id"],
|
||||
"n_name" => $arrNovelInfo["n_name"],
|
||||
];
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
152
code/app/task/crawler/zw74/page/Site.php
Normal file
152
code/app/task/crawler/zw74/page/Site.php
Normal file
@@ -0,0 +1,152 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace app\task\crawler\zw74\page;
|
||||
|
||||
use app\task\crawler\zw\page\Site as ZwSite;
|
||||
use GuzzleHttp\Client;
|
||||
|
||||
/**
|
||||
* @mixin think\Model
|
||||
*/
|
||||
class Site extends ZwSite
|
||||
{
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $strDomain = 'https://www.74zw.cc';
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
*
|
||||
* @var string
|
||||
*/
|
||||
protected $strSiteCode = 'zw';
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*/
|
||||
public function initClient()
|
||||
{
|
||||
$arrConfig = [
|
||||
'http_version' => '1.1', // 强制用 HTTP/1.1，已有，保持
|
||||
'base_uri' => $this->strDomain, // 基础域名，已有，保持
|
||||
'http_errors' => false, // 不抛 HTTP 错误，已有，保持
|
||||
'timeout' => 60, // 从 160 秒改为 60 秒
|
||||
'connect_timeout' => 10, // 从 30 秒改为 10 秒
|
||||
'curl' => [
|
||||
CURLOPT_TCP_KEEPALIVE => 1, // 启用 TCP Keep-Alive，已有，保持
|
||||
CURLOPT_TCP_KEEPIDLE => 120, // Keep-Alive 空闲时间
|
||||
CURLOPT_TCP_KEEPINTVL => 60, // Keep-Alive 探测间隔
|
||||
// CURLOPT_SSL_VERIFYPEER => true, // 验证 SSL，已有，保持
|
||||
],
|
||||
'headers' => [
|
||||
'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36 Edg/135.0.0.0',
|
||||
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
|
||||
'Accept-Language' => 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
|
||||
'Referer' => $this->strDomain,
|
||||
'Connection' => 'keep-alive',
|
||||
'sec-ch-ua' => '"Microsoft Edge";v="135", "Not-A.Brand";v="8", "Chromium";v="135"',
|
||||
'sec-ch-ua-mobile' => '?0',
|
||||
'sec-ch-ua-platform' => '"Windows"',
|
||||
'sec-fetch-dest' => 'document',
|
||||
'sec-fetch-mode' => 'navigate',
|
||||
'sec-fetch-site' => 'same-origin',
|
||||
'sec-fetch-user' => '?1',
|
||||
'upgrade-insecure-requests' => '1',
|
||||
],
|
||||
'pool_size' => 10,
|
||||
'verify' => false,
|
||||
];
|
||||
|
||||
$arrProxy = config('proxy');
|
||||
if ($arrProxy['proxy_status']) {
|
||||
|
||||
$strCode = $arrProxy['proxy_code'];
|
||||
$strProxyHost = $arrProxy['proxy_host'];
|
||||
$strKey = $arrProxy['proxy_key'];
|
||||
$strSecret = $arrProxy['proxy_secret'];
|
||||
|
||||
|
||||
switch ($strCode) {
|
||||
case 'XIONGMAO':
|
||||
$intTime = time();
|
||||
$strTxt = "orderno=" . $strKey . ",secret=" . $strSecret . ",timestamp=" . $intTime;
|
||||
$strSign = strtoupper(md5($strTxt));
|
||||
$strAuth = 'sign=' . $strSign . '&orderno=' . $strKey . '×tamp=' . $intTime;
|
||||
$arrConfig['proxy'] = $strProxyHost;
|
||||
$arrConfig['headers']['Proxy-Authorization'] = $strAuth;
|
||||
break;
|
||||
case 'KUAI':
|
||||
$arrConfig['proxy'] = $strProxyHost;
|
||||
$arrConfig['curl'][CURLOPT_PROXYAUTH] = CURLAUTH_BASIC;
|
||||
$arrConfig['curl'][CURLOPT_PROXYUSERPWD] = "$strKey:$strSecret";
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
$this->Client = new Client($arrConfig);
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @param int $intNSourceId
|
||||
* @param int $intPage
|
||||
* @return string
|
||||
*/
|
||||
public function getNovelPageUri($intNSourceId, $intPage): string
|
||||
{
|
||||
$intUriFirst = intdiv($intNSourceId, 1000);
|
||||
|
||||
$strUri = '';
|
||||
|
||||
if ($intPage == 0) {
|
||||
$strUri = sprintf('/%s/%s/', $intUriFirst, $intNSourceId);
|
||||
} else {
|
||||
$strUri = sprintf('/book/%s/list%s.html', $intNSourceId, $intPage);
|
||||
}
|
||||
|
||||
return $strUri;
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @param object $Site
|
||||
* @param string $strUri
|
||||
* @return NovelPage
|
||||
*/
|
||||
public function getNovelPage($Site, $strUri)
|
||||
{
|
||||
return new NovelPage($Site, $strUri);
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @param object $Site
|
||||
* @param string $strUri
|
||||
* @return ChapterPage
|
||||
*/
|
||||
public function getChapterPage($Site, $strUri)
|
||||
{
|
||||
return new ChapterPage($Site, $strUri);
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @param object $Site
|
||||
* @param string $strUri
|
||||
* @return LatestListPage
|
||||
*/
|
||||
public function getLatestListPage($Site, $strUri)
|
||||
{
|
||||
return new LatestListPage($Site, $strUri);
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,7 @@ namespace app\task\module;
|
||||
|
||||
use app\model\PlanTaskModel;
|
||||
use app\task\crawler\zw630\Scheduler as Zw630Scheduler;
|
||||
use app\task\crawler\zw74\Scheduler as Zw74Scheduler;
|
||||
use microserver\QueueManage;
|
||||
use microserver\ProcessSanitizer;
|
||||
|
||||
@@ -34,6 +35,13 @@ class PlanTask
|
||||
case 'PULL_630_ZW_NOVEL_LATEST':
|
||||
self::pull630ZwNovelLatest();
|
||||
break;
|
||||
|
||||
case 'PULL_74_ZW_NOVEL':
|
||||
self::pull74ZwNovel();
|
||||
break;
|
||||
case 'PULL_74_ZW_NOVEL_LATEST':
|
||||
self::pull74ZwNovelLatest();
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
@@ -71,4 +79,37 @@ class PlanTask
|
||||
$QueueManage = new QueueManage();
|
||||
$QueueManage->set($arrTask);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
public static function pull74ZwNovel()
|
||||
{
|
||||
$arrTask = [
|
||||
'callback' => [Zw74Scheduler::class, 'crawlFull'],
|
||||
'data' => []
|
||||
];
|
||||
|
||||
$QueueManage = new QueueManage();
|
||||
$QueueManage->set($arrTask);
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
public static function pull74ZwNovelLatest()
|
||||
{
|
||||
$arrTask = [
|
||||
'callback' => [Zw74Scheduler::class, 'crawLatest'],
|
||||
'data' => []
|
||||
];
|
||||
|
||||
$QueueManage = new QueueManage();
|
||||
$QueueManage->set($arrTask);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user