admin/SEONexus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'dev' of ssh://git.bgbg.live:18150/root/SEONexus into dev

Browse Source
This commit is contained in:
make
2025-04-11 23:56:36 +08:00
parent bc92b44086 b4b5229d91
commit 1d431ecce5
5 changed files with 200 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
code/app/model/ChapterModel.php
View File

@@ -124,6 +124,8 @@ class ChapterModel extends MongoModel
if ($boolReadContent) {
$arrChapter['content'] = StorageCore::getInstance()->read($arrChapter['c_content_path']);
} else {
$arrChapter['content'] = '';
}
}

84
code/app/task/crawler/zw630/Scheduler.php
View File

@@ -4,6 +4,7 @@ declare(strict_types=1);
namespace app\task\crawler\zw630;
use app\model\ChapterModel;
use app\task\crawler\zw630\page\NovelPage;
use app\task\crawler\zw630\page\Site;
use microserver\QueueManage;
@@ -26,13 +27,13 @@ class Scheduler
}
/**
* 全局扫描任务
* create a site-wide craw task
*
* @return void
*/
public function crawlFull()
{
$intSliceSize = 6;
$intSliceSize = 4;
$arrFilter = [];
// for ($intNId = 1; $intNId <= 531208; $intNId++) {
@@ -54,22 +55,76 @@ class Scheduler
}
/**
* Undocumented function
* create a task to craw the latest chapter
*
* @return void
*/
public function crawLatest()
{
$intSliceSize = 3;
$arrUri = [];
$strUri = '/list/lastupdate_%s_0_0_%s.html';
for ($intCategoryKey = 0; $intCategoryKey < 10; $intCategoryKey++) {
for ($intPage = 1; $intPage <= 10; $intPage++) {
$intEndCategory = $intEndPage = 2;
for ($intCategoryKey = 0; $intCategoryKey < $intEndCategory; $intCategoryKey++) {
for ($intPage = 1; $intPage <= $intEndPage; $intPage++) {
$arrUri[] = sprintf($strUri, $intCategoryKey, $intPage);
}
}
// print_r($arrUri);
$arrUri = array_chunk($arrUri, $intSliceSize);
$arrTask = [
'callback' => [self::class, 'getLatestChapter'],
'data' => [],
];
foreach ($arrUri as $arrUriSlice) {
$arrTask['data'] = $arrUriSlice;
$this->QueueManage->set($arrTask);
}
}
/**
* Undocumented function
*
* @param array $arrData
* @return void
*/
public function getLatestChapter(array $arrFilter)
{
$Site = new Site;
$arrLatestListPageList = $Site->getLatestListPageList($arrFilter);
$arrNSourceId = [];
foreach ($arrLatestListPageList as $LatestListPage) {
$arrResult = $LatestListPage->getNovelFilter();
$arrNSourceId = array_merge($arrNSourceId, $arrResult);
}
$arrNSourceId = array_unique($arrNSourceId);
$arrNSourceId = array_values($arrNSourceId);
$intSliceSize = 4;
$arrFilter = [];
foreach ($arrNSourceId as $intKey => $intNSourceId) {
$arrFilter[] = [
'n_source_id' => $intNSourceId,
'page' => 0,
];
if ($intKey % $intSliceSize == 0) {
$this->fetchNovel($arrFilter);
$arrFilter = [];
}
}
if (!empty($arrFilter)) {
$this->fetchNovel($arrFilter);
}
}
/**
@@ -97,7 +152,7 @@ class Scheduler
*/
public function fetchChapter(int $intNId, int $intPage, array $arrChapterList = [])
{
$intSliceSize = 6;
$intSliceSize = 4;
$intPageSize = 100;
$arrTask = [
@@ -145,7 +200,6 @@ class Scheduler
continue;
}
$arrNextPageData = $NovelPage->getNextPageArgs();
if (!empty($arrNextPageData)) {
$this->fetchNovel([
@@ -175,6 +229,20 @@ class Scheduler
{
$Site = new Site;
$ChapterModel = ChapterModel::getInstance();
foreach ($arrData['filter'] as $intKey => $arrChapterFilter) {
$strUUId = sprintf('%s-chapter-%s-%s', $Site->getSiteCode(), $arrChapterFilter['c_source_id'], $arrChapterFilter['c_page']);
$arrChapter = $ChapterModel->findOne(['c_site_uuid' => $strUUId]);
if (!empty($arrChapter) && !empty($arrChapter['content'])) {
unset($arrData['filter'][$intKey]);
}
}
$arrData['filter'] = array_values($arrData['filter']);
$arrChapterPageList = $Site->getChapterPageList($arrData['filter']);
$arrNextPageFilterList = [];

12
code/app/task/crawler/zw630/page/BasePage.php
View File

@@ -107,6 +107,16 @@ abstract class BasePage
$this->strUri = $strUri;
}
/**
* Undocumented function
*
* @return string
*/
public function getUri(): string
{
return $this->strUri;
}
/**
* Undocumented function
*
@@ -238,6 +248,4 @@ abstract class BasePage
{
return $this->Site;
}
}

66
code/app/task/crawler/zw630/page/LatestListPage.php Normal file
View File

@@ -0,0 +1,66 @@
<?php
declare(strict_types=1);
namespace app\task\crawler\zw630\page;
use app\model\CategoryModel;
use app\model\CountersModel;
use app\model\NovelModel;
class LatestListPage extends BasePage
{
/**
* Undocumented variable
*
* @var NovelModel
*/
protected $NovelModel;
/**
* Undocumented function
*
* @return NovelModel
*/
public function getNovelModel(): NovelModel
{
if ($this->NovelModel == NULL) {
$this->NovelModel = NovelModel::getInstance();
}
return $this->NovelModel;
}
static public $arrOptions = [
'typeMap' => [
'root' => 'array',
'document' => 'array',
'array' => 'array',
],
];
/**
* Undocumented function
*
* @return array
*/
public function getNovelFilter(): array
{
$arrFilter = [];
if ($this->isMobile()) {
$arrFilter = $this->getQueryList()->find('div.wrap-box')->eq(0)->find('ul.sort_list li')->map(function ($li) {
$strUri = $li->find('a')->eq(0)->attr('href');
return supperExtractFilename($strUri);
})->filter()->all();
} else {
$arrFilter = $this->getQueryList()->find('div.layout.layout2.layout-col2.fl ul.txt-list li')->map(function ($li) {
$strUri = $li->find('a')->eq(0)->attr('href');
return supperExtractFilename($strUri);
})->filter()->all();
}
return $arrFilter;
}
}

54
code/app/task/crawler/zw630/page/Site.php
View File

@@ -144,7 +144,7 @@ class Site
}
/**
* Undocumented function
* retrieve novel page model
*
* @param string $strUri
* @return NovelPage
@@ -156,7 +156,7 @@ class Site
/**
* Undocumented function
* retrieve multiple novel page model
*
* @param string $strUri
* @return NovelPage[]
@@ -193,7 +193,7 @@ class Site
continue;
}
$NovelPage = new NovelPage($this, $arrItem['uri']);
$NovelPage = new NovelPage($this, $arrFilter[$intKey]['uri']);
$NovelPage->setContent($strContent);
$NovelPage->setData($arrFilter[$intKey]);
@@ -206,7 +206,7 @@ class Site
}
/**
* Undocumented function
* retrieve multiple chapter page model
*
* @param string $strUri
* @return ChapterPage[]
@@ -233,7 +233,7 @@ class Site
$strContent = $Response->getBody()->getContents();;
$ChapterPage = new ChapterPage($this, $arrItem['uri']);
$ChapterPage = new ChapterPage($this, $arrFilter[$intKey]['uri']);
$ChapterPage->setContent($strContent);
$ChapterPage->setData($arrFilter[$intKey]);
@@ -245,8 +245,48 @@ class Site
return $arrChapterPage;
}
/**
* Undocumented function
* retrieve multiple chapter page model
*
* @param string $strUri
* @return LatestListPage[]
*/
public function getLatestListPageList(array $arrFilter): array
{
/** @var LatestListPage[] */
$arrLatestListPage = [];
$arrPromises = [];
foreach ($arrFilter as $intKey => $strUri) {
$arrPromises[$intKey] = $this->getClient()->getAsync($strUri);
}
$arrResults = Promise\Utils::settle($arrPromises)->wait();
$arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
return $arrResult['state'] === 'fulfilled';
});
foreach ($arrSuccessfulResults as $intKey => $arrResult) {
$Response = $arrResult['value'];
$strContent = $Response->getBody()->getContents();;
$LatestListPage = new LatestListPage($this, $arrFilter[$intKey]);
$LatestListPage->setContent($strContent);
$arrLatestListPage[] = $LatestListPage;
}
return $arrLatestListPage;
}
/**
* download the image to the local system
*
* @param array $arrFilter
* @return void
@@ -272,8 +312,6 @@ class Site
foreach ($arrSuccessfulResults as $intKey => $arrResult) {
$Response = $arrResult['value'];
// $strContent = $Response->getBody()->getContents();
$StorageCore->writeStream($arrFilter[$intKey]['local_path'], $Response->getBody()->detach());;
}
}