admin/SEONexus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

debug

Browse Source
This commit is contained in:
Default
2025-04-15 16:33:25 +08:00
parent ccdae1e854
commit 5893a41166
23 changed files with 1686 additions and 1080 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
code/.example.env
View File

@@ -6,7 +6,7 @@ DEFAULT_LANG = zh-cn
# 数据库 # 数据库
DB_TYPE = mysql DB_TYPE = mysql
DB_HOST = 127.0.0.1 DB_HOST = 127.0.0.1
DB_NAME = seo DB_NAME = novel
DB_USER = root DB_USER = root
DB_PASS = qq123123 DB_PASS = qq123123
DB_PORT = 3306 DB_PORT = 3306
@@ -51,7 +51,14 @@ VIEW_TPL_CACHE = false
VIEW_DATA_CACHE = false VIEW_DATA_CACHE = false
# PROXY # PROXY
PROXY_STATUS = false PROXY_STATUS = true
PROXY_CODE = XIONGMAO # PROXY_CODE = XIONGMAO
PROXY_ORDER = DT20250413204710pFcgjMR9 # PROXY_HOST = dtbf.xiongmaodaili.com:8089
PROXY_SECRET = 583b141edcf65ce94f83591c5cb592ed # PROXY_KEY = DT20250413204710pFcgjMR9
# PROXY_SECRET = 583b141edcf65ce94f83591c5cb592ed
PROXY_CODE = KUAI
#PROXY_HOST = w739.kdltps.com
PROXY_HOST = w740.kdltps.com
PROXY_KEY = t14468368981914
PROXY_SECRET = 03xvfdc0

7
code/app/common.php
View File

@@ -1,4 +1,5 @@
<?php <?php
declare(strict_types=1); declare(strict_types=1);
// 应用公共文件 // 应用公共文件
@@ -750,6 +751,10 @@ namespace {
*/ */
function zhToPinYin(string $strZh): string function zhToPinYin(string $strZh): string
{ {
if (empty($strZh)) {
return $strZh;
}
$strPinYinToolsPath = root_path() . '/extend/tools/pinyin-tool'; $strPinYinToolsPath = root_path() . '/extend/tools/pinyin-tool';
$strPinYin = shell_exec(sprintf("%s %s", $strPinYinToolsPath, escapeshellcmd($strZh))); $strPinYin = shell_exec(sprintf("%s %s", $strPinYinToolsPath, escapeshellcmd($strZh)));
@@ -803,6 +808,4 @@ namespace {
} }
} }
} }
} }

321
code/app/task/crawler/zw/Scheduler.php Normal file
View File

@@ -0,0 +1,321 @@
<?php
declare(strict_types=1);
namespace app\task\crawler\zw;
use app\model\ChapterModel;
use app\task\crawler\zw\page\Site;
use microserver\QueueManage;
use storage\StorageCore;
/**
* @mixin think\Model
*/
class Scheduler
{
protected $intSlice = 8;
protected $intStartNSourceId = 7377;
protected $intEndNSourceId = 7377;
/**
* Undocumented variable
*
* @var \microserver\QueueManage
*/
protected $QueueManage = NULL;
/**
* Undocumented function
*/
public function __construct()
{
$this->QueueManage = new QueueManage();
$this->init();
}
/**
* Undocumented function
*
* @return void
*/
public function init()
{
$this->intSlice = 8;
}
/**
* create a site-wide craw task
*
* @return void
*/
public function crawlFull()
{
$intSliceSize = $this->intSlice;
$arrFilter = [];
for ($intNId = $this->intStartNSourceId; $intNId <= $this->intEndNSourceId; $intNId++) {
$arrFilter[] = [
'n_source_id' => $intNId,
'page' => 0,
];
if ($intNId % $intSliceSize == 0) {
$this->fetchNovel($arrFilter);
$arrFilter = [];
}
}
if (!empty($arrFilter)) {
$this->fetchNovel($arrFilter);
}
}
/**
* Undocumented function
*
* @param array $arrFilter
* @return void
*/
public function fetchNovel(array $arrFilter)
{
$arrTask = [
'callback' => [static::class, 'getNovelInfo'],
'data' => $arrFilter,
];
$this->QueueManage->set($arrTask);
}
/**
* Undocumented function
*
* @param array $arrFilter
* @return void
*/
public function getNovelInfo(array $arrFilter)
{
$Site = $this->getSite();
$arrNovelPageList = $Site->getNovelPageList($arrFilter);
$arrCover = [];
foreach ($arrNovelPageList as $NovelPage) {
$arrNovel = $NovelPage->getNovelInfo();
// $boolResult = $NovelPage->saveNovelInfo();
return;
if ($boolResult == false) {
continue;
}
$arrNextPageData = $NovelPage->getNextPageArgs();
if (!empty($arrNextPageData)) {
$this->fetchNovel([
[
'n_source_id' => $arrNextPageData['n_source_id'],
'page' => $arrNextPageData['page'],
]
]);
}
$arrCoverItem = $NovelPage->getDownCoverInfo();
if (!StorageCore::getInstance()->has($arrCoverItem['local_path'])) {
$arrCover[] = $arrCoverItem;
}
$arrChapterList = $NovelPage->getChapters();
$this->fetchChapter($arrNovel['n_id'], $NovelPage->getData()['page'], $arrChapterList);
}
if ($arrCover) {
$Site->downRemoteImgToLocal($arrCover);
}
}
/**
* create a task to craw the latest chapter
*
* @return void
*/
public function crawLatest()
{
$intSliceSize = $this->intSlice;
$arrUri = [];
$strUri = '/list/lastupdate_%s_0_0_%s.html';
$intEndCategory = $intEndPage = 10;
for ($intCategoryKey = 0; $intCategoryKey < $intEndCategory; $intCategoryKey++) {
for ($intPage = 1; $intPage <= $intEndPage; $intPage++) {
$arrUri[] = sprintf($strUri, $intCategoryKey, $intPage);
}
}
$arrUri = array_chunk($arrUri, $intSliceSize);
$arrTask = [
'callback' => [static::class, 'getLatestChapter'],
'data' => [],
];
foreach ($arrUri as $arrUriSlice) {
$arrTask['data'] = $arrUriSlice;
$this->QueueManage->set($arrTask);
}
}
/**
* Undocumented function
*
* @param array $arrData
* @return void
*/
public function getLatestChapter(array $arrFilter)
{
$Site = $this->getSite();
$arrLatestListPageList = $Site->getLatestListPageList($arrFilter);
$arrNSourceId = [];
foreach ($arrLatestListPageList as $LatestListPage) {
$arrResult = $LatestListPage->getNovelFilter();
$arrNSourceId = array_merge($arrNSourceId, $arrResult);
}
$arrNSourceId = array_unique($arrNSourceId);
$arrNSourceId = array_values($arrNSourceId);
$intSliceSize = $this->intSlice;
$arrFilter = [];
foreach ($arrNSourceId as $intKey => $intNSourceId) {
$arrFilter[] = [
'n_source_id' => $intNSourceId,
'page' => 0,
];
if ($intKey % $intSliceSize == 0) {
$this->fetchNovel($arrFilter);
$arrFilter = [];
}
}
if (!empty($arrFilter)) {
$this->fetchNovel($arrFilter);
}
}
/**
* Undocumented function
*
* @param integer $intNId
* @param integer $intPage
* @param array $arrChapterList
* @return void
*/
public function fetchChapter(int $intNId, int $intPage, array $arrChapterList = [])
{
$intSliceSize = $this->intSlice;
$intPageSize = 100;
$arrTask = [
'callback' => [static::class, 'getChapterInfo'],
'data' => [
'n_id' => $intNId,
]
];
$arrFilter = [];
foreach ($arrChapterList as $intKey => $arrChapter) {
$arrChapter['c_sort_num'] = $intPage * $intPageSize + $intKey + 1;
$arrChapter['c_source_id'] = supperExtractFilename($arrChapter['uri']);
$arrChapter['c_page'] = 0;
$arrFilter[] = $arrChapter;
if ($intKey % $intSliceSize == 0) {
$arrTask['data']['filter'] = $arrFilter;
$this->QueueManage->set($arrTask);
$arrFilter = [];
}
}
if (!empty($arrFilter)) {
$arrTask['data']['filter'] = $arrFilter;
$this->QueueManage->set($arrTask);
}
}
/**
* Undocumented variable
*
* @var Site
*/
public $Site;
/**
* Undocumented function
*
* @return Site
*/
public function getSite()
{
if ($this->Site == NULL) {
$this->Site = new Site;
}
return $this->Site;
}
/**
* Undocumented function
*
* @param array $arrData
* @return void
*/
public function getChapterInfo(array $arrData)
{
$Site = $this->getSite();
$ChapterModel = ChapterModel::getInstance();
foreach ($arrData['filter'] as $intKey => $arrChapterFilter) {
$strUUId = sprintf('%s-chapter-%s-%s', $Site->getSiteCode(), $arrChapterFilter['c_source_id'], $arrChapterFilter['c_page']);
$arrChapter = $ChapterModel->findOne(['c_site_uuid' => $strUUId]);
if (!empty($arrChapter) && !empty($arrChapter['content'])) {
unset($arrData['filter'][$intKey]);
}
}
$arrData['filter'] = array_values($arrData['filter']);
$arrChapterPageList = $Site->getChapterPageList($arrData['filter']);
$arrNextPageFilterList = [];
foreach ($arrChapterPageList as $ChapterPage) {
$ChapterPage->saveChapter($arrData['n_id']);
$arrNextPageFilter = $ChapterPage->getNextPage();
if (!empty($arrNextPageFilter)) {
$arrNextPageFilterList[] = $arrNextPageFilter;
}
}
if (!empty($arrNextPageFilterList)) {
$arrTask = $arrData;
$arrTask['filter'] = $arrNextPageFilterList;
$arrTask = [
'callback' => [static::class, 'getChapterInfo'],
'data' => $arrTask
];
$this->QueueManage->set($arrTask);
}
}
}

4
code/app/task/crawler/zw630/page/BasePage.php → code/app/task/crawler/zw/page/BasePage.php
View File

@@ -2,7 +2,7 @@
declare(strict_types=1); declare(strict_types=1);
namespace app\task\crawler\zw630\page; namespace app\task\crawler\zw\page;
use db\mongo\MongoBase; use db\mongo\MongoBase;
use ddl\DDLManage; use ddl\DDLManage;
@@ -100,7 +100,7 @@ abstract class BasePage
* *
* @param string $strUri * @param string $strUri
*/ */
public function __construct(Site $Site, string $strUri) public function __construct($Site, string $strUri)
{ {
$this->setSite($Site); $this->setSite($Site);

217
code/app/task/crawler/zw/page/ChapterPage.php Normal file
View File

@@ -0,0 +1,217 @@
<?php
declare(strict_types=1);
namespace app\task\crawler\zw\page;
use app\model\ChapterModel;
use storage\StorageCore;
class ChapterPage extends BasePage
{
/**
* Undocumented variable
*
* @var string
*/
protected $strChapterContent = '';
/**
* Undocumented variable
*
* @var ChapterModel
*/
protected $ChapterModel;
/**
* Undocumented function
*
* @return ChapterModel
*/
public function getChapterModel(): ChapterModel
{
if ($this->ChapterModel == NULL) {
$this->ChapterModel = ChapterModel::getInstance();
}
return $this->ChapterModel;
}
public $arrOptions = [
'typeMap' => [
'root' => 'array',
'document' => 'array',
'array' => 'array',
],
];
/**
* Undocumented function
*
* @return array
*/
public function getChapterInfo(): array
{
if ($this->arrChapterInfo === NULL) {
$this->arrChapterInfo = [];
$strPattern = "/qsbs\.bb\('([^']*)'\)/";
preg_match_all($strPattern, $this->getContent(), $arrMatches);
$strContent = '';
foreach ($arrMatches[1] as $str) {
$strContent .= base64_decode($str);
}
$this->strChapterContent = $strContent;
$this->arrChapterInfo['c_page_title'] = $this->getQueryList()->find('title')->text();
$this->arrChapterInfo['c_page_keywords'] = $this->getQueryList()->find('meta[name="keywords"]')->attr('content');
$this->arrChapterInfo['c_page_description'] = $this->getQueryList()->find('meta[name="description"]')->attr('content');
}
return $this->arrChapterInfo;
}
/**
* Undocumented variable
*
* @var array
*/
protected $arrChapterInfo;
public function filterOtherFields(&$arrChapter)
{
$arrFields = [
'n_id',
'name',
'c_sort_num',
'c_page',
'c_source_id',
'c_site_code',
'c_site_uuid',
'c_content_path',
'created_at',
'updated_at',
];
foreach ($arrChapter as $strKey => $strVal) {
if (!in_array($strKey, $arrFields)) {
unset($arrChapter[$strKey]);
}
}
}
public function saveChapter(int $intNId): array
{
try {
$this->getChapterInfo();
$strUUId = $this->getSite()->getSiteUUId('chapter-' . $this->arrData['c_source_id'] . '-' . $this->arrData['c_page']);
$arrExisting = $this->getChapterModel()->findOne(['c_site_uuid' => $strUUId], $this->arrOptions);
if ($arrExisting) {
$arrExisting = (array) $arrExisting;
$this->arrChapterInfo = array_merge($arrExisting, $this->getChapterInfo());
$this->arrChapterInfo['updated_at'] = new \MongoDB\BSON\UTCDateTime();
$arrUpdate = $this->arrChapterInfo;
$this->filterOtherFields($arrUpdate);
$this->getChapterModel()->updateOne(
['c_site_uuid' => $strUUId],
['$set' => $arrUpdate],
);
} else {
$this->arrChapterInfo['n_id'] = $intNId;
$this->arrChapterInfo['name'] = $this->arrData['name'];
$this->arrChapterInfo['c_sort_num'] = $this->arrData['c_sort_num'];
$this->arrChapterInfo['c_page'] = $this->arrData['c_page'];
$this->arrChapterInfo['c_source_id'] = $this->arrData['c_source_id'];
$this->arrChapterInfo['c_site_code'] = $this->getSite()->getSiteCode();
$this->arrChapterInfo['c_site_uuid'] = $strUUId;
$this->arrChapterInfo['c_content_path'] = '/novel/' . ($intNId % 100) . '/' . $intNId . '/' . $this->arrChapterInfo['c_sort_num'] . '_' . $this->arrChapterInfo['c_page'] . '.txt';
$this->arrChapterInfo['created_at'] = new \MongoDB\BSON\UTCDateTime();
$arrInsert = $this->arrChapterInfo;
$this->filterOtherFields($arrInsert);
$this->getChapterModel()->insert($arrInsert);
$this->updateNovelHaveChapter($intNId);
}
if (mb_strlen($this->strChapterContent) > 20) {
if (!StorageCore::getInstance()->has($this->arrChapterInfo['c_content_path'])) {
StorageCore::getInstance()->set($this->arrChapterInfo['c_content_path'], $this->strChapterContent);
}
}
return $this->arrChapterInfo;
} catch (\Exception $e) {
throw new \Exception("Upsert failed: " . $e->getMessage());
}
}
/**
* Undocumented function
*
* @param integer $intNId
* @return void
*/
protected function updateNovelHaveChapter(int $intNId)
{
$arrFilter = [
'n_id' => $intNId,
];
$ChapterModel = ChapterModel::getInstance();
$arrLasteChapter = $ChapterModel->getLatestChapterByNId($intNId, false);
if (!empty($arrLasteChapter)) {
$arrUpdate['$set']['n_have_chapter'] = 1;
$arrUpdate['$set']['og_novel_latest_chapter_name'] = $arrLasteChapter['name'];
$this->getMongoBase()->updateOne('novel', $arrFilter, $arrUpdate);
}
}
protected $arrNextPage;
/**
* Undocumented function
*
* @return array
*/
public function getNextPage(): array
{
if ($this->arrNextPage === NULL) {
$this->arrNextPage = [];
if ($this->isMobile()) {
$strPattern = "/var\s+hhekgsv\s*=\s*['\"](.*?)['\"]/";
preg_match_all($strPattern, $this->getContent(), $arrMatches);
$strNextPageUri = end($arrMatches[1]) ?: '';
} else {
$strHref = $this->getQueryList()->find('.read_btn a:contains("下一章")')->attr('href');
$strNextPageUri = $strHref ?: '';
}
if (is_numeric(strpos($strNextPageUri, '_'))) {
$this->arrNextPage = $this->arrData;
$this->arrNextPage['c_page'] += 1;
$this->arrNextPage['uri'] = $strNextPageUri;
}
}
return $this->arrNextPage;
}
}

64
code/app/task/crawler/zw/page/LatestListPage.php Normal file
View File

@@ -0,0 +1,64 @@
<?php
declare(strict_types=1);
namespace app\task\crawler\zw\page;
use app\model\NovelModel;
class LatestListPage extends BasePage
{
/**
* Undocumented variable
*
* @var NovelModel
*/
protected $NovelModel;
/**
* Undocumented function
*
* @return NovelModel
*/
public function getNovelModel(): NovelModel
{
if ($this->NovelModel == NULL) {
$this->NovelModel = NovelModel::getInstance();
}
return $this->NovelModel;
}
static public $arrOptions = [
'typeMap' => [
'root' => 'array',
'document' => 'array',
'array' => 'array',
],
];
/**
* Undocumented function
*
* @return array
*/
public function getNovelFilter(): array
{
$arrFilter = [];
if ($this->isMobile()) {
$arrFilter = $this->getQueryList()->find('div.wrap-box')->eq(0)->find('ul.sort_list li')->map(function ($li) {
$strUri = $li->find('a')->eq(0)->attr('href');
return supperExtractFilename($strUri);
})->filter()->all();
} else {
$arrFilter = $this->getQueryList()->find('div.layout.layout2.layout-col2.fl ul.txt-list li')->map(function ($li) {
$strUri = $li->find('a')->eq(0)->attr('href');
return supperExtractFilename($strUri);
})->filter()->all();
}
return $arrFilter;
}
}

370
code/app/task/crawler/zw/page/NovelPage.php Normal file
View File

@@ -0,0 +1,370 @@
<?php
declare(strict_types=1);
namespace app\task\crawler\zw\page;
use app\model\CategoryModel;
use app\model\CountersModel;
use app\model\NovelModel;
class NovelPage extends BasePage
{
/**
* Undocumented variable
*
* @var array
*/
protected $arrNovelInfo = [];
/**
* Undocumented variable
*
* @var array
*/
protected $arrRelatedNovel = [];
/**
* Undocumented variable
*
* @var array
*/
protected $arrForgeNovel = [];
/**
* Undocumented variable
*
* @var NovelModel
*/
protected $NovelModel;
/**
* Undocumented function
*
* @return NovelModel
*/
public function getNovelModel(): NovelModel
{
if ($this->NovelModel == NULL) {
$this->NovelModel = NovelModel::getInstance();
}
return $this->NovelModel;
}
public $arrOptions = [
'typeMap' => [
'root' => 'array',
'document' => 'array',
'array' => 'array',
],
];
/**
* 查询看的小说
*
* @param integer $intNSourceId
* @param string $strName
* @return array
*/
protected function findOrCreateNovel(int $intNSourceId, string $strName): array
{
$strUUId = $this->getSite()->getSiteUUId($intNSourceId);
$arrExisting = $this->getNovelModel()->findOne(['n_site_uuid' => $strUUId], $this->arrOptions);
if ($arrExisting) {
$arrNovelInfo = (array) $arrExisting;
} else {
$arrNovelInfo = [
'og_novel_book_name' => $strName,
'og_novel_pin_yin' => zhToPinYin($strName),
'n_site_code' => $this->getSite()->getSiteCode(),
'n_site_uuid' => $strUUId,
'n_source_id' => $intNSourceId,
'n_level' => 1,
'n_have_chapter' => 0,
'n_id' => CountersModel::getInstance()->getNextSequence('novel'),
];
$this->getNovelModel()->insert($arrNovelInfo);
}
return $arrNovelInfo;
}
/**
* 分析关联数据
*
* @return void
*/
public function getReletedNovel() {}
/**
* Undocumented function
*
* @return array
*/
public function getNovelInfo(): array
{
if (empty($this->arrNovelInfo)) {
# 获取head字段
$this->getQueryList()->find('meta[property]')->each(function ($Item) {
$strKey = str_replace(':', '_', $Item->attr('property'));
$this->arrNovelInfo[$strKey] = $Item->attr('content');
});
$this->getReletedNovel();
$this->arrNovelInfo['n_related_novel'] = $this->arrRelatedNovel;
$this->arrNovelInfo['n_forge_novel'] = $this->arrForgeNovel;
// $this->arrNovelInfo['og_novel_pin_yin'] = zhToPinYin($this->arrNovelInfo['og_novel_book_name'] ?? '');
// if (!empty($this->arrNovelInfo['og_novel_category'])) {
// $this->arrNovelInfo['category_pinyin'] = CategoryModel::getPinYinByZh($this->arrNovelInfo['og_novel_category']);
// $this->arrNovelInfo['category_sex'] = CategoryModel::checkSex($this->arrNovelInfo['og_novel_category']);
// $this->arrNovelInfo['category_sex_zh'] = CategoryModel::checkSexZh($this->arrNovelInfo['og_novel_category']);
// }
// $this->arrNovelInfo['n_site_code'] = $this->getSite()->getSiteCode();
// $this->arrNovelInfo['n_source_id'] = $this->arrData['n_source_id'];
}
print_r($this->arrNovelInfo);
return $this->arrNovelInfo;
}
/**
* Undocumented function
*
* @param string $strName
* @return string
*/
public function generateCoverPath($strName): string
{
return sprintf('/img/%s/%s.jpg', date('Y/m/d', time() - 24 * 3600 * 14), $strName);
}
public function filterOtherFields(&$arrNovel)
{
$arrFields = [
'n_id',
'og_novel_book_name',
'og_novel_pin_yin',
'og_novel_status',
'og_novel_author',
'og_description',
'og_novel_category',
'category_pinyin',
'category_sex',
'category_sex_zh',
'n_cover_path',
'og_novel_latest_chapter_name',
'og_novel_update_time',
'n_forge_novel',
'n_related_novel',
'og_site_code',
'n_site_uuid',
'n_source_id',
'n_have_chapter',
'created_at',
'updated_at',
];
foreach ($arrNovel as $strKey => $strVal) {
if (!in_array($strKey, $arrFields)) {
unset($arrNovel[$strKey]);
}
}
}
/**
* Undocumented function
*
* @return void
*/
public function saveNovelInfo()
{
try {
$this->getNovelInfo();
if (!isset($this->arrNovelInfo['og_novel_book_name'])) {
return false;
}
$strUUId = $this->getSite()->getSiteUUId($this->arrData['n_source_id']);
$arrExisting = $this->getNovelModel()->findOne(['n_site_uuid' => $strUUId], $this->arrOptions);
if ($arrExisting) {
$arrExisting = (array) $arrExisting;
$this->arrNovelInfo = array_merge($arrExisting, $this->arrNovelInfo);
if (
!isset($this->arrNovelInfo['n_cover_path']) ||
empty($this->arrNovelInfo['n_cover_path']) ||
!file_exists($this->arrNovelInfo['n_cover_path']) ||
(basename($this->arrNovelInfo['n_cover_path']) == '.jpg')
) {
$this->arrNovelInfo['n_cover_path'] = $this->generateCoverPath($this->arrNovelInfo['og_novel_pin_yin']);
}
$this->arrNovelInfo['updated_at'] = new \MongoDB\BSON\UTCDateTime();
$arrUpdate = $this->arrNovelInfo;
$this->filterOtherFields($arrUpdate);
$this->getNovelModel()->updateOne(
['n_site_uuid' => $strUUId],
['$set' => $arrUpdate],
);
} else {
$this->arrNovelInfo['n_site_uuid'] = $strUUId;
$this->arrNovelInfo['n_id'] = CountersModel::getInstance()->getNextSequence('novel');
$this->arrNovelInfo['n_level'] = 1;
$this->arrNovelInfo['n_have_chapter'] = 0;
$this->arrNovelInfo['n_cover_path'] = $this->generateCoverPath($this->arrNovelInfo['og_novel_pin_yin']);
$arrInsert = $this->arrNovelInfo;
$this->filterOtherFields($arrInsert);
$this->getNovelModel()->insert($arrInsert);
}
return $this->arrNovelInfo;
} catch (\Exception $e) {
throw new \Exception("Upsert failed: " . $e->getMessage());
}
}
/**
* Undocumented function
*
* @return array
*/
public function getDownCoverInfo(): array
{
if (!key_exists('og_image', $this->arrNovelInfo)) {
print_r($this->arrNovelInfo);
echo $this->getContent();
}
return [
'uri' => $this->arrNovelInfo['og_image'],
'local_path' => $this->arrNovelInfo['n_cover_path'],
];
}
/**
* Undocumented variable
*
* @var array
*/
protected $arrLatestChapters = NULL;
/**
* Undocumented variable
*
* @var array
*/
protected $arrChapters = NULL;
public function getLatestChapterByNIdsList(): array
{
if ($this->arrLatestChapters === NULL) {
$this->arrLatestChapters = [];
if ($this->isMobile()) {
$strClass = '.chapter-list';
} else {
$strClass = '.section-list';
}
if ($this->arrData['page'] == 0) {
$this->getQueryList()->find($strClass)->eq(0)->find("a")->map(function ($A) {
$this->arrLatestChapters[] = [
'name' => $A->text(),
'uri' => $A->attr('href'),
];
});
}
}
return $this->arrLatestChapters;
}
/**
* Undocumented function
*
* @return array
*/
public function getChapters(): array
{
if ($this->arrChapters === NULL) {
$this->arrChapters = [];
if ($this->isMobile()) {
$strClass = '.chapter-list';
} else {
$strClass = '.section-list';
}
$intDivIndex = 0;
if ($this->arrData['page'] == 0) {
$intDivIndex = 1;
}
$this->getQueryList()->find($strClass)->eq($intDivIndex)->find("a")->map(function ($A) {
$this->arrChapters[] = [
'name' => $A->text(),
'uri' => $A->attr('href'),
];
});
}
return $this->arrChapters;
}
protected $arrNextPageArgs = NULL;
/**
* Undocumented function
*
* @return array
*/
public function getNextPageArgs(): array
{
$this->arrNextPageArgs = [];
if ($this->arrData['page'] == 0) {
$this->getLatestChapterByNIdsList();
$this->getChapters();
$arrLatestChapterFirstBlock = count($this->arrLatestChapters) > 0 ? $this->arrLatestChapters[0] : null;
$arrLatestChapterSecondBlock = count($this->arrChapters) > 0 ? $this->arrChapters[count($this->arrChapters) - 1] : null;
if (
!empty($arrLatestChapterFirstBlock) && !empty($arrLatestChapterSecondBlock)
&& ($arrLatestChapterFirstBlock['uri'] != $arrLatestChapterSecondBlock['uri'])
) {
$this->arrNextPageArgs = [
'n_source_id' => $this->arrData['n_source_id'],
'page' => $this->arrData['page'] + 1,
];
}
} else {
$strLastUri = $this->getQueryList()->find('select:first option:last')->attr('value');
if ($strLastUri != '' && $strLastUri != $this->strUri) {
$this->arrNextPageArgs = [
'n_source_id' => $this->arrData['n_source_id'],
'page' => $this->arrData['page'] + 1,
];
}
}
return $this->arrNextPageArgs;
}
}

283
code/app/task/crawler/zw/page/Site.php Normal file
View File

@@ -0,0 +1,283 @@
<?php
declare(strict_types=1);
namespace app\task\crawler\zw\page;
use GuzzleHttp\Client;
use GuzzleHttp\Promise;
use storage\StorageCore;
/**
* @mixin think\Model
*/
class Site
{
/**
* Undocumented variable
*
* @var Client
*/
protected $Client;
/**
* Undocumented variable
*
* @var string
*/
protected $strDomain = '';
/**
* Undocumented variable
*
* @var string
*/
protected $strSiteCode = '';
/**
* Undocumented function
*/
public function __construct()
{
$this->initClient();
}
/**
* Undocumented function
*
* @return string
*/
public function getSiteCode(): string
{
return $this->strSiteCode;
}
/**
* Undocumented function
*
* @param integer|string $strSign
* @return string
*/
public function getSiteUUId(int|string $strSign): string
{
return $this->getSiteCode() . '-' . $strSign;
}
/**
* Undocumented function
*
* @return Client
*/
public function getClient(): Client
{
return $this->Client;
}
public function getNovelPageUri($intNSourceId, $intPage): string
{
$intUriFirst = intdiv($intNSourceId, 1000);
$strUri = '';
if ($intPage == 0) {
$strUri = sprintf('/%s/%s/', $intUriFirst, $intNSourceId);
} else {
$strUri = sprintf('/book/%s/list%s.html', $intNSourceId, $intPage);
}
return $strUri;
}
/**
* retrieve multiple novel page model
*
* @param string $strUri
* @return NovelPage[]
*/
public function getNovelPageList(array $arrFilter): array
{
/** @var NovelPage[] */
$arrNovelPage = [];
foreach ($arrFilter as $intKey => $arrItem) {
$arrFilter[$intKey]['uri'] = $this->getNovelPageUri($arrItem['n_source_id'], $arrItem['page']);
$arrPromises[$intKey] = $this->getClient()->getAsync($arrFilter[$intKey]['uri']);
}
$arrResults = Promise\Utils::settle($arrPromises)->wait();
$arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
return $arrResult['state'] === 'fulfilled';
});
foreach ($arrSuccessfulResults as $intKey => $arrResult) {
$Response = $arrResult['value'];
$strContent = $Response->getBody()->getContents();
if ($strContent == '索引文件不存在！') {
continue;
}
$NovelPage = $this->getNovelPage($this, $arrFilter[$intKey]['uri']);
$NovelPage->setContent($strContent);
$NovelPage->setData($arrFilter[$intKey]);
$arrNovelPage[] = $NovelPage;
}
return $arrNovelPage;
}
/**
* retrieve multiple chapter page model
*
* @param string $strUri
* @return ChapterPage[]
*/
public function getChapterPageList(array $arrFilter): array
{
/** @var ChapterPage[] */
$arrChapterPage = [];
$arrPromises = [];
foreach ($arrFilter as $intKey => $arrItem) {
$arrPromises[$intKey] = $this->getClient()->getAsync($arrItem['uri']);
}
$arrResults = Promise\Utils::settle($arrPromises)->wait();
$arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
return $arrResult['state'] === 'fulfilled';
});
foreach ($arrSuccessfulResults as $intKey => $arrResult) {
$Response = $arrResult['value'];
$strContent = $Response->getBody()->getContents();;
$ChapterPage = $this->getChapterPage($this, $arrFilter[$intKey]['uri']);
$ChapterPage->setContent($strContent);
$ChapterPage->setData($arrFilter[$intKey]);
$arrChapterPage[] = $ChapterPage;
}
return $arrChapterPage;
}
/**
* retrieve multiple chapter page model
*
* @param string $strUri
* @return LatestListPage[]
*/
public function getLatestListPageList(array $arrFilter): array
{
/** @var LatestListPage[] */
$arrLatestListPage = [];
$arrPromises = [];
foreach ($arrFilter as $intKey => $strUri) {
$arrPromises[$intKey] = $this->getClient()->getAsync($strUri);
}
$arrResults = Promise\Utils::settle($arrPromises)->wait();
$arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
return $arrResult['state'] === 'fulfilled';
});
foreach ($arrSuccessfulResults as $intKey => $arrResult) {
$Response = $arrResult['value'];
$strContent = $Response->getBody()->getContents();;
$LatestListPage = $this->getLatestListPage($this, $arrFilter[$intKey]);
$LatestListPage->setContent($strContent);
$arrLatestListPage[] = $LatestListPage;
}
return $arrLatestListPage;
}
/**
* download the image to the local system
*
* @param array $arrFilter
* @return void
*/
public function downRemoteImgToLocal(array $arrFilter)
{
$StorageCore = StorageCore::getInstance();
$arrPromises = [];
foreach ($arrFilter as $intKey => $arrItem) {
if (!$StorageCore->has($arrItem['local_path'])) {
$arrPromises[$intKey] = $this->getClient()->getAsync($arrItem['uri']);
}
}
$arrResults = Promise\Utils::settle($arrPromises)->wait();
$arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
return $arrResult['state'] === 'fulfilled';
});
foreach ($arrSuccessfulResults as $intKey => $arrResult) {
$Response = $arrResult['value'];
$StorageCore->writeStream($arrFilter[$intKey]['local_path'], $Response->getBody()->detach());;
}
}
/**
* Undocumented function
*
* @param object $Site
* @param string $strUri
* @return NovelPage
*/
public function getNovelPage($Site, $strUri)
{
return new NovelPage($Site, $strUri);
}
/**
* Undocumented function
*
* @param object $Site
* @param string $strUri
* @return ChapterPage
*/
public function getChapterPage($Site, $strUri)
{
return new ChapterPage($Site, $strUri);
}
/**
* Undocumented function
*
* @param object $Site
* @param string $strUri
* @return LatestListPage
*/
public function getLatestListPage($Site, $strUri)
{
return new LatestListPage($Site, $strUri);
}
}

264
code/app/task/crawler/zw630/Scheduler.php
View File

@@ -4,278 +4,34 @@ declare(strict_types=1);
namespace app\task\crawler\zw630; namespace app\task\crawler\zw630;
use app\model\ChapterModel;
use app\task\crawler\zw630\page\NovelPage;
use app\task\crawler\zw630\page\Site; use app\task\crawler\zw630\page\Site;
use microserver\QueueManage; use app\task\crawler\zw\Scheduler as ZwScheduler;
use storage\StorageCore;
/** /**
* @mixin think\Model * @mixin think\Model
*/ */
class Scheduler class Scheduler extends ZwScheduler
{ {
static $intSlice = 3;
/** /**
* Undocumented variable * Undocumented function
*
* @var \microserver\QueueManage
*/
protected $QueueManage = NULL;
public function __construct()
{
$this->QueueManage = new QueueManage();
}
/**
* create a site-wide craw task
* *
* @return void * @return void
*/ */
public function crawlFull() public function init()
{ {
$intSliceSize = self::$intSlice; $this->intSlice = 8;
$arrFilter = [];
for ($intNId = 1; $intNId <= 531208; $intNId++) {
// for ($intNId = 1; $intNId <= 10000; $intNId++) {
// for ($intNId = 1; $intNId <= 1; $intNId++) {
$arrFilter[] = [
'n_source_id' => $intNId,
'page' => 0,
];
if ($intNId % $intSliceSize == 0) {
$this->fetchNovel($arrFilter);
$arrFilter = [];
}
}
if (!empty($arrFilter)) {
$this->fetchNovel($arrFilter);
}
}
/**
* create a task to craw the latest chapter
*
* @return void
*/
public function crawLatest()
{
$intSliceSize = self::$intSlice;
$arrUri = [];
$strUri = '/list/lastupdate_%s_0_0_%s.html';
$intEndCategory = $intEndPage = 10;
for ($intCategoryKey = 0; $intCategoryKey < $intEndCategory; $intCategoryKey++) {
for ($intPage = 1; $intPage <= $intEndPage; $intPage++) {
$arrUri[] = sprintf($strUri, $intCategoryKey, $intPage);
}
}
$arrUri = array_chunk($arrUri, $intSliceSize);
$arrTask = [
'callback' => [self::class, 'getLatestChapter'],
'data' => [],
];
foreach ($arrUri as $arrUriSlice) {
$arrTask['data'] = $arrUriSlice;
$this->QueueManage->set($arrTask);
}
} }
/** /**
* Undocumented function * Undocumented function
* *
* @param array $arrData * @return Site
* @return void
*/ */
public function getLatestChapter(array $arrFilter) public function getSite()
{ {
$Site = new Site; if ($this->Site == NULL) {
$this->Site = new Site;
$arrLatestListPageList = $Site->getLatestListPageList($arrFilter);
$arrNSourceId = [];
foreach ($arrLatestListPageList as $LatestListPage) {
$arrResult = $LatestListPage->getNovelFilter();
$arrNSourceId = array_merge($arrNSourceId, $arrResult);
}
$arrNSourceId = array_unique($arrNSourceId);
$arrNSourceId = array_values($arrNSourceId);
$intSliceSize = self::$intSlice;
$arrFilter = [];
foreach ($arrNSourceId as $intKey => $intNSourceId) {
$arrFilter[] = [
'n_source_id' => $intNSourceId,
'page' => 0,
];
if ($intKey % $intSliceSize == 0) {
$this->fetchNovel($arrFilter);
$arrFilter = [];
}
}
if (!empty($arrFilter)) {
$this->fetchNovel($arrFilter);
}
}
/**
* Undocumented function
*
* @param array $arrFilter
* @return void
*/
public function fetchNovel(array $arrFilter)
{
$arrTask = [
'callback' => [self::class, 'getNovelInfo'],
'data' => $arrFilter,
];
$this->QueueManage->set($arrTask);
}
/**
* Undocumented function
*
* @param integer $intNId
* @param integer $intPage
* @param array $arrChapterList
* @return void
*/
public function fetchChapter(int $intNId, int $intPage, array $arrChapterList = [])
{
$intSliceSize = self::$intSlice;
$intPageSize = 100;
$arrTask = [
'callback' => [self::class, 'getChapterInfo'],
'data' => [
'n_id' => $intNId,
]
];
$arrFilter = [];
foreach ($arrChapterList as $intKey => $arrChapter) {
$arrChapter['c_sort_num'] = $intPage * $intPageSize + $intKey + 1;
$arrChapter['c_source_id'] = supperExtractFilename($arrChapter['uri']);
$arrChapter['c_page'] = 0;
$arrFilter[] = $arrChapter;
if ($intKey % $intSliceSize == 0) {
$arrTask['data']['filter'] = $arrFilter;
$this->QueueManage->set($arrTask);
$arrFilter = [];
}
}
if (!empty($arrFilter)) {
$arrTask['data']['filter'] = $arrFilter;
$this->QueueManage->set($arrTask);
}
}
/**
* Undocumented function
*
* @param array $arrFilter
* @return void
*/
public function getNovelInfo(array $arrFilter)
{
$Site = new Site;
$arrNovelPageList = $Site->getNovelPageList($arrFilter);
$arrCover = [];
foreach ($arrNovelPageList as $NovelPage) {
$arrNovel = $NovelPage->getNovelInfo();
if (empty($arrNovel)) {
continue;
}
$arrNextPageData = $NovelPage->getNextPageArgs();
if (!empty($arrNextPageData)) {
$this->fetchNovel([
[
'n_source_id' => $arrNextPageData['n_source_id'],
'page' => $arrNextPageData['page'],
]
]);
}
$arrCoverItem = $NovelPage->getDownCoverInfo();
if (!StorageCore::getInstance()->has($arrCoverItem['local_path'])) {
$arrCover[] = $arrCoverItem;
}
$arrChapterList = $NovelPage->getChapters();
$this->fetchChapter($arrNovel['n_id'], $NovelPage->getData()['page'], $arrChapterList);
}
if ($arrCover) {
$Site->downRemoteImgToLocal($arrCover);
}
}
/**
* Undocumented function
*
* @param array $arrData
* @return void
*/
public function getChapterInfo(array $arrData)
{
$Site = new Site;
$ChapterModel = ChapterModel::getInstance();
foreach ($arrData['filter'] as $intKey => $arrChapterFilter) {
$strUUId = sprintf('%s-chapter-%s-%s', $Site->getSiteCode(), $arrChapterFilter['c_source_id'], $arrChapterFilter['c_page']);
$arrChapter = $ChapterModel->findOne(['c_site_uuid' => $strUUId]);
if (!empty($arrChapter) && !empty($arrChapter['content'])) {
unset($arrData['filter'][$intKey]);
}
}
$arrData['filter'] = array_values($arrData['filter']);
$arrChapterPageList = $Site->getChapterPageList($arrData['filter']);
$arrNextPageFilterList = [];
foreach ($arrChapterPageList as $ChapterPage) {
$ChapterPage->saveChapter($arrData['n_id']);
$arrNextPageFilter = $ChapterPage->getNextPage();
if (!empty($arrNextPageFilter)) {
$arrNextPageFilterList[] = $arrNextPageFilter;
}
}
if (!empty($arrNextPageFilterList)) {
$arrTask = $arrData;
$arrTask['filter'] = $arrNextPageFilterList;
$arrTask = [
'callback' => [self::class, 'getChapterInfo'],
'data' => $arrTask
];
$this->QueueManage->set($arrTask);
} }
return $this->Site;
} }
} }

212
code/app/task/crawler/zw630/page/ChapterPage.php
View File

@@ -4,214 +4,6 @@ declare(strict_types=1);
namespace app\task\crawler\zw630\page; namespace app\task\crawler\zw630\page;
use app\model\ChapterModel; use app\task\crawler\zw\page\ChapterPage as ZwChapterPage;
use storage\StorageCore;
class ChapterPage extends BasePage class ChapterPage extends ZwChapterPage {}
{
/**
* Undocumented variable
*
* @var string
*/
protected $strChapterContent = '';
/**
* Undocumented variable
*
* @var ChapterModel
*/
protected $ChapterModel;
/**
* Undocumented function
*
* @return ChapterModel
*/
public function getChapterModel(): ChapterModel
{
if ($this->ChapterModel == NULL) {
$this->ChapterModel = ChapterModel::getInstance();
}
return $this->ChapterModel;
}
static public $arrOptions = [
'typeMap' => [
'root' => 'array',
'document' => 'array',
'array' => 'array',
],
];
/**
* Undocumented function
*
* @return array
*/
public function getChapterInfo(): array
{
if ($this->arrChapterInfo === NULL) {
$this->arrChapterInfo = [];
$strPattern = "/qsbs\.bb\('([^']*)'\)/";
preg_match_all($strPattern, $this->getContent(), $arrMatches);
$strContent = '';
foreach ($arrMatches[1] as $str) {
$strContent .= base64_decode($str);
}
$this->strChapterContent = $strContent;
$this->arrChapterInfo['c_page_title'] = $this->getQueryList()->find('title')->text();
$this->arrChapterInfo['c_page_keywords'] = $this->getQueryList()->find('meta[name="keywords"]')->attr('content');
$this->arrChapterInfo['c_page_description'] = $this->getQueryList()->find('meta[name="description"]')->attr('content');
}
return $this->arrChapterInfo;
}
/**
* Undocumented variable
*
* @var array
*/
protected $arrChapterInfo;
public function filterOtherFields(&$arrChapter)
{
$arrFields = [
'n_id',
'name',
'c_sort_num',
'c_page',
'c_source_id',
'c_site_code',
'c_site_uuid',
'c_content_path',
'created_at',
'updated_at',
];
foreach ($arrChapter as $strKey => $strVal) {
if (!in_array($strKey, $arrFields)) {
unset($arrChapter[$strKey]);
}
}
}
public function saveChapter(int $intNId): array
{
try {
$this->getChapterInfo();
$strUUId = $this->getSite()->getSiteUUId('chapter-' . $this->arrData['c_source_id'] . '-' . $this->arrData['c_page']);
$arrExisting = $this->getChapterModel()->findOne(['c_site_uuid' => $strUUId], self::$arrOptions);
if ($arrExisting) {
$arrExisting = (array) $arrExisting;
$this->arrChapterInfo = array_merge($arrExisting, $this->getChapterInfo());
$this->arrChapterInfo['updated_at'] = new \MongoDB\BSON\UTCDateTime();
$arrUpdate = $this->arrChapterInfo;
$this->filterOtherFields($arrUpdate);
$this->getChapterModel()->updateOne(
['c_site_uuid' => $strUUId],
['$set' => $arrUpdate],
);
} else {
$this->arrChapterInfo['n_id'] = $intNId;
$this->arrChapterInfo['name'] = $this->arrData['name'];
$this->arrChapterInfo['c_sort_num'] = $this->arrData['c_sort_num'];
$this->arrChapterInfo['c_page'] = $this->arrData['c_page'];
$this->arrChapterInfo['c_source_id'] = $this->arrData['c_source_id'];
$this->arrChapterInfo['c_site_code'] = $this->getSite()->getSiteCode();
$this->arrChapterInfo['c_site_uuid'] = $strUUId;
$this->arrChapterInfo['c_content_path'] = '/novel/' . ($intNId % 100) . '/' . $intNId . '/' . $this->arrChapterInfo['c_sort_num'] . '_' . $this->arrChapterInfo['c_page'] . '.txt';
$this->arrChapterInfo['created_at'] = new \MongoDB\BSON\UTCDateTime();
$arrInsert = $this->arrChapterInfo;
$this->filterOtherFields($arrInsert);
$this->getChapterModel()->insert($arrInsert);
$this->updateNovelHaveChapter($intNId);
}
if (mb_strlen($this->strChapterContent) > 20) {
if (!StorageCore::getInstance()->has($this->arrChapterInfo['c_content_path'])) {
StorageCore::getInstance()->set($this->arrChapterInfo['c_content_path'], $this->strChapterContent);
}
}
return $this->arrChapterInfo;
} catch (\Exception $e) {
throw new \Exception("Upsert failed: " . $e->getMessage());
}
}
/**
* Undocumented function
*
* @param integer $intNId
* @return void
*/
protected function updateNovelHaveChapter(int $intNId)
{
$arrFilter = [
'n_id' => $intNId,
];
$ChapterModel = ChapterModel::getInstance();
$arrLasteChapter = $ChapterModel->getLatestChapterByNId($intNId, false);
if (!empty($arrLasteChapter)) {
$arrUpdate['$set']['n_have_chapter'] = 1;
$arrUpdate['$set']['og_novel_latest_chapter_name'] = $arrLasteChapter['name'];
$this->getMongoBase()->updateOne('novel', $arrFilter, $arrUpdate);
}
}
protected $arrNextPage;
/**
* Undocumented function
*
* @return array
*/
public function getNextPage(): array
{
if ($this->arrNextPage === NULL) {
$this->arrNextPage = [];
if ($this->isMobile()) {
$strPattern = "/var\s+hhekgsv\s*=\s*['\"](.*?)['\"]/";
preg_match_all($strPattern, $this->getContent(), $arrMatches);
$strNextPageUri = end($arrMatches[1]) ?: '';
} else {
$strHref = $this->getQueryList()->find('.read_btn a:contains("下一章")')->attr('href');
$strNextPageUri = $strHref ?: '';
}
if (is_numeric(strpos($strNextPageUri, '_'))) {
$this->arrNextPage = $this->arrData;
$this->arrNextPage['c_page'] += 1;
$this->arrNextPage['uri'] = $strNextPageUri;
}
}
return $this->arrNextPage;
}
}

61
code/app/task/crawler/zw630/page/LatestListPage.php
View File

@@ -4,63 +4,8 @@ declare(strict_types=1);
namespace app\task\crawler\zw630\page; namespace app\task\crawler\zw630\page;
use app\model\CategoryModel;
use app\model\CountersModel;
use app\model\NovelModel; use app\model\NovelModel;
use app\task\crawler\zw\page\LatestListPage as ZwLatestListPage;
class LatestListPage extends ZwLatestListPage {}
class LatestListPage extends BasePage
{
/**
* Undocumented variable
*
* @var NovelModel
*/
protected $NovelModel;
/**
* Undocumented function
*
* @return NovelModel
*/
public function getNovelModel(): NovelModel
{
if ($this->NovelModel == NULL) {
$this->NovelModel = NovelModel::getInstance();
}
return $this->NovelModel;
}
static public $arrOptions = [
'typeMap' => [
'root' => 'array',
'document' => 'array',
'array' => 'array',
],
];
/**
* Undocumented function
*
* @return array
*/
public function getNovelFilter(): array
{
$arrFilter = [];
if ($this->isMobile()) {
$arrFilter = $this->getQueryList()->find('div.wrap-box')->eq(0)->find('ul.sort_list li')->map(function ($li) {
$strUri = $li->find('a')->eq(0)->attr('href');
return supperExtractFilename($strUri);
})->filter()->all();
} else {
$arrFilter = $this->getQueryList()->find('div.layout.layout2.layout-col2.fl ul.txt-list li')->map(function ($li) {
$strUri = $li->find('a')->eq(0)->attr('href');
return supperExtractFilename($strUri);
})->filter()->all();
}
return $arrFilter;
}
}

343
code/app/task/crawler/zw630/page/NovelPage.php
View File

@@ -4,113 +4,24 @@ declare(strict_types=1);
namespace app\task\crawler\zw630\page; namespace app\task\crawler\zw630\page;
use app\model\CategoryModel;
use app\model\CountersModel;
use app\model\NovelModel;
class NovelPage extends BasePage use app\task\crawler\zw\page\NovelPage as ZwNovelPage;
class NovelPage extends ZwNovelPage
{ {
/** /**
* Undocumented variable * 分析关联数据
* *
* @var array * @return void
*/ */
protected $arrNovelInfo = []; public function getReletedNovel()
/**
* Undocumented variable
*
* @var array
*/
protected $arrRelatedNovel = [];
/**
* Undocumented variable
*
* @var array
*/
protected $arrForgeNovel = [];
/**
* Undocumented variable
*
* @var NovelModel
*/
protected $NovelModel;
/**
* Undocumented function
*
* @return NovelModel
*/
public function getNovelModel(): NovelModel
{ {
if ($this->NovelModel == NULL) { if (empty($this->arrRelatedNovel) || empty($this->arrForgeNovel)) {
$this->NovelModel = NovelModel::getInstance();
}
return $this->NovelModel;
}
static public $arrOptions = [
'typeMap' => [
'root' => 'array',
'document' => 'array',
'array' => 'array',
],
];
/**
* 查询看的小说
*
* @param integer $intNSourceId
* @param string $strName
* @return array
*/
private function findOrCreateNovel(int $intNSourceId, string $strName): array
{
$strUUId = $this->getSite()->getSiteUUId($intNSourceId);
$arrExisting = $this->getNovelModel()->findOne(['n_site_uuid' => $strUUId], self::$arrOptions);
if ($arrExisting) {
$arrNovelInfo = (array) $arrExisting;
} else {
$arrNovelInfo = [
'og_novel_book_name' => $strName,
'og_novel_pin_yin' => zhToPinYin($strName),
'n_site_code' => $this->getSite()->getSiteCode(),
'n_site_uuid' => $strUUId,
'n_source_id' => $intNSourceId,
'n_level' => 1,
'n_have_chapter' => 0,
'n_id' => CountersModel::getInstance()->getNextSequence('novel'),
];
$this->getNovelModel()->insert($arrNovelInfo);
}
return $arrNovelInfo;
}
/**
* Undocumented function
*
* @return array
*/
public function getNovelInfo($boolSave = true): array
{
if (empty($this->arrNovelInfo)) {
# 获取head字段
$this->getQueryList()->find('meta[property]')->each(function ($Item) {
$strKey = str_replace(':', '_', $Item->attr('property'));
$this->arrNovelInfo[$strKey] = $Item->attr('content');
});
# 获取 kan 和 shu 的关联数据
$this->getQueryList()->find('.first_txt')->eq(0)->find("a")->map(function ($A) { $this->getQueryList()->find('.first_txt')->eq(0)->find("a")->map(function ($A) {
$strName = $A->text(); $strName = $A->text();
$strUri = $A->attr('href'); $strUri = $A->attr('href');
if (is_numeric(strpos($strUri, 'shu'))) { if (is_numeric(strpos($strUri, 'shu'))) {
$intNSourceId = (int)supperExtractFilename($strUri); $intNSourceId = (int)supperExtractFilename($strUri);
$arrNovelInfo = $this->findOrCreateNovel($intNSourceId, $strName); $arrNovelInfo = $this->findOrCreateNovel($intNSourceId, $strName);
@@ -127,244 +38,6 @@ class NovelPage extends BasePage
]; ];
} }
}); });
$this->arrNovelInfo['n_related_novel'] = $this->arrRelatedNovel;
$this->arrNovelInfo['n_forge_novel'] = $this->arrForgeNovel;
$this->arrNovelInfo['og_novel_pin_yin'] = zhToPinYin($this->arrNovelInfo['og_novel_book_name'] ?? '');
if (!empty($this->arrNovelInfo['og_novel_category'])) {
$this->arrNovelInfo['category_pinyin'] = CategoryModel::getPinYinByZh($this->arrNovelInfo['og_novel_category']);
$this->arrNovelInfo['category_sex'] = CategoryModel::checkSex($this->arrNovelInfo['og_novel_category']);
$this->arrNovelInfo['category_sex_zh'] = CategoryModel::checkSexZh($this->arrNovelInfo['og_novel_category']);
}
$this->arrNovelInfo['n_site_code'] = $this->getSite()->getSiteCode();
$this->arrNovelInfo['n_source_id'] = $this->arrData['n_source_id'];
} }
if ($boolSave && !empty($this->arrNovelInfo)) {
$this->saveNovelInfo();
}
return $this->arrNovelInfo;
}
/**
* Undocumented function
*
* @param string $strName
* @return string
*/
public function generateCoverPath($strName): string
{
return sprintf('/img/%s/%s.jpg', date('Y/m/d', time() - 24 * 3600 * 14), $strName);
}
public function filterOtherFields(&$arrNovel)
{
$arrFields = [
'n_id',
'og_novel_book_name',
'og_novel_pin_yin',
'og_novel_status',
'og_novel_author',
'og_description',
'og_novel_category',
'category_pinyin',
'category_sex',
'category_sex_zh',
'n_cover_path',
'og_novel_latest_chapter_name',
'og_novel_update_time',
'n_forge_novel',
'n_related_novel',
'og_site_code',
'n_site_uuid',
'n_source_id',
'n_have_chapter',
'created_at',
'updated_at',
];
foreach ($arrNovel as $strKey => $strVal) {
if (!in_array($strKey, $arrFields)) {
unset($arrNovel[$strKey]);
}
}
}
/**
* Undocumented function
*
* @return void
*/
public function saveNovelInfo()
{
try {
$strUUId = $this->getSite()->getSiteUUId($this->arrData['n_source_id']);
$arrExisting = $this->getNovelModel()->findOne(['n_site_uuid' => $strUUId], self::$arrOptions);
if ($arrExisting) {
$arrExisting = (array) $arrExisting;
$this->arrNovelInfo = array_merge($arrExisting, $this->arrNovelInfo);
if (
!isset($this->arrNovelInfo['n_cover_path']) ||
empty($this->arrNovelInfo['n_cover_path']) ||
!file_exists($this->arrNovelInfo['n_cover_path']) ||
(basename($this->arrNovelInfo['n_cover_path']) == '.jpg')
) {
$this->arrNovelInfo['n_cover_path'] = $this->generateCoverPath($this->arrNovelInfo['og_novel_pin_yin']);
}
$this->arrNovelInfo['updated_at'] = new \MongoDB\BSON\UTCDateTime();
$arrUpdate = $this->arrNovelInfo;
$this->filterOtherFields($arrUpdate);
$this->getNovelModel()->updateOne(
['n_site_uuid' => $strUUId],
['$set' => $arrUpdate],
);
} else {
$this->arrNovelInfo['n_site_uuid'] = $strUUId;
$this->arrNovelInfo['n_id'] = CountersModel::getInstance()->getNextSequence('novel');
$this->arrNovelInfo['n_level'] = 1;
$this->arrNovelInfo['n_have_chapter'] = 0;
$this->arrNovelInfo['n_cover_path'] = $this->generateCoverPath($this->arrNovelInfo['og_novel_pin_yin']);
$arrInsert = $this->arrNovelInfo;
$this->filterOtherFields($arrInsert);
$this->getNovelModel()->insert($arrInsert);
}
return $this->arrNovelInfo;
} catch (\Exception $e) {
throw new \Exception("Upsert failed: " . $e->getMessage());
}
}
/**
* Undocumented function
*
* @return array
*/
public function getDownCoverInfo(): array
{
return [
'uri' => $this->arrNovelInfo['og_image'],
'local_path' => $this->arrNovelInfo['n_cover_path'],
];
}
/**
* Undocumented variable
*
* @var array
*/
protected $arrLatestChapters = NULL;
/**
* Undocumented variable
*
* @var array
*/
protected $arrChapters = NULL;
public function getLatestChapterByNIdsList(): array
{
if ($this->arrLatestChapters === NULL) {
$this->arrLatestChapters = [];
if ($this->isMobile()) {
$strClass = '.chapter-list';
} else {
$strClass = '.section-list';
}
if ($this->arrData['page'] == 0) {
$this->getQueryList()->find($strClass)->eq(0)->find("a")->map(function ($A) {
$this->arrLatestChapters[] = [
'name' => $A->text(),
'uri' => $A->attr('href'),
];
});
}
}
return $this->arrLatestChapters;
}
/**
* Undocumented function
*
* @return array
*/
public function getChapters(): array
{
if ($this->arrChapters === NULL) {
$this->arrChapters = [];
if ($this->isMobile()) {
$strClass = '.chapter-list';
} else {
$strClass = '.section-list';
}
$intDivIndex = 0;
if ($this->arrData['page'] == 0) {
$intDivIndex = 1;
}
$this->getQueryList()->find($strClass)->eq($intDivIndex)->find("a")->map(function ($A) {
$this->arrChapters[] = [
'name' => $A->text(),
'uri' => $A->attr('href'),
];
});
}
return $this->arrChapters;
}
protected $arrNextPageArgs = NULL;
/**
* Undocumented function
*
* @return array
*/
public function getNextPageArgs(): array
{
$this->arrNextPageArgs = [];
if ($this->arrData['page'] == 0) {
$this->getLatestChapterByNIdsList();
$this->getChapters();
$arrLatestChapterFirstBlock = count($this->arrLatestChapters) > 0 ? $this->arrLatestChapters[0] : null;
$arrLatestChapterSecondBlock = count($this->arrChapters) > 0 ? $this->arrChapters[count($this->arrChapters) - 1] : null;
if (
!empty($arrLatestChapterFirstBlock) && !empty($arrLatestChapterSecondBlock)
&& ($arrLatestChapterFirstBlock['uri'] != $arrLatestChapterSecondBlock['uri'])
) {
$this->arrNextPageArgs = [
'n_source_id' => $this->arrData['n_source_id'],
'page' => $this->arrData['page'] + 1,
];
}
} else {
$strLastUri = $this->getQueryList()->find('select:first option:last')->attr('value');
if ($strLastUri != '' && $strLastUri != $this->strUri) {
$this->arrNextPageArgs = [
'n_source_id' => $this->arrData['n_source_id'],
'page' => $this->arrData['page'] + 1,
];
}
}
return $this->arrNextPageArgs;
} }
} }

260
code/app/task/crawler/zw630/page/Site.php
View File

@@ -4,22 +4,14 @@ declare(strict_types=1);
namespace app\task\crawler\zw630\page; namespace app\task\crawler\zw630\page;
use GuzzleHttp\Client; use app\task\crawler\zw\page\Site as ZwSite;
use GuzzleHttp\Promise; use GuzzleHttp\Promise;
use storage\StorageCore; use GuzzleHttp\Client;
/** /**
* @mixin think\Model * @mixin think\Model
*/ */
class Site class Site extends ZwSite
{ {
/**
* Undocumented variable
*
* @var Client
*/
private $Client;
/** /**
* Undocumented variable * Undocumented variable
* *
@@ -27,33 +19,17 @@ class Site
*/ */
protected $strDomain = 'https://www.630zw.com'; protected $strDomain = 'https://www.630zw.com';
/**
* Undocumented variable
*
* @var string
*/
protected $strSiteCode = 'zw630'; protected $strSiteCode = 'zw630';
/** /**
* Undocumented function * Undocumented function
*
* @return string
*/ */
public function getSiteCode(): string public function initClient()
{
return $this->strSiteCode;
}
/**
* Undocumented function
*
* @param integer|string $strSign
* @return string
*/
public function getSiteUUId(int|string $strSign): string
{
return $this->getSiteCode() . '-' . $strSign;
}
/**
* Undocumented function
*/
public function __construct()
{ {
$arrConfig = [ $arrConfig = [
'http_version' => '1.1', // 强制用 HTTP/1.1，已有，保持 'http_version' => '1.1', // 强制用 HTTP/1.1，已有，保持
@@ -82,208 +58,90 @@ class Site
'sec-fetch-user' => '?1', 'sec-fetch-user' => '?1',
'upgrade-insecure-requests' => '1', 'upgrade-insecure-requests' => '1',
], ],
'pool_size' => 5, 'pool_size' => 10,
'verify' => false, 'verify' => false,
]; ];
$arrProxy = config('proxy'); $arrProxy = config('proxy');
if ($arrProxy['proxy_status']) { if ($arrProxy['proxy_status']) {
$intTime = time();
$strOrderNo = $arrProxy['proxy_order']; $strCode = $arrProxy['proxy_code'];
$strProxyHost = $arrProxy['proxy_host'];
$strKey = $arrProxy['proxy_key'];
$strSecret = $arrProxy['proxy_secret']; $strSecret = $arrProxy['proxy_secret'];
$strTxt = "orderno=" . $strOrderNo . ",secret=" . $strSecret . ",timestamp=" . $intTime;
$strSign = strtoupper(md5($strTxt));
$strAuth = 'sign=' . $strSign . '&orderno=' . $strOrderNo . '&timestamp=' . $intTime;
$arrConfig['proxy'] = 'http://dtbf.xiongmaodaili.com:8089'; switch ($strCode) {
$arrConfig['headers']['Proxy-Authorization'] = $strAuth; case 'XIONGMAO':
$intTime = time();
$strTxt = "orderno=" . $strKey . ",secret=" . $strSecret . ",timestamp=" . $intTime;
$strSign = strtoupper(md5($strTxt));
$strAuth = 'sign=' . $strSign . '&orderno=' . $strKey . '&timestamp=' . $intTime;
$arrConfig['proxy'] = $strProxyHost;
$arrConfig['headers']['Proxy-Authorization'] = $strAuth;
break;
case 'KUAI':
$arrConfig['proxy'] = $strProxyHost;
$arrConfig['curl'][CURLOPT_PROXYAUTH] = CURLAUTH_BASIC;
$arrConfig['curl'][CURLOPT_PROXYUSERPWD] = "$strKey:$strSecret";
break;
}
} }
$this->Client = new Client($arrConfig); $this->Client = new Client($arrConfig);
} }
public function getNovelPageUri($intNSourceId, $intPage): string
{
$strUri = '';
if ($intPage == 0) {
$arrItem['uri'] = sprintf('/shu/%s.html', $intNSourceId);
} else {
$arrItem['uri'] = sprintf('/shu/%s_%s.html', $intNSourceId, $intPage);
}
return $strUri;
}
/** /**
* Undocumented function * Undocumented function
* *
* @return Client * @param object $Site
*/
public function getClient(): Client
{
return $this->Client;
}
/**
* retrieve novel page model
*
* @param string $strUri * @param string $strUri
* @return NovelPage * @return NovelPage
*/ */
public function getNovelPage(string $strUri): NovelPage public function getNovelPage($Site, $strUri)
{ {
return new NovelPage($this, $strUri); return new NovelPage($Site, $strUri);
} }
/** /**
* retrieve multiple novel page model * Undocumented function
* *
* @param object $Site
* @param string $strUri * @param string $strUri
* @return NovelPage[] * @return ChapterPage
*/ */
public function getNovelPageList(array $arrFilter): array public function getChapterPage($Site, $strUri)
{ {
return new ChapterPage($Site, $strUri);
/** @var NovelPage[] */
$arrNovelPage = [];
foreach ($arrFilter as $intKey => $arrItem) {
if ($arrItem['page'] == 0) {
$arrItem['uri'] = sprintf('/shu/%s.html', $arrItem['n_source_id']);
} else {
$arrItem['uri'] = sprintf('/shu/%s_%s.html', $arrItem['n_source_id'], $arrItem['page']);
}
$arrFilter[$intKey]['uri'] = $arrItem['uri'];
$arrPromises[$intKey] = $this->getClient()->getAsync($arrItem['uri']);
}
$arrResults = Promise\Utils::settle($arrPromises)->wait();
$arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
return $arrResult['state'] === 'fulfilled';
});
foreach ($arrSuccessfulResults as $intKey => $arrResult) {
$Response = $arrResult['value'];
$strContent = $Response->getBody()->getContents();
if ($strContent == '索引文件不存在！') {
continue;
}
$NovelPage = new NovelPage($this, $arrFilter[$intKey]['uri']);
$NovelPage->setContent($strContent);
$NovelPage->setData($arrFilter[$intKey]);
$arrNovelPage[] = $NovelPage;
}
return $arrNovelPage;
} }
/** /**
* retrieve multiple chapter page model * Undocumented function
* *
* @param object $Site
* @param string $strUri * @param string $strUri
* @return ChapterPage[] * @return LatestListPage
*/ */
public function getChapterPageList(array $arrFilter): array public function getLatestListPage($Site, $strUri)
{ {
return new LatestListPage($Site, $strUri);
/** @var ChapterPage[] */
$arrChapterPage = [];
$arrPromises = [];
foreach ($arrFilter as $intKey => $arrItem) {
$arrPromises[$intKey] = $this->getClient()->getAsync($arrItem['uri']);
}
$arrResults = Promise\Utils::settle($arrPromises)->wait();
$arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
return $arrResult['state'] === 'fulfilled';
});
foreach ($arrSuccessfulResults as $intKey => $arrResult) {
$Response = $arrResult['value'];
$strContent = $Response->getBody()->getContents();;
$ChapterPage = new ChapterPage($this, $arrFilter[$intKey]['uri']);
$ChapterPage->setContent($strContent);
$ChapterPage->setData($arrFilter[$intKey]);
$arrChapterPage[] = $ChapterPage;
}
return $arrChapterPage;
}
/**
* retrieve multiple chapter page model
*
* @param string $strUri
* @return LatestListPage[]
*/
public function getLatestListPageList(array $arrFilter): array
{
/** @var LatestListPage[] */
$arrLatestListPage = [];
$arrPromises = [];
foreach ($arrFilter as $intKey => $strUri) {
$arrPromises[$intKey] = $this->getClient()->getAsync($strUri);
}
$arrResults = Promise\Utils::settle($arrPromises)->wait();
$arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
return $arrResult['state'] === 'fulfilled';
});
foreach ($arrSuccessfulResults as $intKey => $arrResult) {
$Response = $arrResult['value'];
$strContent = $Response->getBody()->getContents();;
$LatestListPage = new LatestListPage($this, $arrFilter[$intKey]);
$LatestListPage->setContent($strContent);
$arrLatestListPage[] = $LatestListPage;
}
return $arrLatestListPage;
}
/**
* download the image to the local system
*
* @param array $arrFilter
* @return void
*/
public function downRemoteImgToLocal(array $arrFilter)
{
$StorageCore = StorageCore::getInstance();
$arrPromises = [];
foreach ($arrFilter as $intKey => $arrItem) {
if (!$StorageCore->has($arrItem['local_path'])) {
$arrPromises[$intKey] = $this->getClient()->getAsync($arrItem['uri']);
}
}
$arrResults = Promise\Utils::settle($arrPromises)->wait();
$arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
return $arrResult['state'] === 'fulfilled';
});
foreach ($arrSuccessfulResults as $intKey => $arrResult) {
$Response = $arrResult['value'];
$StorageCore->writeStream($arrFilter[$intKey]['local_path'], $Response->getBody()->detach());;
}
} }
} }

37
code/app/task/crawler/zw74/Scheduler.php Normal file
View File

@@ -0,0 +1,37 @@
<?php
declare(strict_types=1);
namespace app\task\crawler\zw74;
use app\task\crawler\zw74\page\Site;
use app\task\crawler\zw\Scheduler as ZwScheduler;
/**
* @mixin think\Model
*/
class Scheduler extends ZwScheduler
{
/**
* Undocumented function
*
* @return void
*/
public function init()
{
$this->intSlice = 8;
}
/**
* Undocumented function
*
* @return Site
*/
public function getSite()
{
if ($this->Site == NULL) {
$this->Site = new Site;
}
return $this->Site;
}
}

9
code/app/task/crawler/zw74/page/ChapterPage.php Normal file
View File

@@ -0,0 +1,9 @@
<?php
declare(strict_types=1);
namespace app\task\crawler\zw74\page;
use app\task\crawler\zw\page\ChapterPage as ZwChapterPage;
class ChapterPage extends ZwChapterPage {}

10
code/app/task/crawler/zw74/page/LatestListPage.php Normal file
View File

@@ -0,0 +1,10 @@
<?php
declare(strict_types=1);
namespace app\task\crawler\zw74\page;
use app\model\NovelModel;
use app\task\crawler\zw\page\LatestListPage as ZwLatestListPage;
class LatestListPage extends ZwLatestListPage {}

48
code/app/task/crawler/zw74/page/NovelPage.php Normal file
View File

@@ -0,0 +1,48 @@
<?php
declare(strict_types=1);
namespace app\task\crawler\zw74\page;
use app\task\crawler\zw\page\NovelPage as ZwNovelPage;
class NovelPage extends ZwNovelPage
{
/**
* 分析关联数据
*
* @return void
*/
public function getReletedNovel()
{
if (empty($this->arrRelatedNovel) || empty($this->arrForgeNovel)) {
$this->getQueryList()->find('.first_txt')->eq(0)->find("a")->map(function ($A) {
$strName = $A->text();
$strUri = $A->attr('href');
if (is_numeric(strpos($strUri, 'http'))) {
return;
}
$strPattern = '#^/\d+/(\d+)/$#';
if (is_numeric(strpos($strUri, 'info'))) {
$intNForgeId = (int)supperExtractFilename($strUri);
$this->arrForgeNovel[] = [
"n_forge_id" => $intNForgeId,
"n_forge_title" => $strName,
];
} elseif (preg_match($strPattern, $strUri, $arrMatches)) {
$intNSourceId = (int)$arrMatches[1];
$arrNovelInfo = $this->findOrCreateNovel($intNSourceId, $strName);
$this->arrRelatedNovel[] = [
"n_id" => $arrNovelInfo["n_id"],
"n_source_id" => $arrNovelInfo["n_source_id"],
"og_novel_book_name" => $arrNovelInfo["og_novel_book_name"],
];
}
});
}
}
}

152
code/app/task/crawler/zw74/page/Site.php Normal file
View File

@@ -0,0 +1,152 @@
<?php
declare(strict_types=1);
namespace app\task\crawler\zw74\page;
use app\task\crawler\zw\page\Site as ZwSite;
use GuzzleHttp\Client;
/**
* @mixin think\Model
*/
class Site extends ZwSite
{
/**
* Undocumented variable
*
* @var string
*/
protected $strDomain = 'https://www.74zw.cc';
/**
* Undocumented variable
*
* @var string
*/
protected $strSiteCode = 'zw';
/**
* Undocumented function
*/
public function initClient()
{
$arrConfig = [
'http_version' => '1.1', // 强制用 HTTP/1.1，已有，保持
'base_uri' => $this->strDomain, // 基础域名，已有，保持
'http_errors' => false, // 不抛 HTTP 错误，已有，保持
'timeout' => 60, // 从 160 秒改为 60 秒
'connect_timeout' => 10, // 从 30 秒改为 10 秒
'curl' => [
CURLOPT_TCP_KEEPALIVE => 1, // 启用 TCP Keep-Alive，已有，保持
CURLOPT_TCP_KEEPIDLE => 120, // Keep-Alive 空闲时间
CURLOPT_TCP_KEEPINTVL => 60, // Keep-Alive 探测间隔
// CURLOPT_SSL_VERIFYPEER => true, // 验证 SSL，已有，保持
],
'headers' => [
'User-Agent' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36 Edg/135.0.0.0',
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'Accept-Language' => 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
'Referer' => $this->strDomain,
'Connection' => 'keep-alive',
'sec-ch-ua' => '"Microsoft Edge";v="135", "Not-A.Brand";v="8", "Chromium";v="135"',
'sec-ch-ua-mobile' => '?0',
'sec-ch-ua-platform' => '"Windows"',
'sec-fetch-dest' => 'document',
'sec-fetch-mode' => 'navigate',
'sec-fetch-site' => 'same-origin',
'sec-fetch-user' => '?1',
'upgrade-insecure-requests' => '1',
],
'pool_size' => 10,
'verify' => false,
];
$arrProxy = config('proxy');
if ($arrProxy['proxy_status']) {
$strCode = $arrProxy['proxy_code'];
$strProxyHost = $arrProxy['proxy_host'];
$strKey = $arrProxy['proxy_key'];
$strSecret = $arrProxy['proxy_secret'];
switch ($strCode) {
case 'XIONGMAO':
$intTime = time();
$strTxt = "orderno=" . $strKey . ",secret=" . $strSecret . ",timestamp=" . $intTime;
$strSign = strtoupper(md5($strTxt));
$strAuth = 'sign=' . $strSign . '&orderno=' . $strKey . '&timestamp=' . $intTime;
$arrConfig['proxy'] = $strProxyHost;
$arrConfig['headers']['Proxy-Authorization'] = $strAuth;
break;
case 'KUAI':
$arrConfig['proxy'] = $strProxyHost;
$arrConfig['curl'][CURLOPT_PROXYAUTH] = CURLAUTH_BASIC;
$arrConfig['curl'][CURLOPT_PROXYUSERPWD] = "$strKey:$strSecret";
break;
}
}
$this->Client = new Client($arrConfig);
}
/**
* Undocumented function
*
* @param int $intNSourceId
* @param int $intPage
* @return string
*/
public function getNovelPageUri($intNSourceId, $intPage): string
{
$intUriFirst = intdiv($intNSourceId, 1000);
$strUri = '';
if ($intPage == 0) {
$strUri = sprintf('/%s/%s/', $intUriFirst, $intNSourceId);
} else {
$strUri = sprintf('/book/%s/list%s.html', $intNSourceId, $intPage);
}
return $strUri;
}
/**
* Undocumented function
*
* @param object $Site
* @param string $strUri
* @return NovelPage
*/
public function getNovelPage($Site, $strUri)
{
return new NovelPage($Site, $strUri);
}
/**
* Undocumented function
*
* @param object $Site
* @param string $strUri
* @return ChapterPage
*/
public function getChapterPage($Site, $strUri)
{
return new ChapterPage($Site, $strUri);
}
/**
* Undocumented function
*
* @param object $Site
* @param string $strUri
* @return LatestListPage
*/
public function getLatestListPage($Site, $strUri)
{
return new LatestListPage($Site, $strUri);
}
}

41
code/app/task/module/PlanTask.php
View File

@@ -6,6 +6,7 @@ namespace app\task\module;
use app\model\PlanTaskModel; use app\model\PlanTaskModel;
use app\task\crawler\zw630\Scheduler as Zw630Scheduler; use app\task\crawler\zw630\Scheduler as Zw630Scheduler;
use app\task\crawler\zw74\Scheduler as Zw74Scheduler;
use microserver\QueueManage; use microserver\QueueManage;
use microserver\ProcessSanitizer; use microserver\ProcessSanitizer;
@@ -34,6 +35,13 @@ class PlanTask
case 'PULL_630_ZW_NOVEL_LATEST': case 'PULL_630_ZW_NOVEL_LATEST':
self::pull630ZwNovelLatest(); self::pull630ZwNovelLatest();
break; break;
case 'PULL_74_ZW_NOVEL':
self::pull74ZwNovel();
break;
case 'PULL_74_ZW_NOVEL_LATEST':
self::pull74ZwNovelLatest();
break;
default: default:
break; break;
} }
@@ -71,4 +79,37 @@ class PlanTask
$QueueManage = new QueueManage(); $QueueManage = new QueueManage();
$QueueManage->set($arrTask); $QueueManage->set($arrTask);
} }
/**
* Undocumented function
*
* @return void
*/
public static function pull74ZwNovel()
{
$arrTask = [
'callback' => [Zw74Scheduler::class, 'crawlFull'],
'data' => []
];
$QueueManage = new QueueManage();
$QueueManage->set($arrTask);
}
/**
* Undocumented function
*
* @return void
*/
public static function pull74ZwNovelLatest()
{
$arrTask = [
'callback' => [Zw74Scheduler::class, 'crawLatest'],
'data' => []
];
$QueueManage = new QueueManage();
$QueueManage->set($arrTask);
}
} }

3
code/config/proxy.php
View File

@@ -6,6 +6,7 @@
return [ return [
'proxy_status' => env('PROXY_STATUS', false), 'proxy_status' => env('PROXY_STATUS', false),
'proxy_code' => env('PROXY_CODE', ''), 'proxy_code' => env('PROXY_CODE', ''),
'proxy_order' => env('PROXY_ORDER', ''), 'proxy_host' => env('PROXY_HOST', ''),
'proxy_key' => env('PROXY_KEY', ''),
'proxy_secret' => env('PROXY_SECRET', ''), 'proxy_secret' => env('PROXY_SECRET', ''),
]; ];

10
code/database/mongo/migrations/novel.php
View File

@@ -4,14 +4,14 @@ return [
'indexes' => [ 'indexes' => [
[ [
'key' => ['n_id' => 1], 'key' => ['n_id' => 1],
'options' => ['unique' => true]
],
[
'key' => ['n_name' => 1],
'options' => ['unique' => false] 'options' => ['unique' => false]
], ],
[ [
'key' => ['og_novel_book_name' => 1], 'key' => ['n_status' => 1],
'options' => ['unique' => false]
],
[
'key' => ['og_novel_status' => 1],
'options' => ['unique' => false] 'options' => ['unique' => false]
], ],
[ [

19
code/database/seeders/PlanTaskSeeder.php
View File

@@ -20,13 +20,28 @@ class PlanTaskSeeder
[ [
'pt_name' => '630小说全量扫描', 'pt_name' => '630小说全量扫描',
'pt_code' => 'PULL_630_ZW_NOVEL', 'pt_code' => 'PULL_630_ZW_NOVEL',
'pt_enable' => 1, 'pt_enable' => 0,
'pt_limit' => 10 * 24 * 3600 * 3, 'pt_limit' => 10 * 24 * 3600 * 3,
], ],
[ [
'pt_name' => '630小说每日增量', 'pt_name' => '630小说每日增量',
'pt_code' => 'PULL_630_ZW_NOVEL_LATEST', 'pt_code' => 'PULL_630_ZW_NOVEL_LATEST',
'pt_enable' => 1, 'pt_enable' => 0,
'pt_limit' => 1 * 24 * 3600,
],
[
'pt_name' => '74小说全量扫描',
'pt_code' => 'PULL_74_ZW_NOVEL',
'pt_enable' => 0,
'pt_limit' => 10 * 24 * 3600 * 3,
],
[
'pt_name' => '74小说每日增量',
'pt_code' => 'PULL_74_ZW_NOVEL_LATEST',
'pt_enable' => 0,
'pt_limit' => 1 * 24 * 3600, 'pt_limit' => 1 * 24 * 3600,
], ],
]; ];

12
code/extend/storage/StorageCore.php
View File

@@ -75,10 +75,14 @@ class StorageCore
public function read($strKey) public function read($strKey)
{ {
$Stream = $this->Filesystem->readStream($strKey); try {
$strContent = stream_get_contents($Stream); $Stream = $this->Filesystem->readStream($strKey);
$strContent = zstd_uncompress($strContent); $strContent = stream_get_contents($Stream);
return $strContent; $strContent = zstd_uncompress($strContent);
return $strContent;
} catch (\Exception $e) {
return '';
}
} }
public function has($strKey) public function has($strKey)