debug
This commit is contained in:
@@ -16,9 +16,9 @@ class Scheduler
|
||||
{
|
||||
protected $intSlice = 8;
|
||||
|
||||
protected $intStartNSourceId = 7377;
|
||||
protected $intStartNSourceId = 1;
|
||||
|
||||
protected $intEndNSourceId = 7377;
|
||||
protected $intEndNSourceId = 100;
|
||||
|
||||
/**
|
||||
* Undocumented variable
|
||||
@@ -102,11 +102,9 @@ class Scheduler
|
||||
|
||||
$arrCover = [];
|
||||
foreach ($arrNovelPageList as $NovelPage) {
|
||||
$arrNovel = $NovelPage->getNovelInfo();
|
||||
// $boolResult = $NovelPage->saveNovelInfo();
|
||||
|
||||
|
||||
return;
|
||||
// $arrNovel = $NovelPage->getNovelInfo();
|
||||
$boolResult = $NovelPage->saveNovelInfo();
|
||||
// return;
|
||||
|
||||
if ($boolResult == false) {
|
||||
continue;
|
||||
@@ -129,7 +127,7 @@ class Scheduler
|
||||
}
|
||||
|
||||
$arrChapterList = $NovelPage->getChapters();
|
||||
$this->fetchChapter($arrNovel['n_id'], $NovelPage->getData()['page'], $arrChapterList);
|
||||
$this->fetchChapter($NovelPage->intNId, $NovelPage->getData()['page'], $arrChapterList);
|
||||
}
|
||||
|
||||
if ($arrCover) {
|
||||
@@ -284,7 +282,7 @@ class Scheduler
|
||||
|
||||
foreach ($arrData['filter'] as $intKey => $arrChapterFilter) {
|
||||
$strUUId = sprintf('%s-chapter-%s-%s', $Site->getSiteCode(), $arrChapterFilter['c_source_id'], $arrChapterFilter['c_page']);
|
||||
$arrChapter = $ChapterModel->findOne(['c_site_uuid' => $strUUId]);
|
||||
$arrChapter = $ChapterModel->findOne(['c_source_uuid' => $strUUId]);
|
||||
|
||||
if (!empty($arrChapter) && !empty($arrChapter['content'])) {
|
||||
unset($arrData['filter'][$intKey]);
|
||||
|
||||
@@ -70,6 +70,22 @@ class ChapterPage extends BasePage
|
||||
$this->arrChapterInfo['c_page_title'] = $this->getQueryList()->find('title')->text();
|
||||
$this->arrChapterInfo['c_page_keywords'] = $this->getQueryList()->find('meta[name="keywords"]')->attr('content');
|
||||
$this->arrChapterInfo['c_page_description'] = $this->getQueryList()->find('meta[name="description"]')->attr('content');
|
||||
|
||||
$strPattern = "/lastread\.set\((.*?)\);/";
|
||||
|
||||
if (preg_match($strPattern, $this->getContent(), $arrMatches)) {
|
||||
$strMatch = $arrMatches[1]; // 提取括号内的内容
|
||||
|
||||
$arrData = array_map('trim', explode(",", $strMatch));
|
||||
|
||||
$arrData = array_map(function ($item) {
|
||||
return trim($item, "'");
|
||||
}, $arrData);
|
||||
|
||||
$this->arrChapterInfo['c_page_data'] = $arrData;
|
||||
} else {
|
||||
$this->arrChapterInfo['c_page_data'] = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
return $this->arrChapterInfo;
|
||||
@@ -83,85 +99,78 @@ class ChapterPage extends BasePage
|
||||
protected $arrChapterInfo;
|
||||
|
||||
|
||||
public function filterOtherFields(&$arrChapter)
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @param integer $intNId
|
||||
* @return boolean
|
||||
*/
|
||||
public function saveChapter(int $intNId): bool
|
||||
{
|
||||
$arrFields = [
|
||||
'n_id',
|
||||
'name',
|
||||
'c_sort_num',
|
||||
'c_page',
|
||||
'c_source_id',
|
||||
'c_site_code',
|
||||
'c_site_uuid',
|
||||
'c_content_path',
|
||||
'created_at',
|
||||
'updated_at',
|
||||
$arrPageChapter = $this->getChapterInfo();
|
||||
|
||||
if (
|
||||
empty($arrPageChapter) ||
|
||||
empty($arrPageChapter['c_page_title']) ||
|
||||
empty($arrPageChapter['c_page_data']) ||
|
||||
count($arrPageChapter['c_page_data']) < 5
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$strUUId = $this->getSite()->getSiteUUId('chapter-' . $this->arrData['c_source_id'] . '-' . $this->arrData['c_page']);
|
||||
|
||||
$arrChapter = [
|
||||
'n_id' => $intNId,
|
||||
'c_name' => $arrPageChapter['c_page_data'][3],
|
||||
'c_sort_num' => $this->arrData['c_sort_num'],
|
||||
'c_page' => $this->arrData['c_page'],
|
||||
'c_source_id' => $this->arrData['c_source_id'],
|
||||
'c_site_code' => $this->getSite()->getSiteCode(),
|
||||
'c_source_uuid' => $strUUId,
|
||||
];
|
||||
|
||||
foreach ($arrChapter as $strKey => $strVal) {
|
||||
if (!in_array($strKey, $arrFields)) {
|
||||
unset($arrChapter[$strKey]);
|
||||
}
|
||||
}
|
||||
}
|
||||
$arrExistsChapter = $this->getChapterModel()->findOne(['c_source_uuid' => $strUUId], $this->arrOptions);
|
||||
|
||||
public function saveChapter(int $intNId): array
|
||||
{
|
||||
if ($arrExistsChapter) {
|
||||
|
||||
try {
|
||||
$this->getChapterInfo();
|
||||
$arrExistsChapter = (array) $arrExistsChapter;
|
||||
|
||||
$strUUId = $this->getSite()->getSiteUUId('chapter-' . $this->arrData['c_source_id'] . '-' . $this->arrData['c_page']);
|
||||
$arrExisting = $this->getChapterModel()->findOne(['c_site_uuid' => $strUUId], $this->arrOptions);
|
||||
$arrUpdate = [
|
||||
'updated_at' => new \MongoDB\BSON\UTCDateTime(),
|
||||
];
|
||||
|
||||
if ($arrExisting) {
|
||||
$this->getChapterModel()->updateOne(
|
||||
['c_source_uuid' => $strUUId],
|
||||
['$set' => $arrUpdate],
|
||||
);
|
||||
$this->strCContentPath = $arrExistsChapter['c_content_path'];
|
||||
} else {
|
||||
|
||||
$arrExisting = (array) $arrExisting;
|
||||
$this->arrChapterInfo = array_merge($arrExisting, $this->getChapterInfo());
|
||||
$arrInsertNovel = $arrChapter;
|
||||
|
||||
$this->arrChapterInfo['updated_at'] = new \MongoDB\BSON\UTCDateTime();
|
||||
|
||||
$arrUpdate = $this->arrChapterInfo;
|
||||
|
||||
$this->filterOtherFields($arrUpdate);
|
||||
|
||||
$this->getChapterModel()->updateOne(
|
||||
['c_site_uuid' => $strUUId],
|
||||
['$set' => $arrUpdate],
|
||||
);
|
||||
} else {
|
||||
|
||||
$this->arrChapterInfo['n_id'] = $intNId;
|
||||
$this->arrChapterInfo['name'] = $this->arrData['name'];
|
||||
$this->arrChapterInfo['c_sort_num'] = $this->arrData['c_sort_num'];
|
||||
$this->arrChapterInfo['c_page'] = $this->arrData['c_page'];
|
||||
$this->arrChapterInfo['c_source_id'] = $this->arrData['c_source_id'];
|
||||
$this->arrChapterInfo['c_site_code'] = $this->getSite()->getSiteCode();
|
||||
$this->arrChapterInfo['c_site_uuid'] = $strUUId;
|
||||
$this->arrChapterInfo['c_content_path'] = '/novel/' . ($intNId % 100) . '/' . $intNId . '/' . $this->arrChapterInfo['c_sort_num'] . '_' . $this->arrChapterInfo['c_page'] . '.txt';
|
||||
$this->arrChapterInfo['created_at'] = new \MongoDB\BSON\UTCDateTime();
|
||||
$arrInsert = $this->arrChapterInfo;
|
||||
|
||||
$this->filterOtherFields($arrInsert);
|
||||
|
||||
$this->getChapterModel()->insert($arrInsert);
|
||||
$arrInsertNovel['c_content_path'] = '/novel/' . ($intNId % 100) . '/' . $intNId . '/' . $arrChapter['c_sort_num'] . '_' . $arrChapter['c_page'] . '.txt';
|
||||
$arrInsertNovel['created_at'] = new \MongoDB\BSON\UTCDateTime();
|
||||
|
||||
try {
|
||||
$this->getChapterModel()->insert($arrInsertNovel);
|
||||
$this->updateNovelHaveChapter($intNId);
|
||||
}
|
||||
|
||||
if (mb_strlen($this->strChapterContent) > 20) {
|
||||
if (!StorageCore::getInstance()->has($this->arrChapterInfo['c_content_path'])) {
|
||||
StorageCore::getInstance()->set($this->arrChapterInfo['c_content_path'], $this->strChapterContent);
|
||||
$this->strCContentPath = $arrInsertNovel['c_content_path'];
|
||||
} catch (\Exception $e) {
|
||||
if ($e->getCode() === 11000) {
|
||||
return $this->saveChapter($intNId);
|
||||
} else {
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
return $this->arrChapterInfo;
|
||||
} catch (\Exception $e) {
|
||||
throw new \Exception("Upsert failed: " . $e->getMessage());
|
||||
}
|
||||
|
||||
StorageCore::getInstance()->put($this->strCContentPath, $this->strChapterContent);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
public $strCContentPath;
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
@@ -179,7 +188,7 @@ class ChapterPage extends BasePage
|
||||
|
||||
if (!empty($arrLasteChapter)) {
|
||||
$arrUpdate['$set']['n_have_chapter'] = 1;
|
||||
$arrUpdate['$set']['og_novel_latest_chapter_name'] = $arrLasteChapter['name'];
|
||||
$arrUpdate['$set']['n_latest_chapter_name'] = $arrLasteChapter['c_name'];
|
||||
$this->getMongoBase()->updateOne('novel', $arrFilter, $arrUpdate);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,7 +7,7 @@ namespace app\task\crawler\zw\page;
|
||||
use app\model\CategoryModel;
|
||||
use app\model\CountersModel;
|
||||
use app\model\NovelModel;
|
||||
|
||||
use storage\StorageCore;
|
||||
|
||||
class NovelPage extends BasePage
|
||||
{
|
||||
@@ -71,22 +71,31 @@ class NovelPage extends BasePage
|
||||
{
|
||||
$strUUId = $this->getSite()->getSiteUUId($intNSourceId);
|
||||
|
||||
$arrExisting = $this->getNovelModel()->findOne(['n_site_uuid' => $strUUId], $this->arrOptions);
|
||||
$arrExistsNovel = $this->getNovelModel()->findOne(['n_source_uuid' => $strUUId], $this->arrOptions);
|
||||
|
||||
if ($arrExisting) {
|
||||
$arrNovelInfo = (array) $arrExisting;
|
||||
if ($arrExistsNovel) {
|
||||
$arrNovelInfo = (array) $arrExistsNovel;
|
||||
} else {
|
||||
$arrNovelInfo = [
|
||||
'og_novel_book_name' => $strName,
|
||||
'og_novel_pin_yin' => zhToPinYin($strName),
|
||||
'n_id' => CountersModel::getInstance()->getNextSequence('novel'),
|
||||
'n_name' => $strName,
|
||||
'n_name_pinyin' => zhToPinYin($strName),
|
||||
'n_site_code' => $this->getSite()->getSiteCode(),
|
||||
'n_site_uuid' => $strUUId,
|
||||
'n_source_uuid' => $strUUId,
|
||||
'n_source_id' => $intNSourceId,
|
||||
'n_level' => 1,
|
||||
'n_have_chapter' => 0,
|
||||
'n_id' => CountersModel::getInstance()->getNextSequence('novel'),
|
||||
'created_at' => new \MongoDB\BSON\UTCDateTime(),
|
||||
];
|
||||
$this->getNovelModel()->insert($arrNovelInfo);
|
||||
try {
|
||||
$this->getNovelModel()->insert($arrNovelInfo);
|
||||
} catch (\Exception $e) {
|
||||
if ($e->getCode() === 11000) {
|
||||
return $this->findOrCreateNovel($intNSourceId, $strName);
|
||||
} else {
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
}
|
||||
return $arrNovelInfo;
|
||||
}
|
||||
@@ -116,24 +125,8 @@ class NovelPage extends BasePage
|
||||
});
|
||||
|
||||
$this->getReletedNovel();
|
||||
|
||||
$this->arrNovelInfo['n_related_novel'] = $this->arrRelatedNovel;
|
||||
$this->arrNovelInfo['n_forge_novel'] = $this->arrForgeNovel;
|
||||
|
||||
// $this->arrNovelInfo['og_novel_pin_yin'] = zhToPinYin($this->arrNovelInfo['og_novel_book_name'] ?? '');
|
||||
|
||||
// if (!empty($this->arrNovelInfo['og_novel_category'])) {
|
||||
// $this->arrNovelInfo['category_pinyin'] = CategoryModel::getPinYinByZh($this->arrNovelInfo['og_novel_category']);
|
||||
// $this->arrNovelInfo['category_sex'] = CategoryModel::checkSex($this->arrNovelInfo['og_novel_category']);
|
||||
// $this->arrNovelInfo['category_sex_zh'] = CategoryModel::checkSexZh($this->arrNovelInfo['og_novel_category']);
|
||||
// }
|
||||
|
||||
// $this->arrNovelInfo['n_site_code'] = $this->getSite()->getSiteCode();
|
||||
// $this->arrNovelInfo['n_source_id'] = $this->arrData['n_source_id'];
|
||||
}
|
||||
|
||||
print_r($this->arrNovelInfo);
|
||||
|
||||
return $this->arrNovelInfo;
|
||||
}
|
||||
|
||||
@@ -145,29 +138,31 @@ class NovelPage extends BasePage
|
||||
*/
|
||||
public function generateCoverPath($strName): string
|
||||
{
|
||||
return sprintf('/img/%s/%s.jpg', date('Y/m/d', time() - 24 * 3600 * 14), $strName);
|
||||
$intTime = time() - mt_rand(1, 2 * 24 * 3600 * 360);
|
||||
|
||||
return sprintf('/img/%s/%s.jpg', date('Y/m/d/H', $intTime), $strName);
|
||||
}
|
||||
|
||||
public function filterOtherFields(&$arrNovel)
|
||||
{
|
||||
$arrFields = [
|
||||
'n_id',
|
||||
'og_novel_book_name',
|
||||
'og_novel_pin_yin',
|
||||
'og_novel_status',
|
||||
'og_novel_author',
|
||||
'og_description',
|
||||
'og_novel_category',
|
||||
'category_pinyin',
|
||||
'category_sex',
|
||||
'category_sex_zh',
|
||||
'n_name',
|
||||
'n_name_pinyin',
|
||||
'n_status',
|
||||
'n_author',
|
||||
'n_description',
|
||||
'n_category',
|
||||
'n_category_pinyin',
|
||||
'n_category_sex_en',
|
||||
'n_category_sex_zh',
|
||||
'n_cover_path',
|
||||
'og_novel_latest_chapter_name',
|
||||
'og_novel_update_time',
|
||||
'n_latest_chapter_name',
|
||||
'n_update_time',
|
||||
'n_forge_novel',
|
||||
'n_related_novel',
|
||||
'og_site_code',
|
||||
'n_site_uuid',
|
||||
'n_site_code',
|
||||
'n_source_uuid',
|
||||
'n_source_id',
|
||||
'n_have_chapter',
|
||||
'created_at',
|
||||
@@ -184,65 +179,123 @@ class NovelPage extends BasePage
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
* @return void
|
||||
* @return bool
|
||||
*/
|
||||
public function saveNovelInfo()
|
||||
public function saveNovelInfo(): bool
|
||||
{
|
||||
try {
|
||||
|
||||
$this->getNovelInfo();
|
||||
|
||||
if (!isset($this->arrNovelInfo['og_novel_book_name'])) {
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
$strUUId = $this->getSite()->getSiteUUId($this->arrData['n_source_id']);
|
||||
|
||||
$arrExisting = $this->getNovelModel()->findOne(['n_site_uuid' => $strUUId], $this->arrOptions);
|
||||
|
||||
if ($arrExisting) {
|
||||
|
||||
$arrExisting = (array) $arrExisting;
|
||||
$this->arrNovelInfo = array_merge($arrExisting, $this->arrNovelInfo);
|
||||
|
||||
if (
|
||||
!isset($this->arrNovelInfo['n_cover_path']) ||
|
||||
empty($this->arrNovelInfo['n_cover_path']) ||
|
||||
!file_exists($this->arrNovelInfo['n_cover_path']) ||
|
||||
|
||||
(basename($this->arrNovelInfo['n_cover_path']) == '.jpg')
|
||||
) {
|
||||
$this->arrNovelInfo['n_cover_path'] = $this->generateCoverPath($this->arrNovelInfo['og_novel_pin_yin']);
|
||||
}
|
||||
|
||||
$this->arrNovelInfo['updated_at'] = new \MongoDB\BSON\UTCDateTime();
|
||||
|
||||
$arrUpdate = $this->arrNovelInfo;
|
||||
$this->filterOtherFields($arrUpdate);
|
||||
|
||||
$this->getNovelModel()->updateOne(
|
||||
['n_site_uuid' => $strUUId],
|
||||
['$set' => $arrUpdate],
|
||||
);
|
||||
} else {
|
||||
$this->arrNovelInfo['n_site_uuid'] = $strUUId;
|
||||
$this->arrNovelInfo['n_id'] = CountersModel::getInstance()->getNextSequence('novel');
|
||||
$this->arrNovelInfo['n_level'] = 1;
|
||||
$this->arrNovelInfo['n_have_chapter'] = 0;
|
||||
$this->arrNovelInfo['n_cover_path'] = $this->generateCoverPath($this->arrNovelInfo['og_novel_pin_yin']);
|
||||
|
||||
$arrInsert = $this->arrNovelInfo;
|
||||
$this->filterOtherFields($arrInsert);
|
||||
|
||||
$this->getNovelModel()->insert($arrInsert);
|
||||
}
|
||||
return $this->arrNovelInfo;
|
||||
} catch (\Exception $e) {
|
||||
throw new \Exception("Upsert failed: " . $e->getMessage());
|
||||
$arrPageInfo = $this->getNovelInfo();
|
||||
if (
|
||||
empty($arrPageInfo) ||
|
||||
empty($arrPageInfo['og_title']) ||
|
||||
empty($arrPageInfo['og_image']) ||
|
||||
empty($arrPageInfo['og_novel_category'])
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$arrNovelInfo = [
|
||||
// 'n_id'=> $arrPageInfo['xxxxx'],
|
||||
'n_name' => $arrPageInfo['og_title'],
|
||||
'n_name_pinyin' => zhToPinYin($arrPageInfo['og_title']),
|
||||
'n_status' => $arrPageInfo['og_novel_status'],
|
||||
'n_author' => $arrPageInfo['og_novel_author'],
|
||||
'n_description' => $arrPageInfo['og_description'],
|
||||
'n_category' => $arrPageInfo['og_novel_category'],
|
||||
// 'n_category_pinyin' => $arrPageInfo['xxxxx'],
|
||||
// 'n_category_sex_en' => $arrPageInfo['xxxxx'],
|
||||
// 'n_category_sex_zh' => $arrPageInfo['xxxxx'],
|
||||
// 'n_cover_path' => $arrPageInfo['xxxxx'],
|
||||
'n_latest_chapter_name' => $arrPageInfo['og_novel_latest_chapter_name'],
|
||||
'n_update_time' => $arrPageInfo['og_novel_update_time'],
|
||||
'n_forge_novel' => $this->arrForgeNovel,
|
||||
'n_related_novel' => $this->arrRelatedNovel,
|
||||
|
||||
'n_site_code' => $this->getSite()->getSiteCode(),
|
||||
'n_source_id' => $this->arrData['n_source_id'],
|
||||
'n_source_uuid' => $this->getSite()->getSiteUUId($this->arrData['n_source_id']),
|
||||
|
||||
// 'n_have_chapter' => $arrPageInfo['xxxxx'],
|
||||
// 'created_at' => $arrPageInfo['xxxxx'],
|
||||
// 'updated_at' => $arrPageInfo['xxxxx'],
|
||||
|
||||
];
|
||||
|
||||
$arrExistsNovel = $this->getNovelModel()->findOne(['n_source_uuid' => $arrNovelInfo['n_source_uuid']], $this->arrOptions);
|
||||
|
||||
if ($arrExistsNovel) {
|
||||
|
||||
$arrExistsNovel = (array) $arrExistsNovel;
|
||||
|
||||
$arrUpdate = [
|
||||
'n_latest_chapter_name' => $arrNovelInfo['n_latest_chapter_name'],
|
||||
'n_update_time' => $arrNovelInfo['n_update_time'],
|
||||
'updated_at' => new \MongoDB\BSON\UTCDateTime(),
|
||||
];
|
||||
|
||||
# 如果没有分类,说明这条数据是最初从关联shu那里采集的,需要补全全部信息
|
||||
if (empty($arrExistsNovel['n_category'])) {
|
||||
$arrUpdate = array_merge($arrUpdate, $arrNovelInfo);
|
||||
$arrUpdate['n_related_novel'] = $this->arrRelatedNovel;
|
||||
$arrUpdate['n_forge_novel'] = $this->arrForgeNovel;
|
||||
$arrUpdate['n_category_pinyin'] = CategoryModel::getPinYin($this->arrNovelInfo['og_novel_category']);
|
||||
$arrUpdate['n_category_sex_en'] = CategoryModel::getSexEn($this->arrNovelInfo['og_novel_category']);
|
||||
$arrUpdate['n_category_sex_zh'] = CategoryModel::getSexZh($this->arrNovelInfo['og_novel_category']);
|
||||
$arrUpdate['n_related_novel'] = $this->arrRelatedNovel;
|
||||
$arrUpdate['n_forge_novel'] = $this->arrForgeNovel;
|
||||
}
|
||||
|
||||
# 图片不存在的时候,需要重新生成
|
||||
if (
|
||||
!isset($arrExistsNovel['n_cover_path']) ||
|
||||
empty($arrExistsNovel['n_cover_path']) ||
|
||||
!StorageCore::getInstance()->has($arrExistsNovel['n_cover_path']) ||
|
||||
(basename($arrExistsNovel['n_cover_path']) == '.jpg')
|
||||
) {
|
||||
$arrUpdate['n_cover_path'] = $this->generateCoverPath($arrNovelInfo['n_name_pinyin']);
|
||||
$this->strCoverLocalPath = $arrUpdate['n_cover_path'];
|
||||
} else {
|
||||
$this->strCoverLocalPath = $arrExistsNovel['n_cover_path'];
|
||||
}
|
||||
|
||||
$this->getNovelModel()->updateOne(
|
||||
['n_source_uuid' => $arrNovelInfo['n_source_uuid']],
|
||||
['$set' => $arrUpdate],
|
||||
);
|
||||
$this->intNId = $arrExistsNovel['n_id'];
|
||||
} else {
|
||||
|
||||
$arrNovelInfo['n_category_pinyin'] = CategoryModel::getPinYin($this->arrNovelInfo['og_novel_category']);
|
||||
$arrNovelInfo['n_category_sex_en'] = CategoryModel::getSexEn($this->arrNovelInfo['og_novel_category']);
|
||||
$arrNovelInfo['n_category_sex_zh'] = CategoryModel::getSexZh($this->arrNovelInfo['og_novel_category']);
|
||||
$arrNovelInfo['n_id'] = CountersModel::getInstance()->getNextSequence('novel');
|
||||
$arrNovelInfo['n_level'] = 1;
|
||||
$arrNovelInfo['n_have_chapter'] = 0;
|
||||
$arrNovelInfo['n_cover_path'] = $this->generateCoverPath($arrNovelInfo['n_name_pinyin']);
|
||||
$arrNovelInfo['n_related_novel'] = $this->arrRelatedNovel;
|
||||
$arrNovelInfo['n_forge_novel'] = $this->arrForgeNovel;
|
||||
$arrNovelInfo['created_at'] = new \MongoDB\BSON\UTCDateTime();
|
||||
|
||||
try {
|
||||
$this->getNovelModel()->insert($arrNovelInfo);
|
||||
$this->intNId = $arrNovelInfo['n_id'];
|
||||
$this->strCoverLocalPath = $arrNovelInfo['n_cover_path'];
|
||||
} catch (\Exception $e) {
|
||||
if ($e->getCode() === 11000) {
|
||||
return $this->saveNovelInfo();
|
||||
} else {
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
public $strCoverLocalPath = '';
|
||||
|
||||
public $intNId;
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*
|
||||
@@ -250,14 +303,9 @@ class NovelPage extends BasePage
|
||||
*/
|
||||
public function getDownCoverInfo(): array
|
||||
{
|
||||
if (!key_exists('og_image', $this->arrNovelInfo)) {
|
||||
print_r($this->arrNovelInfo);
|
||||
echo $this->getContent();
|
||||
}
|
||||
|
||||
return [
|
||||
'uri' => $this->arrNovelInfo['og_image'],
|
||||
'local_path' => $this->arrNovelInfo['n_cover_path'],
|
||||
'local_path' => $this->strCoverLocalPath,
|
||||
];
|
||||
}
|
||||
|
||||
|
||||
@@ -42,7 +42,71 @@ class Site
|
||||
$this->initClient();
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
*/
|
||||
public function initClient()
|
||||
{
|
||||
$arrConfig = [
|
||||
'http_version' => '1.1', // 强制用 HTTP/1.1,已有,保持
|
||||
'base_uri' => $this->strDomain, // 基础域名,已有,保持
|
||||
'http_errors' => false, // 不抛 HTTP 错误,已有,保持
|
||||
'timeout' => 60, // 从 160 秒改为 60 秒
|
||||
'connect_timeout' => 10, // 从 30 秒改为 10 秒
|
||||
'curl' => [
|
||||
CURLOPT_TCP_KEEPALIVE => 1, // 启用 TCP Keep-Alive,已有,保持
|
||||
CURLOPT_TCP_KEEPIDLE => 120, // Keep-Alive 空闲时间
|
||||
CURLOPT_TCP_KEEPINTVL => 60, // Keep-Alive 探测间隔
|
||||
// CURLOPT_SSL_VERIFYPEER => true, // 验证 SSL,已有,保持
|
||||
],
|
||||
'headers' => [
|
||||
'User-Agent' => 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36',
|
||||
'Accept' => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
|
||||
'Accept-Language' => 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
|
||||
'Referer' => $this->strDomain,
|
||||
'Connection' => 'keep-alive',
|
||||
'sec-ch-ua' => '"Not/A)Brand";v="8", "Chromium";v="130", "Google Chrome";v="130"',
|
||||
'sec-ch-ua-mobile' => '?1',
|
||||
'sec-ch-ua-platform' => '"Android"',
|
||||
'sec-fetch-dest' => 'document',
|
||||
'sec-fetch-mode' => 'navigate',
|
||||
'sec-fetch-site' => 'same-origin',
|
||||
'sec-fetch-user' => '?1',
|
||||
'upgrade-insecure-requests' => '1',
|
||||
],
|
||||
'pool_size' => 10,
|
||||
'verify' => false,
|
||||
];
|
||||
|
||||
$arrProxy = config('proxy');
|
||||
if ($arrProxy['proxy_status']) {
|
||||
|
||||
$strCode = $arrProxy['proxy_code'];
|
||||
$strProxyHost = $arrProxy['proxy_host'];
|
||||
$strKey = $arrProxy['proxy_key'];
|
||||
$strSecret = $arrProxy['proxy_secret'];
|
||||
|
||||
|
||||
switch ($strCode) {
|
||||
case 'XIONGMAO':
|
||||
$intTime = time();
|
||||
$strTxt = "orderno=" . $strKey . ",secret=" . $strSecret . ",timestamp=" . $intTime;
|
||||
$strSign = strtoupper(md5($strTxt));
|
||||
$strAuth = 'sign=' . $strSign . '&orderno=' . $strKey . '×tamp=' . $intTime;
|
||||
$arrConfig['proxy'] = $strProxyHost;
|
||||
$arrConfig['headers']['Proxy-Authorization'] = $strAuth;
|
||||
break;
|
||||
case 'KUAI':
|
||||
$arrConfig['proxy'] = $strProxyHost;
|
||||
$arrConfig['curl'][CURLOPT_PROXYAUTH] = CURLAUTH_BASIC;
|
||||
$arrConfig['curl'][CURLOPT_PROXYUSERPWD] = "$strKey:$strSecret";
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
$this->Client = new Client($arrConfig);
|
||||
}
|
||||
|
||||
/**
|
||||
* Undocumented function
|
||||
|
||||
@@ -28,7 +28,7 @@ class NovelPage extends ZwNovelPage
|
||||
$this->arrRelatedNovel[] = [
|
||||
"n_id" => $arrNovelInfo["n_id"],
|
||||
"n_source_id" => $arrNovelInfo["n_source_id"],
|
||||
"og_novel_book_name" => $arrNovelInfo["og_novel_book_name"],
|
||||
"n_name" => $arrNovelInfo["n_name"],
|
||||
];
|
||||
} elseif (is_numeric(strpos($strUri, 'kan'))) {
|
||||
$intNForgeId = (int)supperExtractFilename($strUri);
|
||||
|
||||
@@ -39,7 +39,7 @@ class NovelPage extends ZwNovelPage
|
||||
$this->arrRelatedNovel[] = [
|
||||
"n_id" => $arrNovelInfo["n_id"],
|
||||
"n_source_id" => $arrNovelInfo["n_source_id"],
|
||||
"og_novel_book_name" => $arrNovelInfo["og_novel_book_name"],
|
||||
"n_name" => $arrNovelInfo["n_name"],
|
||||
];
|
||||
}
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user