diff --git a/code/app/model/ChapterModel.php b/code/app/model/ChapterModel.php index 3e6534c..9e68c5d 100644 --- a/code/app/model/ChapterModel.php +++ b/code/app/model/ChapterModel.php @@ -124,6 +124,8 @@ class ChapterModel extends MongoModel if ($boolReadContent) { $arrChapter['content'] = StorageCore::getInstance()->read($arrChapter['c_content_path']); + } else { + $arrChapter['content'] = ''; } } diff --git a/code/app/task/crawler/zw630/Scheduler.php b/code/app/task/crawler/zw630/Scheduler.php index 30bf921..4efb66d 100644 --- a/code/app/task/crawler/zw630/Scheduler.php +++ b/code/app/task/crawler/zw630/Scheduler.php @@ -4,6 +4,7 @@ declare(strict_types=1); namespace app\task\crawler\zw630; +use app\model\ChapterModel; use app\task\crawler\zw630\page\NovelPage; use app\task\crawler\zw630\page\Site; use microserver\QueueManage; @@ -26,13 +27,13 @@ class Scheduler } /** - * 全局扫描任务 + * create a site-wide craw task * * @return void */ public function crawlFull() { - $intSliceSize = 6; + $intSliceSize = 4; $arrFilter = []; // for ($intNId = 1; $intNId <= 531208; $intNId++) { @@ -54,22 +55,76 @@ class Scheduler } /** - * Undocumented function + * create a task to craw the latest chapter * * @return void */ public function crawLatest() { + $intSliceSize = 3; $arrUri = []; $strUri = '/list/lastupdate_%s_0_0_%s.html'; - for ($intCategoryKey = 0; $intCategoryKey < 10; $intCategoryKey++) { - for ($intPage = 1; $intPage <= 10; $intPage++) { + $intEndCategory = $intEndPage = 2; + + for ($intCategoryKey = 0; $intCategoryKey < $intEndCategory; $intCategoryKey++) { + for ($intPage = 1; $intPage <= $intEndPage; $intPage++) { $arrUri[] = sprintf($strUri, $intCategoryKey, $intPage); } } - // print_r($arrUri); + $arrUri = array_chunk($arrUri, $intSliceSize); + + $arrTask = [ + 'callback' => [self::class, 'getLatestChapter'], + 'data' => [], + ]; + + foreach ($arrUri as $arrUriSlice) { + $arrTask['data'] = $arrUriSlice; + $this->QueueManage->set($arrTask); + } + } + + /** + * Undocumented function + * + * @param array $arrData + * @return void + */ + public function getLatestChapter(array $arrFilter) + { + $Site = new Site; + + $arrLatestListPageList = $Site->getLatestListPageList($arrFilter); + + $arrNSourceId = []; + + foreach ($arrLatestListPageList as $LatestListPage) { + $arrResult = $LatestListPage->getNovelFilter(); + $arrNSourceId = array_merge($arrNSourceId, $arrResult); + } + + $arrNSourceId = array_unique($arrNSourceId); + $arrNSourceId = array_values($arrNSourceId); + + $intSliceSize = 4; + $arrFilter = []; + + foreach ($arrNSourceId as $intKey => $intNSourceId) { + $arrFilter[] = [ + 'n_source_id' => $intNSourceId, + 'page' => 0, + ]; + + if ($intKey % $intSliceSize == 0) { + $this->fetchNovel($arrFilter); + $arrFilter = []; + } + } + if (!empty($arrFilter)) { + $this->fetchNovel($arrFilter); + } } /** @@ -97,7 +152,7 @@ class Scheduler */ public function fetchChapter(int $intNId, int $intPage, array $arrChapterList = []) { - $intSliceSize = 6; + $intSliceSize = 4; $intPageSize = 100; $arrTask = [ @@ -145,7 +200,6 @@ class Scheduler continue; } - $arrNextPageData = $NovelPage->getNextPageArgs(); if (!empty($arrNextPageData)) { $this->fetchNovel([ @@ -175,6 +229,20 @@ class Scheduler { $Site = new Site; + $ChapterModel = ChapterModel::getInstance(); + + foreach ($arrData['filter'] as $intKey => $arrChapterFilter) { + $strUUId = sprintf('%s-chapter-%s-%s', $Site->getSiteCode(), $arrChapterFilter['c_source_id'], $arrChapterFilter['c_page']); + $arrChapter = $ChapterModel->findOne(['c_site_uuid' => $strUUId]); + + if (!empty($arrChapter) && !empty($arrChapter['content'])) { + unset($arrData['filter'][$intKey]); + } + } + + $arrData['filter'] = array_values($arrData['filter']); + + $arrChapterPageList = $Site->getChapterPageList($arrData['filter']); $arrNextPageFilterList = []; diff --git a/code/app/task/crawler/zw630/page/BasePage.php b/code/app/task/crawler/zw630/page/BasePage.php index eb54f09..bf00316 100644 --- a/code/app/task/crawler/zw630/page/BasePage.php +++ b/code/app/task/crawler/zw630/page/BasePage.php @@ -107,6 +107,16 @@ abstract class BasePage $this->strUri = $strUri; } + /** + * Undocumented function + * + * @return string + */ + public function getUri(): string + { + return $this->strUri; + } + /** * Undocumented function * @@ -238,6 +248,4 @@ abstract class BasePage { return $this->Site; } - - } diff --git a/code/app/task/crawler/zw630/page/LatestListPage.php b/code/app/task/crawler/zw630/page/LatestListPage.php new file mode 100644 index 0000000..1c06477 --- /dev/null +++ b/code/app/task/crawler/zw630/page/LatestListPage.php @@ -0,0 +1,66 @@ +NovelModel == NULL) { + $this->NovelModel = NovelModel::getInstance(); + } + return $this->NovelModel; + } + + static public $arrOptions = [ + 'typeMap' => [ + 'root' => 'array', + 'document' => 'array', + 'array' => 'array', + ], + ]; + + /** + * Undocumented function + * + * @return array + */ + public function getNovelFilter(): array + { + $arrFilter = []; + if ($this->isMobile()) { + $arrFilter = $this->getQueryList()->find('div.wrap-box')->eq(0)->find('ul.sort_list li')->map(function ($li) { + $strUri = $li->find('a')->eq(0)->attr('href'); + return supperExtractFilename($strUri); + })->filter()->all(); + } else { + + $arrFilter = $this->getQueryList()->find('div.layout.layout2.layout-col2.fl ul.txt-list li')->map(function ($li) { + $strUri = $li->find('a')->eq(0)->attr('href'); + return supperExtractFilename($strUri); + })->filter()->all(); + } + + return $arrFilter; + } +} diff --git a/code/app/task/crawler/zw630/page/Site.php b/code/app/task/crawler/zw630/page/Site.php index b55bcb1..364bf99 100644 --- a/code/app/task/crawler/zw630/page/Site.php +++ b/code/app/task/crawler/zw630/page/Site.php @@ -144,7 +144,7 @@ class Site } /** - * Undocumented function + * retrieve novel page model * * @param string $strUri * @return NovelPage @@ -156,7 +156,7 @@ class Site /** - * Undocumented function + * retrieve multiple novel page model * * @param string $strUri * @return NovelPage[] @@ -193,7 +193,7 @@ class Site continue; } - $NovelPage = new NovelPage($this, $arrItem['uri']); + $NovelPage = new NovelPage($this, $arrFilter[$intKey]['uri']); $NovelPage->setContent($strContent); $NovelPage->setData($arrFilter[$intKey]); @@ -206,7 +206,7 @@ class Site } /** - * Undocumented function + * retrieve multiple chapter page model * * @param string $strUri * @return ChapterPage[] @@ -233,7 +233,7 @@ class Site $strContent = $Response->getBody()->getContents();; - $ChapterPage = new ChapterPage($this, $arrItem['uri']); + $ChapterPage = new ChapterPage($this, $arrFilter[$intKey]['uri']); $ChapterPage->setContent($strContent); $ChapterPage->setData($arrFilter[$intKey]); @@ -245,8 +245,48 @@ class Site return $arrChapterPage; } + /** - * Undocumented function + * retrieve multiple chapter page model + * + * @param string $strUri + * @return LatestListPage[] + */ + public function getLatestListPageList(array $arrFilter): array + { + + /** @var LatestListPage[] */ + $arrLatestListPage = []; + + $arrPromises = []; + foreach ($arrFilter as $intKey => $strUri) { + $arrPromises[$intKey] = $this->getClient()->getAsync($strUri); + } + + $arrResults = Promise\Utils::settle($arrPromises)->wait(); + + $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) { + return $arrResult['state'] === 'fulfilled'; + }); + + foreach ($arrSuccessfulResults as $intKey => $arrResult) { + $Response = $arrResult['value']; + + $strContent = $Response->getBody()->getContents();; + + $LatestListPage = new LatestListPage($this, $arrFilter[$intKey]); + + $LatestListPage->setContent($strContent); + + $arrLatestListPage[] = $LatestListPage; + } + + + return $arrLatestListPage; + } + + /** + * download the image to the local system * * @param array $arrFilter * @return void @@ -272,8 +312,6 @@ class Site foreach ($arrSuccessfulResults as $intKey => $arrResult) { $Response = $arrResult['value']; - // $strContent = $Response->getBody()->getContents(); - $StorageCore->writeStream($arrFilter[$intKey]['local_path'], $Response->getBody()->detach());; } }