Merge branch 'dev' of ssh://git.bgbg.live:18150/root/SEONexus into dev

code/app/model/ChapterModel.php

@@ -124,6 +124,8 @@ class ChapterModel extends MongoModel
 
             if ($boolReadContent) {
                 $arrChapter['content'] = StorageCore::getInstance()->read($arrChapter['c_content_path']);
+            } else {
+                $arrChapter['content'] = '';
             }
         }
 

code/app/task/crawler/zw630/Scheduler.php

@@ -4,6 +4,7 @@ declare(strict_types=1);
 
 namespace app\task\crawler\zw630;
 
+use app\model\ChapterModel;
 use app\task\crawler\zw630\page\NovelPage;
 use app\task\crawler\zw630\page\Site;
 use microserver\QueueManage;
@@ -26,13 +27,13 @@ class Scheduler
     }
 
     /**
-     * 全局扫描任务
+     * create a site-wide craw task
      *
      * @return void
      */
     public function crawlFull()
     {
-        $intSliceSize = 6;
+        $intSliceSize = 4;
 
         $arrFilter = [];
         // for ($intNId = 1; $intNId <= 531208; $intNId++) {
@@ -54,22 +55,76 @@ class Scheduler
     }
 
     /**
-     * Undocumented function
+     * create a task to craw the latest chapter
      *
      * @return void
      */
     public function crawLatest()
     {
+        $intSliceSize = 3;
         $arrUri = [];
         $strUri = '/list/lastupdate_%s_0_0_%s.html';
 
-        for ($intCategoryKey = 0; $intCategoryKey < 10; $intCategoryKey++) {
+        $intEndCategory = $intEndPage = 2;
-            for ($intPage = 1; $intPage <= 10; $intPage++) {
+
+        for ($intCategoryKey = 0; $intCategoryKey < $intEndCategory; $intCategoryKey++) {
+            for ($intPage = 1; $intPage <= $intEndPage; $intPage++) {
                 $arrUri[] = sprintf($strUri, $intCategoryKey, $intPage);
             }
         }
 
-        // print_r($arrUri);
+        $arrUri = array_chunk($arrUri, $intSliceSize);
+
+        $arrTask = [
+            'callback' => [self::class, 'getLatestChapter'],
+            'data' => [],
+        ];
+
+        foreach ($arrUri as $arrUriSlice) {
+            $arrTask['data'] = $arrUriSlice;
+            $this->QueueManage->set($arrTask);
+        }
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @param array $arrData
+     * @return void
+     */
+    public function getLatestChapter(array $arrFilter)
+    {
+        $Site = new Site;
+
+        $arrLatestListPageList = $Site->getLatestListPageList($arrFilter);
+
+        $arrNSourceId = [];
+
+        foreach ($arrLatestListPageList as $LatestListPage) {
+            $arrResult = $LatestListPage->getNovelFilter();
+            $arrNSourceId = array_merge($arrNSourceId, $arrResult);
+        }
+
+        $arrNSourceId = array_unique($arrNSourceId);
+        $arrNSourceId  = array_values($arrNSourceId);
+
+        $intSliceSize = 4;
+        $arrFilter = [];
+
+        foreach ($arrNSourceId as $intKey => $intNSourceId) {
+            $arrFilter[] = [
+                'n_source_id' => $intNSourceId,
+                'page' => 0,
+            ];
+
+            if ($intKey % $intSliceSize == 0) {
+                $this->fetchNovel($arrFilter);
+                $arrFilter = [];
+            }
+        }
+        if (!empty($arrFilter)) {
+            $this->fetchNovel($arrFilter);
+        }
     }
 
     /**
@@ -97,7 +152,7 @@ class Scheduler
      */
     public function fetchChapter(int $intNId, int $intPage, array $arrChapterList = [])
     {
-        $intSliceSize = 6;
+        $intSliceSize = 4;
         $intPageSize = 100;
 
         $arrTask = [
@@ -145,7 +200,6 @@ class Scheduler
                 continue;
             }
 
-
             $arrNextPageData = $NovelPage->getNextPageArgs();
             if (!empty($arrNextPageData)) {
                 $this->fetchNovel([
@@ -175,6 +229,20 @@ class Scheduler
     {
         $Site = new Site;
 
+        $ChapterModel = ChapterModel::getInstance();
+
+        foreach ($arrData['filter'] as $intKey => $arrChapterFilter) {
+            $strUUId = sprintf('%s-chapter-%s-%s',  $Site->getSiteCode(),  $arrChapterFilter['c_source_id'],  $arrChapterFilter['c_page']);
+            $arrChapter = $ChapterModel->findOne(['c_site_uuid' => $strUUId]);
+
+            if (!empty($arrChapter) && !empty($arrChapter['content'])) {
+                unset($arrData['filter'][$intKey]);
+            }
+        }
+
+        $arrData['filter'] = array_values($arrData['filter']);
+
+
         $arrChapterPageList = $Site->getChapterPageList($arrData['filter']);
 
         $arrNextPageFilterList = [];

code/app/task/crawler/zw630/page/BasePage.php

@@ -107,6 +107,16 @@ abstract class BasePage
         $this->strUri = $strUri;
     }
 
+    /**
+     * Undocumented function
+     *
+     * @return string
+     */
+    public function getUri(): string
+    {
+        return $this->strUri;
+    }
+
     /**
      * Undocumented function
      *
@@ -238,6 +248,4 @@ abstract class BasePage
     {
         return  $this->Site;
     }
-
-
 }

code/app/task/crawler/zw630/page/LatestListPage.php

@@ -0,0 +1,66 @@
+<?php
+
+declare(strict_types=1);
+
+namespace app\task\crawler\zw630\page;
+
+use app\model\CategoryModel;
+use app\model\CountersModel;
+use app\model\NovelModel;
+
+
+class LatestListPage extends BasePage
+{
+
+    /**
+     * Undocumented variable
+     *
+     * @var NovelModel
+     */
+    protected $NovelModel;
+
+    /**
+     * Undocumented function
+     *
+     * @return NovelModel
+     */
+    public function getNovelModel(): NovelModel
+    {
+        if ($this->NovelModel == NULL) {
+            $this->NovelModel = NovelModel::getInstance();
+        }
+        return  $this->NovelModel;
+    }
+
+    static public $arrOptions = [
+        'typeMap' => [
+            'root' => 'array',
+            'document' => 'array',
+            'array' => 'array',
+        ],
+    ];
+
+    /**
+     * Undocumented function
+     *
+     * @return array
+     */
+    public function getNovelFilter(): array
+    {
+        $arrFilter = [];
+        if ($this->isMobile()) {
+            $arrFilter =  $this->getQueryList()->find('div.wrap-box')->eq(0)->find('ul.sort_list li')->map(function ($li) {
+                $strUri = $li->find('a')->eq(0)->attr('href');
+                return supperExtractFilename($strUri);
+            })->filter()->all();
+        } else {
+
+            $arrFilter =  $this->getQueryList()->find('div.layout.layout2.layout-col2.fl ul.txt-list li')->map(function ($li) {
+                $strUri = $li->find('a')->eq(0)->attr('href');
+                return supperExtractFilename($strUri);
+            })->filter()->all();
+        }
+
+        return $arrFilter;
+    }
+}

code/app/task/crawler/zw630/page/Site.php

@@ -144,7 +144,7 @@ class Site
     }
 
     /**
-     * Undocumented function
+     * retrieve novel page model
      *
      * @param string $strUri
      * @return NovelPage
@@ -156,7 +156,7 @@ class Site
 
 
     /**
-     * Undocumented function
+     * retrieve multiple novel page model
      *
      * @param string $strUri
      * @return NovelPage[]
@@ -193,7 +193,7 @@ class Site
                 continue;
             }
 
-            $NovelPage = new NovelPage($this, $arrItem['uri']);
+            $NovelPage = new NovelPage($this, $arrFilter[$intKey]['uri']);
 
             $NovelPage->setContent($strContent);
             $NovelPage->setData($arrFilter[$intKey]);
@@ -206,7 +206,7 @@ class Site
     }
 
     /**
-     * Undocumented function
+     * retrieve multiple chapter page model
      *
      * @param string $strUri
      * @return ChapterPage[]
@@ -233,7 +233,7 @@ class Site
 
             $strContent =  $Response->getBody()->getContents();;
 
-            $ChapterPage = new ChapterPage($this, $arrItem['uri']);
+            $ChapterPage = new ChapterPage($this, $arrFilter[$intKey]['uri']);
 
             $ChapterPage->setContent($strContent);
             $ChapterPage->setData($arrFilter[$intKey]);
@@ -245,8 +245,48 @@ class Site
         return $arrChapterPage;
     }
 
+    
     /**
-     * Undocumented function
+     * retrieve multiple chapter page model
+     *
+     * @param string $strUri
+     * @return LatestListPage[]
+     */
+    public function getLatestListPageList(array $arrFilter): array
+    {
+
+        /** @var LatestListPage[] */
+        $arrLatestListPage = [];
+
+        $arrPromises = [];
+        foreach ($arrFilter as $intKey => $strUri) {
+            $arrPromises[$intKey] = $this->getClient()->getAsync($strUri);
+        }
+
+        $arrResults = Promise\Utils::settle($arrPromises)->wait();
+
+        $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
+            return $arrResult['state'] === 'fulfilled';
+        });
+
+        foreach ($arrSuccessfulResults as $intKey => $arrResult) {
+            $Response = $arrResult['value'];
+
+            $strContent =  $Response->getBody()->getContents();;
+
+            $LatestListPage = new LatestListPage($this, $arrFilter[$intKey]);
+
+            $LatestListPage->setContent($strContent);
+
+            $arrLatestListPage[] =  $LatestListPage;
+        }
+
+
+        return $arrLatestListPage;
+    }
+
+    /**
+     * download the image to the local system
      *
      * @param array $arrFilter
      * @return void
@@ -272,8 +312,6 @@ class Site
         foreach ($arrSuccessfulResults as $intKey => $arrResult) {
             $Response = $arrResult['value'];
 
-            // $strContent =  $Response->getBody()->getContents();
-
             $StorageCore->writeStream($arrFilter[$intKey]['local_path'], $Response->getBody()->detach());;
         }
     }