debug

code/app/task/crawler/custom/Scheduler.php

@@ -47,14 +47,15 @@ class Scheduler
      */
     public function crawlFull()
     {
+        var_dump('custom-crawlFull');
         $AllCustomCrawlerModel = CustomCrawlerModel::select();
 
         foreach ($AllCustomCrawlerModel as $CustomCrawlerModel) {
-
             if (
                 $CustomCrawlerModel->pt_last_exec != 0 &&
                 ($CustomCrawlerModel->pt_last_exec + $CustomCrawlerModel->pt_limit)  > time()
             ) {
+                var_dump('continue');
                 continue;
             }
 

code/app/task/crawler/custom/WenXueGuan.php

@@ -46,6 +46,7 @@ class WenXueGuan
     public function createCrawlWenXueGuanTask(array $arrData)
     {
         $arrCcData = json_decode($arrData['cc_data'], true);
+        // var_dump($arrCcData);
         foreach ($arrCcData as $intNSourceId) {
             $arrTask = [
                 'callback' => [WxgScheduler::class, 'getNovelInfo'],

code/app/task/crawler/wxg/Scheduler.php

@@ -107,6 +107,7 @@ class Scheduler
             // return;
 
             $boolResult = $NovelPage->saveNovelInfo();
+            // var_dump($boolResult);
             // return;
 
             if ($boolResult == false) {

code/app/task/crawler/wxg/page/NovelPage.php

@@ -124,11 +124,13 @@ class NovelPage extends BasePage
                 $this->arrNovelInfo[$strKey] = $Item->attr('content');
             });
 
-            if (!empty($this->arrNovelInfo['og_novel_category']) && $this->arrNovelInfo['og_novel_category'] == '玄幻小说') {
-                $this->arrNovelInfo['og_novel_category'] = '玄幻修真';
+            if (!empty($this->arrNovelInfo['og_novel_category']) ) {
+                if($this->arrNovelInfo['og_novel_category'] == '玄幻小说' || $this->arrNovelInfo['og_novel_category'] == '武侠小说'){
+                    $this->arrNovelInfo['og_novel_category'] = '玄幻修真';
+                }
+                
             }
 
-
             $this->getReletedNovel();
         }
 
@@ -242,6 +244,7 @@ class NovelPage extends BasePage
                 $arrUpdate = array_merge($arrUpdate, $arrNovelInfo);
                 $arrUpdate['n_related_novel'] = $this->arrRelatedNovel;
                 $arrUpdate['n_forge_novel'] = $this->arrForgeNovel;
+
                 $arrUpdate['n_category_pinyin'] = CategoryModel::getPinYin($this->arrNovelInfo['og_novel_category']);
                 $arrUpdate['n_category_sex_en'] = CategoryModel::getSexEn($this->arrNovelInfo['og_novel_category']);
                 $arrUpdate['n_category_sex_zh'] = CategoryModel::getSexZh($this->arrNovelInfo['og_novel_category']);

code/app/task/crawler/zw/Scheduler.php

@@ -96,17 +96,20 @@ class Scheduler
      */
     public function getNovelInfo(array $arrFilter)
     {
+        // var_dump('getNovelInfo------------------------------------------------------------');
         $Site = $this->getSite();
 
         $arrNovelPageList = $Site->getNovelPageList($arrFilter);
-
+        // var_dump($arrNovelPageList);
+        // var_dump('$arrNovelPageList------------------------------------------------------------');
         $arrCover = [];
         foreach ($arrNovelPageList as $NovelPage) {
             // $arrNovel = $NovelPage->getNovelInfo();
             // print_r($arrNovel);
             // return;
-
+            // var_dump('$NovelPage------------------------------------------------------------');
             $boolResult = $NovelPage->saveNovelInfo();
+            // var_dump($boolResult);
             // return;
 
             if ($boolResult == false) {
@@ -130,6 +133,7 @@ class Scheduler
             }
 
             $arrChapterList = $NovelPage->getChapters();
+            // var_dump($arrChapterList);
             $this->fetchChapter($NovelPage->intNId, $NovelPage->getData()['page'], $arrChapterList);
         }
 
@@ -147,9 +151,13 @@ class Scheduler
     {
         $intSliceSize =  $this->intSlice;
         $arrUri = [];
-        $strUri = '/list/lastupdate_%s_0_0_%s.html';
+        // $strUri = '/list/lastupdate_%s_0_0_%s.html';
+        $strUri = '/sort/%s_%s/';
 
-        $intEndCategory = $intEndPage = 10;
+        $intEndCategory = 10;
+        $intEndPage = 10;
+        // $intEndCategory = 2;
+        // $intEndPage = 2;
 
         for ($intCategoryKey = 0; $intCategoryKey < $intEndCategory; $intCategoryKey++) {
             for ($intPage = 1; $intPage <= $intEndPage; $intPage++) {
@@ -159,6 +167,8 @@ class Scheduler
 
         $arrUri = array_chunk($arrUri, $intSliceSize);
 
+        // var_dump($arrUri);
+        // return ;
         $arrTask = [
             'callback' => [static::class, 'getLatestChapter'],
             'data' => [],
@@ -179,29 +189,36 @@ class Scheduler
     public function getLatestChapter(array $arrFilter)
     {
         $Site = $this->getSite();
-
+     
         $arrLatestListPageList = $Site->getLatestListPageList($arrFilter);
-
         $arrNSourceId = [];
 
         foreach ($arrLatestListPageList as $LatestListPage) {
             $arrResult = $LatestListPage->getNovelFilter();
+            // var_dump($arrResult);
             $arrNSourceId = array_merge($arrNSourceId, $arrResult);
         }
-
+        // var_dump($arrNSourceId);
         $arrNSourceId = array_unique($arrNSourceId);
         $arrNSourceId  = array_values($arrNSourceId);
 
         $intSliceSize =  $this->intSlice;
         $arrFilter = [];
 
+        // var_dump($arrNSourceId);
+        // return ;
+
         foreach ($arrNSourceId as $intKey => $intNSourceId) {
             $arrFilter[] = [
                 'n_source_id' => $intNSourceId,
                 'page' => 0,
             ];
-
+            // var_dump($intKey);
+            // var_dump($intSliceSize);
+            // var_dump($intKey % $intSliceSize);
+            // var_dump('---------------------------------------------');
             if ($intKey % $intSliceSize == 0) {
+                // var_dump('$intKey % $intSliceSize == 0');
                 $this->fetchNovel($arrFilter);
                 $arrFilter = [];
             }

code/app/task/crawler/zw/page/LatestListPage.php

@@ -49,13 +49,16 @@ class LatestListPage extends BasePage
         if ($this->isMobile()) {
             $arrFilter =  $this->getQueryList()->find('div.wrap-box')->eq(0)->find('ul.sort_list li')->map(function ($li) {
                 $strUri = $li->find('a')->eq(0)->attr('href');
-                return supperExtractFilename($strUri);
+                $strNovelId = extractNovelId($strUri);
+                return (int)$strNovelId;
             })->filter()->all();
         } else {
-
+            
             $arrFilter =  $this->getQueryList()->find('div.layout.layout2.layout-col2.fl ul.txt-list li')->map(function ($li) {
                 $strUri = $li->find('a')->eq(0)->attr('href');
-                return supperExtractFilename($strUri);
+                $strNovelId = extractNovelId($strUri);
+                return (int)$strNovelId;
+                return extractNovelId($strUri);
             })->filter()->all();
         }
 

code/app/task/crawler/zw/page/Site.php

@@ -163,6 +163,7 @@ class Site
 
     public function getNovelPageUri($intNSourceId, $intPage): string
     {
+        $intNSourceId = int($intNSourceId);
         $intUriFirst = intdiv($intNSourceId, 1000);
 
         $strUri = '';
@@ -275,12 +276,16 @@ class Site
 
         $arrPromises = [];
         foreach ($arrFilter as $intKey => $strUri) {
+            
             $arrPromises[$intKey] = $this->getClient()->getAsync($strUri);
         }
-
+        
         $arrResults = Promise\Utils::settle($arrPromises)->wait();
+        // var_dump($strUri);
+        // var_dump($arrResults);
 
         $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
+            // var_dump($arrResult);
             return $arrResult['state'] === 'fulfilled';
         });
 
@@ -296,7 +301,7 @@ class Site
             $arrLatestListPage[] =  $LatestListPage;
         }
 
-
+        // var_dump($arrLatestListPage);
         return $arrLatestListPage;
     }