debug

code/app/common.php

@@ -716,6 +716,24 @@ namespace {
 
         return false;
     }
+    /**
+     * 从 URL、URI 或路径中提取小说id（不带后缀）  
+     *
+     * @param [type] $path  /30/30345/
+     * @return void
+     */
+    function extractNovelId($path)
+    {
+        if (!is_string($path) || empty($path)) {
+            return null;
+        }
+
+        $segments = explode('/', trim($path, '/'));
+        $last = end($segments);
+
+        return is_numeric($last) ? $last : null;
+    }
+
 
     /**
      * 从 URL、URI 或路径中提取文件名（不带后缀）

code/app/task/crawler/custom/Scheduler.php

@@ -47,14 +47,15 @@ class Scheduler
      */
     public function crawlFull()
     {
+        var_dump('custom-crawlFull');
         $AllCustomCrawlerModel = CustomCrawlerModel::select();
 
         foreach ($AllCustomCrawlerModel as $CustomCrawlerModel) {
-
             if (
                 $CustomCrawlerModel->pt_last_exec != 0 &&
                 ($CustomCrawlerModel->pt_last_exec + $CustomCrawlerModel->pt_limit)  > time()
             ) {
+                var_dump('continue');
                 continue;
             }
 

code/app/task/crawler/custom/WenXueGuan.php

@@ -46,6 +46,7 @@ class WenXueGuan
     public function createCrawlWenXueGuanTask(array $arrData)
     {
         $arrCcData = json_decode($arrData['cc_data'], true);
+        // var_dump($arrCcData);
         foreach ($arrCcData as $intNSourceId) {
             $arrTask = [
                 'callback' => [WxgScheduler::class, 'getNovelInfo'],

code/app/task/crawler/wxg/Scheduler.php

@@ -107,6 +107,7 @@ class Scheduler
             // return;
 
             $boolResult = $NovelPage->saveNovelInfo();
+            // var_dump($boolResult);
             // return;
 
             if ($boolResult == false) {

code/app/task/crawler/wxg/page/NovelPage.php

@@ -124,10 +124,12 @@ class NovelPage extends BasePage
                 $this->arrNovelInfo[$strKey] = $Item->attr('content');
             });
 
-            if (!empty($this->arrNovelInfo['og_novel_category']) && $this->arrNovelInfo['og_novel_category'] == '玄幻小说') {
+            if (!empty($this->arrNovelInfo['og_novel_category']) ) {
+                if($this->arrNovelInfo['og_novel_category'] == '玄幻小说' || $this->arrNovelInfo['og_novel_category'] == '武侠小说'){
                     $this->arrNovelInfo['og_novel_category'] = '玄幻修真';
                 }
                 
+            }
 
             $this->getReletedNovel();
         }
@@ -242,6 +244,7 @@ class NovelPage extends BasePage
                 $arrUpdate = array_merge($arrUpdate, $arrNovelInfo);
                 $arrUpdate['n_related_novel'] = $this->arrRelatedNovel;
                 $arrUpdate['n_forge_novel'] = $this->arrForgeNovel;
+
                 $arrUpdate['n_category_pinyin'] = CategoryModel::getPinYin($this->arrNovelInfo['og_novel_category']);
                 $arrUpdate['n_category_sex_en'] = CategoryModel::getSexEn($this->arrNovelInfo['og_novel_category']);
                 $arrUpdate['n_category_sex_zh'] = CategoryModel::getSexZh($this->arrNovelInfo['og_novel_category']);

code/app/task/crawler/zw/Scheduler.php

@@ -96,17 +96,20 @@ class Scheduler
      */
     public function getNovelInfo(array $arrFilter)
     {
+        // var_dump('getNovelInfo------------------------------------------------------------');
         $Site = $this->getSite();
 
         $arrNovelPageList = $Site->getNovelPageList($arrFilter);
-
+        // var_dump($arrNovelPageList);
+        // var_dump('$arrNovelPageList------------------------------------------------------------');
         $arrCover = [];
         foreach ($arrNovelPageList as $NovelPage) {
             // $arrNovel = $NovelPage->getNovelInfo();
             // print_r($arrNovel);
             // return;
-
+            // var_dump('$NovelPage------------------------------------------------------------');
             $boolResult = $NovelPage->saveNovelInfo();
+            // var_dump($boolResult);
             // return;
 
             if ($boolResult == false) {
@@ -130,6 +133,7 @@ class Scheduler
             }
 
             $arrChapterList = $NovelPage->getChapters();
+            // var_dump($arrChapterList);
             $this->fetchChapter($NovelPage->intNId, $NovelPage->getData()['page'], $arrChapterList);
         }
 
@@ -147,9 +151,13 @@ class Scheduler
     {
         $intSliceSize =  $this->intSlice;
         $arrUri = [];
-        $strUri = '/list/lastupdate_%s_0_0_%s.html';
+        // $strUri = '/list/lastupdate_%s_0_0_%s.html';
+        $strUri = '/sort/%s_%s/';
 
-        $intEndCategory = $intEndPage = 10;
+        $intEndCategory = 10;
+        $intEndPage = 10;
+        // $intEndCategory = 2;
+        // $intEndPage = 2;
 
         for ($intCategoryKey = 0; $intCategoryKey < $intEndCategory; $intCategoryKey++) {
             for ($intPage = 1; $intPage <= $intEndPage; $intPage++) {
@@ -159,6 +167,8 @@ class Scheduler
 
         $arrUri = array_chunk($arrUri, $intSliceSize);
 
+        // var_dump($arrUri);
+        // return ;
         $arrTask = [
             'callback' => [static::class, 'getLatestChapter'],
             'data' => [],
@@ -181,27 +191,34 @@ class Scheduler
         $Site = $this->getSite();
      
         $arrLatestListPageList = $Site->getLatestListPageList($arrFilter);
-
         $arrNSourceId = [];
 
         foreach ($arrLatestListPageList as $LatestListPage) {
             $arrResult = $LatestListPage->getNovelFilter();
+            // var_dump($arrResult);
             $arrNSourceId = array_merge($arrNSourceId, $arrResult);
         }
-
+        // var_dump($arrNSourceId);
         $arrNSourceId = array_unique($arrNSourceId);
         $arrNSourceId  = array_values($arrNSourceId);
 
         $intSliceSize =  $this->intSlice;
         $arrFilter = [];
 
+        // var_dump($arrNSourceId);
+        // return ;
+
         foreach ($arrNSourceId as $intKey => $intNSourceId) {
             $arrFilter[] = [
                 'n_source_id' => $intNSourceId,
                 'page' => 0,
             ];
-
+            // var_dump($intKey);
+            // var_dump($intSliceSize);
+            // var_dump($intKey % $intSliceSize);
+            // var_dump('---------------------------------------------');
             if ($intKey % $intSliceSize == 0) {
+                // var_dump('$intKey % $intSliceSize == 0');
                 $this->fetchNovel($arrFilter);
                 $arrFilter = [];
             }

code/app/task/crawler/zw/page/LatestListPage.php

@@ -49,13 +49,16 @@ class LatestListPage extends BasePage
         if ($this->isMobile()) {
             $arrFilter =  $this->getQueryList()->find('div.wrap-box')->eq(0)->find('ul.sort_list li')->map(function ($li) {
                 $strUri = $li->find('a')->eq(0)->attr('href');
-                return supperExtractFilename($strUri);
+                $strNovelId = extractNovelId($strUri);
+                return (int)$strNovelId;
             })->filter()->all();
         } else {
             
             $arrFilter =  $this->getQueryList()->find('div.layout.layout2.layout-col2.fl ul.txt-list li')->map(function ($li) {
                 $strUri = $li->find('a')->eq(0)->attr('href');
-                return supperExtractFilename($strUri);
+                $strNovelId = extractNovelId($strUri);
+                return (int)$strNovelId;
+                return extractNovelId($strUri);
             })->filter()->all();
         }
 

code/app/task/crawler/zw/page/Site.php

@@ -163,6 +163,7 @@ class Site
 
     public function getNovelPageUri($intNSourceId, $intPage): string
     {
+        $intNSourceId = int($intNSourceId);
         $intUriFirst = intdiv($intNSourceId, 1000);
 
         $strUri = '';
@@ -275,12 +276,16 @@ class Site
 
         $arrPromises = [];
         foreach ($arrFilter as $intKey => $strUri) {
+            
             $arrPromises[$intKey] = $this->getClient()->getAsync($strUri);
         }
         
         $arrResults = Promise\Utils::settle($arrPromises)->wait();
+        // var_dump($strUri);
+        // var_dump($arrResults);
 
         $arrSuccessfulResults = array_filter($arrResults, function ($arrResult) {
+            // var_dump($arrResult);
             return $arrResult['state'] === 'fulfilled';
         });
 
@@ -296,7 +301,7 @@ class Site
             $arrLatestListPage[] =  $LatestListPage;
         }
 
-
+        // var_dump($arrLatestListPage);
         return $arrLatestListPage;
     }
 

code/app/task/module/PlanTask.php

@@ -12,6 +12,7 @@ use microserver\ProcessSanitizer;
 
 use app\task\crawler\zw630\Scheduler as Zw630Scheduler;
 use app\task\crawler\zw74\Scheduler as Zw74Scheduler;
+use app\task\crawler\xbiqugu\Scheduler as XBiQuGeScheduler;
 use app\task\crawler\youzhi\Scheduler as YouZhiScheduler;
 use app\task\crawler\fengchao\Scheduler as FengChaoScheduler;
 use app\task\crawler\wuxian\Scheduler as WuXianScheduler;
@@ -59,6 +60,13 @@ class PlanTask
                 case 'PULL_74_ZW_NOVEL_LATEST':
                     self::pull74ZwNovelLatest();
                     break;
+                case 'PULL_XBIQUGE_NOVEL':
+                    self::pullXBiQuGeNovel();
+                    break;
+                case 'PULL_XBIQUGE_NOVEL_LATEST':
+                    self::pullXBiQuGeNovelLatest();
+                    break;
+                      
                 case 'PULL_YOU_ZHI_SHI_PIN':
                     self::pullYouZhiShiPin();
                     break;
@@ -207,6 +215,39 @@ class PlanTask
         $QueueManage->set($arrTask);
     }
 
+
+    /**
+     * Undocumented function
+     *
+     * @return void
+     */
+    public static function pullXBiQuGeNovel()
+    {
+        $arrTask = [
+            'callback' => [XBiQuGeScheduler::class, 'crawlFull'],
+            'data' => []
+        ];
+
+        $QueueManage =  QueueManage::getInstance();
+        $QueueManage->set($arrTask);
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @return void
+     */
+    public static function pullXBiQuGeNovelLatest()
+    {
+        $arrTask = [
+            'callback' => [XBiQuGeScheduler::class, 'crawLatest'],
+            'data' => []
+        ];
+
+        $QueueManage =  QueueManage::getInstance();
+        $QueueManage->set($arrTask);
+    }
+
     /**
      * Undocumented function
      *

code/database/seeders/PlanTaskSeeder.php

@@ -46,6 +46,20 @@ class PlanTaskSeeder
             ],
 
 
+            // [
+            //     'pt_name' => 'xbiqugu小说全量扫描',
+            //     'pt_code' => 'PULL_XBIQUGE_NOVEL',
+            //     'pt_enable' => 0,
+            //     'pt_limit' => 10 * 24 * 3600 * 3,
+            // ],
+            // [
+            //     'pt_name' => 'xbiqugu小说每日增量',
+            //     'pt_code' => 'PULL_XBIQUGE_NOVEL_LATEST',
+            //     'pt_enable' => 0,
+            //     'pt_limit' => 1 * 24 * 3600,
+            // ],
+
+            
             [
                 'pt_name' => '优质资源库全量采集',
                 'pt_code' => 'PULL_YOU_ZHI_SHI_PIN',