debug

code/app/home/controller/Novel.php

@@ -244,7 +244,7 @@ class Novel extends BaseController
    
         // echo "<pre>";
         // var_dump($arrNewsUpdateNovel);exit;
-        // TODO   n_page_description  改成 og_description
+        // TODO   og_description  改成 og_description
         // 更新值
         ConverterMovel::setVal([
             'intPage' => 1,
@@ -428,7 +428,7 @@ class Novel extends BaseController
             'strNovelCategoryName' => $arrNovel['og_novel_category'],
             'strNovelSex' => $arrNovel['category_sex'] == 'man' ? '男生' : '女生',
             'strNovelStatus' => $arrNovel['og_novel_status'],
-            'strNovelDescription' => $arrNovel['n_page_description'],
+            'strNovelDescription' => $arrNovel['og_description'],
         ]);
 
         $strTitleTemplate =  $Request->DomainModel->getFomartSubject('NOVEL@GETNOVELINFO@TITLE');
@@ -441,7 +441,7 @@ class Novel extends BaseController
 
         // js使用到
         $arrNovelJs = $arrNovel;
-        unset($arrNovelJs['n_related_novel'], $arrNovelJs['n_forge_novel'], $arrNovelJs['n_page_description']);
+        unset($arrNovelJs['n_related_novel'], $arrNovelJs['n_forge_novel'], $arrNovelJs['og_description']);
 
         View::assign([
             'arrRelateveNovel' => $arrRelateveNovel,
@@ -523,7 +523,7 @@ class Novel extends BaseController
             'strNovelCategoryName' => $arrNovel['og_novel_category'],
             'strNovelSex' => $arrNovel['category_sex'] == 'man' ? '男生' : '女生',
             'strNovelStatus' => $arrNovel['og_novel_status'],
-            'strNovelDescription' => $arrNovel['n_page_description'],
+            'strNovelDescription' => $arrNovel['og_description'],
         ]);
 
         $strTitleTemplate =  $Request->DomainModel->getFomartSubject(getFomartSubjectKey(__METHOD__) . '@TITLE');
@@ -603,7 +603,7 @@ class Novel extends BaseController
             'strNovelAuthor' => $arrNovel['og_novel_author'],
             'strNovelCategoryName' => $arrNovel['og_novel_category'],
             'strNovelStatus' => $arrNovel['og_novel_status'],
-            'strNovelDescription' => $arrNovel['n_page_description'],
+            'strNovelDescription' => $arrNovel['og_description'],
             'strNovelChapterName' => $arrChapter['name'] ?? '',
             'strNovelChapterSort' => $arrChapter['c_sort_num'] ?? '',
         ]);
@@ -683,7 +683,7 @@ class Novel extends BaseController
             'strNovelAuthor' => $arrNovel['og_novel_author'],
             'strNovelCategoryName' => $arrNovel['og_novel_category'],
             'strNovelStatus' => $arrNovel['og_novel_status'],
-            'strNovelDescription' => $arrNovel['n_page_description'],
+            'strNovelDescription' => $arrNovel['og_description'],
             'strNovelChapterName' => $arrChapter['name'],
             'strNovelChapterSort' => $arrChapter['c_sort_num'],
         ]);
@@ -766,7 +766,7 @@ class Novel extends BaseController
             'strNovelAuthor' => $arrNovel['og_novel_author'],
             'strNovelCategoryName' => $arrNovel['og_novel_category'],
             'strNovelStatus' => $arrNovel['og_novel_status'],
-            'strNovelDescription' => $arrNovel['n_page_description'],
+            'strNovelDescription' => $arrNovel['og_description'],
         ]);
 
         $strTitleTemplate =  $Request->DomainModel->getFomartSubject(getFomartSubjectKey(__METHOD__) . '@TITLE');

code/app/model/ChapterModel.php

@@ -35,12 +35,7 @@ class ChapterModel extends MongoModel
         ],
         'projection' => [
             '_id' => 0,
-            'c_page_title' => 0,
-            'c_page_keywords' => 0,
-            'c_page_description' => 0,
-            'c_source_uri' => 0,
             'c_source_id' => 0,
-            'c_site_name' => 0,
             'c_site_code' => 0,
             'c_site_uuid' => 0,
 

code/app/model/CountersModel.php

@@ -0,0 +1,63 @@
+<?php
+
+declare(strict_types=1);
+
+namespace app\model;
+
+
+/**
+ * @mixin think\Model
+ */
+class CountersModel extends MongoModel
+{
+    /**
+     * Undocumented variable
+     *
+     * @var self
+     */
+    static public $obj;
+
+    static public function getInstance(): self
+    {
+        if (self::$obj == NULL) {
+            self::$obj = new self;
+        }
+        return  self::$obj;
+    }
+
+    /**
+     * Undocumented variable
+     *
+     * @var string
+     */
+    protected $strCollection = 'counters';
+
+    static public $arrOptions = [
+        'typeMap' => [
+            'root' => 'array',
+            'document' => 'array',
+            'array' => 'array',
+        ],
+        'projection' => [
+            '_id' => 0,
+        ],
+    ];
+
+    /**
+     * 获取下一个自增序列值
+     * @param string $strSequenceName 序列名称
+     * @return int 自增 ID
+     */
+    public function getNextSequence(string $strSequenceName): int
+    {
+        $Result = $this->getCol()->findOneAndUpdate(
+            ['_id' => $strSequenceName],
+            ['$inc' => ['sequence_value' => 1]],
+            [
+                'upsert' => true,
+                'returnDocument' => \MongoDB\Operation\FindOneAndUpdate::RETURN_DOCUMENT_AFTER
+            ]
+        );
+        return $Result->sequence_value ?? 1;
+    }
+}

code/app/model/MongoModel.php

@@ -129,4 +129,19 @@ abstract class MongoModel
         $Result = $this->getCol()->insertOne($arrData);
         return $Result->getInsertedId();
     }
+
+
+    /**
+     * 更新单条数据
+     * @param string $strColName 集合名称
+     * @param array $arrFilter 查询条件
+     * @param array $arrUpdate 更新内容
+     * @param array $arrOptions 更新选项
+     * @return int 受影响的文档数
+     */
+    public function updateOne(array $arrFilter, array $arrUpdate, array $arrOptions = []): int
+    {
+        $Result = $this->getCol()->updateOne($arrFilter, $arrUpdate, $arrOptions);
+        return $Result->getModifiedCount();
+    }
 }

code/app/model/NovelModel.php

@@ -43,17 +43,9 @@ class NovelModel extends MongoModel
         ],
         'projection' => [
             '_id' => 0,
-            'og_title' => 0,
-            'n_site_name' => 0,
             'n_site_code' => 0,
             'n_site_uuid' => 0,
             'n_source_id' => 0,
-            'n_page_title' => 0,
-            'og_image' => 0,
-            'og_novel_latest_chapter_url' => 0,
-            'og_novel_read_url' => 0,
-            'og_type' => 0,
-            'og_url' => 0,
         ],
     ];
 
@@ -171,7 +163,7 @@ class NovelModel extends MongoModel
             $arrOptions = self::$arrOptions;
         }
 
-        $arrNovel =  $this->getOne($arrFilter, $arrOptions);
+        $arrNovel = $this->getOne($arrFilter, $arrOptions);
 
         if (!empty($arrNovel)) {
             applyToKeys($arrNovel, ['created_at', 'updated_at'], 'formatMongoDate');

code/app/task/crawler/zw630/Scheduler.php

@@ -37,7 +37,7 @@ class Scheduler
         $arrFilter = [];
         // for ($intNId = 1; $intNId <= 531208; $intNId++) {
         // for ($intNId = 1; $intNId <= 10000; $intNId++) {
-        for ($intNId = 1; $intNId <= 6; $intNId++) {
+        for ($intNId = 1; $intNId <= 1; $intNId++) {
             $arrFilter[] = [
                 'n_source_id' => $intNId,
                 'page' => 0,

code/app/task/crawler/zw630/page/BasePage.php

@@ -239,20 +239,5 @@ abstract class BasePage
         return  $this->Site;
     }
 
-    /**
-     * Undocumented variable
-     *
-     * @var boolean
-     */
-    protected $boolIsFirstSave = false;
 
-    /**
-     * Undocumented function
-     *
-     * @return boolean
-     */
-    public function getIsFirstSave(): bool
-    {
-        return $this->boolIsFirstSave;
-    }
 }

code/app/task/crawler/zw630/page/ChapterPage.php

@@ -16,6 +16,34 @@ class ChapterPage extends BasePage
      */
     protected $strChapterContent = '';
 
+    /**
+     * Undocumented variable
+     *
+     * @var ChapterModel
+     */
+    protected $ChapterModel;
+
+    /**
+     * Undocumented function
+     *
+     * @return ChapterModel
+     */
+    public function getChapterModel(): ChapterModel
+    {
+        if ($this->ChapterModel == NULL) {
+            $this->ChapterModel = ChapterModel::getInstance();
+        }
+        return  $this->ChapterModel;
+    }
+
+    static public $arrOptions = [
+        'typeMap' => [
+            'root' => 'array',
+            'document' => 'array',
+            'array' => 'array',
+        ],
+    ];
+
     /**
      * Undocumented function
      *
@@ -37,7 +65,6 @@ class ChapterPage extends BasePage
                 $strContent  .= base64_decode($str);
             }
 
-            // $this->arrChapterInfo['c_content'] = $strContent;
             $this->strChapterContent =  $strContent;
 
             $this->arrChapterInfo['c_page_title'] = $this->getQueryList()->find('title')->text();
@@ -55,16 +82,38 @@ class ChapterPage extends BasePage
      */
     protected $arrChapterInfo;
 
+
+    public function filterOtherFields(&$arrChapter)
+    {
+        $arrFields = [
+            'n_id',
+            'name',
+            'c_sort_num',
+            'c_page',
+            'c_source_id',
+            'c_site_code',
+            'c_site_uuid',
+            'c_content_path',
+            'created_at',
+            'updated_at',
+
+        ];
+
+        foreach ($arrChapter as $strKey => $strVal) {
+            if (!in_array($strKey, $arrFields)) {
+                unset($arrChapter[$strKey]);
+            }
+        }
+    }
+
     public function saveChapter(int $intNId): array
     {
 
         try {
             $this->getChapterInfo();
 
-            $strUUId = 'zw630-chapter-' . $this->arrData['c_source_id'] . '-' . $this->arrData['c_page'];
+            $strUUId = $this->getSite()->getSiteUUId('chapter-' . $this->arrData['c_source_id'] . '-' . $this->arrData['c_page']);
-            $Col = $this->getMongoBase()->getDb()->selectCollection('chapter');
+            $arrExisting = $this->getChapterModel()->findOne(['c_site_uuid' => $strUUId], self::$arrOptions);
-
-            $arrExisting = $Col->findOne(['c_site_uuid' => $strUUId]);
 
             if ($arrExisting) {
 
@@ -72,39 +121,42 @@ class ChapterPage extends BasePage
                 $this->arrChapterInfo = array_merge($arrExisting,   $this->getChapterInfo());
 
                 $this->arrChapterInfo['updated_at'] = new \MongoDB\BSON\UTCDateTime();
-                $Col->updateOne(
+
+                $arrUpdate = $this->arrChapterInfo;
+
+                $this->filterOtherFields($arrUpdate);
+
+                $this->getChapterModel()->updateOne(
                     ['c_site_uuid' => $strUUId],
-                    ['$set' =>   $this->arrChapterInfo],
+                    ['$set' =>   $arrUpdate],
-                    // ['upsert' => false]
                 );
             } else {
 
                 $this->arrChapterInfo['n_id'] =  $intNId;
-
                 $this->arrChapterInfo['name'] =  $this->arrData['name'];
-
                 $this->arrChapterInfo['c_sort_num'] =  $this->arrData['c_sort_num'];
                 $this->arrChapterInfo['c_page'] =  $this->arrData['c_page'];
-
                 $this->arrChapterInfo['c_source_id'] = $this->arrData['c_source_id'];
-                $this->arrChapterInfo['c_source_uri'] =  $this->arrData['uri'];
+                $this->arrChapterInfo['c_site_code'] =  $this->getSite()->getSiteCode();
-                $this->arrChapterInfo['c_site_name'] = '恋上你中文';
-                $this->arrChapterInfo['c_site_code'] = 'zw630';
                 $this->arrChapterInfo['c_site_uuid'] = $strUUId;
                 $this->arrChapterInfo['c_content_path'] = '/novel/' . ($intNId % 100) . '/' . $intNId . '/' . $this->arrChapterInfo['c_sort_num'] . '_' .  $this->arrChapterInfo['c_page'] . '.txt';
 
-                $this->getMongoBase()->insertOne('chapter',   $this->arrChapterInfo);
+                $arrInsert = $this->arrChapterInfo;
+
+                $this->filterOtherFields($arrInsert);
+
+                $this->getChapterModel()->insert($arrInsert);
 
                 $this->updateNovelHaveChapter($intNId);
+            }
 
+            if (mb_strlen($this->strChapterContent) > 20) {
                 if (!StorageCore::getInstance()->has($this->arrChapterInfo['c_content_path'])) {
                     StorageCore::getInstance()->set($this->arrChapterInfo['c_content_path'], $this->strChapterContent);
                 }
-
-                $this->boolIsFirstSave = true;
             }
             return   $this->arrChapterInfo;
-        } catch (\MongoDB\Exception\Exception $e) {
+        } catch (\Exception $e) {
             throw new \Exception("Upsert failed: " . $e->getMessage());
         }
     }

code/app/task/crawler/zw630/page/NovelPage.php

@@ -5,8 +5,8 @@ declare(strict_types=1);
 namespace app\task\crawler\zw630\page;
 
 use app\model\CategoryModel;
-use ddl\DDLManage;
+use app\model\CountersModel;
-use QL\QueryList;
+use app\model\NovelModel;
 
 
 class NovelPage extends BasePage
@@ -33,60 +33,65 @@ class NovelPage extends BasePage
      */
     protected $arrForgeNovel = [];
 
+    /**
+     * Undocumented variable
+     *
+     * @var NovelModel
+     */
+    protected $NovelModel;
+
+    /**
+     * Undocumented function
+     *
+     * @return NovelModel
+     */
+    public function getNovelModel(): NovelModel
+    {
+        if ($this->NovelModel == NULL) {
+            $this->NovelModel = NovelModel::getInstance();
+        }
+        return  $this->NovelModel;
+    }
+
+    static public $arrOptions = [
+        'typeMap' => [
+            'root' => 'array',
+            'document' => 'array',
+            'array' => 'array',
+        ],
+    ];
+
+    /**
+     * 查询看的小说
+     *
+     * @param integer $intNSourceId
+     * @param string $strName
+     * @return array
+     */
     private function findOrCreateNovel(int $intNSourceId, string $strName): array
     {
+        $strUUId = $this->getSite()->getSiteUUId($intNSourceId);
 
-        $strUUId = 'zw630-' . $intNSourceId;
+        $arrExisting = $this->getNovelModel()->findOne(['n_site_uuid' => $strUUId], self::$arrOptions);
-        $Col = $this->getMongoBase()->getDb()->selectCollection('novel');
-
-        $arrExisting = $Col->findOne(['n_site_uuid' => $strUUId]);
 
         if ($arrExisting) {
             $arrNovelInfo = (array) $arrExisting;
         } else {
-            $arrNovelInfo['og_novel_book_name'] = $strName;
+            $arrNovelInfo = [
-            $arrNovelInfo['og_novel_pin_yin'] = zhToPinYin($strName);
+                'og_novel_book_name' => $strName,
-            $arrNovelInfo['n_site_name'] = '恋上你中文';
+                'og_novel_pin_yin' => zhToPinYin($strName),
-            $arrNovelInfo['n_site_code'] = 'zw630';
+                'n_site_code' =>  $this->getSite()->getSiteCode(),
-            $arrNovelInfo['n_site_uuid'] = $strUUId;
+                'n_site_uuid' => $strUUId,
-            $arrNovelInfo['n_source_id'] = $intNSourceId;
+                'n_source_id' => $intNSourceId,
-            $arrNovelInfo['n_level'] = 1;
+                'n_level' => 1,
-            $arrNovelInfo['n_have_chapter'] = 0;
+                'n_have_chapter' => 0,
-            $arrNovelInfo['n_id'] =  $this->getMongoBase()->getNextSequence('novel');
+                'n_id' =>  CountersModel::getInstance()->getNextSequence('novel'),
-            $this->getMongoBase()->insertOne('novel',   $arrNovelInfo);
+            ];
+            $this->getNovelModel()->insert($arrNovelInfo);
         }
         return $arrNovelInfo;
     }
 
-    /**
-     * Find or create a category by name, handling concurrency safely.
-     *
-     * @param string $strCategory
-     * @return array|null The category document (existing or newly created)
-     */
-    public function findOrCreateCategory(string $strCategory)
-    {
-        if (empty($strCategory)) {
-            return false;
-        }
-        $Col = $this->getMongoBase()->getDb()->selectCollection('category');
-
-        $arrCategory = ['ca_name' => $strCategory];
-        try {
-            $Result = $Col->findOneAndUpdate(
-                $arrCategory,
-                ['$set' => $arrCategory],
-                [
-                    'upsert' => true,
-                    'returnDocument' => \MongoDB\Operation\FindOneAndUpdate::RETURN_DOCUMENT_AFTER
-                ]
-            );
-            return $Result;
-        } catch (\MongoDB\Driver\Exception\Exception $e) {
-            return null;
-        }
-    }
-
 
     /**
      * Undocumented function
@@ -101,10 +106,6 @@ class NovelPage extends BasePage
                 $this->arrNovelInfo[$strKey] = $Item->attr('content');
             });
 
-            // $this->arrNovelInfo['n_page_title'] = $this->getQueryList()->find('title')->text();
-            // $this->arrNovelInfo['n_page_keywords'] = $this->getQueryList()->find('meta[name="keywords"]')->attr('content');
-            // $this->arrNovelInfo['n_page_description'] = $this->getQueryList()->find('meta[name="description"]')->attr('content');
-
             $this->getQueryList()->find('.first_txt')->eq(0)->find("a")->map(function ($A) {
                 $strName =  $A->text();
                 $strUri =  $A->attr('href');
@@ -131,12 +132,54 @@ class NovelPage extends BasePage
 
         if ($boolSave && !empty($this->arrNovelInfo)) {
             $this->saveNovelInfo();
-
-            $this->findOrCreateCategory($this->arrNovelInfo['og_novel_category']);
         }
         return  $this->arrNovelInfo;
     }
 
+    /**
+     * Undocumented function
+     *
+     * @param [type] $strName
+     * @return string
+     */
+    public function generateCoverPath($strName): string
+    {
+        return  '/img/' . date('Y/m/d', time() - 24 * 3600 * 14) . '/' . $strName . '.jpg';
+    }
+
+    public function filterOtherFields(&$arrNovel)
+    {
+        $arrFields = [
+            'n_id',
+            'og_novel_book_name',
+            'og_novel_pin_yin',
+            'og_novel_status',
+            'og_novel_author',
+            'og_description',
+            'og_novel_category',
+            'category_pinyin',
+            'category_sex',
+            'category_sex_zh',
+            'n_cover_path',
+            'og_novel_latest_chapter_name',
+            'og_novel_update_time',
+            'n_forge_novel',
+            'n_related_novel',
+            'og_site_code',
+            'n_site_uuid',
+            'n_source_id',
+            'n_have_chapter',
+            'created_at',
+            'updated_at',
+        ];
+
+        foreach ($arrNovel as $strKey => $strVal) {
+            if (!in_array($strKey, $arrFields)) {
+                unset($arrNovel[$strKey]);
+            }
+        }
+    }
+
     /**
      * Undocumented function
      *
@@ -145,10 +188,9 @@ class NovelPage extends BasePage
     public function saveNovelInfo()
     {
         try {
-            $strUUId = 'zw630-' . $this->arrData['n_source_id'];
+            $strUUId = $this->getSite()->getSiteUUId($this->arrData['n_source_id']);
-            $Col = $this->getMongoBase()->getDb()->selectCollection('novel');
 
-            $arrExisting = $Col->findOne(['n_site_uuid' => $strUUId]);
+            $arrExisting = $this->getNovelModel()->findOne(['n_site_uuid' => $strUUId], self::$arrOptions);
 
             if ($arrExisting) {
 
@@ -160,25 +202,29 @@ class NovelPage extends BasePage
                 }
 
                 if (!isset($this->arrNovelInfo['n_cover_path']) || empty($this->arrNovelInfo['n_cover_path'])) {
-                    $this->arrNovelInfo['n_cover_path'] = '/img/' . date('Y/m/d', time() - 24 * 3600 * 14) . '/' . $this->arrNovelInfo['n_id'] . '.jpg';
+                    $this->arrNovelInfo['n_cover_path'] = $this->generateCoverPath($this->arrNovelInfo['og_novel_pin_yin']);
                 }
 
                 $this->arrNovelInfo['updated_at'] = new \MongoDB\BSON\UTCDateTime();
-                $Col->updateOne(
+
+
+                $arrUpdate =  $this->arrNovelInfo;
+                $this->filterOtherFields($arrUpdate);
+
+                $this->getNovelModel()->updateOne(
                     ['n_site_uuid' => $strUUId],
-                    ['$set' =>   $this->arrNovelInfo],
+                    ['$set' => $arrUpdate],
-                    // ['upsert' => false]
                 );
             } else {
                 $this->arrNovelInfo['og_novel_pin_yin'] = zhToPinYin($this->arrNovelInfo['og_novel_book_name'] ?? '');
-                $this->arrNovelInfo['n_site_name'] = '恋上你中文';
+
-                $this->arrNovelInfo['n_site_code'] = 'zw630';
+                $this->arrNovelInfo['n_site_code'] =  $this->getSite()->getSiteCode();
                 $this->arrNovelInfo['n_site_uuid'] = $strUUId;
                 $this->arrNovelInfo['n_source_id'] = $this->arrData['n_source_id'];
-                $this->arrNovelInfo['n_id'] =  $this->getMongoBase()->getNextSequence('novel');
+                $this->arrNovelInfo['n_id'] =  CountersModel::getInstance()->getNextSequence('novel');
                 $this->arrNovelInfo['n_level'] = 1;
                 $this->arrNovelInfo['n_have_chapter'] = 0;
-                $this->arrNovelInfo['n_cover_path'] = '/img/' . date('Y/m/d', time() - 24 * 3600 * 14) . '/' . $this->arrNovelInfo['n_id'] . '.jpg';
+                $this->arrNovelInfo['n_cover_path'] =  $this->generateCoverPath($this->arrNovelInfo['og_novel_pin_yin']);
 
 
                 if (!empty($this->arrNovelInfo['og_novel_category'])) {
@@ -187,13 +233,13 @@ class NovelPage extends BasePage
                     $this->arrNovelInfo['category_sex_zh'] = CategoryModel::checkSexZh($this->arrNovelInfo['og_novel_category']);
                 }
 
+                $arrInsert =  $this->arrNovelInfo;
+                $this->filterOtherFields($arrInsert);
 
-                $this->getMongoBase()->insertOne('novel',   $this->arrNovelInfo);
+                $this->getNovelModel()->insert($arrInsert);
-
-                $this->boolIsFirstSave = true;
             }
-            return   $this->arrNovelInfo;
+            return $this->arrNovelInfo;
-        } catch (\MongoDB\Exception\Exception $e) {
+        } catch (\Exception $e) {
             throw new \Exception("Upsert failed: " . $e->getMessage());
         }
     }

code/app/task/crawler/zw630/page/Site.php

@@ -27,6 +27,29 @@ class Site
      */
     protected $strDomain = 'https://www.630zw.com';
 
+    protected $strSiteCode = 'zw630';
+
+    /**
+     * Undocumented function
+     *
+     * @return string
+     */
+    public function getSiteCode(): string
+    {
+        return $this->strSiteCode;
+    }
+
+    /**
+     * Undocumented function
+     *
+     * @param integer|string $strSign
+     * @return string
+     */
+    public function getSiteUUId(int|string $strSign): string
+    {
+        return $this->getSiteCode() . '-' . $strSign;
+    }
+
     /**
      * Undocumented function
      */