Merge branch 'dev' of ssh://git.bgbg.live:18150/root/SEONexus into dev

code/app/model/ConfigSubjectFomartModel.php

@@ -0,0 +1,147 @@
+<?php
+
+declare(strict_types=1);
+
+namespace app\model;
+
+
+use Exception;
+
+
+/**
+ * @mixin think\Model
+ */
+class ConfigSubjectFomartModel extends MongoModel
+{
+    /**
+     * Undocumented variable
+     *
+     * @var self
+     */
+    static public $obj;
+
+    static public function getInstance(): self
+    {
+        if (self::$obj == NULL) {
+            self::$obj = new self;
+        }
+        return  self::$obj;
+    }
+
+    /**
+     * Undocumented variable
+     *
+     * @var string
+     */
+    protected $strCollection = 'config_subject_fomart';
+
+    static public $arrOptions = [
+        'typeMap' => [
+            'root' => 'array',
+            'document' => 'array',
+            'array' => 'array',
+        ],
+        'projection' => [
+            '_id' => 0,
+        ],
+    ];
+
+
+    /**
+     * 直接读库查询大量小说
+     *
+     * @param integer $intNId
+     * @return null|array
+     */
+    public function findMany(array $arrFilter, int $intCount = 10, array $arrSort =  [], array  $arrOptions = []): null|array
+    {
+        if (empty($arrOptions)) {
+            $arrOptions = self::$arrOptions;
+        }
+
+        if ($intCount > 0) {
+            $arrOptions['limit'] = $intCount;
+        }
+
+        if (empty($arrSort)) {
+            $arrOptions['sort'] = $arrSort;
+        }
+
+        $Cursor = $this->getCol()->find($arrFilter, $arrOptions);
+
+        $arrResult = iterator_to_array($Cursor);
+
+        return  $arrResult;
+    }
+
+
+    /**
+     * 查询多条数据并缓存
+     *
+     * @param array $arrFilter
+     * @param integer $intCount
+     * @param array $arrSort
+     * @param array $arrOptions
+     * @param string $strKey
+     * @param integer $intLifeTime
+     * @return null|array
+     */
+    public function findManyWithCache(
+        array $arrFilter,
+        int $intCount = 0,
+        array $arrSort =  [],
+        array  $arrOptions = [],
+        string $strKey = '',
+        int $intLifeTime = 24 * 3600,
+    ): null|array {
+        try {
+
+            $arrResult =  $this->checkCacheByKey($strKey);
+            if ($arrResult !== NULL) {
+                return $arrResult;
+            }
+
+            $arrResult = $this->findMany($arrFilter,  $intCount, $arrSort,   $arrOptions);
+
+            $this->setCacheByKey($strKey, $arrResult, $intLifeTime);
+
+            return $arrResult;
+        } catch (\Exception $e) {
+            return [];
+        }
+    }
+
+
+    /**
+     * find one novel
+     *
+     * @param array $arrFilter
+     * @param array $arrOptions
+     * @return null|array
+     */
+    public function findOne(array $arrFilter, array  $arrOptions = []): null|array
+    {
+        if (empty($arrOptions)) {
+            $arrOptions = self::$arrOptions;
+        }
+
+        $arrNovel = $this->getOne($arrFilter, $arrOptions);
+
+
+        return  $arrNovel;
+    }
+
+    /**
+     * 根据小说ID查询指定小说
+     *
+     * @param integer $intNId
+     * @return array
+     */
+    public function getConfigSubjectFomartByCfsKey(string $intNId): null|array
+    {
+        $arrFilter = ['cfs_key' => $intNId];
+        return $this->findOne($arrFilter);
+    }
+
+
+}

code/app/model/DomainModel.php

@@ -94,22 +94,46 @@ class DomainModel extends BaseModel
      * @param string $strKey
      * @return void
      */
-    public function getFomartSubject($strKey) //, $intNum = 1, $boolRand = false)
+    public function getFomartSubject($strKey, $strDiff = '')
     {
         $intSfgId = $this->t_cfg->{$strKey}['sfg_id'];
 
-        // $SubjectFomartModel = SubjectFomartModel::where('sfg_id', $intSfgId)->select()->first();
+        $SubjectFomartModel = NULL;
+
         $arrAllSubjectFomartModel = SubjectFomartModel::getAllSubjectFomartOnCacheBySfgId($intSfgId);
+        if (empty($arrAllSubjectFomartModel)) {
+            return '';
+        }
 
-        $SubjectFomartModel =  (is_array($arrAllSubjectFomartModel) && !empty($arrAllSubjectFomartModel)) ? reset($arrAllSubjectFomartModel) : null;
+        $strDomain =  request()->DomainModel->d_domain;
+        $strController = request()->controller();
+        $strAction = request()->action();
 
-        return empty($SubjectFomartModel) ? '' : $SubjectFomartModel->sf_val;
+        $strUUId = sprintf('%s-%s-%s-%s-%s', $strDomain, $strController, $strAction, $strKey, $strDiff);
 
-        // echo '<pre>';
+        $arrConfigSubjectFomart = ConfigSubjectFomartModel::getInstance()->getConfigSubjectFomartByCfsKey($strUUId);
-        // print_r($SubjectFomartModel);
-        // exit;
 
-        // return  $SubjectFomartModel->sf_val;
+        if (empty($arrConfigSubjectFomart)) {
+
+            $intKey =  array_rand($arrAllSubjectFomartModel);
+
+            $SubjectFomartModel = $arrAllSubjectFomartModel[$intKey];
+
+            ConfigSubjectFomartModel::getInstance()->insert([
+                'cfs_key' => $strUUId,
+                'sf_id' =>  $SubjectFomartModel->sf_id,
+            ]);
+        } else {
+            $intSfId = $arrConfigSubjectFomart['sf_id'];
+
+            foreach ($arrAllSubjectFomartModel as $objTemp) {
+                if ($objTemp->sf_id == $intSfId) {
+                    $SubjectFomartModel = $objTemp;
+                }
+            }
+        }
+
+        return  $SubjectFomartModel->sf_val;
     }
 
 

code/app/task/crawler/zw630/Scheduler.php

@@ -53,13 +53,32 @@ class Scheduler
         }
     }
 
+    /**
+     * Undocumented function
+     *
+     * @return void
+     */
+    public function crawLatest()
+    {
+        $arrUri = [];
+        $strUri = '/list/lastupdate_%s_0_0_%s.html';
+
+        for ($intCategoryKey = 0; $intCategoryKey < 10; $intCategoryKey++) {
+            for ($intPage = 1; $intPage <= 10; $intPage++) {
+                $arrUri[] = sprintf($strUri, $intCategoryKey, $intPage);
+            }
+        }
+
+        print_r($arrUri);
+    }
+
     /**
      * Undocumented function
      *
      * @param array $arrFilter
      * @return void
      */
-    public function fetchNovel(array $arrFilter) //(int $intNSourceId, int $intPage = 0)
+    public function fetchNovel(array $arrFilter)
     {
         $arrTask = [
             'callback' => [self::class, 'getNovelInfo'],

code/app/task/module/PlanTask.php

@@ -31,13 +31,20 @@ class PlanTask
                 case 'PULL_630_ZW_NOVEL':
                     self::pull630ZwNovel();
                     break;
-
+                case 'PULL_630_ZW_NOVEL_LATEST':
+                    self::pull630ZwNovelLatest();
+                    break;
                 default:
                     break;
             }
         }
     }
 
+    /**
+     * Undocumented function
+     *
+     * @return void
+     */
     public static function pull630ZwNovel()
     {
         $arrTask = [
@@ -48,4 +55,20 @@ class PlanTask
         $QueueManage = new QueueManage();
         $QueueManage->set($arrTask);
     }
+
+    /**
+     * Undocumented function
+     *
+     * @return void
+     */
+    public static function pull630ZwNovelLatest()
+    {
+        $arrTask = [
+            'callback' => [Zw630Scheduler::class, 'crawLatest'],
+            'data' => []
+        ];
+
+        $QueueManage = new QueueManage();
+        $QueueManage->set($arrTask);
+    }
 }

code/database/mongo/migrations/config_subject_fomart.php

@@ -0,0 +1,10 @@
+<?php
+return [
+    'collection' => 'config_subject_fomart',
+    'indexes' => [
+        [
+            'key' => ['csf_key' => 1],
+            'options' => ['unique' => true]
+        ],
+    ]
+];

code/database/schema/mysql-schema.dump

@@ -71,8 +71,8 @@ CREATE TABLE `domain` (
   `d_index_description` varchar(1000) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT '首页description模板',
   `d_statis` text CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT '单域名统计',
   `d_logo_type` tinyint(1) DEFAULT NULL,
-  `d_text_log` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `d_text_logo` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT NULL,
-  `d_img_log` varchar(500) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `d_img_logo` varchar(500) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT NULL,
   `d_content_encode` tinyint unsigned DEFAULT '0',
   `t_cfg` json DEFAULT NULL,
   PRIMARY KEY (`d_id`),

code/database/seeders/PlanTaskSeeder.php

@@ -9,12 +9,10 @@ class PlanTaskSeeder
 {
     static public function handle()
     {
-
         self::addNovelTypePlanTask();
     }
 
 
-    // 添加小说类型站点 爬虫任务
     static public function addNovelTypePlanTask()
     {
         $arrDataNovel = [
@@ -25,8 +23,14 @@ class PlanTaskSeeder
                 'pt_enable' => 1,
                 'pt_limit' => 10 * 24 * 3600 * 3,
             ],
-
+            [
+                'pt_name' => '630小说每日增量',
+                'pt_code' => 'PULL_630_ZW_NOVEL_LATEST',
+                'pt_enable' => 1,
+                'pt_limit' => 1 * 24 * 3600,
+            ],
         ];
+
         foreach ($arrDataNovel as $arrPlanTask) {
             $NovelOnePlanTaskModel = PlanTaskModel::where('pt_code', $arrPlanTask['pt_code'])->find();