From 44eb45793b508467078081aeeb8fe65f5b938f04 Mon Sep 17 00:00:00 2001 From: Default Date: Thu, 10 Apr 2025 15:55:53 +0800 Subject: [PATCH 1/2] debug --- code/app/task/crawler/zw630/Scheduler.php | 21 ++++++++++++++++++- code/app/task/module/PlanTask.php | 25 ++++++++++++++++++++++- code/database/seeders/PlanTaskSeeder.php | 10 ++++++--- 3 files changed, 51 insertions(+), 5 deletions(-) diff --git a/code/app/task/crawler/zw630/Scheduler.php b/code/app/task/crawler/zw630/Scheduler.php index 03091ac..abb3dd4 100644 --- a/code/app/task/crawler/zw630/Scheduler.php +++ b/code/app/task/crawler/zw630/Scheduler.php @@ -53,13 +53,32 @@ class Scheduler } } + /** + * Undocumented function + * + * @return void + */ + public function crawLatest() + { + $arrUri = []; + $strUri = '/list/lastupdate_%s_0_0_%s.html'; + + for ($intCategoryKey = 0; $intCategoryKey < 10; $intCategoryKey++) { + for ($intPage = 1; $intPage <= 10; $intPage++) { + $arrUri[] = sprintf($strUri, $intCategoryKey, $intPage); + } + } + + print_r($arrUri); + } + /** * Undocumented function * * @param array $arrFilter * @return void */ - public function fetchNovel(array $arrFilter) //(int $intNSourceId, int $intPage = 0) + public function fetchNovel(array $arrFilter) { $arrTask = [ 'callback' => [self::class, 'getNovelInfo'], diff --git a/code/app/task/module/PlanTask.php b/code/app/task/module/PlanTask.php index db73150..8807682 100644 --- a/code/app/task/module/PlanTask.php +++ b/code/app/task/module/PlanTask.php @@ -31,13 +31,20 @@ class PlanTask case 'PULL_630_ZW_NOVEL': self::pull630ZwNovel(); break; - + case 'PULL_630_ZW_NOVEL_LATEST': + self::pull630ZwNovelLatest(); + break; default: break; } } } + /** + * Undocumented function + * + * @return void + */ public static function pull630ZwNovel() { $arrTask = [ @@ -48,4 +55,20 @@ class PlanTask $QueueManage = new QueueManage(); $QueueManage->set($arrTask); } + + /** + * Undocumented function + * + * @return void + */ + public static function pull630ZwNovelLatest() + { + $arrTask = [ + 'callback' => [Zw630Scheduler::class, 'crawLatest'], + 'data' => [] + ]; + + $QueueManage = new QueueManage(); + $QueueManage->set($arrTask); + } } diff --git a/code/database/seeders/PlanTaskSeeder.php b/code/database/seeders/PlanTaskSeeder.php index 9808864..e59e5ca 100644 --- a/code/database/seeders/PlanTaskSeeder.php +++ b/code/database/seeders/PlanTaskSeeder.php @@ -9,12 +9,10 @@ class PlanTaskSeeder { static public function handle() { - self::addNovelTypePlanTask(); } - // 添加小说类型站点 爬虫任务 static public function addNovelTypePlanTask() { $arrDataNovel = [ @@ -25,8 +23,14 @@ class PlanTaskSeeder 'pt_enable' => 1, 'pt_limit' => 10 * 24 * 3600 * 3, ], - + [ + 'pt_name' => '630小说每日增量', + 'pt_code' => 'PULL_630_ZW_NOVEL_LATEST', + 'pt_enable' => 1, + 'pt_limit' => 1 * 24 * 3600, + ], ]; + foreach ($arrDataNovel as $arrPlanTask) { $NovelOnePlanTaskModel = PlanTaskModel::where('pt_code', $arrPlanTask['pt_code'])->find(); From 4d60f87c45ab99da4bbfafa4b44bf408e7160217 Mon Sep 17 00:00:00 2001 From: Default Date: Thu, 10 Apr 2025 17:05:46 +0800 Subject: [PATCH 2/2] debug --- code/app/model/ConfigSubjectFomartModel.php | 147 ++++++++++++++++++ code/app/model/DomainModel.php | 40 ++++- .../migrations/config_subject_fomart.php | 10 ++ code/database/schema/mysql-schema.dump | 4 +- 4 files changed, 191 insertions(+), 10 deletions(-) create mode 100644 code/app/model/ConfigSubjectFomartModel.php create mode 100644 code/database/mongo/migrations/config_subject_fomart.php diff --git a/code/app/model/ConfigSubjectFomartModel.php b/code/app/model/ConfigSubjectFomartModel.php new file mode 100644 index 0000000..6e89ed5 --- /dev/null +++ b/code/app/model/ConfigSubjectFomartModel.php @@ -0,0 +1,147 @@ + [ + 'root' => 'array', + 'document' => 'array', + 'array' => 'array', + ], + 'projection' => [ + '_id' => 0, + ], + ]; + + + /** + * 直接读库查询大量小说 + * + * @param integer $intNId + * @return null|array + */ + public function findMany(array $arrFilter, int $intCount = 10, array $arrSort = [], array $arrOptions = []): null|array + { + if (empty($arrOptions)) { + $arrOptions = self::$arrOptions; + } + + if ($intCount > 0) { + $arrOptions['limit'] = $intCount; + } + + if (empty($arrSort)) { + $arrOptions['sort'] = $arrSort; + } + + $Cursor = $this->getCol()->find($arrFilter, $arrOptions); + + $arrResult = iterator_to_array($Cursor); + + return $arrResult; + } + + + /** + * 查询多条数据并缓存 + * + * @param array $arrFilter + * @param integer $intCount + * @param array $arrSort + * @param array $arrOptions + * @param string $strKey + * @param integer $intLifeTime + * @return null|array + */ + public function findManyWithCache( + array $arrFilter, + int $intCount = 0, + array $arrSort = [], + array $arrOptions = [], + string $strKey = '', + int $intLifeTime = 24 * 3600, + ): null|array { + try { + + $arrResult = $this->checkCacheByKey($strKey); + if ($arrResult !== NULL) { + return $arrResult; + } + + $arrResult = $this->findMany($arrFilter, $intCount, $arrSort, $arrOptions); + + $this->setCacheByKey($strKey, $arrResult, $intLifeTime); + + return $arrResult; + } catch (\Exception $e) { + return []; + } + } + + + /** + * find one novel + * + * @param array $arrFilter + * @param array $arrOptions + * @return null|array + */ + public function findOne(array $arrFilter, array $arrOptions = []): null|array + { + if (empty($arrOptions)) { + $arrOptions = self::$arrOptions; + } + + $arrNovel = $this->getOne($arrFilter, $arrOptions); + + + return $arrNovel; + } + + /** + * 根据小说ID查询指定小说 + * + * @param integer $intNId + * @return array + */ + public function getConfigSubjectFomartByCfsKey(string $intNId): null|array + { + $arrFilter = ['cfs_key' => $intNId]; + return $this->findOne($arrFilter); + } + + +} diff --git a/code/app/model/DomainModel.php b/code/app/model/DomainModel.php index 5157848..b5f5e78 100644 --- a/code/app/model/DomainModel.php +++ b/code/app/model/DomainModel.php @@ -94,22 +94,46 @@ class DomainModel extends BaseModel * @param string $strKey * @return void */ - public function getFomartSubject($strKey) //, $intNum = 1, $boolRand = false) + public function getFomartSubject($strKey, $strDiff = '') { $intSfgId = $this->t_cfg->{$strKey}['sfg_id']; - // $SubjectFomartModel = SubjectFomartModel::where('sfg_id', $intSfgId)->select()->first(); + $SubjectFomartModel = NULL; + $arrAllSubjectFomartModel = SubjectFomartModel::getAllSubjectFomartOnCacheBySfgId($intSfgId); + if (empty($arrAllSubjectFomartModel)) { + return ''; + } - $SubjectFomartModel = (is_array($arrAllSubjectFomartModel) && !empty($arrAllSubjectFomartModel)) ? reset($arrAllSubjectFomartModel) : null; + $strDomain = request()->DomainModel->d_domain; + $strController = request()->controller(); + $strAction = request()->action(); - return empty($SubjectFomartModel) ? '' : $SubjectFomartModel->sf_val; + $strUUId = sprintf('%s-%s-%s-%s-%s', $strDomain, $strController, $strAction, $strKey, $strDiff); - // echo '
';
-        // print_r($SubjectFomartModel);
-        // exit;
+        $arrConfigSubjectFomart = ConfigSubjectFomartModel::getInstance()->getConfigSubjectFomartByCfsKey($strUUId);
 
-        // return  $SubjectFomartModel->sf_val;
+        if (empty($arrConfigSubjectFomart)) {
+
+            $intKey =  array_rand($arrAllSubjectFomartModel);
+
+            $SubjectFomartModel = $arrAllSubjectFomartModel[$intKey];
+
+            ConfigSubjectFomartModel::getInstance()->insert([
+                'cfs_key' => $strUUId,
+                'sf_id' =>  $SubjectFomartModel->sf_id,
+            ]);
+        } else {
+            $intSfId = $arrConfigSubjectFomart['sf_id'];
+
+            foreach ($arrAllSubjectFomartModel as $objTemp) {
+                if ($objTemp->sf_id == $intSfId) {
+                    $SubjectFomartModel = $objTemp;
+                }
+            }
+        }
+
+        return  $SubjectFomartModel->sf_val;
     }
 
 
diff --git a/code/database/mongo/migrations/config_subject_fomart.php b/code/database/mongo/migrations/config_subject_fomart.php
new file mode 100644
index 0000000..2277de8
--- /dev/null
+++ b/code/database/mongo/migrations/config_subject_fomart.php
@@ -0,0 +1,10 @@
+ 'config_subject_fomart',
+    'indexes' => [
+        [
+            'key' => ['csf_key' => 1],
+            'options' => ['unique' => true]
+        ],
+    ]
+];
diff --git a/code/database/schema/mysql-schema.dump b/code/database/schema/mysql-schema.dump
index 44572e0..3339c1e 100644
--- a/code/database/schema/mysql-schema.dump
+++ b/code/database/schema/mysql-schema.dump
@@ -71,8 +71,8 @@ CREATE TABLE `domain` (
   `d_index_description` varchar(1000) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT '首页description模板',
   `d_statis` text CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT '单域名统计',
   `d_logo_type` tinyint(1) DEFAULT NULL,
-  `d_text_log` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT NULL,
-  `d_img_log` varchar(500) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `d_text_logo` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `d_img_logo` varchar(500) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT NULL,
   `d_content_encode` tinyint unsigned DEFAULT '0',
   `t_cfg` json DEFAULT NULL,
   PRIMARY KEY (`d_id`),