diff --git a/code/app/model/ConfigSubjectFomartModel.php b/code/app/model/ConfigSubjectFomartModel.php new file mode 100644 index 0000000..6e89ed5 --- /dev/null +++ b/code/app/model/ConfigSubjectFomartModel.php @@ -0,0 +1,147 @@ + [ + 'root' => 'array', + 'document' => 'array', + 'array' => 'array', + ], + 'projection' => [ + '_id' => 0, + ], + ]; + + + /** + * 直接读库查询大量小说 + * + * @param integer $intNId + * @return null|array + */ + public function findMany(array $arrFilter, int $intCount = 10, array $arrSort = [], array $arrOptions = []): null|array + { + if (empty($arrOptions)) { + $arrOptions = self::$arrOptions; + } + + if ($intCount > 0) { + $arrOptions['limit'] = $intCount; + } + + if (empty($arrSort)) { + $arrOptions['sort'] = $arrSort; + } + + $Cursor = $this->getCol()->find($arrFilter, $arrOptions); + + $arrResult = iterator_to_array($Cursor); + + return $arrResult; + } + + + /** + * 查询多条数据并缓存 + * + * @param array $arrFilter + * @param integer $intCount + * @param array $arrSort + * @param array $arrOptions + * @param string $strKey + * @param integer $intLifeTime + * @return null|array + */ + public function findManyWithCache( + array $arrFilter, + int $intCount = 0, + array $arrSort = [], + array $arrOptions = [], + string $strKey = '', + int $intLifeTime = 24 * 3600, + ): null|array { + try { + + $arrResult = $this->checkCacheByKey($strKey); + if ($arrResult !== NULL) { + return $arrResult; + } + + $arrResult = $this->findMany($arrFilter, $intCount, $arrSort, $arrOptions); + + $this->setCacheByKey($strKey, $arrResult, $intLifeTime); + + return $arrResult; + } catch (\Exception $e) { + return []; + } + } + + + /** + * find one novel + * + * @param array $arrFilter + * @param array $arrOptions + * @return null|array + */ + public function findOne(array $arrFilter, array $arrOptions = []): null|array + { + if (empty($arrOptions)) { + $arrOptions = self::$arrOptions; + } + + $arrNovel = $this->getOne($arrFilter, $arrOptions); + + + return $arrNovel; + } + + /** + * 根据小说ID查询指定小说 + * + * @param integer $intNId + * @return array + */ + public function getConfigSubjectFomartByCfsKey(string $intNId): null|array + { + $arrFilter = ['cfs_key' => $intNId]; + return $this->findOne($arrFilter); + } + + +} diff --git a/code/app/model/DomainModel.php b/code/app/model/DomainModel.php index 5157848..b5f5e78 100644 --- a/code/app/model/DomainModel.php +++ b/code/app/model/DomainModel.php @@ -94,22 +94,46 @@ class DomainModel extends BaseModel * @param string $strKey * @return void */ - public function getFomartSubject($strKey) //, $intNum = 1, $boolRand = false) + public function getFomartSubject($strKey, $strDiff = '') { $intSfgId = $this->t_cfg->{$strKey}['sfg_id']; - // $SubjectFomartModel = SubjectFomartModel::where('sfg_id', $intSfgId)->select()->first(); + $SubjectFomartModel = NULL; + $arrAllSubjectFomartModel = SubjectFomartModel::getAllSubjectFomartOnCacheBySfgId($intSfgId); + if (empty($arrAllSubjectFomartModel)) { + return ''; + } - $SubjectFomartModel = (is_array($arrAllSubjectFomartModel) && !empty($arrAllSubjectFomartModel)) ? reset($arrAllSubjectFomartModel) : null; + $strDomain = request()->DomainModel->d_domain; + $strController = request()->controller(); + $strAction = request()->action(); - return empty($SubjectFomartModel) ? '' : $SubjectFomartModel->sf_val; + $strUUId = sprintf('%s-%s-%s-%s-%s', $strDomain, $strController, $strAction, $strKey, $strDiff); - // echo '
';
- // print_r($SubjectFomartModel);
- // exit;
+ $arrConfigSubjectFomart = ConfigSubjectFomartModel::getInstance()->getConfigSubjectFomartByCfsKey($strUUId);
- // return $SubjectFomartModel->sf_val;
+ if (empty($arrConfigSubjectFomart)) {
+
+ $intKey = array_rand($arrAllSubjectFomartModel);
+
+ $SubjectFomartModel = $arrAllSubjectFomartModel[$intKey];
+
+ ConfigSubjectFomartModel::getInstance()->insert([
+ 'cfs_key' => $strUUId,
+ 'sf_id' => $SubjectFomartModel->sf_id,
+ ]);
+ } else {
+ $intSfId = $arrConfigSubjectFomart['sf_id'];
+
+ foreach ($arrAllSubjectFomartModel as $objTemp) {
+ if ($objTemp->sf_id == $intSfId) {
+ $SubjectFomartModel = $objTemp;
+ }
+ }
+ }
+
+ return $SubjectFomartModel->sf_val;
}
diff --git a/code/app/task/crawler/zw630/Scheduler.php b/code/app/task/crawler/zw630/Scheduler.php
index 03091ac..abb3dd4 100644
--- a/code/app/task/crawler/zw630/Scheduler.php
+++ b/code/app/task/crawler/zw630/Scheduler.php
@@ -53,13 +53,32 @@ class Scheduler
}
}
+ /**
+ * Undocumented function
+ *
+ * @return void
+ */
+ public function crawLatest()
+ {
+ $arrUri = [];
+ $strUri = '/list/lastupdate_%s_0_0_%s.html';
+
+ for ($intCategoryKey = 0; $intCategoryKey < 10; $intCategoryKey++) {
+ for ($intPage = 1; $intPage <= 10; $intPage++) {
+ $arrUri[] = sprintf($strUri, $intCategoryKey, $intPage);
+ }
+ }
+
+ print_r($arrUri);
+ }
+
/**
* Undocumented function
*
* @param array $arrFilter
* @return void
*/
- public function fetchNovel(array $arrFilter) //(int $intNSourceId, int $intPage = 0)
+ public function fetchNovel(array $arrFilter)
{
$arrTask = [
'callback' => [self::class, 'getNovelInfo'],
diff --git a/code/app/task/module/PlanTask.php b/code/app/task/module/PlanTask.php
index db73150..8807682 100644
--- a/code/app/task/module/PlanTask.php
+++ b/code/app/task/module/PlanTask.php
@@ -31,13 +31,20 @@ class PlanTask
case 'PULL_630_ZW_NOVEL':
self::pull630ZwNovel();
break;
-
+ case 'PULL_630_ZW_NOVEL_LATEST':
+ self::pull630ZwNovelLatest();
+ break;
default:
break;
}
}
}
+ /**
+ * Undocumented function
+ *
+ * @return void
+ */
public static function pull630ZwNovel()
{
$arrTask = [
@@ -48,4 +55,20 @@ class PlanTask
$QueueManage = new QueueManage();
$QueueManage->set($arrTask);
}
+
+ /**
+ * Undocumented function
+ *
+ * @return void
+ */
+ public static function pull630ZwNovelLatest()
+ {
+ $arrTask = [
+ 'callback' => [Zw630Scheduler::class, 'crawLatest'],
+ 'data' => []
+ ];
+
+ $QueueManage = new QueueManage();
+ $QueueManage->set($arrTask);
+ }
}
diff --git a/code/database/mongo/migrations/config_subject_fomart.php b/code/database/mongo/migrations/config_subject_fomart.php
new file mode 100644
index 0000000..2277de8
--- /dev/null
+++ b/code/database/mongo/migrations/config_subject_fomart.php
@@ -0,0 +1,10 @@
+ 'config_subject_fomart',
+ 'indexes' => [
+ [
+ 'key' => ['csf_key' => 1],
+ 'options' => ['unique' => true]
+ ],
+ ]
+];
diff --git a/code/database/schema/mysql-schema.dump b/code/database/schema/mysql-schema.dump
index 44572e0..3339c1e 100644
--- a/code/database/schema/mysql-schema.dump
+++ b/code/database/schema/mysql-schema.dump
@@ -71,8 +71,8 @@ CREATE TABLE `domain` (
`d_index_description` varchar(1000) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT '首页description模板',
`d_statis` text CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci COMMENT '单域名统计',
`d_logo_type` tinyint(1) DEFAULT NULL,
- `d_text_log` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT NULL,
- `d_img_log` varchar(500) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+ `d_text_logo` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+ `d_img_logo` varchar(500) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`d_content_encode` tinyint unsigned DEFAULT '0',
`t_cfg` json DEFAULT NULL,
PRIMARY KEY (`d_id`),
diff --git a/code/database/seeders/PlanTaskSeeder.php b/code/database/seeders/PlanTaskSeeder.php
index 9808864..e59e5ca 100644
--- a/code/database/seeders/PlanTaskSeeder.php
+++ b/code/database/seeders/PlanTaskSeeder.php
@@ -9,12 +9,10 @@ class PlanTaskSeeder
{
static public function handle()
{
-
self::addNovelTypePlanTask();
}
- // 添加小说类型站点 爬虫任务
static public function addNovelTypePlanTask()
{
$arrDataNovel = [
@@ -25,8 +23,14 @@ class PlanTaskSeeder
'pt_enable' => 1,
'pt_limit' => 10 * 24 * 3600 * 3,
],
-
+ [
+ 'pt_name' => '630小说每日增量',
+ 'pt_code' => 'PULL_630_ZW_NOVEL_LATEST',
+ 'pt_enable' => 1,
+ 'pt_limit' => 1 * 24 * 3600,
+ ],
];
+
foreach ($arrDataNovel as $arrPlanTask) {
$NovelOnePlanTaskModel = PlanTaskModel::where('pt_code', $arrPlanTask['pt_code'])->find();