diff --git a/code/app/task/crawler/zw/Scheduler.php b/code/app/task/crawler/zw/Scheduler.php index a94854c..0f71746 100644 --- a/code/app/task/crawler/zw/Scheduler.php +++ b/code/app/task/crawler/zw/Scheduler.php @@ -218,9 +218,9 @@ class Scheduler /** * Undocumented function * - * @param integer $intNId - * @param integer $intPage - * @param array $arrChapterList + * @param integer $intNId 小说id + * @param integer $intPage 分页 + * @param array $arrChapterList 章节列表 * @return void */ public function fetchChapter(int $intNId, int $intPage, array $arrChapterList = []) @@ -276,6 +276,7 @@ class Scheduler /** * Undocumented function + * 获取每个章节详情 * * @param array $arrData * @return void @@ -284,63 +285,107 @@ class Scheduler { $Site = $this->getSite(); - $ChapterModel = ChapterModel::getInstance(); - #FIXED 这里增加临时代码，将章节任务拆分为一个章节用一个进程请求，防止代理同时请求多个异常问题。 - $arrTask = [ - 'callback' => [static::class, 'getChapterInfo'], - 'data' => [ - 'n_id' => $arrData['n_id'], - 'filter' => [], - ] - ]; + // 过滤已有章节内容 + $arrData['filter'] = array_filter($arrData['filter'], function ($filter) use ($Site, $ChapterModel) { + $strUUId = sprintf('%s-chapter-%s-%s', $Site->getSiteCode(), $filter['c_source_id'], $filter['c_page']); + $chapter = $ChapterModel->findOne(['c_source_uuid' => $strUUId]); + return empty($chapter) || empty($chapter['c_content']); + }); - foreach ($arrData['filter'] as $intKey => $arrChapterFilter) { - $strUUId = sprintf('%s-chapter-%s-%s', $Site->getSiteCode(), $arrChapterFilter['c_source_id'], $arrChapterFilter['c_page']); - $arrChapter = $ChapterModel->findOne(['c_source_uuid' => $strUUId]); + $arrData['filter'] = array_values($arrData['filter']); // 重新索引 - if (!empty($arrChapter) && !empty($arrChapter['content'])) { - unset($arrData['filter'][$intKey]); - } else { - #FIXED 这里增加临时代码，将章节任务拆分为一个章节用一个进程请求，防止代理同时请求多个异常问题。 - - $arrFilter = [$arrChapterFilter]; - $arrTask['data']['filter'] = $arrFilter; - $this->QueueManage->set($arrTask); - } + // 如果全部都已存在，直接返回 + if (empty($arrData['filter'])) { + return; } - if (count($arrData['filter']) > 1 || count($arrData['filter']) <= 0) { - #FIXED 这里增加临时代码，将章节任务拆分为一个章节用一个进程请求，防止代理同时请求多个异常问题。 - return false; - } - - - $arrData['filter'] = array_values($arrData['filter']); $arrChapterPageList = $Site->getChapterPageList($arrData['filter']); $arrNextPageFilterList = []; foreach ($arrChapterPageList as $ChapterPage) { - $ChapterPage->saveChapter($arrData['n_id']); - $arrNextPageFilter = $ChapterPage->getNextPage(); - - if (!empty($arrNextPageFilter)) { - $arrNextPageFilterList[] = $arrNextPageFilter; + $next = $ChapterPage->getNextPage(); + if (!empty($next)) { + $arrNextPageFilterList[] = $next; } } if (!empty($arrNextPageFilterList)) { - - $arrTask = $arrData; - $arrTask['filter'] = $arrNextPageFilterList; - $arrTask = [ + $this->QueueManage->set([ 'callback' => [static::class, 'getChapterInfo'], - 'data' => $arrTask - ]; - $this->QueueManage->set($arrTask); + 'data' => [ + 'n_id' => $arrData['n_id'], + 'filter' => $arrNextPageFilterList + ] + ]); } } + // public function getChapterInfo(array $arrData) + // { + + // $Site = $this->getSite(); + + // $ChapterModel = ChapterModel::getInstance(); + + // #FIXED 这里增加临时代码，将章节任务拆分为一个章节用一个进程请求，防止代理同时请求多个异常问题。 + // $arrTask = [ + // 'callback' => [static::class, 'getChapterInfo'], + // 'data' => [ + // 'n_id' => $arrData['n_id'], + // 'filter' => [], + // ] + // ]; + + // foreach ($arrData['filter'] as $intKey => $arrChapterFilter) { + // $strUUId = sprintf('%s-chapter-%s-%s', $Site->getSiteCode(), $arrChapterFilter['c_source_id'], $arrChapterFilter['c_page']); + + // $arrChapter = $ChapterModel->findOne(['c_source_uuid' => $strUUId]); + + // // 如果数据存在，且 内容不为空，删除任务 + // if (!empty($arrChapter) && !empty($arrChapter['c_content'])) { + // unset($arrData['filter'][$intKey]); + // } else { + // #FIXED 这里增加临时代码，将章节任务拆分为一个章节用一个进程请求，防止代理同时请求多个异常问题。 + + // $arrFilter = [$arrChapterFilter]; + // $arrTask['data']['filter'] = $arrFilter; + // $this->QueueManage->set($arrTask); + // } + // } + // if (count($arrData['filter']) > 1 || count($arrData['filter']) <= 0) { + // #FIXED 这里增加临时代码，将章节任务拆分为一个章节用一个进程请求，防止代理同时请求多个异常问题。 + // return false; + // } + + + // $arrData['filter'] = array_values($arrData['filter']); + + // $arrChapterPageList = $Site->getChapterPageList($arrData['filter']); + + // $arrNextPageFilterList = []; + // foreach ($arrChapterPageList as $ChapterPage) { + + // $ChapterPage->saveChapter($arrData['n_id']); + + // $arrNextPageFilter = $ChapterPage->getNextPage(); + + // if (!empty($arrNextPageFilter)) { + // $arrNextPageFilterList[] = $arrNextPageFilter; + // } + // } + + // if (!empty($arrNextPageFilterList)) { + + // $arrTask = $arrData; + // $arrTask['filter'] = $arrNextPageFilterList; + // $arrTask = [ + // 'callback' => [static::class, 'getChapterInfo'], + // 'data' => $arrTask + // ]; + // $this->QueueManage->set($arrTask); + // } + // } } diff --git a/code/app/task/crawler/zw/page/ChapterPage.php b/code/app/task/crawler/zw/page/ChapterPage.php index 72f87a7..d12f099 100644 --- a/code/app/task/crawler/zw/page/ChapterPage.php +++ b/code/app/task/crawler/zw/page/ChapterPage.php @@ -71,6 +71,7 @@ class ChapterPage extends BasePage $this->arrChapterInfo['c_page_keywords'] = $this->getQueryList()->find('meta[name="keywords"]')->attr('content'); $this->arrChapterInfo['c_page_description'] = $this->getQueryList()->find('meta[name="description"]')->attr('content'); + // var lastread=new LastRead();lastread.set('529620', '184472933', '身为妖族的我，却在人族成家立业', '第321章 一人一兽联手对敌', '0','眯着眼','0'); $strPattern = "/lastread\.set\((.*?)\);/"; if (preg_match($strPattern, $this->getContent(), $arrMatches)) { @@ -117,6 +118,14 @@ class ChapterPage extends BasePage ) { $incKey = config('task.app_name').'_empty_saveChapter:' . date('Y-m-d'); $this->redisDailyInc($incKey); + + if(config('log.task_debug_log_open_status')){ + // 记录失败原因 + $strTitle = $arrPageChapter['c_page_title'] ?? ''; + $strPageData = $arrPageChapter['c_page_data'] ?? ''; + \think\facade\Log::error("Request empty_saveChapter , strTitle: {$strTitle }, strPageData: {$strPageData}"); + var_dump($strTitle .' ---- '.$strPageData); + } return false; }