diff --git a/code/app/task/crawler/douban/Scheduler.php b/code/app/task/crawler/douban/Scheduler.php index 4d3c846..0edc116 100644 --- a/code/app/task/crawler/douban/Scheduler.php +++ b/code/app/task/crawler/douban/Scheduler.php @@ -111,8 +111,8 @@ class Scheduler continue; } - $intTotalPage = $arrCategory['total_page']; + // $intTotalPage = 2 ; // test $arrFilter = []; for ($intStartPage = 1; $intStartPage <= $intTotalPage; $intStartPage++) { diff --git a/code/app/task/crawler/douban/page/VideoInfoPage.php b/code/app/task/crawler/douban/page/VideoInfoPage.php index 776a6ff..8ee48e4 100644 --- a/code/app/task/crawler/douban/page/VideoInfoPage.php +++ b/code/app/task/crawler/douban/page/VideoInfoPage.php @@ -67,14 +67,22 @@ class VideoInfoPage extends BasePage if (empty($this->arrVideo)) { $arrVideo = json_decode($this->getContent(), true); - // $arrVideo = $arrVideo['list'][0]; // 这个日志会报错 - $arrVideo = $arrVideo['list'][0] ?? []; - - $arrVideo = $arrVideo['list'][0]; - - // print_r($arrVideo); - $strPlayUrl = $arrVideo['vod_play_url']; + $arrVideo = $arrVideo['list'][0] ?? []; + // 如果结构不正确 → 返回空数组 + if (!$arrVideo) { + return []; + } + + // 2. 检查播放地址 + $strPlayUrl = $arrVideo['vod_play_url'] ?? null; + + // 2. 判断是否有播放地址 + if (empty($arrVideo['vod_play_url'])) { + // 无播放地址 → 丢弃该数据 + return []; + } + unset($arrVideo['vod_play_url']); $arrPlayUrl = str_replace('#', '$', $strPlayUrl); diff --git a/code/app/task/crawler/maotai/Scheduler.php b/code/app/task/crawler/maotai/Scheduler.php index 3eca4aa..359de02 100644 --- a/code/app/task/crawler/maotai/Scheduler.php +++ b/code/app/task/crawler/maotai/Scheduler.php @@ -113,6 +113,7 @@ class Scheduler $intTotalPage = $arrCategory['total_page']; + // $intTotalPage = 2 ; // test $arrFilter = []; for ($intStartPage = 1; $intStartPage <= $intTotalPage; $intStartPage++) { diff --git a/code/app/task/crawler/maotai/page/VideoInfoPage.php b/code/app/task/crawler/maotai/page/VideoInfoPage.php index 4f8735a..88b1710 100644 --- a/code/app/task/crawler/maotai/page/VideoInfoPage.php +++ b/code/app/task/crawler/maotai/page/VideoInfoPage.php @@ -162,7 +162,7 @@ class VideoInfoPage extends BasePage empty($arrPageVideo['v_name']) || empty($arrPageVideo['v_category']) || empty($arrPageVideo['v_play_url']) || - empty($arrPageVideo['v_play_url']['douban']) + empty($arrPageVideo['v_play_url']['maotai']) ) { return false; } @@ -175,7 +175,7 @@ class VideoInfoPage extends BasePage $arrUpdate = [ 'updated_at' => new \MongoDB\BSON\UTCDateTime(), - 'v_play_url.douban' => $arrPageVideo['v_play_url']['douban'], + 'v_play_url.maotai' => $arrPageVideo['v_play_url']['maotai'], ]; $this->getVideoModel()->updateOne( diff --git a/doc/mongodb.md b/doc/mongodb.md index 10500f0..62fba7b 100644 --- a/doc/mongodb.md +++ b/doc/mongodb.md @@ -40,4 +40,18 @@ db.category.createIndex({"ca_name":1},{unique:true}) // 统计 db.novel_clicks.createIndex({ "n_id": 1, "nc_date": 1, "nc_type": 1 },{ unique: true }); + + + +// 查询某个资源线路的数据 +db.video.find( + { "v_play_url.douban": { $exists: true } } +).limit(10) + +// 删除某个资源线路的数据 +db.video.updateMany( + {}, + { $unset: { "v_play_url.douban": "" } } +) + ```` \ No newline at end of file