From 240fe900fb9d9dec9e97725ff467beb2694f4aea Mon Sep 17 00:00:00 2001 From: Yarik Date: Thu, 18 May 2017 16:16:11 +0300 Subject: [PATCH] Sitemap --- console/SiteMapController.php | 220 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++------------------------------------------------------------------------- 1 file changed, 147 insertions(+), 73 deletions(-) diff --git a/console/SiteMapController.php b/console/SiteMapController.php index edc7afe..2677750 100755 --- a/console/SiteMapController.php +++ b/console/SiteMapController.php @@ -25,31 +25,46 @@ public $mapNumber = 1; public $mainMap = ''; public $content = ''; + public $robots = []; public function getAddStatic() { return [ - Yii::$app->urlManager->baseUrl.'/ru', + Yii::$app->urlManager->baseUrl . '/ru', ]; } - - - public function getHost(){ - return Yii::$app->urlManager->baseUrl.'/ru'; + + public function getHost() + { + return Yii::$app->urlManager->baseUrl . '/ru'; } public function getVariants() { return ProductVariant::find() ->with('lang') - ->with('product.lang'); + ->with('product.lang') + ->select('*', 'DISTINCT ON (product_id)') + ->andWhere( + [ + '!=', + 'stock', + 0, + ] + ); } public function getSeoLinks() { return Seo::find() - ->where(['!=', 'meta', 'noindex,nofollow' ]) + ->where( + [ + '!=', + 'meta', + 'noindex,nofollow', + ] + ) ->all(); } @@ -64,8 +79,27 @@ public function getCategories() { return Category::find() - ->with('lang') - ->where(['!=', 'parent_id', 0 ]) + ->joinWith('lang') + ->where( + [ + '!=', + 'parent_id', + 0, + ] + ) + ->andWhere( + [ + 'not', + [ + 'meta_robots' => [ + 'noindex,nofollow', + 'noindex, nofollow', + 'noindex,follow', + 'noindex, follow', + ], + ], + ] + ) ->all(); } @@ -75,10 +109,27 @@ ->with('lang') ->joinWith('taxGroups.lang') ->with('taxGroups.taxOptions.lang') - ->where(['!=', 'parent_id', 0 ]) - ->andWhere(['!=', 'tax_group.meta_robots', 'noindex,nofollow' ]) - ->andWhere(['!=', 'tax_group.meta_robots', 'noindex, nofollow' ]) - ->andWhere(['tax_group.is_filter'=>true ]) + ->where( + [ + '!=', + 'parent_id', + 0, + ] + ) + ->andWhere( + [ + 'not', + [ + 'tax_group.meta_robots' => [ + 'noindex,nofollow', + 'noindex, nofollow', + 'noindex,follow', + 'noindex, follow', + ], + ], + ] + ) + ->andWhere([ 'tax_group.is_filter' => true ]) ->all(); } @@ -86,6 +137,19 @@ { return Brand::find() ->joinWith('lang') + ->andWhere( + [ + 'not', + [ + 'meta_robots' => [ + 'noindex,nofollow', + 'noindex, nofollow', + 'noindex,follow', + 'noindex, follow', + ], + ], + ] + ) ->all(); } @@ -99,68 +163,74 @@ } } - public function createRow($url, $priority) + public function createRow($url, $priority, $freq = 'Weekly') { -// if ($this->checkUrl($url)) { - if($this->count % 500 == 0) { + if(in_array($url, $this->robots)) { + return; + } + // if ($this->checkUrl($url)) { + if ($this->count % 500 == 0) { $this->stdout($this->count . " : ", Console::BOLD); $this->stdout($url . "\n", Console::FG_YELLOW); } - $this->content .= '' . '' . $url . '' . '' . date( + $this->content .= '' . '' . $url . '' . '' . date( + 'Y-m-d' + ) . '' . '' . $freq . '' . '' . $priority . '' . ''; + $this->count++; + if ($this->count % 10000 == 0) { + $this->content .= ''; + $this->stdout('Added unset' . "\n", Console::FG_CYAN); + fwrite($this->handle, $this->content); + fclose($this->handle); + $this->mapNumber++; + + $this->mainMap .= '' . 'https://extremstyle.ua/' . $this->fileName . '' . '' . date( 'Y-m-d' - ) . '' . 'Weekly' . '' . $priority . '' . ''; - $this->count++; - if ($this->count % 10000 == 0) { - $this->content .= ''; - $this->stdout('Added unset' . "\n", Console::FG_CYAN); - fwrite($this->handle, $this->content); - fclose($this->handle); - $this->mapNumber++; - - $this->mainMap .= ''. - 'https://extremstyle.ua/' . $this->fileName . ''. - '' . date('Y-m-d') . ''. - ''; - - $this->fileName = 'sitemap' . $this->mapNumber . '.xml'; - $this->handle = fopen(Yii::getAlias('@frontend') . '/web' . '/' . $this->fileName, "w"); - - $this->content = ''; - } -// } + ) . '' . ''; + + $this->fileName = 'sitemap' . $this->mapNumber . '.xml'; + $this->handle = fopen(Yii::getAlias('@frontend') . '/web' . '/' . $this->fileName, "w"); + + $this->content = ''; + } + // } } public function actionProcess() { - + Language::setCurrent('ru'); - - + $config = ArrayHelper::merge( require( \Yii::getAlias('@frontend/config/') . 'main.php' ), require( \Yii::getAlias('@common/config/') . 'main.php' ), - ['components'=>['urlManager'=>['hostInfo'=>'https://extremstyle.ua']]] + [ 'components' => [ 'urlManager' => [ 'hostInfo' => 'https://extremstyle.ua' ] ] ] ); - - if(isset($config['components']['urlManager']['class'])){ - unset($config['components']['urlManager']['class']); + + if (isset($config[ 'components' ][ 'urlManager' ][ 'class' ])) { + unset($config[ 'components' ][ 'urlManager' ][ 'class' ]); } //Yii::$app->urlManager = new LanguageUrlManager($config['components']['urlManager']); - - $urlManager = new LanguageUrlManager($config['components']['urlManager']); - + + $urlManager = new LanguageUrlManager($config[ 'components' ][ 'urlManager' ]); + $this->mainMap = ''; $this->mainMap .= ''; $this->fileName = 'sitemap' . $this->mapNumber . '.xml'; setlocale(LC_ALL, 'ru_RU.CP1251'); $this->handle = fopen(Yii::getAlias('@frontend') . '/web' . '/' . $this->fileName, "w"); - - - + $this->content = ''; + $robots = file_get_contents(\Yii::getAlias('@frontend/web/robots.txt')); + if (preg_match_all('/Disallow: (.*)\n/', $robots, $matches)) { + foreach ($matches[ 1 ] as $match) { + $this->robots[] = $this->getHost() . $match; + } + } + foreach ($this->getAddStatic() as $page) { - $this->createRow($page, 1); + $this->createRow($page, 1, 'Daily'); } foreach ($this->getStaticPages() as $page) { @@ -170,32 +240,33 @@ 'slug' => $page->lang->alias, ] ); - $this->createRow($url, 1); + $this->createRow($url, 0.5); } foreach ($this->getCategories() as $category) { - $url = $urlManager->createAbsoluteUrl( + $url = $urlManager->createAbsoluteUrl( [ 'catalog/category', 'category' => $category->lang->alias, ] ); - $this->createRow($url, 0.8); + $this->createRow($url, 0.9, 'Daily'); } - foreach ($this->getVariants()->batch(1000) as $rows) { + foreach ($this->getVariants() + ->batch(1000) as $rows) { foreach ($rows as $row) { - if(!preg_match("@^[a-zA-Z\d]+$@i", $row->sku)) { + if (!preg_match("@^[a-zA-Z\d]+$@i", $row->sku)) { continue; } - $url = $urlManager->createAbsoluteUrl( + $url = $urlManager->createAbsoluteUrl( [ 'catalog/product', 'product' => $row->product->lang->alias, 'variant' => $row->sku, ] ); - $this->createRow($url, 0.9); + $this->createRow($url, 0.7, 'Daily'); } } @@ -207,30 +278,35 @@ 'slug' => $brand->lang->alias, ] ); - $this->createRow($url, 0.7); + $this->createRow($url, 0.5); } - + foreach ($this->getCategoriesWithFilters() as $category) { foreach ($category->taxGroups as $group) { - if($group->meta_robots == 'noindex,nofollow') { + if (in_array( + $group->meta_robots, + [ + 'noindex,nofollow', + 'noindex, nofollow', + 'noindex,follow', + 'noindex, follow', + ] + )) { continue; } - if($group->is_filter){ + if ($group->is_filter) { foreach ($group->options as $option) { - $url = $urlManager->createAbsoluteUrl( + $url = $urlManager->createAbsoluteUrl( [ 'catalog/category', 'category' => $category, 'filters' => [ $group->lang->alias => [ $option->lang->alias ] ], ] ); - $this->createRow($url, 0.8); + $this->createRow($url, 0.8, 'Daily'); } } - - - } } @@ -245,12 +321,10 @@ fwrite($this->handle, $this->content); fclose($this->handle); - - $this->mainMap .= ''. - ''.'https://extremstyle.ua/'. $this->fileName . ''. - '' . date('Y-m-d') . ''. - ''. - ''; + + $this->mainMap .= '' . '' . 'https://extremstyle.ua/' . $this->fileName . '' . '' . date( + 'Y-m-d' + ) . '' . '' . ''; $mainHandle = fopen(Yii::getAlias('@frontend') . '/web/sitemap.xml', "w"); fwrite($mainHandle, $this->mainMap); -- libgit2 0.21.4