Yarik / artbox-ecommerce

Commit 240fe900fb9d9dec9e97725ff467beb2694f4aea

Authored by Yarik
1 parent 16aaf6ac

Sitemap

Showing 1 changed file with 147 additions and 73 deletions   Show diff stats
Inline Side-by-side
console/SiteMapController.php
  Show/Hide comments   View file @240fe90
@@ -25,31 +25,46 @@ @@ -25,31 +25,46 @@
25 public $mapNumber = 1; 25 public $mapNumber = 1;
26 public $mainMap = ''; 26 public $mainMap = '';
27 public $content = ''; 27 public $content = '';
  28 + public $robots = [];
28 29
29 public function getAddStatic() 30 public function getAddStatic()
30 { 31 {
31 return [ 32 return [
32 - Yii::$app->urlManager->baseUrl.'/ru', 33 + Yii::$app->urlManager->baseUrl . '/ru',
33 ]; 34 ];
34 } 35 }
35 -  
36 -  
37 - public function getHost(){  
38 - return Yii::$app->urlManager->baseUrl.'/ru'; 36 +
  37 + public function getHost()
  38 + {
  39 + return Yii::$app->urlManager->baseUrl . '/ru';
39 } 40 }
40 41
41 public function getVariants() 42 public function getVariants()
42 { 43 {
43 return ProductVariant::find() 44 return ProductVariant::find()
44 ->with('lang') 45 ->with('lang')
45 - ->with('product.lang'); 46 + ->with('product.lang')
  47 + ->select('*', 'DISTINCT ON (product_id)')
  48 + ->andWhere(
  49 + [
  50 + '!=',
  51 + 'stock',
  52 + 0,
  53 + ]
  54 + );
46 55
47 } 56 }
48 57
49 public function getSeoLinks() 58 public function getSeoLinks()
50 { 59 {
51 return Seo::find() 60 return Seo::find()
52 - ->where(['!=', 'meta', 'noindex,nofollow' ]) 61 + ->where(
  62 + [
  63 + '!=',
  64 + 'meta',
  65 + 'noindex,nofollow',
  66 + ]
  67 + )
53 ->all(); 68 ->all();
54 69
55 } 70 }
@@ -64,8 +79,27 @@ @@ -64,8 +79,27 @@
64 public function getCategories() 79 public function getCategories()
65 { 80 {
66 return Category::find() 81 return Category::find()
67 - ->with('lang')  
68 - ->where(['!=', 'parent_id', 0 ]) 82 + ->joinWith('lang')
  83 + ->where(
  84 + [
  85 + '!=',
  86 + 'parent_id',
  87 + 0,
  88 + ]
  89 + )
  90 + ->andWhere(
  91 + [
  92 + 'not',
  93 + [
  94 + 'meta_robots' => [
  95 + 'noindex,nofollow',
  96 + 'noindex, nofollow',
  97 + 'noindex,follow',
  98 + 'noindex, follow',
  99 + ],
  100 + ],
  101 + ]
  102 + )
69 ->all(); 103 ->all();
70 } 104 }
71 105
@@ -75,10 +109,27 @@ @@ -75,10 +109,27 @@
75 ->with('lang') 109 ->with('lang')
76 ->joinWith('taxGroups.lang') 110 ->joinWith('taxGroups.lang')
77 ->with('taxGroups.taxOptions.lang') 111 ->with('taxGroups.taxOptions.lang')
78 - ->where(['!=', 'parent_id', 0 ])  
79 - ->andWhere(['!=', 'tax_group.meta_robots', 'noindex,nofollow' ])  
80 - ->andWhere(['!=', 'tax_group.meta_robots', 'noindex, nofollow' ])  
81 - ->andWhere(['tax_group.is_filter'=>true ]) 112 + ->where(
  113 + [
  114 + '!=',
  115 + 'parent_id',
  116 + 0,
  117 + ]
  118 + )
  119 + ->andWhere(
  120 + [
  121 + 'not',
  122 + [
  123 + 'tax_group.meta_robots' => [
  124 + 'noindex,nofollow',
  125 + 'noindex, nofollow',
  126 + 'noindex,follow',
  127 + 'noindex, follow',
  128 + ],
  129 + ],
  130 + ]
  131 + )
  132 + ->andWhere([ 'tax_group.is_filter' => true ])
82 ->all(); 133 ->all();
83 } 134 }
84 135
@@ -86,6 +137,19 @@ @@ -86,6 +137,19 @@
86 { 137 {
87 return Brand::find() 138 return Brand::find()
88 ->joinWith('lang') 139 ->joinWith('lang')
  140 + ->andWhere(
  141 + [
  142 + 'not',
  143 + [
  144 + 'meta_robots' => [
  145 + 'noindex,nofollow',
  146 + 'noindex, nofollow',
  147 + 'noindex,follow',
  148 + 'noindex, follow',
  149 + ],
  150 + ],
  151 + ]
  152 + )
89 ->all(); 153 ->all();
90 } 154 }
91 155
@@ -99,68 +163,74 @@ @@ -99,68 +163,74 @@
99 } 163 }
100 } 164 }
101 165
102 - public function createRow($url, $priority) 166 + public function createRow($url, $priority, $freq = 'Weekly')
103 { 167 {
104 -// if ($this->checkUrl($url)) {  
105 - if($this->count % 500 == 0) { 168 + if(in_array($url, $this->robots)) {
  169 + return;
  170 + }
  171 + // if ($this->checkUrl($url)) {
  172 + if ($this->count % 500 == 0) {
106 $this->stdout($this->count . " : ", Console::BOLD); 173 $this->stdout($this->count . " : ", Console::BOLD);
107 $this->stdout($url . "\n", Console::FG_YELLOW); 174 $this->stdout($url . "\n", Console::FG_YELLOW);
108 } 175 }
109 - $this->content .= '<url>' . '<loc>' . $url . '</loc>' . '<lastmod>' . date( 176 + $this->content .= '<url>' . '<loc>' . $url . '</loc>' . '<lastmod>' . date(
  177 + 'Y-m-d'
  178 + ) . '</lastmod>' . '<changefreq>' . $freq . '</changefreq>' . '<priority>' . $priority . '</priority>' . '</url>';
  179 + $this->count++;
  180 + if ($this->count % 10000 == 0) {
  181 + $this->content .= '</urlset>';
  182 + $this->stdout('Added unset' . "\n", Console::FG_CYAN);
  183 + fwrite($this->handle, $this->content);
  184 + fclose($this->handle);
  185 + $this->mapNumber++;
  186 +
  187 + $this->mainMap .= '<sitemap>' . '<loc>https://extremstyle.ua/' . $this->fileName . '</loc>' . '<lastmod>' . date(
110 'Y-m-d' 188 'Y-m-d'
111 - ) . '</lastmod>' . '<changefreq>Weekly</changefreq>' . '<priority>' . $priority . '</priority>' . '</url>';  
112 - $this->count++;  
113 - if ($this->count % 10000 == 0) {  
114 - $this->content .= '</urlset>';  
115 - $this->stdout('Added unset' . "\n", Console::FG_CYAN);  
116 - fwrite($this->handle, $this->content);  
117 - fclose($this->handle);  
118 - $this->mapNumber++;  
119 -  
120 - $this->mainMap .= '<sitemap>'.  
121 - '<loc>https://extremstyle.ua/' . $this->fileName . '</loc>'.  
122 - '<lastmod>' . date('Y-m-d') . '</lastmod>'.  
123 - '</sitemap>';  
124 -  
125 - $this->fileName = 'sitemap' . $this->mapNumber . '.xml';  
126 - $this->handle = fopen(Yii::getAlias('@frontend') . '/web' . '/' . $this->fileName, "w");  
127 -  
128 - $this->content = '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">';  
129 - }  
130 -// } 189 + ) . '</lastmod>' . '</sitemap>';
  190 +
  191 + $this->fileName = 'sitemap' . $this->mapNumber . '.xml';
  192 + $this->handle = fopen(Yii::getAlias('@frontend') . '/web' . '/' . $this->fileName, "w");
  193 +
  194 + $this->content = '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">';
  195 + }
  196 + // }
131 } 197 }
132 198
133 public function actionProcess() 199 public function actionProcess()
134 { 200 {
135 - 201 +
136 Language::setCurrent('ru'); 202 Language::setCurrent('ru');
137 -  
138 - 203 +
139 $config = ArrayHelper::merge( 204 $config = ArrayHelper::merge(
140 require( \Yii::getAlias('@frontend/config/') . 'main.php' ), 205 require( \Yii::getAlias('@frontend/config/') . 'main.php' ),
141 require( \Yii::getAlias('@common/config/') . 'main.php' ), 206 require( \Yii::getAlias('@common/config/') . 'main.php' ),
142 - ['components'=>['urlManager'=>['hostInfo'=>'https://extremstyle.ua']]] 207 + [ 'components' => [ 'urlManager' => [ 'hostInfo' => 'https://extremstyle.ua' ] ] ]
143 ); 208 );
144 -  
145 - if(isset($config['components']['urlManager']['class'])){  
146 - unset($config['components']['urlManager']['class']); 209 +
  210 + if (isset($config[ 'components' ][ 'urlManager' ][ 'class' ])) {
  211 + unset($config[ 'components' ][ 'urlManager' ][ 'class' ]);
147 } 212 }
148 //Yii::$app->urlManager = new LanguageUrlManager($config['components']['urlManager']); 213 //Yii::$app->urlManager = new LanguageUrlManager($config['components']['urlManager']);
149 -  
150 - $urlManager = new LanguageUrlManager($config['components']['urlManager']);  
151 - 214 +
  215 + $urlManager = new LanguageUrlManager($config[ 'components' ][ 'urlManager' ]);
  216 +
152 $this->mainMap = '<?xml version="1.0" encoding="UTF-8"?>'; 217 $this->mainMap = '<?xml version="1.0" encoding="UTF-8"?>';
153 $this->mainMap .= '<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">'; 218 $this->mainMap .= '<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">';
154 $this->fileName = 'sitemap' . $this->mapNumber . '.xml'; 219 $this->fileName = 'sitemap' . $this->mapNumber . '.xml';
155 setlocale(LC_ALL, 'ru_RU.CP1251'); 220 setlocale(LC_ALL, 'ru_RU.CP1251');
156 $this->handle = fopen(Yii::getAlias('@frontend') . '/web' . '/' . $this->fileName, "w"); 221 $this->handle = fopen(Yii::getAlias('@frontend') . '/web' . '/' . $this->fileName, "w");
157 -  
158 -  
159 - 222 +
160 $this->content = '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">'; 223 $this->content = '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">';
161 224
  225 + $robots = file_get_contents(\Yii::getAlias('@frontend/web/robots.txt'));
  226 + if (preg_match_all('/Disallow: (.*)\n/', $robots, $matches)) {
  227 + foreach ($matches[ 1 ] as $match) {
  228 + $this->robots[] = $this->getHost() . $match;
  229 + }
  230 + }
  231 +
162 foreach ($this->getAddStatic() as $page) { 232 foreach ($this->getAddStatic() as $page) {
163 - $this->createRow($page, 1); 233 + $this->createRow($page, 1, 'Daily');
164 } 234 }
165 235
166 foreach ($this->getStaticPages() as $page) { 236 foreach ($this->getStaticPages() as $page) {
@@ -170,32 +240,33 @@ @@ -170,32 +240,33 @@
170 'slug' => $page->lang->alias, 240 'slug' => $page->lang->alias,
171 ] 241 ]
172 ); 242 );
173 - $this->createRow($url, 1); 243 + $this->createRow($url, 0.5);
174 } 244 }
175 245
176 foreach ($this->getCategories() as $category) { 246 foreach ($this->getCategories() as $category) {
177 - $url = $urlManager->createAbsoluteUrl( 247 + $url = $urlManager->createAbsoluteUrl(
178 [ 248 [
179 'catalog/category', 249 'catalog/category',
180 'category' => $category->lang->alias, 250 'category' => $category->lang->alias,
181 ] 251 ]
182 ); 252 );
183 - $this->createRow($url, 0.8); 253 + $this->createRow($url, 0.9, 'Daily');
184 } 254 }
185 255
186 - foreach ($this->getVariants()->batch(1000) as $rows) { 256 + foreach ($this->getVariants()
  257 + ->batch(1000) as $rows) {
187 foreach ($rows as $row) { 258 foreach ($rows as $row) {
188 - if(!preg_match("@^[a-zA-Z\d]+$@i", $row->sku)) { 259 + if (!preg_match("@^[a-zA-Z\d]+$@i", $row->sku)) {
189 continue; 260 continue;
190 } 261 }
191 - $url = $urlManager->createAbsoluteUrl( 262 + $url = $urlManager->createAbsoluteUrl(
192 [ 263 [
193 'catalog/product', 264 'catalog/product',
194 'product' => $row->product->lang->alias, 265 'product' => $row->product->lang->alias,
195 'variant' => $row->sku, 266 'variant' => $row->sku,
196 ] 267 ]
197 ); 268 );
198 - $this->createRow($url, 0.9); 269 + $this->createRow($url, 0.7, 'Daily');
199 } 270 }
200 } 271 }
201 272
@@ -207,30 +278,35 @@ @@ -207,30 +278,35 @@
207 'slug' => $brand->lang->alias, 278 'slug' => $brand->lang->alias,
208 ] 279 ]
209 ); 280 );
210 - $this->createRow($url, 0.7); 281 + $this->createRow($url, 0.5);
211 282
212 } 283 }
213 - 284 +
214 foreach ($this->getCategoriesWithFilters() as $category) { 285 foreach ($this->getCategoriesWithFilters() as $category) {
215 foreach ($category->taxGroups as $group) { 286 foreach ($category->taxGroups as $group) {
216 - if($group->meta_robots == 'noindex,nofollow') { 287 + if (in_array(
  288 + $group->meta_robots,
  289 + [
  290 + 'noindex,nofollow',
  291 + 'noindex, nofollow',
  292 + 'noindex,follow',
  293 + 'noindex, follow',
  294 + ]
  295 + )) {
217 continue; 296 continue;
218 } 297 }
219 - if($group->is_filter){ 298 + if ($group->is_filter) {
220 foreach ($group->options as $option) { 299 foreach ($group->options as $option) {
221 - $url = $urlManager->createAbsoluteUrl( 300 + $url = $urlManager->createAbsoluteUrl(
222 [ 301 [
223 'catalog/category', 302 'catalog/category',
224 'category' => $category, 303 'category' => $category,
225 'filters' => [ $group->lang->alias => [ $option->lang->alias ] ], 304 'filters' => [ $group->lang->alias => [ $option->lang->alias ] ],
226 ] 305 ]
227 ); 306 );
228 - $this->createRow($url, 0.8); 307 + $this->createRow($url, 0.8, 'Daily');
229 } 308 }
230 } 309 }
231 -  
232 -  
233 -  
234 310
235 } 311 }
236 } 312 }
@@ -245,12 +321,10 @@ @@ -245,12 +321,10 @@
245 321
246 fwrite($this->handle, $this->content); 322 fwrite($this->handle, $this->content);
247 fclose($this->handle); 323 fclose($this->handle);
248 -  
249 - $this->mainMap .= '<sitemap>'.  
250 - '<loc>'.'https://extremstyle.ua/'. $this->fileName . '</loc>'.  
251 - '<lastmod>' . date('Y-m-d') . '</lastmod>'.  
252 - '</sitemap>'.  
253 - '</sitemapindex>'; 324 +
  325 + $this->mainMap .= '<sitemap>' . '<loc>' . 'https://extremstyle.ua/' . $this->fileName . '</loc>' . '<lastmod>' . date(
  326 + 'Y-m-d'
  327 + ) . '</lastmod>' . '</sitemap>' . '</sitemapindex>';
254 328
255 $mainHandle = fopen(Yii::getAlias('@frontend') . '/web/sitemap.xml', "w"); 329 $mainHandle = fopen(Yii::getAlias('@frontend') . '/web/sitemap.xml', "w");
256 fwrite($mainHandle, $this->mainMap); 330 fwrite($mainHandle, $this->mainMap);