Найден корректный robots.txt
Файл управления сканированием сайта роботами
?
Robots.txt указывает поисковым роботам, какие страницы сканировать, а какие — нет. Ошибки в файле могут случайно закрыть важные разделы от индексации.
Robots.txt настроен корректно. Размер файла: 16475 байт. Загружен за: 1сек.
Проверяемая страница не запрещена в robots.txt.
Цепочка редиректов для файла robots.txt:
http://yuiblog.com/robots.txt
301 MovedPermanently
https://clarle.github.io/yui3/
200 OK
Показать содержимое robots.txt
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>YUI Library</title>
<meta name="viewport" content="width=960" id="meta-viewport">
<script>
if (screen.width < 768) {
document.getElementById('meta-viewport').setAttribute('content', 'width=768');
}
</script>
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=PT+Sans:400,700,400italic,700italic">
<link rel="stylesheet" href="https://yui-s.yahooapis.com/3.9.0/build/cssgrids/cssgrids-min.css">
<link rel="stylesheet" href="/yui3/combo/css-main-min.css-home-min.css">
<meta http-equiv="X-UA-Compatible" content="IE=Edge,chrome=1">
<meta http-equiv="X-XRDS-Location" content="/xrds.xml">
<link rel="shortcut icon" type="image/png" href="/yui3/img/favicon.png">
<script>
var _gaq = _gaq || [];
_gaq.push(['_setAccount', 'UA-29453705-2']);
_gaq.push(['_trackPageview']);
</script>
<script>
var YUI_config={"filter":"min","maxURLLength":1024,"root":"3.18.1/","groups":{"site":{"combine":true,"comboBase":"/combo/js?","root":"","modules":{"hoverable":{"path":"hoverable-min.js","requires":["event-hover","node-base","node-event-delegate"]},"search":{"path":"search-min.js","requires":["autocomplete","autocomplete-highlighters","node-pluginhost"]},"api-filter":{"path":"apidocs/api-filter-min.js","requires":["autocomplete-base","autocomplete-highlighters","autocomplete-sources"]},"api-list":{"path":"apidocs/api-list-min.js","requires":["api-filter","api-search","event-key","node-focusmanager","tabview"]},"api-search":{"path":"apidocs/api-search-min.js","requires":["autocomplete-base","autocomplete-highlighters","autocomplete-sources","escape"]}}}}};
if (location.protocol.indexOf('https') > -1) {
YUI_config.comboBase = 'https://yui-s.yahooapis.com/combo?';
YUI_config.combine = true;
}
</script>
<script src="https://yui-s.yahooapis.com/3.18.1/build/yui/yui-min.js"></script>
</head>
<body class="yui3-skin-sam">
<!--
<a href="/yuiconf/" class="yuiconf ribbon">
<img src="/yui3/img/yuiconf/2013/ribbon.png" alt="YUIConf 2013" width="139" height="139">
</a>
-->
<div id="doc">
<div id="hd" class="yui3-g"><header>
<div class="content">
<div class="yui3-u-3-4">
<a href="/yui3" rel="home"><div id="hd-logo" role="image" aria-label="YUI logo"></div></a>
</div>
<div class="yui3-u-1-4">
<form id="main-search" class="search" action="https://search.yahoo.com/search"
method="get" role="search">
<input type="search" class="search-input" name="q"
aria-label="Search APIs"
placeholder="Search APIs">
<input type="hidden" name="vs" value="yuilibrary.com">
<input type="hidden" name="vs" value="yuiblog.com">
</form>
</div>
<div id="main-nav" class="yui3-u"><nav>
<ul class="nav hoverable-group">
<li class="nav-tab hoverable nav-tab-active">
<a href="/yui3">Home</a>
</li>
<li class="nav-tab hoverable ">
<a href="https://github.com/yui/yui3/wiki/Quick-Start">Quick Start</a>
<ul class="nav-submenu">
<li class="nav-child ">
<a href="/download/yui3/">Download YUI 3</a>
</li>
</ul>
</li>
<li class="nav-tab hoverable ">
<a href="https://github.com/yui/yui3/wiki/YUI-Documentation">Documentation</a>
<ul class="nav-submenu">
<li class="nav-child ">
<a href="/yui3/yui/docs/guides/">User Guides</a>
</li>
<li class="nav-child ">
<a href="/yui3/yui/docs/examples/">Examples</a>
</li>
<li class="nav-child ">
<a href="/yui3/yui/docs/api/">API Docs</a>
</li>
<li class="nav-child ">
<a href="/yui3/yui/environments/">Environments</a>
</li>
<li class="nav-child ">
<a href="/yui3/yui/docs/tutorials/">Tutorials</a>
</li>
<li class="nav-child ">
<a href="/yui3/yui/docs/tutorials/faq/">FAQ</a>
</li>
</ul>
</li>
<li class="nav-tab hoverable nav-break">
<a href="/community/">Community</a>
<ul class="nav-submenu">
<li class="nav-child ">
<a href="/gallery/">Gallery</a>
</li>
<li class="nav-child ">
<a href="http://yuiblog.com/">Blog</a>
</li>
<li class="nav-child ">
<a href="/forum/">Forums</a>
</li>
<li class="nav-child ">
<a href="/theater/">YUI Theater</a>
</li>
</ul>
</li>
<li class="nav-tab hoverable ">
<a href="/projects/">Other Projects</a>
<ul class="nav-submenu">
<li class="nav-child ">
<a href="http://yui.github.com/shifter/" target="_blank" title="External link">Shifter »</a>
</li>
<li class="nav-child ">
<a href="http://yui.github.com/yogi/" target="_blank" title="External link">Yogi »</a>
</li>
<li class="nav-child ">
<a href="/projects/yui2/">YUI 2</a>
</li>
<li class="nav-child ">
<a href="http://yui.github.com/yuidoc/" target="_blank" title="External link">YUI Doc »</a>
</li>
<li class="nav-child ">
<a href="/projects/yuitest/">YUI Test</a>
</li>
<li class="nav-child ">
<a href="http://yui.github.com/yuicompressor/" target="_blank" title="External link">YUI Compressor »</a>
</li>
<li class="nav-child ">
<a href="http://github.com/yui/builder" target="_blank" title="External link">YUI Builder »</a>
</li>
<li class="nav-child ">
<a href="/projects/phploader/">YUI PHP Loader</a>
</li>
<li class="nav-child ">
<a href="http://yui.github.com/gridbuilder/" target="_blank" title="External link">Grid Builder »</a>
</li>
<li class="nav-child ">
<a href="http://yui.github.com/skinbuilder/" target="_blank" title="External link">Skin Builder »</a>
</li>
</ul>
</li>
</ul>
</nav></div>
</div>
</header></div>
<div id="bd" class="clearfix yui3-g">
<div id="deprecated" class="yui3-u-1 notice">
<p>IMPORTANT: The YUI library is no longer actively maintained. Please see this <a href="http://yahooeng.tumblr.com/post/96098168666/important-announcement-regarding-yui/">announcement</a> for more information.</p>
</div>
<div class="yui3-u-1">
<div class="content">
<div class="intro">
<p class="headline">
YUI is a free, open source JavaScript and CSS library for building richly interactive web applications.
</p>
<p>
<a href="https://github.com/yui/yui3/wiki/Quick-Start" class="button get-started">
Get Started
<span class="extra">
Start using YUI 3.18.1 in two easy steps.
</span>
</a>
</p>
</div>
<div class="divider">
<hr>
<h2>Why YUI?</h2>
</div>
<div class="yui3-g">
<div id="reasons" class="reasons">
<div class="yui3-u-1-4">
<div class="reason">
<h3>Fast</h3>
<p>YUI's lightweight core and <a href="/yui3/yui/docs/guides">modular architecture</a> make it scalable, fast, and robust. Built by frontend engineers at Yahoo!, YUI powers the most popular websites in the world.</p>
</div>
</div>
<div class="yui3-u-1-4">
<div class="reason">
<h3>Complete</h3>
<p>YUI's <a href="https://github.com/yui/yui3/wiki/YUI-Documentation">intuitive and well-documented API</a> takes you from basic DOM handling to building performant and maintainable applications on desktop browsers, mobile devices, and servers.</p>
</div>
</div>
<div class="yui3-u-1-4">
<div class="reason">
<h3>Industrial Strength</h3>
<p>A <a href="/community/">thriving community</a>, a carefully architected infrastructure, and a comprehensive suite of tools help you code like a pro, from simple web pages to complex web applications.</p>
</div>
</div>
<div class="yui3-u-1-4">
<div class="reason">
<h3>Free & Open</h3>
<p>YUI is <a href="/license/">free for all uses</a> and is <a href="https://github.com/yui">developed in the open</a> on GitHub. Core team members can always be found in the <a href="/forum/">forums</a> and the <a href="irc://irc.freenode.net#yui">#yui</a> IRC channel on <a href="http://freenode.net/">Freenode</a>. Pull requests welcome!</p>
</div>
</div>
</div>
</div>
<div class="divider">
<hr>
<h2>A Few <a href="/yui3/yui/docs/examples/">Examples</a></h2>
</div>
<div class="examples-showcase">
<div class="yui3-g">
<ul class="examples">
<li class="yui3-u-1-3">
<div class="example-item">
<h3 class="example-title"><a href="/yui3/yui/docs/dial/dial-interactive.html">Dial with Interactive UI</a></h3>
<a href="/yui3/yui/docs/dial/dial-interactive.html"><img src="/yui3/img/home/example-dial-interactive.jpg" alt="Screenshot of a Dial widget." class="hover-border"></a>
</div>
</li>
<li class="yui3-u-1-3">
<div class="example-item">
<h3 class="example-title"><a href="/yui3/yui/docs/cssgrids/cssgrids-units.html">CSS Grids: Using Grid Units</a></h3>
<a href="/yui3/yui/docs/cssgrids/cssgrids-units.html"><img src="/yui3/img/home/example-grids.png" alt="Screenshot of a page layout using CSS grids." class="hover-border"></a>
</div>
</li>
<li class="yui3-u-1-3">
<div class="example-item">
<h3 class="example-title"><a href="/yui3/yui/docs/dd/groups-drag.html">Drag & Drop: Interaction Groups</a></h3>
<a href="/yui3/yui/docs/dd/groups-drag.html"><img src="/yui3/img/home/example-dd.png" alt="Screenshot of a mouse cursor dragging a box onto a drop target." class="hover-border"></a>
</div>
</li>
</ul>
</div>
</div>
<div class="yui3-g">
<div class="yui3-u-2-3">
<div class="column-left">
<div class="divider">
<hr>
<h2><a href="http://youtube.com/yuilibrary/">YUI Theater</a> Videos</h2>
</div>
<div class="theater">
<p class="youtube-link">
<a class="button" href="http://youtube.com/yuilibrary">
<img src="/yui3/img/misc/youtube-small.png" alt="YouTube icon"
class="youtube-icon">
YUI Theater on YouTube
</a>
</p>
</div>
</div>
</div>
<div class="yui3-u-1-3">
<div class="column-right">
<div class="divider">
<hr>
<h2><a href="http://twitter.com/yuilibrary">@yuilibrary</a> on Twitter</h2>
</div>
<div class="twitter">
<p class="twitter-link">
<a class="button" href="http://twitter.com/yuilibrary">
<img src="/yui3/img/misc/twitter-small.png" alt="Twitter icon"
class="twitter-icon">
Follow YUI on Twitter
</a>
</p>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
<div id="ft"><footer>
<p class="copyright">
All code on this site is licensed under the <a href="/license/">BSD License</a> unless stated otherwise.<br>
<span class="links"><a href="/security/">Security Contact Info</a></span>
</p>
</footer></div>
</div>
<script src="/yui3/combo/js-search-min.js-hoverable-min.js-main-min.js"></script>
<script>
(function() {
var ga = document.createElement('script'); ga.async = true;
ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
}());
</script>
</body>
</html>