Найден корректный robots.txt
Файл управления сканированием сайта роботами
?
Robots.txt указывает поисковым роботам, какие страницы сканировать, а какие — нет. Ошибки в файле могут случайно закрыть важные разделы от индексации.
Robots.txt настроен корректно. Размер файла: 15570 байт. Загружен за: 1сек.
Проверяемая страница не запрещена в robots.txt.
Цепочка редиректов для файла robots.txt:
http://oeit.mit.edu/robots.txt
302 Found
https://oeit.mit.edu/
200 OK
Показать содержимое robots.txt
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en" dir="ltr">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<title>Home | OEIT</title>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<link rel="EditURI" type="application/rsd+xml" title="RSD" href="blogapi/rsd/index.html" />
<link rel="shortcut icon" href="sites/default/files/favicon_1.png" type="image/x-icon" />
<link rel="alternate" type="application/rss+xml" title="Blog RSS Feed" href="blog/feed/index.html" />
<link type="text/css" rel="stylesheet" media="all" href="modules/node/node-J.css" />
<link type="text/css" rel="stylesheet" media="all" href="modules/system/defaults-J.css" />
<link type="text/css" rel="stylesheet" media="all" href="modules/system/system-J.css" />
<link type="text/css" rel="stylesheet" media="all" href="modules/system/system-menus-J.css" />
<link type="text/css" rel="stylesheet" media="all" href="modules/user/user-J.css" />
<link type="text/css" rel="stylesheet" media="all" href="sites/all/modules/cck/theme/content-module-J.css" />
<link type="text/css" rel="stylesheet" media="all" href="sites/all/modules/ckeditor/ckeditor-J.css" />
<link type="text/css" rel="stylesheet" media="all" href="sites/all/modules/date/date-J.css" />
<link type="text/css" rel="stylesheet" media="all" href="sites/all/modules/date/date_popup/themes/datepicker-J.css" />
<link type="text/css" rel="stylesheet" media="all" href="sites/all/modules/date/date_popup/themes/jquery.timeentry-J.css" />
<link type="text/css" rel="stylesheet" media="all" href="sites/all/modules/filefield/filefield-J.css" />
<link type="text/css" rel="stylesheet" media="all" href="sites/all/modules/views_slideshow/contrib/views_slideshow_singleframe/views_slideshow-J.css" />
<link type="text/css" rel="stylesheet" media="all" href="sites/all/modules/cck/modules/fieldgroup/fieldgroup-J.css" />
<link type="text/css" rel="stylesheet" media="all" href="sites/all/modules/views/css/views-J.css" />
<link type="text/css" rel="stylesheet" media="all" href="sites/all/themes/oeit/style-J.css" />
<link type="text/css" rel="stylesheet" media="all" href="sites/all/themes/oeit/lev2-J.css" />
<script type="text/javascript" src="misc/jquery.js?J"></script>
<script type="text/javascript" src="misc/drupal.js?J"></script>
<script type="text/javascript" src="sites/all/modules/google_analytics/googleanalytics.js?J"></script>
<script type="text/javascript" src="sites/all/modules/swftools/shared/swfobject2/swfobject.js?J"></script>
<script type="text/javascript" src="sites/all/modules/views_slideshow/js/jquery.cycle.all.min.js?J"></script>
<script type="text/javascript" src="sites/all/modules/views_slideshow/contrib/views_slideshow_singleframe/views_slideshow.js?J"></script>
<script type="text/javascript">
<!--//--><![CDATA[//><!--
jQuery.extend(Drupal.settings, { "basePath": "/", "googleanalytics": { "trackOutbound": 1, "trackMailto": 1, "trackDownload": 1, "trackDownloadExtensions": "7z|aac|arc|arj|asf|asx|avi|bin|csv|doc|exe|flv|gif|gz|gzip|hqx|jar|jpe?g|js|mp(2|3|4|e?g)|mov(ie)?|msi|msp|pdf|phps|png|ppt|qtm?|ra(m|r)?|sea|sit|tar|tgz|torrent|txt|wav|wma|wmv|wpd|xls|xml|z|zip" } });
//--><!]]>
</script>
<meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1" />
<meta http-equiv="Content-Language" content="en-us" />
<meta name="robots" content="all" />
<meta name="author" content="OEIT" />
<meta name="description" content="OEIT | " />
<link rel="stylesheet" href="sites/all/themes/oeit/home.css" type="text/css" />
</head>
<body>
<div class="offscreen">
<a href="#startcontent">
<!--<img src="/images/spacer.gif" width="1" height="1" border="0" alt="Skip to content" />
-->
</a>
</div>
<!-- header -->
<div id="divheader">
<!-- top stuff - long MIT and search tool -->
<!-- top stuff - long mit and search tool -->
<div id="divtop">
<p id="divmitlong"><a href="http://web.mit.edu/"><img src="images/mitlong.gif" width="220" height="23" border="0" alt="Massachusetts Institute of Technology" title="MIT home" /></a></p>
<!-- search -->
<div id="divsearch">
<form id="searchform" method="get" action="http://search.mit.edu/search">
<input type='hidden' name='site' value='mit' />
<input type='hidden' name='client' value='mit' />
<input type='hidden' name='proxystylesheet' value='mit' />
<input type='hidden' name='output' value='xml_no_dtd' />
<input type='hidden' name='as_dt' value='i' />
<!--<input type='hidden' name='proxystylesheet'
value='http://web.mit.edu/oeit/styles/google-oeit.xsl'/>-->
<!--<input type="hidden" name="proxystylesheet" value="http://web.mit.edu/oeit/google-oeit.v4.xsl" />-->
<input type='hidden' name='reload' value='1'/>
<input type='hidden' name='debug' value='1'/>
<input type='hidden' name='as_sitesearch' value='oeit.mit.edu' />
<table cellspacing="0" cellpadding="0" border="0" class="layouttable">
<tr>
<td class="label"><label for="q"><img src="images/search.gif" width="49" height="11" border="0" alt="Search" title="" /></label></td>
<td class="input"><input type="text" name="q" id="q" size="8" value="" maxlength="255" class="inputbox" /></td>
<td class="button"><input type="image" name="gosearch" id="gosearch" src="images/go.gif" class="gobutton" alt="Go" title="" /></td>
</tr>
</table>
</form>
</div><!-- divsearch -->
</div><!-- divtop -->
<!-- primary navigation -->
<!-- primary navigation -->
<div id="divprimarynav">
<ul class="links primary-links"><li class="menu-162 first"><a href="opportunities-you/index.html" title="">Opportunities</a></li>
<li class="menu-178"><a href="gallery/index.html" title="">Gallery</a></li>
<li class="menu-191"><a href="community/index.html" title="">Community</a></li>
<li class="menu-192"><a href="resources/index.html" title="">Resources</a></li>
<li class="menu-193 last"><a href="about/index.html" title="">About OEIT</a></li>
</ul> </div><!-- divprimarynav -->
<!-- banner - logo and photo -->
<div id="divbanner">
<h1><a href="index.html"><img src="images/oeitlogo-home.gif" border="0" alt="OEIT - Office of Educational Innovation and Technology" title="OEIT home" /></a></h1>
<div id="divphoto">
<div id="block-block-16" class="block block-block">
<div class="content">
<p>
<a href="http://www.facebook.com/pages/Office-of-Educational-Innovation-and-Technology/193527290762667"><img alt="Facebooks" src="sites/default/files/facebook.png" style="width: 24px; height: 24px; " /></a> <a href="https://twitter.com/#!/OEIT_MIT"><img alt="Twitter" src="sites/default/files/twitter.gif" style="width: 23px; height: 24px; " /></a></p>
<style>
#block-block-16 { margin-top:-35px; float:right; }
#block-block-16 p { margin:0px; }
#block-block-16 a { display:inline-block; margin-left:5px; }
</style> </div>
</div>
<div id="block-block-6" class="block block-block">
<div class="content">
<p>
<img alt="students in Haynes Miller's mathematics class" height="170" src="images/home-header.jpg" title="students in Haynes Miller's mathematics class" width="295" /></p>
</div>
</div>
</div>
</div>
</div><!-- divbanner -->
</div><!-- divheader -->
<!-- body of page - secondary navigation, main content, related links -->
<div id="divbody">
<div id="node-15" class="node ">
<div class="content clear-block">
</div>
<div class="meta">
</div>
</div><div id="block-block-8" class="block block-block">
<div class="content">
<h4 class="rteindent1"> </h4>
<h4 class="rteindent1"><span style="font-size: 1em;">Please note: </span></h4>
<h4 class="rteindent1"><span style="font-size: 1em;">This is an archive site for a department that is no longer active.</span></h4>
<p class="rteindent4"><em><span style="font-size: 1em;">- March 31, 2015</span></em></p>
<address class="rteindent1"> </address>
<address class="rteindent1"> </address>
<p> </p>
</div>
</div>
<div id="block-views-news-block_1" class="block block-views">
<div class="content">
<div class="view view-news view-id-news view-display-id-block_1 view-dom-id-1">
<div class="view-header">
<h2><a href="drupal/community/news/index.html"><img src="drupal/images/newsheading.gif" border="0" alt="News and Announcements:" title="" /></a></h2>
</div>
<div class="view-content">
<div class="views-row views-row-1 views-row-odd views-row-first">
<div class='newslistitem'><i><span class="date-display-single">March 12, 2015</span></i><br /><span><a href="http://campaign.r20.constantcontact.com/render?ca=d014f769-3b65-4887-8e52-a64c63734513&c=ee7b7e60-5fef-11e3-b089-d4ae5292c3f3&ch=ef7d0fe0-5fef-11e3-b0ff-d4ae5292c3f3">Register for NMC Summer Conference</a>, t<span style="font-size: 1em;">he Global Ed Tech Forum for Higher Ed, Museums, Libraries, and Schools, <strong>June 9-11, Washington D.C.</strong></span></span>...<a href="community/news/register-nmc/index.html">read more</a></div> </div>
<div class="views-row views-row-2 views-row-even views-row-last">
<div class='newslistitem'><i><span class="date-display-single">March 10, 2015</span></i><br /><span><a href="http://web.mit.edu/macvicar/macvicarday.html">MacVicar Day</a> at MIT celebrating excellence in teaching: Friday, March 13, 2015 Symposium: 2:00 – 4:00 PM Bartos Theater, E15-070, 20 Ames Street, Cambridge, MA 02139 </span>...<a href="community/news/macvicar-day-313-2-4/index.html">read more</a></div> </div>
</div>
</div> </div>
</div>
<div id="block-views-projects-block_4" class="block block-views">
<div class="content">
<div class="view view-projects view-id-projects view-display-id-block_4 view-dom-id-2">
<div class="view-header">
<h4><img src="drupal/images/featuredprojectheading.gif" alt="Featured Project" /></h4>
</div>
<div class="view-content">
<div class="views-row views-row-1 views-row-odd views-row-first views-row-last">
<div id="featured-project"><div id="fp-content">
<h4>StarMolsim Makes Students Better Engineers</h4><p><strong>Introduction and Educational Need</strong><br />
Students in civil engineering usually explore the components of buildings and other structures from a macro viewpoint. However, many critical elements of good design and safe structural engineering happen at the atomic level, such as material bonding and chemical properties. The StarMolsim tool allows students to learn these concepts through hands-on simulations. </p>
<p>The cross-discipline MIT Course 3.021 uses StarMolsim as...</p><i><a href="gallery/projects/starmolsim/index.html">read more</a></i></div>
<div id="featured-quote"><span class="fqquote-left"><img src="sites/default/files/quotes-left.gif"></span><p>Students quickly understand the tools' feature-set without having to acquire any extra programming or operating system skills. </p><span class="fqquote-right"><img src="sites/default/files/quotes-right.gif"></span><p style="text-align:right">—Justin Riley, OEIT</p></div><div id="featured-project-image"><a href="gallery/projects/starmolsim/index.html" class="imagecache imagecache-medium imagecache-linked imagecache-medium_linked"><img src="sites/default/files/imagecache/medium/staff/J-Mol.png" alt="" title="" width="250" height="214" class="imagecache imagecache-medium"/></a></div>
</div> </div>
</div>
</div> </div>
</div>
<div id="block-views-blog_home-block_2" class="block block-views">
<div class="content">
<div class="view view-blog-home view-id-blog_home view-display-id-block_2 view-dom-id-3">
<div class="view-header">
<h4><a href="drupal/blog/index.html"><img src="drupal/images/blogheading.gif" alt="Blog" /></a></h4>
</div>
<div class="view-content">
<div class="views-row views-row-1 views-row-odd views-row-first">
<div class="views-field-field-date-value">
<span class="field-content"><span class="date-display-single">March 20, 2015</span></span>
</div>
<div class="views-field-title">
<span class="field-content"><a href="blog/nmc-conference-promises-multiple/675/index.html">NMC Conference promises multiple tracks, rich program</a></span>
</div>
<div class="views-field-body">
<span class="field-content">
Learn new skills at NMC Academy Workshops for the
2015...</span>
</div>
</div>
<div class="views-row views-row-2 views-row-even views-row-last">
<div class="views-field-field-date-value">
<span class="field-content"><span class="date-display-single">February 11, 2015</span></span>
</div>
<div class="views-field-title">
<span class="field-content"><a href="blog/nmc-releases-2015-horizon-report/671/index.html">NMC releases 2015 Horizon Report</a></span>
</div>
<div class="views-field-body">
<span class="field-content">
NMC and EDUCAUSE Learning Initiative Release the
NMC Horizon Report
2015 Higher Ed Edit...</span>
</div>
</div>
</div>
<div class="feed-icon">
<a href="blog/feed/index.html" class="feed-icon"><img src="misc/feed.png" alt="Syndicate content" title="Blog RSS Feed" width="16" height="16" /></a> </div>
</div> </div>
</div>
<div class="tinyclear"></div>
</div><!-- divbody -->
<div id="divfooter">
<div id="block-block-11" class="block block-block">
<div class="content">
<p id="divmitlogo"><a href="http://web.mit.edu/"><img alt="MIT home" border="0" height="20" src="images/mit.gif" title="MIT home" width="36" /></a></p>
<p id="divaddress">Office of Educational Innovation and Technology<br />
<a href="http://whereis.mit.edu/map-jpg?selection=NE48">Building NE48-308</a>, Cambridge, MA 02139-4307<br />
Phone: (617) 252-1981; Fax: (617) 452-4044</p>
<div id="divtoolnav">
<ul>
<li><a href="about/contact/index.html">Contact</a> | <a href="sitemap/index.html">Site map</a> | <a href="http://web.mit.edu/star">STAR</a> | <a href="http://web.mit.edu/cet">MITCET</a> | <a href="resources/learning-environments/index.html" title="Experimental Learning Environments">ELE</a> | <a href="about/tsa/index.html">Transmedia</a></li>
<li><a href="http://odl.mit.edu">Office of Digital Learning (ODL)</a></li>
</ul>
<div class="tinyclear"> </div>
</div>
</div>
</div>
</div><!-- divfooter -->
<script type="text/javascript">
<!--//--><![CDATA[//><!--
var _gaq = _gaq || [];_gaq.push(["_setAccount", "UA-10400361-1"]);_gaq.push(["_trackPageview"]);(function() {var ga = document.createElement("script");ga.type = "text/javascript";ga.async = true;ga.src = ("https:" == document.location.protocol ? "https://ssl" : "http://www") + ".google-analytics.com/ga.js";var s = document.getElementsByTagName("script")[0];s.parentNode.insertBefore(ga, s);})();
//--><!]]>
</script>
</body>
</html>
<!-- Localized -->
PrestaShop
Apache