Найден корректный robots.txt
Файл управления сканированием сайта роботами
?
Robots.txt указывает поисковым роботам, какие страницы сканировать, а какие — нет. Ошибки в файле могут случайно закрыть важные разделы от индексации.
Robots.txt настроен корректно. Размер файла: 10375 байт. Загружен за: 0сек.
Проверяемая страница не запрещена в robots.txt.
Цепочка редиректов для файла robots.txt:
http://lists.ut.ee/robots.txt
301 MovedPermanently
https://lists.ut.ee/wws/robots.txt
200 OK
Показать содержимое robots.txt
<!DOCTYPE html>
<html class="no-js" lang="en-US" xml:lang="en-US">
<!-- main.tt2 -->
<head>
<meta charset="UTF-8" />
<meta name="generator" content="Sympa 6.2.70" />
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title> UT Sympa - home </title>
<!-- head_ui.tt2 -->
<link rel="stylesheet" href="/static-sympa/js/jquery-ui/jquery-ui.css?v1.12.1" />
<link rel="stylesheet" href="/static-sympa/js/jqplot/jquery.jqplot.css?v1.0.8" />
<link rel="stylesheet" href="/static-sympa/js/jquery-minicolors/jquery.minicolors.css?v2.3.1">
<link rel="stylesheet" href="/static-sympa/fonts/font-awesome/css/font-awesome.css?v4.3.0" />
<link rel="stylesheet" href="/static-sympa/fonts/foundation-icons/foundation-icons.css?v3.0" />
<link rel="stylesheet" href="/static-sympa/js/foundation/css/foundation-float.css?v6.4.2" />
<style>
select[multiple], select[size] { height: auto; background-image: none; }
</style>
<link rel="stylesheet" href="/css-sympa/lists.ut.ee/style.css?h=fe1a2a3550a7ebda8bd16fcde485f3b3" title="Sympa CSS" />
<link rel="shortcut icon" href="/static-sympa/icons/favicon_sympa.png" />
<!-- end head_ui.tt2 -->
<!-- head_javascript.tt2 -->
<script>
<!--
var sympa = {
backText: 'Back',
calendarButtonText: 'Calendar',
calendarFirstDay: 0,
closeText: 'Close',
dayNames: 'Sunday:Monday:Tuesday:Wednesday:Thursday:Friday:Saturday'.split(":"),
dayNamesMin: 'Su:Mo:Tu:We:Th:Fr:Sa'.split(":"),
home_url: '/wws/',
icons_url: '/static-sympa/icons',
lang: 'en-US',
loadingText: 'Please Wait...',
monthNamesShort: 'Jan:Feb:Mar:Apr:May:Jun:Jul:Aug:Sep:Oct:Nov:Dec'.split(":"),
openInNewWinText: 'Open in a new window',
resetText: 'Reset'
};
var lang = 'en-US';
//-->
</script>
<script src="/static-sympa/js/jquery.js?v3.6.0"></script>
<script src="/static-sympa/js/jquery-migrate.js?v1.4.1"></script>
<script src="/static-sympa/js/jquery-ui/jquery-ui.js?v1.12.1"></script>
<script src="/static-sympa/js/jqplot/jquery.jqplot.min.js?v1.0.8"></script>
<script src="/static-sympa/js/jqplot/jqplot.categoryAxisRenderer.min.js?v1.0.8"></script>
<script src="/static-sympa/js/jqplot/jqplot.barRenderer.min.js?v1.0.8"></script>
<script src="/static-sympa/js/jqplot/jqplot.canvasAxisTickRenderer.min.js?v1.0.8"></script>
<script src="/static-sympa/js/jqplot/jqplot.canvasTextRenderer.min.js?v1.0.8"></script>
<script src="/static-sympa/js/jquery-minicolors/jquery.minicolors.min.js?v2.3.1"></script>
<script src="/static-sympa/js/sympa.js?v6.2.70"></script>
<!--[if lt IE 9]>
<script type="text/javascript" src="/static-sympa/js/html5shiv/html5shiv.js?v3.7.2"></script>
<![endif]-->
<script type="text/javascript" src="/static-sympa/js/respondjs/respond.min.js?v1.4.2"></script>
<script src="/static-sympa/js/foundation/js/vendor/what-input.js?v4.2.0"></script>
<script src="/static-sympa/js/foundation/js/foundation.min.js?v6.4.2"></script>
<!-- end head_javascript.tt2 -->
<!-- RSS chanel -->
<link rel="alternate" type="application/xml" title="RSS Latest lists" href="https://lists.ut.ee/wws/rss/latest_lists?count=20"/>
<link rel="alternate" type="application/xml" title="RSS Active lists" href="https://lists.ut.ee/wws/rss/active_lists?count=20"/>
</head>
<body>
<div id="top">
<a class="displayNone" name="top" href="#content-inner">
Skip to Content.
</a>
</div>
<noscript>
<em>Please activate JavaScript in your web browser</em>
</noscript>
<!-- error.tt2 -->
<div id="ErrorMsg" class="reveal medium" data-reveal
aria-labelledby="Error" aria-hidden="true" role="dialog">
<div class="alert-box alert radius">
<i class="fi-x"></i>
ERROR () -
Unknown action
<br />
</div>
</div>
<!-- end error.tt2 -->
<!-- header.tt2 -->
<header>
<div class="title-bar"
data-responsive-toggle="sympa-top-bar" data-hide-for="medium">
<span class="menu-text" aria-hidden="true">
<a href="https://lists.ut.ee/wws" title="https://lists.ut.ee/wws">
<img src="/static-sympa/icons/logo_sympa.png" alt="SYMPA logo" />
</a>
</span>
<div class="title-bar-right">
<div class="title-bar-title">
<span>Sympa Menu</span>
</div>
<a class="menu-icon" href="#" data-toggle="sympa-top-bar"></a>
</div>
</div>
<nav class="top-bar" id="sympa-top-bar" role="navigation">
<ul class="top-bar-left medium-2 columns show-for-medium">
<li class="name">
<span class="menu-text">
<a href="https://lists.ut.ee/wws" title="https://lists.ut.ee/wws">
<img src="/static-sympa/icons/logo_sympa.png" alt="SYMPA logo" />
</a>
</span>
</li>
</ul>
<menu class="top-bar-right">
<ul class="vertical medium-horizontal menu" data-responsive-menu="drilldown medium-dropdown">
<!-- sympa_menu.tt2 -->
<li class="active"><a href="/wws/home"><i class="fi-home"></i> Home</a></li>
<li class="has-dropdown ">
<a href="#"><i class="fa fa-search-plus"></i> Search for List(s)</a>
<ul class="dropdown">
<li class=""><a href="/wws/search_list_request"><i class="fa fa-search"></i> Search form</a></li>
<li class=""><a class="heavyWork" href="/wws/lists"><i class="fa fa-list"></i> Index of Lists</a></li>
<li class=""><a href="/wws/lists_categories" target=""><i class="fa fa-sitemap"></i> Browse lists by categories</a>
</ul>
</li>
<li class="has-dropdown ">
<a href="#"><i class="fa fa-flag"></i> Language</a>
<ul class="dropdown">
<li class=""><a href="/wws/set_lang/et"><i class="fa fa-flag-checkered"></i> Eesti keel</a></li>
<li class=""><a href="/wws/set_lang/en"><i class="fa fa-flag-checkered"></i> English</a></li>
</ul>
</li>
<li class="has-dropdown ">
<a href="#"><i class="fa fa-life-ring"></i> Support</a>
<ul class="dropdown">
<li class=""><a target="_blank" href="https://wiki.ut.ee/display/AA/Meililistid" title="UT Wiki"><i class="fa fa-info-circle"></i> UT Wiki</a></li>
<li class=""><a href="/wws/help" title="Documentation"><i class="fa fa-book"></i> Documentation</a></li>
<li class=""><a href="/wws/help/faq" title="Frequently asked questions"><i class="fa fa-question"></i> FAQ</a></li>
</ul>
</li>
<!-- end sympa_menu.tt2 -->
<!-- login_menu.tt2 -->
<li>
<form id="use-sso" action="/wws" method="post" autocomplete="off">
<fieldset><input type="hidden" name="csrftoken" value="dc96d2862893ba53ba5ed303df6d40ff" />
<input type="hidden" name="list" value="" />
<input type="hidden" name="action" value="sso_login" />
<input type="hidden" name="auth_service_name" value="sympa" />
<span>
<button type="submit" name="action_sso_login" value="SSO Login">
<i class="fa fa-user"></i>
SSO Login
</button>
</span>
</fieldset>
</form>
</li>
<!-- end login_menu.tt2 -->
</ul>
</menu>
</nav>
<h1 class="row">UT Sympa</h1>
</header>
<!-- end header.tt2 -->
<div id="loading"></div>
<!-- aside_menu.tt2 -->
<aside class="medium-3 columns">
</aside>
<!-- end aside_menu.tt2 -->
<!-- home.tt2 -->
<main class="row home" role="main">
<article >
<h1>Welcome</h1>
<p>
This server provides you access to your environment on mailing list server. Starting from this web page, you can perform subscription options, unsubscription, archives, list management and so on.
</p>
</article>
<article>
<h2>Logging In</h2>
<p>Many functions in Sympa require you to identify yourself to the system by logging in, using the login form in the top right menu.</p>
<h2>What would you like to do ?</h2>
</article>
</main>
<section class="home-submenu">
<h1 class="hide">Submenu</h1>
<ul class="row">
<li class="small-12 columns">
<a href="/wws/search_list_request" title="Search for List(s)">
<span class="fa-stack fa-5x">
<i class="fa fa-circle fa-stack-2x "></i>
<i class="fa fa-search fa-stack-1x"></i>
</span>
Search for List(s)
</a>
</li>
</ul>
</section>
<!-- end home.tt2 -->
<div class="displayNone"><a href="#top">Top of Page</a></div>
<div class="scroll-top-wrapper "></div>
<!-- footer.tt2 -->
<footer role="contentinfo">
<img src="/static-sympa/icons/favicon_sympa.png" alt="SYMPA logo" width="20" height="20"/>
<a href="https://www.sympa.community/">Powered by Sympa 6.2.70</a>
</footer>
<!-- end footer.tt2 -->
</body>
<!-- end main.tt2 -->
</html>
Apache