Найден корректный robots.txt
Файл управления сканированием сайта роботами
?
Robots.txt указывает поисковым роботам, какие страницы сканировать, а какие — нет. Ошибки в файле могут случайно закрыть важные разделы от индексации.
Кол-во редиректов для файла robots.txt 3 слишком большое! Это может вызывать затруднение при индексации поисковыми роботами!
Проверяемая страница не запрещена в robots.txt.
Цепочка редиректов для файла robots.txt:
http://fairwarning.org/robots.txt
301 MovedPermanently
http://www.fairwarning.org/robots.txt
301 MovedPermanently
https://archon.library.illinois.edu/archives/index.php?p=collections/controlcard&id=13014
200 OK
Показать содержимое robots.txt
<!DOCTYPE html>
<html lang="en">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
<meta name="og:site_name" content="University of Illinois Archives Holdings Database"/>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>FairWarning Web Archives, 2010-2021 | University of Illinois Archives</title>
<link rel="stylesheet" type="text/css" href="themes/library_web/style.css?v=20240717" />
<link rel="stylesheet" type="text/css" href="themes/library_web/js/cluetip/jquery.cluetip.css" />
<link rel="stylesheet" type="text/css" href="themes/library_web/js/jgrowl/jquery.jgrowl.css" />
<link rel="icon" type="image/ico" href="themes/library_web/images/favicon.ico"/>
<!--[if lte IE 7]>
<link rel="stylesheet" type="text/css" href="themes/library_web/ie.css" />
<link rel="stylesheet" type="text/css" href="themes/themes/library_web/js/cluetip/jquery.cluetip.ie.css" />
<![endif]-->
<script type='text/javascript' src='packages/core/js/jquery.min.js'></script> <script type='text/javascript' src='packages/core/js/jquery-ui.custom.min.js'></script> <script type='text/javascript' src='packages/core/js/jquery-expander.js'></script> <script type="text/javascript" src="themes/library_web/js/jquery.hoverIntent.js"></script>
<script type="text/javascript" src="themes/library_web/js/cluetip/jquery.cluetip.js"></script>
<script type="text/javascript" src="themes/library_web/js/jquery.scrollTo-min.js"></script>
<script type='text/javascript' src='packages/core/js/jquery.jgrowl.min.js'></script> <script type='text/javascript' src='packages/core/js/archon.js'></script><script type='text/javascript' src='packages/digitallibrary/js/archon.js'></script> <script type="text/javascript">
/* <![CDATA[ */
imagePath = 'themes/library_web/images';
jQuery(document).ready(function($) {
$('div.listitem:nth-child(even)').addClass('evenlistitem');
$('div.listitem:last-child').addClass('lastlistitem');
$('#locationtable tr:nth-child(odd)').addClass('oddtablerow');
$('.expandable').expander({
slicePoint: 600, // make expandable if over this x chars
widow: 100, // do not make expandable unless total length > slicePoint + widow
expandPrefix: '. . . ', // text to come before the expand link
expandText: 'more', //text to use for expand link
expandEffect: 'fadeIn', // or slideDown
expandSpeed: 0, // in milliseconds
collapseTimer: 0, // milliseconds before auto collapse; default is 0 (don't re-collape)
userCollapseText: '[collapse]' // text for collaspe link
});
$('.expandablesmall').expander({
slicePoint: 100, // make expandable if over this x chars
widow: 10, // do not make expandable unless total length > slicePoint + widow
expandPrefix: '. . . ', // text to come before the expand link
expandText: 'more', //text to use for expand link
expandEffect: 'fadeIn', // or slideDown
expandSpeed: 0, // in milliseconds
collapseTimer: 0, // milliseconds before auto collapse; default is 0 (don't re-collape)
userCollapseText: '[collapse]' // text for collaspe link
});
});
function js_highlighttoplink(selectedSpan)
{
$('.currentBrowseLink').toggleClass('browseLink').toggleClass('currentBrowseLink');
$(selectedSpan).toggleClass('currentBrowseLink');
$(selectedSpan).effect('highlight', {}, 400);
}
$(document).ready(function() {externalLinks();});
$(window).unload(function() {});
/* ]]> */
</script>
<script src="https://ajax.googleapis.com/ajax/libs/jquery/3.5.1/jquery.min.js"></script>
<script>
var $k =jQuery.noConflict();
$k(document).ready(function(){
$k("#ccardstaff .locationFilter").on("keyup", function() {
var value = $k(this).val().toLowerCase();
$k("#ccardstaff .locationTableBody tr").filter(function() {
var $t = $(this).children().last();
$k(this).toggle($k($t).text().toLowerCase().indexOf(value) > -1)
});
});
$k("#ccardstaff .staffBoxFilter").on("keyup", function() {
var value = $k(this).val().toLowerCase();
$k("#ccardstaff .locationTableBody tr").filter(function() {
var $h = $(this).children().first();
$k(this).toggle($k($h).text().toLowerCase().indexOf(value) > -1)
});
});
$k("#ccardstaff .staffLocationFilter").on("keyup", function() {
var value = $k(this).val().toLowerCase();
$k("#ccardstaff .locationTableBody tr").filter(function() {
$k(this).toggle($k(this).text().toLowerCase().indexOf(value) > -1)
});
});
$k("#stafflocationtable .staffBoxFilter").on("keyup", function() {
var value = $k(this).val().toLowerCase();
$k("#stafflocationtable .locationTableBody tr").filter(function() {
var $h = $(this).children().first();
$k(this).toggle($k($h).text().toLowerCase().indexOf(value) > -1)
});
});
$k("#stafflocationtable .staffLocationFilter").on("keyup", function() {
var value = $k(this).val().toLowerCase();
$k("#stafflocationtable .locationTableBody tr").filter(function() {
$k(this).toggle($k(this).text().toLowerCase().indexOf(value) > -1)
});
});
});
</script>
</head>
<body>
<div id='top'>
<div id="logosearchwrapper">
<div id="sitetitleblock"><a href="https://archives.library.illinois.edu">University of Illinois Archives</a> –> <a href="index.php">Holdings Database</a></div>
<div id="searchblock">
<form action="index.php" accept-charset="UTF-8" method="get" onsubmit="if(!this.q.value) { alert('Please enter search terms.'); return false; } else { return true; }">
<div>
<input type="hidden" name="p" value="core/search" />
<input type="text" size="25" maxlength="150" name="q" id="q" title="input box for search field" value="" tabindex="100" />
<input type="submit" value="Search" tabindex="300" class='button' title="Search" /> <a class='bold pdfsearchlink' href='?p=core/index&f=pdfsearch'>Search PDF lists</a>
<input type="hidden" name="content" value="0" />
</div></form></div>
</div>
<div id="browsebyblock">
<div id="browseblockcontent">
<span class="browseLink">
<a href="?p=collections/classifications" onclick="js_highlighttoplink(this.parentNode); return true;">Campus Units/Record Groups</a>
</span>
<span class="currentBrowseLink">
<a href="?p=collections/collections" onclick="js_highlighttoplink(this.parentNode); return true;">Physical Collections: A-Z</a>
</span>
<span class="browseLink">
<a href="?p=digitallibrary/digitallibrary" onclick="js_highlighttoplink(this.parentNode); return true;">Digital Materials: A-Z</a>
</span>
<span class="browseLink">
<a href="?p=subjects/subjects" onclick="js_highlighttoplink(this.parentNode); return true;">Subject Headings</a>
</span>
<span class="browseLink">
<a href="?p=creators/creators" onclick="js_highlighttoplink(this.parentNode); return true;">Record Creators</a>
</span>
</div>
</div>
</div>
<div id="breadcrumbblock">
<a href='https://archives.library.illinois.edu'>University Archives</a> <img src='themes/library_web/images/arrowright.gif' alt='right-pointing arrow' /> <a href='index.php'>Holdings</a> <img src='themes/library_web/images/arrowright.gif' alt='right-pointing arrow' /> <a href='?p=collections/classifications&id=36'>University Library</a> <img src='themes/library_web/images/arrowright.gif' alt='right-pointing arrow' /> <a href='?p=collections/classifications&id=563'>University and Non-Univeris...</a> <img src='themes/library_web/images/arrowright.gif' alt='right-pointing arrow' /> FairWarning Web Archives </div>
<div id="breadcrumbclearblock">.</div>
<script type="text/javascript">
/* <![CDATA[ */
if ($.browser.msie && parseInt($.browser.version, 10) <= 8){
$.getScript('packages/core/js/jquery.corner.js', function(){
$("#searchblock").corner("5px");
$("#browsebyblock").corner("tl 10px");
$(function(){
$(".bground").corner("20px");
$(".mdround").corner("10px");
$(".smround").corner("5px");
$("#dlsearchblock").corner("bottom 10px");
});
});
}
/* ]]> */
</script> <div id="main">
<h1 id='titleheader'>FairWarning Web Archives, 2010-2021 | University of Illinois Archives</h1>
<div id='ccardleft'>
<div id="ccardpublic" class='mdround'>
<div class='ccardcontent'><span class='ccardlabel'>Title:</span> FairWarning Web Archives, 2010-2021</div>
<div class='ccardcontent'><span class='ccardlabel'>Series Number:</span> 35/4/5</div>
<div class='ccardcontent'><span class='ccardlabel'>Acquired:</span>
12/07/2021. </div>
<div class='ccardcontent'><span class='ccardlabel'>Volume:</span> 113800.0 megabytes </div>
<div class='ccardcontent'><span class='ccardlabel'><a href='#' onclick="toggleDisplay('creators'); return false;"><img id='creatorsImage' src='themes/library_web/images/plus.gif' alt='expand icon'/> Creator(s)</a></span><br/>
<div class='ccardshowlist' style="display: none;" id="creatorsResults"><a href='?p=creators/creator&id=3260'>FairWarning (2010-2021)</a></div></div>
<div class='ccardcontent'><span class='ccardlabel'><a href='#' onclick="toggleDisplay('BiogHist'); return false;"><img id='BiogHistImage' src='themes/library_web/images/plus.gif' alt='expand icon'/>
Administrative History of Creating Unit </a></span><br/>
<div class='ccardshowlist' style='display:none' id='BiogHistResults'>
FairWarning was a nonprofit investigative news organization with a focus on public health, consumer, workplace, and environmental issues, and related topics, and related topics of government and business accountability. FairWarning began publication in March 2010 and was a member of the Institute for Nonprofit News. </div>
</div>
<div class='ccardcontent'><span class='ccardlabel'><a href='#' onclick="toggleDisplay('restriction'); return false;"><img id='restrictionImage' src='themes/library_web/images/plus.gif' alt='expand icon'/> Access Restrictions</a></span><br/>
<div class='ccardshowlist' style="display: none;" id="restrictionResults">None.</div>
</div>
<div class='ccardcontent'><span class='ccardlabel'><a href='#' onclick="toggleDisplay('subjects'); return false;"><img id='subjectsImage' src='themes/library_web/images/plus.gif' alt='expand icon'/> Subjects</a> <span style='font-size:80%'>(links to similar materials)</span></span><br/>
<div class='ccardshowlist' style='display: none' id='subjectsResults'><a href='?p=subjects/subjects&id=5753'>FairWarning</a><br/>
<a href='?p=subjects/subjects&id=2571'>Journalism</a><br/>
<a href='?p=subjects/subjects&id=5563'>Web Archives</a></div>
</div>
</div><div id='ccardstaff' class='mdround'> <span class='ccardlabel' id='requestlocations'>Locations for this record series:</span><br/>
<ul><li>Digital files: May require 24 hours notice for retrieval</li></ul><a id='requestModalLink2' tabindex='0' role='button'><img src='themes/library_web/images/box.png' alt='Request' style='padding-right:2px'/>Submit request (Aeon)</a>
<script>
var modal = document.getElementById("requestModal");
var btn2 = document.getElementById("requestModalLink2");
btn2.addEventListener('click', function() {
openRequestModal();
});
btn2.addEventListener("keypress", function(event) {
if (event.key === "Enter") {
openRequestModal();
}
});
btn2.addEventListener("keydown", function(event) {
if (event.keyCode == 32) {
event.preventDefault();
openRequestModal();
}
});
</script>
</div>
</div> <!--end ccardstaffdiv -->
</div><div id='ccardprintcontact' class='smround'><span id='requestModalLink' tabindex='0' role='button'><img src='themes/library_web/images/box.png' alt='Request' style='padding-right:2px'/>Submit request (Aeon)</span> |
<!-- The Modal to show request locations -->
<div id="requestModal" class="request-modal" style="display:none">
<!-- Modal content -->
<div class="request-modal-content" aria-label="Submit request options" role="dialog" aria-modal="true">
<span class="request-modal-close" aria-label="close" tabindex="0" role='button'>×</span>
<span class='ccardlabel' id='requestlocations'>Locations for this record series:</span><br/>
<ul class='locationsummary'><li>Digital files: May require 24 hours notice for retrieval</li></ul> <table id='locationtable' border='1' style='margin-left:0'>
<thread><tr>
<th style='width:400px'>Service Location</th>
<th style='width:100px'>Boxes</th>
<th style='width:150px'>Request</th> </tr></thread>
<tbody class='locationTableBody'>
<tr><td>Digital files: May require 24 hours notice for retrieval</td><td>e-records<td><a href='https://armarium.library.illinois.edu/logon?Action=10&Form=20&Value=GenericRequestUAArchives&ItemTitle=FairWarning+Web+Archives%2C+2010-2021&CallNumber=RS 35/4/5&ItemDate=2010-2021&ItemInfo4=113800.0 megabytes&SpecialRequest=Access restrictions: None.&DocumentType=Publications&Site=ILLINOISARCHIVES&Location=DIG&ItemISxN=e-records' target='_blank' class='request-button'>Request e-records</a></td></td></tr></tbody></table> </div>
</div>
<script>
var modal = document.getElementById("requestModal");
var btn = document.getElementById("requestModalLink");
var span = document.getElementsByClassName("request-modal-close")[0];
function openRequestModal() {
modal.style.display = "block";
span.focus();
document.addEventListener('keydown', addESC);
}
function closeRequestModal(){
modal.style.display = "none";
btn.focus();
document.removeEventListener('keydown', addESC);
}
var addESC = function(e) {
if (e.keyCode == 27) {
closeRequestModal();
}
};
const focusableElements =
'button, [href], input, select, textarea, [tabindex]:not([tabindex="-1"])';
const firstFocusableElement = modal.querySelectorAll(focusableElements)[0]; // get first element to be focused inside modal
const focusableContent = modal.querySelectorAll(focusableElements);
const lastFocusableElement = focusableContent[focusableContent.length - 1]; // get last element to be focused inside modal
document.addEventListener('keydown', function(e) {
let isTabPressed = e.key === 'Tab' || e.keyCode === 9;
if (!isTabPressed) {
return;
}
if (e.shiftKey) { // if shift key pressed for shift + tab combination
if (document.activeElement === firstFocusableElement) {
lastFocusableElement.focus(); // add focus for the last focusable element
e.preventDefault();
}
} else { // if tab key is pressed
if (document.activeElement === lastFocusableElement) { // if focused has reached to last focusable element then focus first focusable element after pressing tab
firstFocusableElement.focus(); // add focus for the first focusable element
e.preventDefault();
}
}
});
btn.addEventListener('click', function() {
openRequestModal();
});
btn.addEventListener("keypress", function(event) {
if (event.key === "Enter") {
openRequestModal();
}
});
btn.addEventListener("keydown", function(event) {
if (event.keyCode == 32) {
event.preventDefault();
openRequestModal();
}
});
span.addEventListener('click', function() {
closeRequestModal();
});
span.addEventListener("keypress", function(event) {
if (event.key === "Enter") {
closeRequestModal();
}
});
span.addEventListener("keydown", function(event) {
if (event.keyCode == 32) {
event.preventDefault();
closeRequestModal();
}
});
window.onclick = function(event) {
if (event.target == modal) {
closeRequestModal();
}
}
const filterBox = document.querySelector('#requestModal .locationFilter');
var $k =jQuery.noConflict();
filterBox.addEventListener('input', filterBoxes);
function filterBoxes(e) {
var filterValue = (e.target.value).toLowerCase();
$k("#requestModal .locationTableBody tr").filter(function() {
var $t = $k(this).children().last();
$k(this).toggle($k($t).text().toLowerCase().indexOf(filterValue) > -1)
});
}
const filterSelect = document.querySelector('#requestModal #locations');
if (filterSelect){
filterSelect.addEventListener('input',filterSelection);
}
function filterSelection(f) {
var selectValue = $k('#requestModal #locations option:selected').text().toLowerCase();
if(selectValue =="all locations"){
selectValue =" ";
}
$k("#requestModal .locationTableBody tr").filter(function() {
$k(this).toggle($k(this).text().toLowerCase().indexOf(selectValue) > -1)
});
}
</script>
<a href='https://archives.library.illinois.edu/email-ahx.php?this_page=https://archon.library.illinois.edu%2Farchives%2Findex.php%3Fp%3Dcollections%2Fcontrolcard%26id%3D13014' target='_blank'><img src='themes/library_web/images/email.png' alt='email' /> Email us about these publications</a> | <a href='?p=collections/controlcard&id=13014&templateset=print&disabletheme=1'><img src='themes/library_web/images/printer.png' alt='printer' /> Print this information</a></div> <div id="ccardscope" class="mdround">
<div class='ccardcontent expandable' style='padding-left:.2em'><span class='ccardlabel'>Description:</span> <p>The FairWarning web archives includes 113.8 gigabytes of online news articles relating to consumer protection, environment, labor, public health, and transportation safety. Articles written span from 2010-2021 and were collected using Archive-It, the web archiving service built by the Internet Archive.</p><p> FairWarning was a nonprofit investigative news organization with a focus on public health, consumer, workplace, and environmental issues, and related topics, and related topics of government and business accountability. FairWarning began publication in March 2010 and was a member of the Institute for Nonprofit News.</p><p> FairWarning donated its website to the University of Illinois at Urbana-Champaign University Archives in the Fall of 2021.</p><p> The web archives are avaiable at <a href="https://wayback.archive-it.org/16877/20210929142827/https://www.fairwarning.org/" rel="external">https://wayback.archive-it.org/16877/20210929142827/https://www.fairwarning.org/</a></p></div>
<div class='ccardcontent expandablesmall' style="padding-left:.2em" id="AccessRestrictResults"><span class='ccardlabel'>Access Restriction:</span> None.</div>
<div style='clear:both'><pre></pre></div> </div> <!--end ccard scope -->
</div>
<div id="bottom">
<br/>
<hr id="footerhr" />
<div id="userbox" class="smround">
<div id="userlogincontrols">
<a id="loginlink" href="index.php?p=admin/core/login&go=" onclick="if($('#userlogin').is(':visible')) {this.innerHTML = 'Log In (Staff)';} else {this.innerHTML = 'Hide';} $('#userlogin').slideToggle('normal'); return false;">Log In (Staff)</a>
</div>
<div id="userlogin" class="mdround" style="display:none">
<form action="/archives/index.php?p=collections/controlcard&id=13014" accept-charset="UTF-8" method="post">
<div class='loginpair'>
<div class='loginlabel'><label for="ArchonLoginField">Login/E-mail:</label></div>
<div class='logininput'><input id="ArchonLoginField" type="text" name="ArchonLogin" size="20" tabindex="400" /></div>
</div>
<div class='loginpair'>
<div class='loginlabel'><label for="ArchonPasswordField">Password:</label></div>
<div class='logininput'><input id="ArchonPasswordField" type="password" name="ArchonPassword" size="20" tabindex="500" /></div>
</div>
<div id='loginsubmit'>
<input type="submit" value="Log In" class="button" tabindex="700" /> <label for="RememberMeField"><input id="RememberMeField" type="checkbox" name="RememberMe" value="1" tabindex="600" />Remember me</label>
</div>
<div id='registerlink'>
<a href="?p=core/register" tabindex="800">Register an Account</a>
</div>
</form>
</div>
</div>
<div id='contactcontainer'><div id='repositorylink'><a href='http://archives.library.illinois.edu'>University of Illinois Archives</a></div>
<div id='emaillink'>Contact Us: <a href='http://www.library.uiuc.edu/archives/email-ahx.php?this_page=https://archon.library.illinois.edu%2Farchives%2Findex.php%3Fp%3Dcollections%2Fcontrolcard%26id%3D13014'>Email Form</a></div>
</div></div>
<div id="archoninfo">
Page Generated in: 0.167 seconds (using 63 queries).<br/>
Using 6.43MB of memory. (Peak of 6.83MB.)<br/>
<br/>
Powered by <a href='https://web.archive.org/web/20170612215939/http://archon.org/'>Archon</a> Version 3.21 rev-3<br/>
Copyright ©2017 <a href="http://www.uiuc.edu/" rel="external">The University of Illinois at Urbana-Champaign</a> <br/>
</div>
</body>
</html>
PrestaShop
Apache