User:Magicpiano/NRBot/UpdateNRHPProgress2.js
< User:Magicpiano | NRBot
| Code that you insert on this page could contain malicious content capable of compromising your account. If you import a script from another page with "importScript", "mw.loader.load", "iusc", or "lusc", take note that this causes you to dynamically load a remote script, which could be changed by others. Editors are responsible for all edits and actions they perform, including by scripts. User scripts are not centrally supported and may malfunction or become inoperable due to software changes. A guide to help you find broken scripts is available. If you are unsure whether code you are adding to this page is safe, you can ask at the appropriate village pump. This code will be executed when previewing this page. |
 | Documentation for this user script can be added at User:Magicpiano/NRBot/UpdateNRHPProgress2. |
/* jshint maxerr: 3000 */
/*
The following script places a button at the top of the NRHP Progress page [[WP:NRHPPROGRESS]]. When the button is clicked,
the script begins to load each county list linked from the Progress page in the background, extract statistics about
sites in each list, and updates the Progress page with the fetched data.
*/
//var wikitext = 'error';
var ProgressStructure=[]; // ProgressStructure[table][row].StatName
var TotalToQuery=0;
var TotalQueried=0;
var ErrorCount=0;
var WarningCount=[["",0]]; // 0=status, 1=count
var InitialTime=0;
var ProgressDivTimer=0; // timer for updating ProgressDiv
var DefaultQueryPause=1; // number of milliseconds to wait between each API query; increased by code if rate limit reached
var RestrictedImageCount=0;
var bIgnoreTableCount = false;
// Users authorized to run this script
var AuthorizedUsers = [ "Magicpiano" ]; // nb this is for use only when g_disabled is true
var CurrentMaintainer = "Magicpiano"; // User who is currently maintaining the script
function getRandomIntInclusive(min, max) {
min = Math.ceil(min);
max = Math.floor(max);
return Math.floor(Math.random() * (max - min + 1) + min); //The maximum is inclusive and the minimum is inclusive
}
//
// WorkQueue helper
//
// run up to maxActiveSize work items at a time
//
function WorkQueue(maxActiveSize = 1, initialWorkQueue = [])
{
this.maxActiveSize = maxActiveSize;
this.pendingQueue = initialWorkQueue;
this.activeQueue = [];
this.completedQueue = [];
this.isRunning = false;
if (initialWorkQueue.length > 0) this.run();
}
// determine if we have room to run something more, and if there is something more to run
WorkQueue.prototype.runNext = function()
{
return ((this.activeQueue.length < this.maxActiveSize) && this.pendingQueue.length > 0);
};
WorkQueue.prototype.createPromise = function(work, retryCount=0)
{
var p = new Promise((result, reject) => work().then(result).catch(reject));
p.work = work;
p.retryCount = retryCount;
}
// add a work item
WorkQueue.prototype.push = function(p, retryCount=0)
{
this.pendingQueue.push(this.createPromise(p,retryCount));
};
// add a list of work items
WorkQueue.prototype.pushList = function(plist, retryCount=0)
{
for (var idx in plist)
{
const p = plist[idx];
this.pendingQueue.push(this.createPromise(p,retryCount));
}
};
WorkQueue.prototype.getCompletedCount = function()
{
return this.completedQueue.length;
}
// run (or attempt to) the queue
WorkQueue.prototype.run = function()
{
if (this.isRunning) return;
this.isRunning = true;
this.nFailed = 0;
this.nSucceeded = 0;
this.nCompleted = 0;
this.nEntries = this.pendingQueue.length;
var ranAtLeastOnce = false;
const wq = this;
try {
while (this.runNext())
{
if (!ranAtLeastOnce)
{
console.log('Start running queue',this);
ranAtLeastOnce = true;
}
const promise = this.pendingQueue.shift();
console.log('Running promise',promise);
promise
.then((result) => {
wq.nSucceeded++;
})
.catch((e) => {
console.log('Promise failed',promise,e);
if (promise.retryCount > 0)
{
console.log('Retrying');
wq.push(promise.work, promise.retryCount-1);
}
})
.finally(() => {
wq.completedQueue.push(removeFromArray(wq.activeQueue, promise));
console.log('Finished promise',promise);
wq.run();
});
this.activeQueue.push(promise);
}
}
finally {
this.isRunning = false;
if (ranAtLeastOnce) console.log('Stop running queue',this);
}
};
function removeFromArray(arr, item)
{
var idx = arr.indexOf(item);
if (idx < 0) return;
arr.splice(idx,1);
}
async function makeJob(work, state)
{
return await work(state);
}
function ProgressButton() {
var bNotHere = true;
if (mw.config.get('wgPageName')=="Wikipedia:WikiProject_National_Register_of_Historic_Places/Progress") bNotHere = false;
if (mw.config.get('wgPageName')=="User:Magicpiano/NRBot/UpdateNRHPProgressTester")
{
bNotHere = false;
bIgnoreTableCount = true;
}
// the above notwithstanding, if in edit mode we also ignore
if (location.href.indexOf('action')!=-1) bNotHere = true;
if (bNotHere) return;
var button = document.getElementById("NRHPProgressUpdateButton2");
if (button !== null) return;
button=document.createElement("input");
button.setAttribute("type", "button");
button.setAttribute("value", "Update Statistics 2");
button.setAttribute("onclick", "ClickUpdateNRHPProgress()");
button.id = "NRHPProgressUpdateButton2";
var content=document.getElementById('mw-content-text');
content.parentNode.insertBefore(button, content);
}
var g_disabled = true;
function CheckPermission() {
var username = mw.user.getName();
console.log("UpdateNRHPProgress: run by user "+username);
if (username == CurrentMaintainer) return true;
for (var i=0; i< AuthorizedUsers.length; i++) {
if (username == AuthorizedUsers[i]) {
return true;
}
}
if (g_disabled) {
alert("Script is a work in progress.");
//alert("Script is currently disabled for maintenance.");
return false;
}
}
async function ClickUpdateNRHPProgress() { // after button is clicked, disable it and fetch source of Progress page
if (!CheckPermission()) {
return;
}
var button2 = document.getElementById("NRHPProgressUpdateButton2");
button2.disabled = true;
var ProgressDiv = document.getElementById("ProgressDiv");
if (ProgressDiv === null) {
ProgressDiv = document.createElement("div");
ProgressDiv.setAttribute("id", "ProgressDiv");
ProgressDiv.setAttribute("style", "width:500px; border:1px solid black; padding:5px; background:#ffffff");
button2.parentNode.insertBefore(ProgressDiv, button2);
}
ProgressDiv.innerHTML = "Initializing...";
var pagesource = await getPageSource(mw.config.get('wgPageName')); // TODO not really needed here?
SetupTables();
InitialTime=new Date(); // record starting time
UpdateProgressDiv();
processTableList(ProgressStructure);
//LoadList(1,0); // begin querying first page
}
// create array of table structure to be populated later
function SetupTables() {
var table=document.getElementsByClassName('wikitable sortable');
// initialize globals
TotalToQuery=0;
TotalQueried=0;
ErrorCount=0;
WarningCount=[["",0]]; // 0=status, 1=count
InitialTime=0;
ProgressDivTimer=0; // timer for updating ProgressDiv
DefaultQueryPause=1; // number of milliseconds to wait between each API query; increased by code if rate limit reached
RestrictedImageCount=0;
// Expected table count:
// There should be 61 tables, as follows:
// The national table, 50 states, and the following 10 non-state entities:
// District of Columbia, Puerto Rico, Virgin Islands, Guam, American Samoa,
// Northern Mariana Islands, Federated States of Micronesia, Palau, Marshall Islands
// Minor Outlying Islands
// This number should only need adjustmnent if the number of states or non-state entities having
// separate tables is changed. An alert here is more likely caused by some sort of formatting change
// or error in the table.
//
// Ideally, the above search should be restrictable (by class in some way?) to only the specific tables of interest.
//
if ((!bIgnoreTableCount) && table.length !== 61) {
alert('Incorrect table count in progress page: expected 61, saw '+table.length+'.');
return;
}
// set up national totals
var tr=table[0].getElementsByTagName("tr");
var i,j, td;
ProgressStructure[0]=[];
for (j=1; j<tr.length-3; j++) {
td=tr[j].getElementsByTagName("td");
ProgressStructure[0][j-1]={
ID: td[0].innerHTML,
Total: 0,
Illustrated: 0,
Articled: 0,
Stubs: 0,
NRISonly: 0,
StartPlus: 0,
Unassessed: 0,
Untagged: 0
};
}
// special row for Tangier, Morocco
td=tr[tr.length-3].getElementsByTagName("td");
/* This code copies the existing settings for the American Legation in Tangier into the table.
This process is prone to error if it is modified. Because the article is an illustrated Start+ article,
the settings hardcoded further down are unlikely to change.
*/
/* // To copy previous contents instead of hardcoding:
ProgressStructure[0][tr.length-4]={};
ProgressStructure[0][tr.length-4].ID="Tangier, Morocco";
ProgressStructure[0][tr.length-4].Total=parseFloat(td[1].innerHTML.replace(",",""));
ProgressStructure[0][tr.length-4].Illustrated=parseFloat(td[2].innerHTML.replace(",",""));
ProgressStructure[0][tr.length-4].Articled=parseFloat(td[4].innerHTML.replace(",",""));
ProgressStructure[0][tr.length-4].Stubs=parseFloat(td[6].innerHTML.replace(",",""));
ProgressStructure[0][tr.length-4].NRISonly=parseFloat(td[7].innerHTML.replace(",",""));
ProgressStructure[0][tr.length-4].StartPlus=parseFloat(td[8].innerHTML.replace(",",""));
ProgressStructure[0][tr.length-4].Unassessed=parseFloat(td[10].innerHTML.replace(",",""));
ProgressStructure[0][tr.length-4].Untagged=parseFloat(td[11].innerHTML.replace(",",""));
*/
/* American Legation: Illustrated, Articled, not stub, not NRIS, Start+, Assessed, Tagged */
ProgressStructure[0][tr.length-4]={
ID: "Tangier, Morocco",
Total: 1,
Illustrated: 1,
Articled: 1,
Stubs: 0,
NRISonly: 0,
StartPlus: 1,
Unassessed: 0,
Untagged: 0
};
// duplicates row
td=tr[tr.length-2].getElementsByTagName("td");
ProgressStructure[0][tr.length-3]={
ID: "National Duplicates",
Total: parseFloat(td[0].innerHTML.replace(",","")),
Illustrated: parseFloat(td[1].innerHTML.replace(",","")),
Articled: parseFloat(td[3].innerHTML.replace(",","")),
Stubs: parseFloat(td[5].innerHTML.replace(",","")),
NRISonly: parseFloat(td[6].innerHTML.replace(",","")),
StartPlus: parseFloat(td[7].innerHTML.replace(",","")),
Unassessed: parseFloat(td[9].innerHTML.replace(",","")),
Untagged: parseFloat(td[10].innerHTML.replace(",",""))
};
// national totals
ProgressStructure[0][tr.length-2]={
ID: "National Totals",
Total: 0,
Illustrated: 0,
Articled:0,
Stubs: 0,
NRISonly: 0,
StartPlus: 0,
Unassessed: 0,
Untagged: 0
};
// now data for each state
for (i=1; i<table.length; i++) {
tr=table[i].getElementsByTagName("tr");
ProgressStructure[i]=[];
for (j=1; j<tr.length-2; j++) { // skip title row, statewide duplicates, and totals row
td=tr[j].getElementsByTagName("td");// fill in existing data in case error
ProgressStructure[i][j-1]={};
ProgressStructure[i][j-1].ID=td[0].innerHTML.substr(0,5);
ProgressStructure[i][j-1].Total=parseFloat(td[3].innerHTML.replace(",",""));
ProgressStructure[i][j-1].Illustrated=parseFloat(td[4].innerHTML.replace(",",""));
ProgressStructure[i][j-1].Articled=parseFloat(td[6].innerHTML.replace(",",""));
ProgressStructure[i][j-1].Stubs=parseFloat(td[8].innerHTML.replace(",",""));
ProgressStructure[i][j-1].NRISonly=parseFloat(td[9].innerHTML.replace(",",""));
ProgressStructure[i][j-1].StartPlus=parseFloat(td[10].innerHTML.replace(",",""));
ProgressStructure[i][j-1].Unassessed=parseFloat(td[12].innerHTML.replace(",",""));
ProgressStructure[i][j-1].Untagged=parseFloat(td[13].innerHTML.replace(",",""));
var link=td[1].getElementsByTagName("a");
if (link.length!==0 && link[0].href.search("#")==-1) {
link=decodeURI(link[0].href).split("/");
link=link[link.length-1].replace(/_/g," ");
ProgressStructure[i][j-1].Link=link;
ProgressStructure[i][j-1].ArticleQueried=0; // for querying later
ProgressStructure[i][j-1].TalkQueried=0;
} else {
if (ProgressStructure[i][j-1].ID!="ddddd") { // if no link and not duplicate, must be totals row, so we can zero it
ProgressStructure[i][j-1].Total=0;
ProgressStructure[i][j-1].Illustrated=0;
ProgressStructure[i][j-1].Articled=0;
ProgressStructure[i][j-1].Stubs=0;
ProgressStructure[i][j-1].NRISonly=0;
ProgressStructure[i][j-1].StartPlus=0;
ProgressStructure[i][j-1].Unassessed=0;
ProgressStructure[i][j-1].Untagged=0;
}
}
}
// duplicates row
td=tr[tr.length-2].getElementsByTagName("td");
ProgressStructure[i][tr.length-3]={
ID: ProgressStructure[0][i-1].ID+" Duplicates",
Total: parseFloat(td[0].innerHTML.replace(",","")),
Illustrated: parseFloat(td[1].innerHTML.replace(",","")),
Articled: parseFloat(td[3].innerHTML.replace(",","")),
Stubs: parseFloat(td[5].innerHTML.replace(",","")),
NRISonly: parseFloat(td[6].innerHTML.replace(",","")),
StartPlus: parseFloat(td[7].innerHTML.replace(",","")),
Unassessed: parseFloat(td[9].innerHTML.replace(",","")),
Untagged: parseFloat(td[10].innerHTML.replace(",",""))
};
// state totals
ProgressStructure[i][tr.length-2]={
ID: ProgressStructure[0][i-1].ID+" Totals",
Total: 0,
Illustrated: 0,
Articled:0,
Stubs: 0,
NRISonly: 0,
StartPlus: 0,
Unassessed: 0,
Untagged: 0
};
}
for (i=1; i<ProgressStructure.length; i++) { // count total number of rows to check
for (j=0; j<ProgressStructure[i].length-2; j++) {
if (typeof ProgressStructure[i][j].Link!="undefined") TotalToQuery++; // don't count duplicates and total rows
}
}
TotalQueried=0;
var ProgressDiv=document.getElementById("ProgressDiv");
ProgressDiv.innerHTML+=" Done!<br>";
var ProgressSpan=document.createElement("span");
ProgressSpan.setAttribute("id", "ProgressSpan");
ProgressDiv.appendChild(ProgressSpan);
ProgressSpan.innerHTML = "Querying county data... 0 (0%) of "+TotalToQuery+" lists checked.";
var TimeSpan=document.createElement("span");
TimeSpan.setAttribute("id", "TimeSpan");
ProgressDiv.appendChild(TimeSpan);
TimeSpan.innerHTML = "";
}
function processTableList(ps)
{
console.log("processTableList");
var curtable;
var tabletitle;
var wq = new WorkQueue(3);
var tablecount = 0;
for (curtable = 0; curtable<2 /*ps.length-1*/; curtable++)
{
tablecount++;
var state = {
progressState: ps,
curtable: curtable
};
wq.push(makeJob(processTable,state));
}
}
async function processTable(state)
{
var curtable = state.curtable;
var ps = state.progressState;
var tabletitle;
console.log('processTable',curtable);
var rowcount = 0;
for (currow = 0; currow<ps[curtable].length-2; currow++)
{
if (typeof ps[curtable][currow].Link=="undefined")
{
// skip duplicate and total rows
continue;
}
rowcount++;
tabletitle=ps[curtable][currow].Link;
console.log('processTable doing '+tabletitle);
}
// TODO return promise that resolves when all rows are processed
return rowcount;
}
/*
for (currow = 0; currow<ps[curtable].length-2; currow++)
{
if (typeof ps[curtable][currow].Link=="undefined") {
// skip duplicate and total rows
continue;
}
tabletitle=ps[curtable][currow].Link;
console.log('processTableList doing '+tabletitle);
var state = {
tabletitle: tabletitle,
curtable: curtable,
currow: currow,
progressState: ps,
promise: null
};
var promise = new Promise((resolve,reject) => {
processTableRow(state, resolve, reject);
});
promise.psState = state;
state.promise = promise;
promise.nTries = 5;
wq.push(promise);
}
}
}
*/
function processTableRow1(ps,tabletitle,curtable,currow, resolve, reject)
{
console.log('processTableRow1',tabletitle,curtable,currow);
resolve(true);
}
async function processTableRow(promise, resolve, reject)
{
var ps = promise.progressState;
var tabletitle = promise.tabletitle;
var curtable = promise.curtable;
var currow = promise.currow;
console.log("processTableRow",tabletitle,curtable,currow);
var listingSource, listingTalkSource;
var results;
try {
listingSource = getPageSource(tabletitle);
listingTalkSource = getPageSource('Talk:'+tabletitle);
results = await Promise.allSettled([listingSource,listingTalkSource]);
console.log('Promise results', results);
listingSource = getPageTextFromQueryResponse(results[0].value);
listingTalkSource = getPageTextFromQueryResponse(results[1].value);
if (listingTalkSource == null) listingTalkSource = "";
// TODO also need query results concerning redirects (see WikiTextFetched)
console.log('Results of fetching listing article source and talk page',tabletitle, listingSource.length, listingTalkSource.length);
//processListingSources(ps,tabletitle,curtable,currow, listingSource, listingTalkSource);
resolve(true);
}
catch (e) {
console.log('Error getting source for listing article or talk page',tabletitle,e);
//ProgressFatalError2(0,tabletitle,curtable,currow,e);
reject(e);
}
}
async function processListingSources(ps,tabletitle, curtable, currow, listingSource, listingTalkSource)
{
console.log('processListingSources', tabletitle);
// TODO WikiTextFetched
}
// load next list to query
function LoadList(currentTable,currentRow) {
// check if we need to go to the next table
if (currentRow>ProgressStructure[currentTable].length-3) {
currentRow=0;
currentTable++;
}
// check if there are no more tables
if (currentTable>ProgressStructure.length-1) return;
if (typeof ProgressStructure[currentTable][currentRow].Link=="undefined") { // skip duplicate and total rows
LoadList(currentTable,currentRow+1);
return;
}
var title=ProgressStructure[currentTable][currentRow].Link;
setTimeout(function(){ // short delay to prevent API overload
getProgressListWikitext(title,currentTable,currentRow);
LoadList(currentTable,currentRow+1);
}, DefaultQueryPause);
return;
}
function WikitextFetched(ajaxResponse,status,title,currentTable,currentRow) {
// console.log("WikitextFetched: table="+currentTable+" row="+currentRow+" title="+title+" status="+status);
if (status!="success") {
NewWarning("Wikitext "+ajaxResponse.errorThrown);
setTimeout(function(){ // try again after delay if rate limit reached
getProgressListWikitext(title,currentTable,currentRow);
}, 250);
return;
}
// won't get here unless successful
var responseText, pagetext;
var tabletext, regex, i, j;
try {
responseText=JSON.parse(ajaxResponse.responseText);
pagetext=responseText.query.pages[responseText.query.pageids[0]].revisions[0]["*"];
}
catch (e) {
console.log("WikiTextFetched: Exception parsing "+title+": "+e);
ProgressFatalError(0,title,currentTable,currentRow);
return;
}
var StartIndex;
// console.log("WikiTextFetched: Parsing out "+title+" to find relevant table");
try {
if (responseText.query.redirects) { // if redirect, find section
var SectionName="Undefined";
for (var r in responseText.query.redirects) {
if (typeof responseText.query.redirects[r].tofragment!="undefined") SectionName=responseText.query.redirects[r].tofragment.replace(/.27/g,"'");
}
regex = new RegExp("=[ ]*(\\[\\[(.*?\\|)?[ ]*)?"+SectionName+"([ ]*\\]\\])?[ ]*=", "g");
var sectionheader=pagetext.match(regex);
if (sectionheader === null || sectionheader === undefined) { // if no section found, check if one of known empty counties
var ID = ProgressStructure[currentTable][currentRow].ID;
//console.log("WikiTextFetched: List appears to be empty: title="+title+" id="+ID);
// list last check date: 2020-11-09
var EmptyCounties=["01061", // Geneva County AL
"02270", // Kusilvak Census Area AK
"08014", // Broomfield County CO
"12067", // Lafayette County FL
"20081", // Haskell County KS
"20175", // Seward County KS
"20187", // Stanton County KS
"20189", // Stevens County KS
"26051", // Gladwin County MI
"26079", // Kalkaska County MI
"26119", // Montmorency County MI
"26129", // Ogemaw County MI
"26133", // Osceola County MI
"31009", // Blaine County NE
"31113", // Logan County NE
"31117", // McPherson County NE
"38085", // Sioux County ND
"42023", // Cameron County PA
"48017", // Bailey County TX
"48023", // Baylor County TX
"48033", // Borden County TX
"48069", // Castro County TX
"48079", // Cochran County TX
"48103", // Crane County TX
"48107", // Crosby County TX
"48119", // Delta County TX
"48131", // Duval County TX
"48155", // Foard County TX
"48165", // Gaines County TX
"48207", // Haskell County TX
"48219", // Hockley County TX
"48247", // Jim Hogg County TX
"48269", // King County TX
"48279", // Lamb County TX
"48341", // Moore County TX
"48389", // Reeves County TX
"48415", // Scurry County TX
"48421", // Sherman County TX
"48433", // Stonewall County TX
"48437", // Swisher County TX
"48445", // Terry County TX
"48461", // Upton County TX
"48475", // Ward County TX
"48501", // Yoakum County TX
"51735" // Poquoson VA
];
var errorcode = 0;
for (var k=0; k<EmptyCounties.length; k++) {
if (ID==EmptyCounties[k]) {errorcode=-1;}
}
if (errorcode!==0) { // must be an empty county
ProgressStructure[currentTable][currentRow].Total=0;
ProgressStructure[currentTable][currentRow].Illustrated=0;
ProgressStructure[currentTable][currentRow].Articled=0;
ProgressStructure[currentTable][currentRow].Stubs=0;
ProgressStructure[currentTable][currentRow].NRISonly=0;
ProgressStructure[currentTable][currentRow].StartPlus=0;
ProgressStructure[currentTable][currentRow].Unassessed=0;
ProgressStructure[currentTable][currentRow].Untagged=0;
ProgressStructure[currentTable][currentRow].Link=title;
TotalQueried++;
if (TotalQueried==TotalToQuery) CalculateProgressTotals();
return;
}
// if we're here, must have been a redirect with no section, and not a known empty county
sectionheader=pagetext.match(/{{NRHP header/g); // then look for tables without a section
if (sectionheader===null||sectionheader.length>1) { // if still can't find a table or find multiple tables, fatal error
ProgressFatalError(0,title,currentTable,currentRow);
}
}
StartIndex=pagetext.indexOf(sectionheader[0]);
var sectiontext=pagetext.substr(StartIndex,pagetext.indexOf("\n==",StartIndex)-StartIndex); // only look at relevant section
StartIndex=sectiontext.indexOf("{{NRHP header");
if (StartIndex==-1) {
if (sectiontext.indexOf("{{NRHP row")!=-1) {
ProgressFatalError(2,title,currentTable,currentRow); // incorrectly formatted table
} else { // must be an empty county
console.log("WikiTextFetched: county appears to be empty: "+title);
ProgressStructure[currentTable][currentRow].Total=0;
ProgressStructure[currentTable][currentRow].Illustrated=0;
ProgressStructure[currentTable][currentRow].Articled=0;
ProgressStructure[currentTable][currentRow].Stubs=0;
ProgressStructure[currentTable][currentRow].NRISonly=0;
ProgressStructure[currentTable][currentRow].StartPlus=0;
ProgressStructure[currentTable][currentRow].Unassessed=0;
ProgressStructure[currentTable][currentRow].Untagged=0;
ProgressStructure[currentTable][currentRow].Link=title;
TotalQueried++;
if (TotalQueried==TotalToQuery) CalculateProgressTotals();
return;
}
}
var EndIndex = sectiontext.indexOf("\n|}",StartIndex);
// EndIndex should exist, but a malformed file may not have it
if (EndIndex === -1) {
console.log("WikiTextFetched: missing end of table in "+title);
ProgressFatalError(2,title,currentTable,currentRow); // list is malformed
return;
}
tabletext=sectiontext.substr(StartIndex,EndIndex-StartIndex);
} else { // if not a redirect, default to first table on page
StartIndex=pagetext.indexOf("{{NRHP header");
if (StartIndex==-1) {
ProgressFatalError(1,title,currentTable,currentRow); // no list found
return;
}
var EndIndex = pagetext.indexOf("\n|}",StartIndex);
// EndIndex should exist, but a malformed file may not have it
if (EndIndex === -1) {
console.log("WikiTextFetched: missing end of table in "+title);
ProgressFatalError(2,title,currentTable,currentRow); // list is malformed
return;
}
tabletext=pagetext.substr(StartIndex,EndIndex-StartIndex);
}
}
catch (e) {
console.log("WikiTextFetched: Exception searching for table in "+title+": "+e);
ProgressFatalError(0,title,currentTable,currentRow);
return;
}
// now that tabletext has only relevant table, extract rows
var Rows=[];
var str = "{{";
var start=0;
var commentstart=0;
//console.log("WikiTextFetched: Extracting rows from "+title);
try {
while (true) {
commentstart=tabletext.indexOf("<!--",start);
start=tabletext.indexOf(str,start);
if (start==-1) break;
while (commentstart<start&&commentstart!=-1) { // skip any commented out rows
start=tabletext.indexOf("-->",commentstart);
commentstart=tabletext.indexOf("<!--",start);
start=tabletext.indexOf(str,start);
}
if (start==-1) break;
var open=1;
var index=start+str.length;
while (open!==0 && index<tabletext.length) { // make sure to find correct matching close brackets for row template
if (tabletext.substr(index,2)=="}}") {
open--;
index++;
} else if (tabletext.substr(index,2)=="{{") {
open++;
index++;
}
index++;
}
var template=tabletext.substr(start,index-start);
regex = new RegExp("{{[\\s]*NRHP row(\\s)*\\|", "g");
if (template.match(regex)!==null) Rows.push(template); // make sure it's the row template and not some other one
else {
console.log("WikiTextFetched: failed to find 'NRHP row' in "+template);
// ProgressFatalError(0,title,currentTable,currentRow);
// return;
}
start++;
}
for (i=0; i<Rows.length; i++) { // get rid of false positives inside nowiki or pre tags
regex=new RegExp("<[ ]*?(nowiki|pre)[ ]*?>((?!<[ ]*?/[ ]*?(nowiki|pre)[ ]*?>)(.|\\n))*?"+Rows[i].replace(/[\-\[\]\/\{\}\(\)\*\+\?\.\\\^\$\|]/g, "\\$&")+"(.|\\n)*?<[ ]*?/[ ]*?(nowiki|pre)[ ]*?>", "g");
if (tabletext.match(regex)!==null) {Rows.splice(i,1); i--;}
}
}
catch (e) {
console.log("WikiTextFetched: Exception extracting rows from "+title+": "+e);
ProgressFatalError(0,title,currentTable,currentRow);
return;
}
// now begin querying statistics
var Stats={"ID":ProgressStructure[currentTable][currentRow].ID, "Total":Rows.length, "Illustrated":0, "Articled":0, "Stubs":0, "NRISonly":0, "StartPlus":0, "Unassessed":0, "Untagged":0, "Link":title};
var Titles=[];
console.log("WikiTextFetched: query statistics for "+title+": "+Rows.length+" rows");
try {
for (i=0; i<Rows.length; i++) { // extract titles for querying
var ThisRow=Rows[i];
// check for illustrated while we're cycling through anyway
var test=ThisRow.match(/\|[ ]*?image[ ]*?=.*?(\n|\||}})/g);
if (test!==null) {
test=test[0].replace(/\|[ ]*?image[ ]*?=/g,"").replace(/(\n|\||}})/g,"").replace(/<\!\-\-(.|[\r\n])*?\-\-\>/g,"").trim();
if (test!=="") {
//console.log('NRHPstats: found image in row '+NRHPstats_currentRow+': '+test);
var test2 = test.match(/Address restricted/gi);
if (test2!==null && test2!=="") {
RestrictedImageCount++;
console.log('NRHPstats: image on row '+i+' appears to be address restricted image (ignoring #'+RestrictedImageCount+'): '+test);
Stats.Illustrated++; // TODO keep for now
} else {
Stats.Illustrated++; // only true if image param there and non-blank
}
}
}
var article=ThisRow.match(/\|[ ]*?article[ ]*?=[ ]*?.*?[\n|\|]/g);
var blank=ThisRow.match(/\|[ ]*?article[ ]*?=[ ]*?[\n|\|]/g); // default to name param if article
if (article===null||blank!==null) article=ThisRow.match(/\|[ ]*?name[ ]*?=[ ]*?.*?[\n|\|]/g); // blank or missing
// strip param name, final line break
article=article[0].replace(/\|[ ]*?(article|name)[ ]*?=[ ]*?/g,"").replace(/[\n|\|]/g,"").replace(/<\!\-\-(.|[\r\n])*?\-\-\>/g,"").trim();
article=decodeURIComponent(article.split("#")[0].trim()); // corrections for weird titles
try {
Titles.push(article);
}
catch (e) {
console.log("WikiTextFetched: Exception parsing "+title+": "+e);
ProgressFatalError(0,title,currentTable,currentRow);
}
}
}
catch (e) {
console.log("WikiTextFetched: Exception extracting statistics from rows from "+title+": "+e);
ProgressFatalError(0,title,currentTable,currentRow);
return;
}
StartIndex=0;
LoadNextProgressQuery(Rows,Stats,Titles,StartIndex,title,currentTable,currentRow);
return;
}
// ready next batch of articles to query
function LoadNextProgressQuery(Rows,Stats,Titles,StartIndex,title,currentTable,currentRow) {
if (StartIndex==Stats.Total) { // all queries begun for this list
return;
}
var TempTitles;
// must have some more rows to query
if (Stats.Total-StartIndex>50) {
TempTitles=Titles.slice(StartIndex,StartIndex+50);
} else {
TempTitles=Titles.slice(StartIndex);
}
StartIndex+=TempTitles.length;
setTimeout(function(){ // short delay to prevent API overload
QueryProgressStats(Rows,Stats,Titles,TempTitles,StartIndex,title,currentTable,currentRow);
LoadNextProgressQuery(Rows,Stats,Titles,StartIndex,title,currentTable,currentRow);
}, DefaultQueryPause);
return;
}
// query next batch of articles
function QueryProgressStats(Rows,Stats,Titles,TempTitles,StartIndex,title,currentTable,currentRow) {
var TitleList=TempTitles.join("|");
// console.log("QueryProgressStats: titles="+TitleList);
$.ajax({
dataType: "json",
url: mw.util.wikiScript('api'),
data: {
format: 'json',
action: 'query',
prop: 'categories',
clcategories: 'Category:All disambiguation pages|Category:All articles sourced only to NRIS',
cllimit: 'max',
titles: TitleList,
redirects: 'true'
},
error: function(ArticlejsonObject,status,errorThrown) {
ArticlejsonObject.errorThrown=errorThrown;
console.log("QueryProgressStats: error fetching "+TempTitles);
},
complete: function(ArticlejsonObject,status) {
try {
ProgressArticleChecked(ArticlejsonObject,status,Rows,Stats,Titles,TempTitles,StartIndex,title,currentTable,currentRow);
}
catch (e) {
console.log("QueryProgressStats: Eception checking article "+TempTitles+": "+e);
ProgressFatalError(0,title,currentTable,currentRow);
}
}
});
return;
}
// parse API response for article query
function ProgressArticleChecked(ArticlejsonObject,status,Rows,Stats,Titles,TempTitles,StartIndex,title,currentTable,currentRow) {
if (status!="success") {
NewWarning("Articles "+ArticlejsonObject.errorThrown);
setTimeout(function(){ // try again after delay if rate limit reached
QueryProgressStats(Rows,Stats,Titles,TempTitles,StartIndex,title,currentTable,currentRow);
}, 250);
return;
}
// won't get here unless successful
ProgressStructure[currentTable][currentRow].ArticleQueried+=TempTitles.length;
var responseText=JSON.parse(ArticlejsonObject.responseText);
var i;
if (responseText.query.normalized) { // normalize any weird titles
for (var n in responseText.query.normalized) {
for (i=0; i<TempTitles.length; i++) {
if (TempTitles[i]==responseText.query.normalized[n].from) TempTitles[i]=responseText.query.normalized[n].to;
}
for (i=0; i<Titles.length; i++) { // also update in Titles array to prepare to query talk pages
if (Titles[i]==responseText.query.normalized[n].from) Titles[i]=responseText.query.normalized[n].to;
}
}
}
if (responseText.query.redirects) { // resolve any redirects also
for (var r in responseText.query.redirects) {
for (i=0; i<TempTitles.length; i++) {
if (TempTitles[i]==responseText.query.redirects[r].from) TempTitles[i]=responseText.query.redirects[r].to;
}
for (i=0; i<Titles.length; i++) { // also update in Titles array to prepare to query talk pages
if (Titles[i]==responseText.query.redirects[r].from) Titles[i]=responseText.query.redirects[r].to;
}
}
}
// now determine the number of bluelinks and NRIS-only articles
for (var page in responseText.query.pages) {
var articled=true; // default to articled, not NRIS-only
var NRISonly=false;
var pagetitle=responseText.query.pages[page].title;
if (typeof responseText.query.pages[page].missing!="undefined") {
// redlink=unarticled
articled=false;
}
if (responseText.query.pages[page].categories) {
for (var category in responseText.query.pages[page].categories) {
if (responseText.query.pages[page].categories[category].title=="Category:All disambiguation pages") {
// dab=unarticled
articled=false;
}
if (responseText.query.pages[page].categories[category].title.indexOf("sourced only to NRIS")!=-1) {
// mark as NRIS-only
NRISonly=true;
}
}
}
for (i=0; i<TempTitles.length; i++) { // if page is duplicated, count it multiple times
if (TempTitles[i]==pagetitle) {
if (articled) Stats.Articled++;
if (NRISonly) Stats.NRISonly++;
}
}
}
if (ProgressStructure[currentTable][currentRow].ArticleQueried==Stats.Total) { // after querying all articles, query talk pages
for (i=0; i<Titles.length; i++) {
Titles[i]="Talk:"+Titles[i];
}
StartIndex=0;
LoadNextProgressTalkQuery(Rows,Stats,Titles,StartIndex,title,currentTable,currentRow);
}
}
// ready next batch of talk pages to query
function LoadNextProgressTalkQuery(Rows,Stats,Titles,StartIndex,title,currentTable,currentRow) {
if (StartIndex==Stats.Total) { // all queries begun for this list
return;
}
var TempTitles;
// must have some more rows to query
if (Stats.Total-StartIndex>50) {
TempTitles=Titles.slice(StartIndex,StartIndex+50);
} else {
TempTitles=Titles.slice(StartIndex);
}
StartIndex+=TempTitles.length;
setTimeout(function(){ // short delay to prevent API overload
QueryProgressTalkStats(Rows,Stats,Titles,TempTitles,StartIndex,title,currentTable,currentRow);
LoadNextProgressTalkQuery(Rows,Stats,Titles,StartIndex,title,currentTable,currentRow);
}, DefaultQueryPause);
return;
}
// query the next batch of talk pages
function QueryProgressTalkStats(Rows,Stats,Titles,TempTitles,StartIndex,title,currentTable,currentRow) {
var catlist='Category:FA-Class National Register of Historic Places articles|Category:A-Class National Register of Historic Places ';
catlist+='articles|Category:GA-Class National Register of Historic Places articles|Category:B-Class National Register of Historic ';
catlist+='Places articles|Category:C-Class National Register of Historic Places articles|Category:Start-Class National Register of ';
catlist+='Historic Places articles|Category:Stub-Class National Register of Historic Places articles|Category:Unassessed National ';
catlist+='Register of Historic Places articles|Category:List-Class National Register of Historic Places articles|Category:Redirect-';
catlist+='Class National Register of Historic Places articles';
var TitleList=TempTitles.join("|");
$.ajax({
dataType: "json",
url: mw.util.wikiScript('api'),
data: {
format: 'json',
action: 'query',
prop: 'categories',
clcategories: catlist,
cllimit: 'max',
titles: TitleList,
redirects: 'true'
},
error: function(ArticlejsonObject,status,errorThrown) {
ArticlejsonObject.errorThrown=errorThrown;
console.log("QueryProgressTalkStats: Error fetching "+TitleList);
},
complete: function(ArticlejsonObject,status) {
try {
ProgressTalkChecked(ArticlejsonObject,status,Rows,Stats,Titles,TempTitles,StartIndex,title,currentTable,currentRow);
}
catch (e) {
alert("QueryProgressTalkStats: Exception processing "+TitleList+": "+e);
ProgressFatalError(0,title,currentTable,currentRow);
}
}
});
return;
}
// parse API response for talk page query
function ProgressTalkChecked(ArticlejsonObject,status,Rows,Stats,Titles,TempTitles,StartIndex,title,currentTable,currentRow) {
if (status!="success") {
NewWarning("Talk "+ArticlejsonObject.errorThrown);
setTimeout(function(){ // try again after delay if rate limit reached
QueryProgressTalkStats(Rows,Stats,Titles,TempTitles,StartIndex,title,currentTable,currentRow);
}, 250);
return;
}
// won't get here unless successful
ProgressStructure[currentTable][currentRow].TalkQueried+=TempTitles.length;
// now determine quality statistics
var responseText=JSON.parse(ArticlejsonObject.responseText);
for (var page in responseText.query.pages) {
if (typeof responseText.query.pages[page].missing!="undefined") continue; // skip talk page if not articled
var untagged=true; // default to untagged
var articled=true; // assume articled to check for link to other county/MPS lists
var stub=false;
var unassessed=false;
var startPlus=false;
var pagetitle=responseText.query.pages[page].title;
if (responseText.query.pages[page].categories) {
untagged=false; // if cat hit, mark as tagged
for (var category in responseText.query.pages[page].categories) {
var CatTitle=responseText.query.pages[page].categories[category].title;
if (CatTitle.indexOf("Stub")!=-1) {
stub=true; // mark as stub
}
if (CatTitle.indexOf("Unassessed")!=-1||CatTitle.indexOf("Redirect")!=-1) {
unassessed=true; // mark as unassessed
}
if (CatTitle.indexOf("List")!=-1) { // count links to other county/MPS lists as unarticled; other list-class as stubs
if (responseText.query.pages[page].title.indexOf("National Register of Historic Places")!=-1){
articled=false;
} else {
stub=true;
}
}
}
}
if (articled&&!untagged&&!unassessed&&!stub) { // if articled, tagged, and assessed, but not stub, must be Start+
startPlus=true;
}
for (var i=0; i<TempTitles.length; i++) {
if (TempTitles[i]==pagetitle) {
if (!articled) Stats.Articled--; // reduce the count of articled if links to lists
if (untagged) Stats.Untagged++;
if (stub) Stats.Stubs++;
if (unassessed) Stats.Unassessed++;
if (startPlus) Stats.StartPlus++;
}
}
}
if (ProgressStructure[currentTable][currentRow].TalkQueried==Stats.Total) {
ProgressStructure[currentTable][currentRow].Total=Stats.Total;
ProgressStructure[currentTable][currentRow].Illustrated=Stats.Illustrated;
ProgressStructure[currentTable][currentRow].Articled=Stats.Articled;
ProgressStructure[currentTable][currentRow].Stubs=Stats.Stubs;
ProgressStructure[currentTable][currentRow].NRISonly=Stats.NRISonly;
ProgressStructure[currentTable][currentRow].StartPlus=Stats.StartPlus;
ProgressStructure[currentTable][currentRow].Unassessed=Stats.Unassessed;
ProgressStructure[currentTable][currentRow].Untagged=Stats.Untagged;
ProgressStructure[currentTable][currentRow].Link=title;
TotalQueried++;
if (TotalQueried==TotalToQuery) CalculateProgressTotals();
}
}
// keep track of warnings encountered while querying API
function NewWarning(warning) {
var i, NewWarning=true;
for (i=0; i<WarningCount.length; i++) { // check if already encountered error
if (warning==WarningCount[i][0]||WarningCount[i][0]==="") {WarningCount[i][0]=warning; WarningCount[i][1]++; NewWarning=false;}
}
if (NewWarning) WarningCount[WarningCount.length]=[warning,1]; // if new warning, make new entry
var test=0;
for (i=0; i<WarningCount.length; i++) {
test+=WarningCount[i][1];
}
if (test%50===0) DefaultQueryPause++; // for every 50 errors encountered, increase time between each query to throttle speed
}
var g_fetchErrors = {};
function ProgressFatalError2(code,title, curtable, currow, ex)
{
console.log('ProgressFatalError2: ',code,title,curtable,currow, ex);
var errorArray = ['No county section found for ','No list found for ','Incorrectly formatted list for '];
if (!g_fetchErrors.hasOwnProperty(title)) g_fetchErrors[title] = 0;
g_fetchErrors[title]++;
var retry = true;
if (g_fetchErrors[title] > 5) {
retry=confirm(errorArray[code]+title+"!\n\nCancel=Skip OK=Retry");
if (retry) g_fetchErrors[title] = 0; // reset counter for this title
}
if (retry) {
processTableRow(title,curtable,currow);
} else {
TotalQueried++;
ErrorCount++;
if (TotalQueried==TotalToQuery) CalculateProgressTotals();
}
return;
}
// these errors require user input; can't just be ignored
function ProgressFatalError(code,title,currentTable,currentRow) {
var errorArray = ['No county section found for ','No list found for ','Incorrectly formatted list for '];
if (!g_fetchErrors.hasOwnProperty(title)) g_fetchErrors[title] = 0;
g_fetchErrors[title]++;
var retry = true;
if (g_fetchErrors[title] > 5) {
retry=confirm(errorArray[code]+title+"!\n\nCancel=Skip OK=Retry");
if (retry) g_fetchErrors[title] = 0; // reset counter for this title
}
if (retry) {
getProgressListWikitext(title,currentTable,currentRow);
} else {
TotalQueried++;
ErrorCount++;
if (TotalQueried==TotalToQuery) CalculateProgressTotals();
}
return;
}
// update ProgressDiv to let user know what's going on
function UpdateProgressDiv() {
var ProgressSpan=document.getElementById("ProgressSpan");
var TimeSpan=document.getElementById("TimeSpan");
var PercentQueried=Math.round(TotalQueried/TotalToQuery*1000)/10;
ProgressSpan.innerHTML = "Querying county data... "+TotalQueried+" ("+PercentQueried+"%) of "+TotalToQuery+" lists checked.";
var TimeRemainingStr="";
if (TotalQueried>100) {
var CurrentTime=new Date();
var SecondsElapsed = (CurrentTime-InitialTime)/1000;
var Average = SecondsElapsed/TotalQueried;
SecondsElapsed=Math.round(SecondsElapsed);
var MinutesElapsed = 0;
while (SecondsElapsed>=60) {
SecondsElapsed-=60;
MinutesElapsed++;
}
var SecondsRemaining = Math.round(Average*(TotalToQuery-TotalQueried));
var MinutesRemaining = 0;
while (SecondsRemaining>=60) {
SecondsRemaining-=60;
MinutesRemaining++;
}
TimeRemainingStr = "";
if (MinutesRemaining!==0) TimeRemainingStr=MinutesRemaining+" min ";
TimeRemainingStr+=SecondsRemaining+" sec";
var TimeElapsedStr = "";
if (MinutesElapsed!==0) TimeElapsedStr=MinutesElapsed+" min ";
TimeElapsedStr+=SecondsElapsed+" sec";
TimeRemainingStr+=" ("+TimeElapsedStr+" elapsed)";
} else {
TimeRemainingStr="Calculating...";
}
TimeSpan.innerHTML="<br>Estimated time remaining: "+TimeRemainingStr;
if (TotalQueried!=TotalToQuery) { // update ProgressDiv only at regular intervals to prevent CPU overload; stop once done
ProgressDivTimer=setTimeout(function(){
UpdateProgressDiv();
}, 500);
}
return;
}
// after all querying complete, calculate totals for states and counties with multiple sublists
function CalculateProgressTotals() {
clearTimeout(ProgressDivTimer);
var ProgressSpan=document.getElementById("ProgressSpan");
var TimeSpan=document.getElementById("TimeSpan");
ProgressSpan.innerHTML = "Querying county data... Done! "+TotalToQuery+" lists checked.";
var CurrentTime=new Date();
var SecondsElapsed = (CurrentTime-InitialTime)/1000;
SecondsElapsed=Math.round(SecondsElapsed);
var MinutesElapsed = 0;
while (SecondsElapsed>=60) {
SecondsElapsed-=60;
MinutesElapsed++;
}
TimeSpan.innerHTML=" Time elapsed: ";
if (MinutesElapsed!==0) TimeSpan.innerHTML+=MinutesElapsed+" min ";
TimeSpan.innerHTML+=SecondsElapsed+" sec";
var TotalsIndex, NationalTotalsIndex;
for (var i=1; i<ProgressStructure.length; i++) { // i=table number; skip national table until end
for (var j=0; j<ProgressStructure[i].length-2; j++) {
TotalsIndex=ProgressStructure[i].length-1;
if (!isNaN(parseFloat(ProgressStructure[i][j].ID))) { // if regular county without sublists, add to totals
ProgressStructure[i][TotalsIndex].Total+=ProgressStructure[i][j].Total;
ProgressStructure[i][TotalsIndex].Illustrated+=ProgressStructure[i][j].Illustrated;
ProgressStructure[i][TotalsIndex].Articled+=ProgressStructure[i][j].Articled;
ProgressStructure[i][TotalsIndex].Stubs+=ProgressStructure[i][j].Stubs;
ProgressStructure[i][TotalsIndex].NRISonly+=ProgressStructure[i][j].NRISonly;
ProgressStructure[i][TotalsIndex].StartPlus+=ProgressStructure[i][j].StartPlus;
ProgressStructure[i][TotalsIndex].Unassessed+=ProgressStructure[i][j].Unassessed;
ProgressStructure[i][TotalsIndex].Untagged+=ProgressStructure[i][j].Untagged;
} else if (ProgressStructure[i][j].ID=="-----") { // if county sublist, find total county row and add there
var CountyTotalsIndex=j+1;
while (CountyTotalsIndex<ProgressStructure[i].length-2) {
if (!isNaN(parseFloat(ProgressStructure[i][CountyTotalsIndex].ID))) break;
CountyTotalsIndex++;
}
ProgressStructure[i][CountyTotalsIndex].Total+=ProgressStructure[i][j].Total;
ProgressStructure[i][CountyTotalsIndex].Illustrated+=ProgressStructure[i][j].Illustrated;
ProgressStructure[i][CountyTotalsIndex].Articled+=ProgressStructure[i][j].Articled;
ProgressStructure[i][CountyTotalsIndex].Stubs+=ProgressStructure[i][j].Stubs;
ProgressStructure[i][CountyTotalsIndex].NRISonly+=ProgressStructure[i][j].NRISonly;
ProgressStructure[i][CountyTotalsIndex].StartPlus+=ProgressStructure[i][j].StartPlus;
ProgressStructure[i][CountyTotalsIndex].Unassessed+=ProgressStructure[i][j].Unassessed;
ProgressStructure[i][CountyTotalsIndex].Untagged+=ProgressStructure[i][j].Untagged;
} else if (ProgressStructure[i][j].ID=="ddddd") { // if county duplicate row, subtract from county total
ProgressStructure[i][j+1].Total-=ProgressStructure[i][j].Total;
ProgressStructure[i][j+1].Illustrated-=ProgressStructure[i][j].Illustrated;
ProgressStructure[i][j+1].Articled-=ProgressStructure[i][j].Articled;
ProgressStructure[i][j+1].Stubs-=ProgressStructure[i][j].Stubs;
ProgressStructure[i][j+1].NRISonly-=ProgressStructure[i][j].NRISonly;
ProgressStructure[i][j+1].StartPlus-=ProgressStructure[i][j].StartPlus;
ProgressStructure[i][j+1].Unassessed-=ProgressStructure[i][j].Unassessed;
ProgressStructure[i][j+1].Untagged-=ProgressStructure[i][j].Untagged;
} else { // unknown ID; skip it
alert("Error! Unknown ID="+ProgressStructure[i][j].ID+" in Table "+i+", Row "+j+". Skipping this list in totals.");
}
}
// subtract state duplicates
ProgressStructure[i][TotalsIndex].Total-=ProgressStructure[i][TotalsIndex-1].Total;
ProgressStructure[i][TotalsIndex].Illustrated-=ProgressStructure[i][TotalsIndex-1].Illustrated;
ProgressStructure[i][TotalsIndex].Articled-=ProgressStructure[i][TotalsIndex-1].Articled;
ProgressStructure[i][TotalsIndex].Stubs-=ProgressStructure[i][TotalsIndex-1].Stubs;
ProgressStructure[i][TotalsIndex].NRISonly-=ProgressStructure[i][TotalsIndex-1].NRISonly;
ProgressStructure[i][TotalsIndex].StartPlus-=ProgressStructure[i][TotalsIndex-1].StartPlus;
ProgressStructure[i][TotalsIndex].Unassessed-=ProgressStructure[i][TotalsIndex-1].Unassessed;
ProgressStructure[i][TotalsIndex].Untagged-=ProgressStructure[i][TotalsIndex-1].Untagged;
// record state totals in national table
ProgressStructure[0][i-1].Total=ProgressStructure[i][TotalsIndex].Total;
ProgressStructure[0][i-1].Illustrated=ProgressStructure[i][TotalsIndex].Illustrated;
ProgressStructure[0][i-1].Articled=ProgressStructure[i][TotalsIndex].Articled;
ProgressStructure[0][i-1].Stubs=ProgressStructure[i][TotalsIndex].Stubs;
ProgressStructure[0][i-1].NRISonly=ProgressStructure[i][TotalsIndex].NRISonly;
ProgressStructure[0][i-1].StartPlus=ProgressStructure[i][TotalsIndex].StartPlus;
ProgressStructure[0][i-1].Unassessed=ProgressStructure[i][TotalsIndex].Unassessed;
ProgressStructure[0][i-1].Untagged=ProgressStructure[i][TotalsIndex].Untagged;
// add state totals to national totals
NationalTotalsIndex=ProgressStructure[0].length-1;
ProgressStructure[0][NationalTotalsIndex].Total+=ProgressStructure[0][i-1].Total;
ProgressStructure[0][NationalTotalsIndex].Illustrated+=ProgressStructure[0][i-1].Illustrated;
ProgressStructure[0][NationalTotalsIndex].Articled+=ProgressStructure[0][i-1].Articled;
ProgressStructure[0][NationalTotalsIndex].Stubs+=ProgressStructure[0][i-1].Stubs;
ProgressStructure[0][NationalTotalsIndex].NRISonly+=ProgressStructure[0][i-1].NRISonly;
ProgressStructure[0][NationalTotalsIndex].StartPlus+=ProgressStructure[0][i-1].StartPlus;
ProgressStructure[0][NationalTotalsIndex].Unassessed+=ProgressStructure[0][i-1].Unassessed;
ProgressStructure[0][NationalTotalsIndex].Untagged+=ProgressStructure[0][i-1].Untagged;
}
// special row for Tangier, Morocco
ProgressStructure[0][NationalTotalsIndex].Total+=ProgressStructure[0][NationalTotalsIndex-2].Total;
ProgressStructure[0][NationalTotalsIndex].Illustrated+=ProgressStructure[0][NationalTotalsIndex-2].Illustrated;
ProgressStructure[0][NationalTotalsIndex].Articled+=ProgressStructure[0][NationalTotalsIndex-2].Articled;
ProgressStructure[0][NationalTotalsIndex].Stubs+=ProgressStructure[0][NationalTotalsIndex-2].Stubs;
ProgressStructure[0][NationalTotalsIndex].NRISonly+=ProgressStructure[0][NationalTotalsIndex-2].NRISonly;
ProgressStructure[0][NationalTotalsIndex].StartPlus+=ProgressStructure[0][NationalTotalsIndex-2].StartPlus;
ProgressStructure[0][NationalTotalsIndex].Unassessed+=ProgressStructure[0][NationalTotalsIndex-2].Unassessed;
ProgressStructure[0][NationalTotalsIndex].Untagged+=ProgressStructure[0][NationalTotalsIndex-2].Untagged;
// subtract national duplicates
ProgressStructure[0][NationalTotalsIndex].Total-=ProgressStructure[0][NationalTotalsIndex-1].Total;
ProgressStructure[0][NationalTotalsIndex].Illustrated-=ProgressStructure[0][NationalTotalsIndex-1].Illustrated;
ProgressStructure[0][NationalTotalsIndex].Articled-=ProgressStructure[0][NationalTotalsIndex-1].Articled;
ProgressStructure[0][NationalTotalsIndex].Stubs-=ProgressStructure[0][NationalTotalsIndex-1].Stubs;
ProgressStructure[0][NationalTotalsIndex].NRISonly-=ProgressStructure[0][NationalTotalsIndex-1].NRISonly;
ProgressStructure[0][NationalTotalsIndex].StartPlus-=ProgressStructure[0][NationalTotalsIndex-1].StartPlus;
ProgressStructure[0][NationalTotalsIndex].Unassessed-=ProgressStructure[0][NationalTotalsIndex-1].Unassessed;
ProgressStructure[0][NationalTotalsIndex].Untagged-=ProgressStructure[0][NationalTotalsIndex-1].Untagged;
setTimeout(function() {ParseProgressSource()},1); // small delay for non-Firefox browsers to update screen
}
// update progress page source with queried totals
async function ParseProgressSource() {
var pagesourceresponse = await getPageSource(mw.config.get('wgPageName'));
console.log("Page source query",pagesourceresponse);
var pagesource = getPageTextFromQueryResponse(pagesourceresponse);
var newpagesource=pagesource;
var TableStartIndex=pagesource.indexOf("==State totals");
var test;
var RowEndIndex;
var oldRow;
var firstColumn;
var newRow;
var temp;
var str;
for (var i=0; i<ProgressStructure.length; i++) {
TableStartIndex=pagesource.indexOf("{|",TableStartIndex+1); // find next table in old page source
var TableEndIndex=pagesource.indexOf("|}",TableStartIndex)+2;
var oldTable=pagesource.substr(TableStartIndex,TableEndIndex-TableStartIndex);
var newTable=oldTable;
var RowStartIndex=0;
for (var j=0; j<ProgressStructure[i].length-2; j++) {
RowStartIndex=oldTable.indexOf("\n|-",RowStartIndex+1); // find next row in old table
if (ProgressStructure[i][j].ID=="ddddd") continue; // skip duplicate rows
RowEndIndex=oldTable.indexOf("\n|-",RowStartIndex+1);
oldRow=oldTable.substr(RowStartIndex,RowEndIndex-RowStartIndex);
firstColumn=oldRow.indexOf("\n|");
var stop=4; // skip one cell for national table, three for states
if (i===0) stop=2;
for (var inc=0; inc<stop; inc++) {
firstColumn=oldRow.indexOf("\n|",firstColumn+1); // find next cell
}
// build up new row
newRow=oldRow.substr(0,firstColumn);
temp=ProgressStructure[i][j];
// total
str=temp.Total.toString();
if (temp.Total>999) str=str.replace(/\B(?=(\d{3})+(?!\d))/g, ","); // add thousands separators
newRow+="\n| "+str;
// illustrated
str=temp.Illustrated.toString();
if (temp.Illustrated>999) str=str.replace(/\B(?=(\d{3})+(?!\d))/g, ",");
newRow+="\n| "+str;
// illustrated percent
if (temp.Total===0) {
str="-";
} else {
str = Math.round(temp.Illustrated/temp.Total*1000)/10;
test = str.toString().indexOf(".");
if (test==-1 && str!=100 && str!==0) str+=".0"; // force decimal
str+="%";
}
newRow+="\n| "+str;
// articled
str=temp.Articled.toString();
if (temp.Articled>999) str=str.replace(/\B(?=(\d{3})+(?!\d))/g, ",");
newRow+="\n| "+str;
// articled percent
if (temp.Total===0) {
str="-";
} else {
str = Math.round(temp.Articled/temp.Total*1000)/10;
test = str.toString().indexOf(".");
if (test==-1 && str!=100 && str!==0) str+=".0";
str+="%";
}
newRow+="\n| "+str;
// stubs
str=temp.Stubs.toString();
if (temp.Stubs>999) str=str.replace(/\B(?=(\d{3})+(?!\d))/g, ",");
newRow+="\n| "+str;
// NRIS-only
str=temp.NRISonly.toString();
if (temp.NRISonly>999) str=str.replace(/\B(?=(\d{3})+(?!\d))/g, ",");
newRow+="\n| "+str;
// Start+
str=temp.StartPlus.toString();
if (temp.StartPlus>999) str=str.replace(/\B(?=(\d{3})+(?!\d))/g, ",");
newRow+="\n| "+str;
// Start+ percent
if (temp.Total===0) {
str="-";
} else {
str = Math.round(temp.StartPlus/temp.Total*1000)/10;
test = str.toString().indexOf(".");
if (test==-1 && str!=100 && str!==0) str+=".0"; // force decimal
str+="%";
}
newRow+="\n| "+str;
// unassessed
str=temp.Unassessed.toString();
if (temp.Unassessed>999) str=str.replace(/\B(?=(\d{3})+(?!\d))/g, ",");
newRow+="\n| "+str;
// untagged
str=temp.Untagged.toString();
if (temp.Untagged>999) str=str.replace(/\B(?=(\d{3})+(?!\d))/g, ",");
newRow+="\n| "+str;
// net quality
if (temp.Total===0) {
str="-";
} else {
str=temp.StartPlus+0.5*temp.Stubs+0.5*temp.Unassessed-0.5*temp.Untagged-0.75*temp.NRISonly;
str=Math.round((0.75*str/temp.Total+0.25*temp.Illustrated/temp.Total)*1000)/10;
if (str<0) str=0;
test=str.toString().indexOf(".");
if (test==-1 && str!=100 && str!==0) str+=".0";
str+="%";
}
newRow+="\n| "+str;
// update new table with new row
newTable=newTable.replace(oldRow,newRow);
}
RowStartIndex=oldTable.indexOf("\n|-",RowStartIndex+1); // skip duplicate row
RowStartIndex=oldTable.indexOf("\n|-",RowStartIndex+1);
RowEndIndex=oldTable.indexOf("\n|}",RowStartIndex+1);
oldRow=oldTable.substr(RowStartIndex,RowEndIndex-RowStartIndex);
firstColumn=oldRow.indexOf("\n!"); // totals row uses ! instead of |
firstColumn=oldRow.indexOf("\n!",firstColumn+1); // skip first cell
// build up new totals row
newRow=oldRow.substr(0,firstColumn);
temp=ProgressStructure[i][ProgressStructure[i].length-1];
// total
str=temp.Total.toString();
if (temp.Total>999) str=str.replace(/\B(?=(\d{3})+(?!\d))/g, ","); // add thousands separators
newRow+="\n! "+str;
// illustrated
str=temp.Illustrated.toString();
if (temp.Illustrated>999) str=str.replace(/\B(?=(\d{3})+(?!\d))/g, ",");
newRow+="\n! "+str;
// illustrated percent
if (temp.Total===0) {
str="-";
} else {
str = Math.round(temp.Illustrated/temp.Total*1000)/10;
test = str.toString().indexOf(".");
if (test==-1 && str!=100 && str!==0) str+=".0"; // force decimal
str+="%";
}
newRow+="\n! "+str;
// articled
str=temp.Articled.toString();
if (temp.Articled>999) str=str.replace(/\B(?=(\d{3})+(?!\d))/g, ",");
newRow+="\n! "+str;
// articled percent
if (temp.Total===0) {
str="-";
} else {
str = Math.round(temp.Articled/temp.Total*1000)/10;
test = str.toString().indexOf(".");
if (test==-1 && str!=100 && str!==0) str+=".0"; // force decimal
str+="%";
}
newRow+="\n! "+str;
// stubs
str=temp.Stubs.toString();
if (temp.Stubs>999) str=str.replace(/\B(?=(\d{3})+(?!\d))/g, ",");
newRow+="\n! "+str;
// NRIS-only
str=temp.NRISonly.toString();
if (temp.NRISonly>999) str=str.replace(/\B(?=(\d{3})+(?!\d))/g, ",");
newRow+="\n! "+str;
// Start+
str=temp.StartPlus.toString();
if (temp.StartPlus>999) str=str.replace(/\B(?=(\d{3})+(?!\d))/g, ",");
newRow+="\n! "+str;
// Start+ percent
if (temp.Total===0) {
str="-";
} else {
str = Math.round(temp.StartPlus/temp.Total*1000)/10;
test = str.toString().indexOf(".");
if (test==-1 && str!=100 && str!==0) str+=".0"; // force decimal
str+="%";
}
newRow+="\n! "+str;
// unassessed
str=temp.Unassessed.toString();
if (temp.Unassessed>999) str=str.replace(/\B(?=(\d{3})+(?!\d))/g, ",");
newRow+="\n! "+str;
// untagged
str=temp.Untagged.toString();
if (temp.Untagged>999) str=str.replace(/\B(?=(\d{3})+(?!\d))/g, ",");
newRow+="\n! "+str;
// net quality
if (temp.Total===0) {
str="-";
} else {
str=temp.StartPlus+0.5*temp.Stubs+0.5*temp.Unassessed-0.5*temp.Untagged-0.75*temp.NRISonly;
str=Math.round((0.75*str/temp.Total+0.25*temp.Illustrated/temp.Total)*1000)/10;
if (str<0) str=0;
test=str.toString().indexOf(".");
if (test==-1 && str!=100 && str!==0) str+=".0";
str+="%";
}
newRow+="\n! "+str;
// update new page source with new table
newTable=newTable.replace(oldRow,newRow);
newpagesource=newpagesource.replace(oldTable,newTable);
}
// now edit page to provide new source
var ProgressDiv=document.getElementById("ProgressDiv");
ProgressDiv.innerHTML+="<br>Editing page (this might take up to one minute)... ";
InitializeEdit(newpagesource);
}
// initialize edit
function InitializeEdit(pagesource) {
var d=new Date();
var months=['January','February','March','April','May','June','July','August','September','October','November','December'];
var year=d.getYear();
if (year < 1000) year += 1900;
var DateStr=months[d.getMonth()]+" "+d.getDate()+", "+year;
regex=/(January|February|March|April|May|June|July|August|September|October|November|December) [0-9]{1,2}, [0-9]{4}/g;
var tempstring=pagesource.split('==County totals=='); // ignore dates in lead (e.g. date of last map update)
pagesource=tempstring[0]+'==County totals=='+tempstring[1].replace(regex,DateStr); // update date strings above tables
var ErrorStr = '';
if (ErrorCount>0) ErrorStr = " Errors encountered for "+ErrorCount+" counties, which were skipped. Human attention needed.";
var summary='Updating county data as of '+DateStr+' using [[User:Magicpiano/NRBot/UpdateNRHPProgress.js|script]].'+ErrorStr;
editPage(pagesource,mw.config.get('wgPageName'),summary);
}
function PageEdited(ajaxResponse,status,pagesource) {
var ProgressDiv=document.getElementById("ProgressDiv");
if (status!="success") {
var retry=confirm("Error: "+ajaxResponse.errorThrown+" while editing page!\n\nCancel=Abort OK=Retry");
if (retry) {
ProgressDiv.innerHTML+="Retrying... ";
InitializeEdit(pagesource); // try again
} else {
ProgressDiv.innerHTML+="Edit failure! Script aborted!";
}
return;
}
var responseText=JSON.parse(ajaxResponse.responseText);
var diff=responseText.edit.newrevid;
var linkStr="//en.wikipedia.org/w/index.php?diff="+diff;
ProgressDiv.innerHTML+="Page edited! Click <a href='"+linkStr+"'>here</a> for diff.";
console.log("Number of restricted images found: "+RestrictedImageCount);
// output technical information to console
var WarningText="NRHP Progress Warnings: ";
for (var i=0; i<WarningCount.length; i++) {
WarningText+=WarningCount[i][0]+" ("+WarningCount[i][1]+"), ";
}
if (WarningCount[0][0]!=="") {
WarningText=WarningText.substr(0,WarningText.length-2);
} else {
WarningText="NRHP Progress Warnings: none";
}
console.log(WarningText);
}
function editPage(text,title,summary) { // edit page when done
var edit_token;
edit_token = mw.user.tokens.get( 'editToken' );
if (edit_token === null) {
edit_token = mw.user.tokens.get( 'csrfToken');
}
$.ajax({
dataType: 'json',
url: mw.util.wikiScript( 'api' ),
type: 'POST',
data: {
format: 'json',
action: 'edit',
title: title,
text: text,
summary: summary,
token: edit_token
},
success: function( data ) {
if (data && data.edit && data.edit.result && data.edit.result == 'Success') {
console.log('Page edit succeeded');
} else {
var msg;
if (data && data.error) {
msg = "Error editing page: ";
msg += " code=" + data.error.code;
msg += " info=" + data.error.info;
} else {
msg = "Error editing page (no data or error)";
}
alert(msg);
}
},
error: function(ajaxResponse,status,errorThrown) {
ajaxResponse.errorThrown=errorThrown;
console.log("EditPage: Error editing "+title);
},
complete: function(ajaxResponse,status) {PageEdited(ajaxResponse,status,text)}
});
}
function getPageTextFromQueryResponse(response)
{
var txt;
var query = response.query;
console.log('getPageText',response);
for (var pageid in query.pages) {
if (pageid == "-1") return null; // this means
txt=query.pages[pageid].revisions[0].slots.main['*'];
}
//console.log('Body:');
//console.log(txt);
return txt;
}
async function getPageSource(page) {
console.log("getPageSource page",page);
var api = new mw.Api();
var rsp;
// nb this will throw an exception if it fails
rsp = await api.get({
format: 'json',
action: 'query',
prop: 'revisions',
rvprop: 'content',
rvslots: '*',
titles: page,
indexpageids: true,
redirects: 'true'
});
console.log("getPageSource response",rsp);
return rsp;
}
function getProgressListWikitext(title,currentTable,currentRow) { // asynchronous fetch of each list's wikitext
console.log("GetProgressListWikiText: fetch "+title);
$.ajax({
dataType: "json",
url: mw.util.wikiScript('api'),
data: {
format: 'json',
action: 'query',
prop: 'revisions',
rvprop: 'content',
titles: title,
indexpageids: true,
redirects: 'true'
},
error: function(ajaxResponse,status,errorThrown) {
console.log("GetProgressListWikiText: Error fetching "+title);
ajaxResponse.errorThrown=errorThrown;
},
complete: function(ajaxResponse,status) {
console.log("GetProgressListWikitext: Status fetching "+title+": "+status);
try {
WikitextFetched(ajaxResponse,status,title,currentTable,currentRow);
}
catch (e) {
alert("Exception after fetching "+title+": "+e);
ProgressFatalError(0,title,currentTable,currentRow);
}
}
});
}
//jQuery(ProgressButton);
$.when($.ready).then(ProgressButton);