Jump to content

User:Magicpiano/NRBot/UpdateNRHPProgress2.js

From Wikipedia, the free encyclopedia
Note: After saving, you have to bypass your browser's cache to see the changes. Google Chrome, Firefox, Microsoft Edge and Safari: Hold down the ⇧ Shift key and click the Reload toolbar button. For details and instructions about other browsers, see Wikipedia:Bypass your cache.
/* jshint maxerr: 3000 */
/*
   The following script places a button at the top of the NRHP Progress page [[WP:NRHPPROGRESS]]. When the button is clicked,
   the script begins to load each county list linked from the Progress page in the background, extract statistics about
   sites in each list, and updates the Progress page with the fetched data.
*/

//var wikitext = 'error';
var ProgressStructure=[]; // ProgressStructure[table][row].StatName
var TotalToQuery=0;
var TotalQueried=0;
var ErrorCount=0;
var WarningCount=[["",0]]; // 0=status, 1=count
var InitialTime=0;
var ProgressDivTimer=0; // timer for updating ProgressDiv
var DefaultQueryPause=1; // number of milliseconds to wait between each API query; increased by code if rate limit reached
var RestrictedImageCount=0;
var bIgnoreTableCount = false;

// Users authorized to run this script
var AuthorizedUsers = [ "Magicpiano" ]; // nb this is for use only when g_disabled is true
var CurrentMaintainer = "Magicpiano"; // User who is currently maintaining the script

function getRandomIntInclusive(min, max) {
  min = Math.ceil(min);
  max = Math.floor(max);
  return Math.floor(Math.random() * (max - min + 1) + min); //The maximum is inclusive and the minimum is inclusive
}
//
// WorkQueue helper
//
// run up to maxActiveSize work items at a time
//
function WorkQueue(maxActiveSize = 1, initialWorkQueue = [])
{
	this.maxActiveSize = maxActiveSize;
	this.pendingQueue = initialWorkQueue;
	this.activeQueue = [];
	this.completedQueue = [];
	this.isRunning = false;
	if (initialWorkQueue.length > 0) this.run();
}

// determine if we have room to run something more, and if there is something more to run
WorkQueue.prototype.runNext = function()
{
	return ((this.activeQueue.length < this.maxActiveSize) && this.pendingQueue.length > 0);
};

WorkQueue.prototype.createPromise = function(work, retryCount=0)
{
	var p = new Promise((result, reject) => work().then(result).catch(reject));
	p.work = work;
	p.retryCount = retryCount;
}

// add a work item
WorkQueue.prototype.push = function(p, retryCount=0)
{
	this.pendingQueue.push(this.createPromise(p,retryCount));
};

// add a list of work items
WorkQueue.prototype.pushList = function(plist, retryCount=0)
{
	for (var idx in plist)
	{
		const p = plist[idx];
		this.pendingQueue.push(this.createPromise(p,retryCount));
	}
};

WorkQueue.prototype.getCompletedCount = function()
{
	return this.completedQueue.length;	
}
// run (or attempt to) the queue
WorkQueue.prototype.run = function()
{
	if (this.isRunning) return;
	this.isRunning = true;
	this.nFailed = 0;
	this.nSucceeded = 0;
	this.nCompleted = 0;
	this.nEntries = this.pendingQueue.length;
	var ranAtLeastOnce = false;
	const wq = this;
	try {
		while (this.runNext())
		{
			if (!ranAtLeastOnce)
			{
				console.log('Start running queue',this);
				ranAtLeastOnce = true;
			}
			const promise = this.pendingQueue.shift();
			console.log('Running promise',promise);
			promise
				.then((result) => {
					wq.nSucceeded++;
				})
				.catch((e) => {
					console.log('Promise failed',promise,e);
					if (promise.retryCount > 0)
					{
						console.log('Retrying');
						wq.push(promise.work, promise.retryCount-1);
					}
				})
				.finally(() => {
					wq.completedQueue.push(removeFromArray(wq.activeQueue, promise));
					console.log('Finished promise',promise);
					wq.run();
			});
			this.activeQueue.push(promise);
		}
	}
	finally {
		this.isRunning = false;
		if (ranAtLeastOnce) console.log('Stop running queue',this);
	}
};

function removeFromArray(arr, item)
{
	var idx = arr.indexOf(item);
	if (idx < 0) return;
	arr.splice(idx,1);
}

async function makeJob(work, state)
{
	return await work(state);
}

function ProgressButton() {
	var bNotHere = true;
    if (mw.config.get('wgPageName')=="Wikipedia:WikiProject_National_Register_of_Historic_Places/Progress") bNotHere = false;
    if (mw.config.get('wgPageName')=="User:Magicpiano/NRBot/UpdateNRHPProgressTester")
    {
    	bNotHere = false;
    	bIgnoreTableCount = true;
    }
	// the above notwithstanding, if in edit mode we also ignore
    if (location.href.indexOf('action')!=-1) bNotHere = true;
    if (bNotHere) return;
    
    var button = document.getElementById("NRHPProgressUpdateButton2");
    if (button !== null) return;
   	button=document.createElement("input");
    button.setAttribute("type", "button");
    button.setAttribute("value", "Update Statistics 2");
    button.setAttribute("onclick", "ClickUpdateNRHPProgress()");
    button.id = "NRHPProgressUpdateButton2";
    var content=document.getElementById('mw-content-text');

    content.parentNode.insertBefore(button, content);
}

var g_disabled = true;

function CheckPermission() {
	var username = mw.user.getName();
	console.log("UpdateNRHPProgress: run by user "+username);
	if (username == CurrentMaintainer) return true;
	for (var i=0; i< AuthorizedUsers.length; i++) {
		if (username == AuthorizedUsers[i]) {
			return true;
		}
	}
	if (g_disabled) {
		alert("Script is a work in progress.");
	    //alert("Script is currently disabled for maintenance.");
		return false;	
	}
}

async function ClickUpdateNRHPProgress() {   // after button is clicked, disable it and fetch source of Progress page
    if (!CheckPermission()) {
    	return;
    }
    var button2 = document.getElementById("NRHPProgressUpdateButton2");
    button2.disabled = true;
    var ProgressDiv = document.getElementById("ProgressDiv");
    if (ProgressDiv === null) {
	    ProgressDiv = document.createElement("div");
	    ProgressDiv.setAttribute("id", "ProgressDiv");
	    ProgressDiv.setAttribute("style", "width:500px; border:1px solid black; padding:5px; background:#ffffff");
	    button2.parentNode.insertBefore(ProgressDiv, button2);
    }
    ProgressDiv.innerHTML = "Initializing...";

    var pagesource = await getPageSource(mw.config.get('wgPageName')); // TODO not really needed here?
    SetupTables();
    
    InitialTime=new Date(); // record starting time
    UpdateProgressDiv();
    processTableList(ProgressStructure);
    //LoadList(1,0); // begin querying first page
}

// create array of table structure to be populated later
function SetupTables() {
    var table=document.getElementsByClassName('wikitable sortable');

	// initialize globals    
	TotalToQuery=0;
	TotalQueried=0;
	ErrorCount=0;
	WarningCount=[["",0]]; // 0=status, 1=count
	InitialTime=0;
	ProgressDivTimer=0; // timer for updating ProgressDiv
	DefaultQueryPause=1; // number of milliseconds to wait between each API query; increased by code if rate limit reached
	RestrictedImageCount=0;

    // Expected table count:
    // There should be 61 tables, as follows:
    // The national table, 50 states, and the following 10 non-state entities:
    //  District of Columbia, Puerto Rico, Virgin Islands, Guam, American Samoa,
    //  Northern Mariana Islands, Federated States of Micronesia, Palau, Marshall Islands
    //  Minor Outlying Islands
    // This number should only need adjustmnent if the number of states or non-state entities having
    // separate tables is changed.  An alert here is more likely caused by some sort of formatting change
    // or error in the table.
    //
    // Ideally, the above search should be restrictable (by class in some way?) to only the specific tables of interest.
    //
    if ((!bIgnoreTableCount) && table.length !== 61) {
    	alert('Incorrect table count in progress page: expected 61, saw '+table.length+'.');
    	return;
    }

    // set up national totals
    var tr=table[0].getElementsByTagName("tr");
    var i,j, td;
    ProgressStructure[0]=[];
    for (j=1; j<tr.length-3; j++) {
        td=tr[j].getElementsByTagName("td");
        ProgressStructure[0][j-1]={
        	ID: td[0].innerHTML,
        	Total: 0,
        	Illustrated: 0,
        	Articled: 0,
        	Stubs: 0,
        	NRISonly: 0,
        	StartPlus: 0,
        	Unassessed: 0,
        	Untagged: 0
        };
    }

    // special row for Tangier, Morocco
    td=tr[tr.length-3].getElementsByTagName("td");
    /* This code copies the existing settings for the American Legation in Tangier into the table.
       This process is prone to error if it is modified. Because the article is an illustrated Start+ article,
       the settings hardcoded further down are unlikely to change.
    */
    /* // To copy previous contents instead of hardcoding:
    ProgressStructure[0][tr.length-4]={};
    ProgressStructure[0][tr.length-4].ID="Tangier, Morocco";
    ProgressStructure[0][tr.length-4].Total=parseFloat(td[1].innerHTML.replace(",",""));
    ProgressStructure[0][tr.length-4].Illustrated=parseFloat(td[2].innerHTML.replace(",",""));
    ProgressStructure[0][tr.length-4].Articled=parseFloat(td[4].innerHTML.replace(",",""));
    ProgressStructure[0][tr.length-4].Stubs=parseFloat(td[6].innerHTML.replace(",",""));
    ProgressStructure[0][tr.length-4].NRISonly=parseFloat(td[7].innerHTML.replace(",",""));
    ProgressStructure[0][tr.length-4].StartPlus=parseFloat(td[8].innerHTML.replace(",",""));
    ProgressStructure[0][tr.length-4].Unassessed=parseFloat(td[10].innerHTML.replace(",",""));
    ProgressStructure[0][tr.length-4].Untagged=parseFloat(td[11].innerHTML.replace(",",""));
	*/
	/* American Legation: Illustrated, Articled, not stub, not NRIS, Start+, Assessed, Tagged */
    ProgressStructure[0][tr.length-4]={
    	ID: "Tangier, Morocco",
    	Total: 1,
    	Illustrated: 1,
    	Articled: 1,
    	Stubs: 0,
    	NRISonly: 0,
    	StartPlus: 1,
    	Unassessed: 0,
    	Untagged: 0
    };

    // duplicates row
    td=tr[tr.length-2].getElementsByTagName("td");
    ProgressStructure[0][tr.length-3]={
    	ID: "National Duplicates",
    	Total: parseFloat(td[0].innerHTML.replace(",","")),
		Illustrated: parseFloat(td[1].innerHTML.replace(",","")),
		Articled: parseFloat(td[3].innerHTML.replace(",","")),
    	Stubs: parseFloat(td[5].innerHTML.replace(",","")),
    	NRISonly: parseFloat(td[6].innerHTML.replace(",","")),
    	StartPlus: parseFloat(td[7].innerHTML.replace(",","")),
    	Unassessed: parseFloat(td[9].innerHTML.replace(",","")),
    	Untagged: parseFloat(td[10].innerHTML.replace(",",""))
    };

    // national totals
    ProgressStructure[0][tr.length-2]={
    	ID: "National Totals",
    	Total: 0,
    	Illustrated: 0,
    	Articled:0,
    	Stubs: 0,
    	NRISonly: 0,
    	StartPlus: 0,
    	Unassessed: 0,
    	Untagged: 0 
    };

    // now data for each state
    for (i=1; i<table.length; i++) {
        tr=table[i].getElementsByTagName("tr");
        ProgressStructure[i]=[];
        for (j=1; j<tr.length-2; j++) { // skip title row, statewide duplicates, and totals row
            td=tr[j].getElementsByTagName("td");// fill in existing data in case error
            ProgressStructure[i][j-1]={};
            ProgressStructure[i][j-1].ID=td[0].innerHTML.substr(0,5);
            ProgressStructure[i][j-1].Total=parseFloat(td[3].innerHTML.replace(",",""));
            ProgressStructure[i][j-1].Illustrated=parseFloat(td[4].innerHTML.replace(",",""));
            ProgressStructure[i][j-1].Articled=parseFloat(td[6].innerHTML.replace(",",""));
            ProgressStructure[i][j-1].Stubs=parseFloat(td[8].innerHTML.replace(",",""));
            ProgressStructure[i][j-1].NRISonly=parseFloat(td[9].innerHTML.replace(",",""));
            ProgressStructure[i][j-1].StartPlus=parseFloat(td[10].innerHTML.replace(",",""));
            ProgressStructure[i][j-1].Unassessed=parseFloat(td[12].innerHTML.replace(",",""));
            ProgressStructure[i][j-1].Untagged=parseFloat(td[13].innerHTML.replace(",",""));
            var link=td[1].getElementsByTagName("a");
            if (link.length!==0 && link[0].href.search("#")==-1) {
                link=decodeURI(link[0].href).split("/");
                link=link[link.length-1].replace(/_/g," ");
                ProgressStructure[i][j-1].Link=link;

                ProgressStructure[i][j-1].ArticleQueried=0; // for querying later
                ProgressStructure[i][j-1].TalkQueried=0;
            } else {
                if (ProgressStructure[i][j-1].ID!="ddddd") { // if no link and not duplicate, must be totals row, so we can zero it

                    ProgressStructure[i][j-1].Total=0;
                    ProgressStructure[i][j-1].Illustrated=0;
                    ProgressStructure[i][j-1].Articled=0;
                    ProgressStructure[i][j-1].Stubs=0;
                    ProgressStructure[i][j-1].NRISonly=0;
                    ProgressStructure[i][j-1].StartPlus=0;
                    ProgressStructure[i][j-1].Unassessed=0;
                    ProgressStructure[i][j-1].Untagged=0;
                }
            }
        }

        // duplicates row
        td=tr[tr.length-2].getElementsByTagName("td");

	    ProgressStructure[i][tr.length-3]={
	    	ID: ProgressStructure[0][i-1].ID+" Duplicates",
	    	Total: parseFloat(td[0].innerHTML.replace(",","")),
			Illustrated: parseFloat(td[1].innerHTML.replace(",","")),
			Articled: parseFloat(td[3].innerHTML.replace(",","")),
	    	Stubs: parseFloat(td[5].innerHTML.replace(",","")),
	    	NRISonly: parseFloat(td[6].innerHTML.replace(",","")),
	    	StartPlus: parseFloat(td[7].innerHTML.replace(",","")),
	    	Unassessed: parseFloat(td[9].innerHTML.replace(",","")),
	    	Untagged: parseFloat(td[10].innerHTML.replace(",",""))
	    };
	
        // state totals
	    ProgressStructure[i][tr.length-2]={
	    	ID: ProgressStructure[0][i-1].ID+" Totals",
	    	Total: 0,
	    	Illustrated: 0,
	    	Articled:0,
	    	Stubs: 0,
	    	NRISonly: 0,
	    	StartPlus: 0,
	    	Unassessed: 0,
	    	Untagged: 0 
	    };
	
    }
    for (i=1; i<ProgressStructure.length; i++) { // count total number of rows to check
        for (j=0; j<ProgressStructure[i].length-2; j++) {
            if (typeof ProgressStructure[i][j].Link!="undefined") TotalToQuery++; // don't count duplicates and total rows
        }
    }
    TotalQueried=0;

    var ProgressDiv=document.getElementById("ProgressDiv");
    ProgressDiv.innerHTML+=" Done!<br>";

    var ProgressSpan=document.createElement("span");
    ProgressSpan.setAttribute("id", "ProgressSpan");
    ProgressDiv.appendChild(ProgressSpan);
    ProgressSpan.innerHTML = "Querying county data... 0 (0%) of "+TotalToQuery+" lists checked.";

    var TimeSpan=document.createElement("span");
    TimeSpan.setAttribute("id", "TimeSpan");
    ProgressDiv.appendChild(TimeSpan);
    TimeSpan.innerHTML = "";

}

function processTableList(ps)
{
	console.log("processTableList");
	var curtable;
	var tabletitle;
	var wq = new WorkQueue(3);
	var tablecount = 0;
	for (curtable = 0; curtable<2 /*ps.length-1*/; curtable++)
	{
		tablecount++;
		var state = {
			progressState: ps,
			curtable: curtable
		};
		wq.push(makeJob(processTable,state));	
	}
}

async function processTable(state)
{
	var curtable = state.curtable;
	var ps = state.progressState;
	var tabletitle;
	console.log('processTable',curtable);
	var rowcount = 0;

	for (currow = 0; currow<ps[curtable].length-2; currow++)
	{
	    if (typeof ps[curtable][currow].Link=="undefined")
	    {
	    	// skip duplicate and total rows
	    	continue;
	    }
	    rowcount++;
	    tabletitle=ps[curtable][currow].Link;
	    console.log('processTable doing '+tabletitle);
	}
	
	// TODO return promise that resolves when all rows are processed	
	return rowcount;
}
/*

		for (currow = 0; currow<ps[curtable].length-2; currow++)
		{
		    if (typeof ps[curtable][currow].Link=="undefined") {
		    	// skip duplicate and total rows
		    	continue;
		    }
		    tabletitle=ps[curtable][currow].Link;
		    console.log('processTableList doing '+tabletitle);
		    var state = {
		    	tabletitle: tabletitle,
		    	curtable: curtable,
		    	currow: currow,
		    	progressState: ps,
		    	promise: null
		    };
			var promise = new Promise((resolve,reject) => {
				processTableRow(state, resolve, reject);	
			});
			promise.psState = state;
			state.promise = promise;
			promise.nTries = 5;
			wq.push(promise);
		}
	}
}
*/

function processTableRow1(ps,tabletitle,curtable,currow, resolve, reject)
{
	console.log('processTableRow1',tabletitle,curtable,currow);
	resolve(true);
}

async function processTableRow(promise, resolve, reject)
{
	var ps = promise.progressState;
	var tabletitle = promise.tabletitle;
	var curtable = promise.curtable;
	var currow = promise.currow;
    console.log("processTableRow",tabletitle,curtable,currow);
    var listingSource, listingTalkSource;
	var results;
	try {
	    listingSource = getPageSource(tabletitle);
	    listingTalkSource = getPageSource('Talk:'+tabletitle);
	    results = await Promise.allSettled([listingSource,listingTalkSource]);
	    console.log('Promise results', results);
	    listingSource = getPageTextFromQueryResponse(results[0].value);
	    listingTalkSource = getPageTextFromQueryResponse(results[1].value);
	    if (listingTalkSource == null) listingTalkSource = "";
	    // TODO also need query results concerning redirects (see WikiTextFetched)
	    console.log('Results of fetching listing article source and talk page',tabletitle, listingSource.length, listingTalkSource.length);
	    //processListingSources(ps,tabletitle,curtable,currow, listingSource, listingTalkSource);
	    resolve(true);
	}
	catch (e) {
		console.log('Error getting source for listing article or talk page',tabletitle,e);
        //ProgressFatalError2(0,tabletitle,curtable,currow,e);
        reject(e);
	}
}

async function processListingSources(ps,tabletitle, curtable, currow, listingSource, listingTalkSource)
{
	console.log('processListingSources', tabletitle);	
	// TODO WikiTextFetched
}

// load next list to query
function LoadList(currentTable,currentRow) {
    // check if we need to go to the next table
    if (currentRow>ProgressStructure[currentTable].length-3) {
        currentRow=0;
        currentTable++;
    }
    // check if there are no more tables
    if (currentTable>ProgressStructure.length-1) return;

    if (typeof ProgressStructure[currentTable][currentRow].Link=="undefined") { // skip duplicate and total rows
        LoadList(currentTable,currentRow+1);
        return;
    }

    var title=ProgressStructure[currentTable][currentRow].Link;

    setTimeout(function(){ // short delay to prevent API overload
        getProgressListWikitext(title,currentTable,currentRow);
        LoadList(currentTable,currentRow+1);
    }, DefaultQueryPause);
    return;
}

function WikitextFetched(ajaxResponse,status,title,currentTable,currentRow) {
	// console.log("WikitextFetched: table="+currentTable+" row="+currentRow+" title="+title+" status="+status);
    if (status!="success") {
        NewWarning("Wikitext "+ajaxResponse.errorThrown);
        setTimeout(function(){ // try again after delay if rate limit reached
            getProgressListWikitext(title,currentTable,currentRow);
        }, 250);
        return;
    }
    // won't get here unless successful
    var responseText, pagetext;
    var tabletext, regex, i, j;
    
    try {
	    responseText=JSON.parse(ajaxResponse.responseText);
    	pagetext=responseText.query.pages[responseText.query.pageids[0]].revisions[0]["*"];
    }
    catch (e) {
    	console.log("WikiTextFetched: Exception parsing "+title+": "+e);
        ProgressFatalError(0,title,currentTable,currentRow);
        return;
    }
    
    var StartIndex;
    // console.log("WikiTextFetched: Parsing out "+title+" to find relevant table");
    try {
	    if (responseText.query.redirects) { // if redirect, find section
	        var SectionName="Undefined";
	        for (var r in responseText.query.redirects) {
	            if (typeof responseText.query.redirects[r].tofragment!="undefined") SectionName=responseText.query.redirects[r].tofragment.replace(/.27/g,"'");
	        }
	
	        regex = new RegExp("=[ ]*(\\[\\[(.*?\\|)?[ ]*)?"+SectionName+"([ ]*\\]\\])?[ ]*=", "g");
	        var sectionheader=pagetext.match(regex);
	        if (sectionheader === null || sectionheader === undefined) { // if no section found, check if one of known empty counties
	            var ID = ProgressStructure[currentTable][currentRow].ID;
	            //console.log("WikiTextFetched: List appears to be empty: title="+title+" id="+ID);
	        	// list last check date: 2020-11-09
	            var EmptyCounties=["01061", // Geneva County AL
	            	"02270", // Kusilvak Census Area AK
	            	"08014", // Broomfield County CO
	            	"12067", // Lafayette County FL
	            	"20081", // Haskell County KS
	            	"20175", // Seward County KS
	            	"20187", // Stanton County KS
	            	"20189", // Stevens County KS
	            	"26051", // Gladwin County MI
	            	"26079", // Kalkaska County MI
	            	"26119", // Montmorency County MI
	            	"26129", // Ogemaw County MI
	            	"26133", // Osceola County MI
	            	"31009", // Blaine County NE
	            	"31113", // Logan County NE
	            	"31117", // McPherson County NE
	            	"38085", // Sioux County ND
	            	"42023", // Cameron County PA
	            	"48017", // Bailey County TX
	            	"48023", // Baylor County TX
	            	"48033", // Borden County TX
	            	"48069", // Castro County TX
	            	"48079", // Cochran County TX
	            	"48103", // Crane County TX
	            	"48107", // Crosby County TX
	            	"48119", // Delta County TX
	            	"48131", // Duval County TX
	            	"48155", // Foard County TX
	            	"48165", // Gaines County TX
	            	"48207", // Haskell County TX
	            	"48219", // Hockley County TX
	            	"48247", // Jim Hogg County TX
	            	"48269", // King County TX
	            	"48279", // Lamb County TX
	            	"48341", // Moore County TX
	            	"48389", // Reeves County TX
	            	"48415", // Scurry County TX
	            	"48421", // Sherman County TX
	            	"48433", // Stonewall County TX
	            	"48437", // Swisher County TX
	            	"48445", // Terry County TX
	            	"48461", // Upton County TX
	            	"48475", // Ward County TX
	            	"48501", // Yoakum County TX
	            	"51735"  // Poquoson VA
	            ];
	            var errorcode = 0;
	            for (var k=0; k<EmptyCounties.length; k++) {
	                if (ID==EmptyCounties[k]) {errorcode=-1;}
	            }
	            if (errorcode!==0) { // must be an empty county
	                ProgressStructure[currentTable][currentRow].Total=0;
	                ProgressStructure[currentTable][currentRow].Illustrated=0;
	                ProgressStructure[currentTable][currentRow].Articled=0;
	                ProgressStructure[currentTable][currentRow].Stubs=0;
	                ProgressStructure[currentTable][currentRow].NRISonly=0;
	                ProgressStructure[currentTable][currentRow].StartPlus=0;
	                ProgressStructure[currentTable][currentRow].Unassessed=0;
	                ProgressStructure[currentTable][currentRow].Untagged=0;
	                ProgressStructure[currentTable][currentRow].Link=title;
	
	                TotalQueried++;
	                if (TotalQueried==TotalToQuery) CalculateProgressTotals();
	                return;
	            }
	            // if we're here, must have been a redirect with no section, and not a known empty county
	            sectionheader=pagetext.match(/{{NRHP header/g); // then look for tables without a section
	            if (sectionheader===null||sectionheader.length>1) { // if still can't find a table or find multiple tables, fatal error
	                ProgressFatalError(0,title,currentTable,currentRow);
	            }
	        }
	        StartIndex=pagetext.indexOf(sectionheader[0]);
	        var sectiontext=pagetext.substr(StartIndex,pagetext.indexOf("\n==",StartIndex)-StartIndex); // only look at relevant section
	
	        StartIndex=sectiontext.indexOf("{{NRHP header");
	        if (StartIndex==-1) {
	            if (sectiontext.indexOf("{{NRHP row")!=-1) {
	                ProgressFatalError(2,title,currentTable,currentRow); // incorrectly formatted table
	            } else { // must be an empty county
	            	console.log("WikiTextFetched: county appears to be empty: "+title);
	                ProgressStructure[currentTable][currentRow].Total=0;
	                ProgressStructure[currentTable][currentRow].Illustrated=0;
	                ProgressStructure[currentTable][currentRow].Articled=0;
	                ProgressStructure[currentTable][currentRow].Stubs=0;
	                ProgressStructure[currentTable][currentRow].NRISonly=0;
	                ProgressStructure[currentTable][currentRow].StartPlus=0;
	                ProgressStructure[currentTable][currentRow].Unassessed=0;
	                ProgressStructure[currentTable][currentRow].Untagged=0;
	                ProgressStructure[currentTable][currentRow].Link=title;
	
	                TotalQueried++;
	                if (TotalQueried==TotalToQuery) CalculateProgressTotals();
	                return;
	            }
	        }

	        var EndIndex = sectiontext.indexOf("\n|}",StartIndex);
	        // EndIndex should exist, but a malformed file may not have it
	        if (EndIndex === -1) {
		        console.log("WikiTextFetched: missing end of table in "+title);
	            ProgressFatalError(2,title,currentTable,currentRow); // list is malformed
	            return;
	        }
	        tabletext=sectiontext.substr(StartIndex,EndIndex-StartIndex);
	    } else { // if not a redirect, default to first table on page
	        StartIndex=pagetext.indexOf("{{NRHP header");
	        if (StartIndex==-1) {
	            ProgressFatalError(1,title,currentTable,currentRow); // no list found
	            return;
	        }
	        var EndIndex = pagetext.indexOf("\n|}",StartIndex);
	        // EndIndex should exist, but a malformed file may not have it
	        if (EndIndex === -1) {
		        console.log("WikiTextFetched: missing end of table in "+title);
	            ProgressFatalError(2,title,currentTable,currentRow); // list is malformed
	            return;
	        }
	        
	        tabletext=pagetext.substr(StartIndex,EndIndex-StartIndex);
	    }
    }
    catch (e) {
    	console.log("WikiTextFetched: Exception searching for table in "+title+": "+e);
        ProgressFatalError(0,title,currentTable,currentRow);
        return;

    }

    // now that tabletext has only relevant table, extract rows
    var Rows=[];
    var str = "{{";
    var start=0;
    var commentstart=0;
    //console.log("WikiTextFetched: Extracting rows from "+title);
    try {
	    while (true) {
	        commentstart=tabletext.indexOf("<!--",start);
	        start=tabletext.indexOf(str,start);
	        if (start==-1) break;
	        while (commentstart<start&&commentstart!=-1) { // skip any commented out rows
	            start=tabletext.indexOf("-->",commentstart);
	            commentstart=tabletext.indexOf("<!--",start);
	            start=tabletext.indexOf(str,start);
	        }
	        if (start==-1) break;
	        var open=1;
	        var index=start+str.length;
	        while (open!==0 && index<tabletext.length) { // make sure to find correct matching close brackets for row template
	            if (tabletext.substr(index,2)=="}}") {
	                open--;
	                index++;
	            } else if (tabletext.substr(index,2)=="{{") {
	                open++;
	                index++;
	            }
	            index++;
	        }
	        var template=tabletext.substr(start,index-start);
	        regex = new RegExp("{{[\\s]*NRHP row(\\s)*\\|", "g");
	        if (template.match(regex)!==null) Rows.push(template); // make sure it's the row template and not some other one
	        else {
	        	console.log("WikiTextFetched: failed to find 'NRHP row' in "+template);
		        // ProgressFatalError(0,title,currentTable,currentRow);
    		    // return;
	        }
	        start++;
	    }
	    for (i=0; i<Rows.length; i++) { // get rid of false positives inside nowiki or pre tags
	        regex=new RegExp("<[ ]*?(nowiki|pre)[ ]*?>((?!<[ ]*?/[ ]*?(nowiki|pre)[ ]*?>)(.|\\n))*?"+Rows[i].replace(/[\-\[\]\/\{\}\(\)\*\+\?\.\\\^\$\|]/g, "\\$&")+"(.|\\n)*?<[ ]*?/[ ]*?(nowiki|pre)[ ]*?>", "g");
	        if (tabletext.match(regex)!==null) {Rows.splice(i,1); i--;}
	    }
    }
    catch (e) {
    	console.log("WikiTextFetched: Exception extracting rows from "+title+": "+e);
        ProgressFatalError(0,title,currentTable,currentRow);
        return;

    }

    // now begin querying statistics
    var Stats={"ID":ProgressStructure[currentTable][currentRow].ID, "Total":Rows.length, "Illustrated":0, "Articled":0, "Stubs":0, "NRISonly":0, "StartPlus":0, "Unassessed":0, "Untagged":0, "Link":title};

    var Titles=[];
    console.log("WikiTextFetched: query statistics for "+title+": "+Rows.length+" rows");
    try {
	    for (i=0; i<Rows.length; i++) { // extract titles for querying
	        var ThisRow=Rows[i];
	
	        // check for illustrated while we're cycling through anyway
	        var test=ThisRow.match(/\|[ ]*?image[ ]*?=.*?(\n|\||}})/g);
	        if (test!==null) {
	            test=test[0].replace(/\|[ ]*?image[ ]*?=/g,"").replace(/(\n|\||}})/g,"").replace(/<\!\-\-(.|[\r\n])*?\-\-\>/g,"").trim();
	            if (test!=="") {
        			//console.log('NRHPstats: found image in row '+NRHPstats_currentRow+': '+test);
        			var test2 = test.match(/Address restricted/gi);
        			if (test2!==null && test2!=="") {
        				RestrictedImageCount++;
        				console.log('NRHPstats: image on row '+i+' appears to be address restricted image (ignoring #'+RestrictedImageCount+'): '+test);
	                	Stats.Illustrated++;  // TODO keep for now
        			} else {
	                	Stats.Illustrated++;  // only true if image param there and non-blank
	            	}
	            }
	        }
	
	        var article=ThisRow.match(/\|[ ]*?article[ ]*?=[ ]*?.*?[\n|\|]/g);
	        var blank=ThisRow.match(/\|[ ]*?article[ ]*?=[ ]*?[\n|\|]/g);                               // default to name param if article
	        if (article===null||blank!==null) article=ThisRow.match(/\|[ ]*?name[ ]*?=[ ]*?.*?[\n|\|]/g); // blank or missing
	        // strip param name, final line break
	        article=article[0].replace(/\|[ ]*?(article|name)[ ]*?=[ ]*?/g,"").replace(/[\n|\|]/g,"").replace(/<\!\-\-(.|[\r\n])*?\-\-\>/g,"").trim();
	        article=decodeURIComponent(article.split("#")[0].trim());     // corrections for weird titles
	        try {
		        Titles.push(article);
	        }
	        catch (e) {
				console.log("WikiTextFetched: Exception parsing "+title+": "+e);
				ProgressFatalError(0,title,currentTable,currentRow);
			}
	    }
    }
    catch (e) {
    	console.log("WikiTextFetched: Exception extracting statistics from rows from "+title+": "+e);
        ProgressFatalError(0,title,currentTable,currentRow);
        return;
    }

    StartIndex=0;
    LoadNextProgressQuery(Rows,Stats,Titles,StartIndex,title,currentTable,currentRow);
    return;
}

// ready next batch of articles to query
function LoadNextProgressQuery(Rows,Stats,Titles,StartIndex,title,currentTable,currentRow) {
    if (StartIndex==Stats.Total) { // all queries begun for this list
        return;
    }
    var TempTitles;
    // must have some more rows to query
    if (Stats.Total-StartIndex>50) {
        TempTitles=Titles.slice(StartIndex,StartIndex+50);
    } else {
        TempTitles=Titles.slice(StartIndex);
    }
    StartIndex+=TempTitles.length;

    setTimeout(function(){ // short delay to prevent API overload
        QueryProgressStats(Rows,Stats,Titles,TempTitles,StartIndex,title,currentTable,currentRow);
        LoadNextProgressQuery(Rows,Stats,Titles,StartIndex,title,currentTable,currentRow);
    }, DefaultQueryPause);
    return;
}

// query next batch of articles
function QueryProgressStats(Rows,Stats,Titles,TempTitles,StartIndex,title,currentTable,currentRow) {
    var TitleList=TempTitles.join("|");
    // console.log("QueryProgressStats: titles="+TitleList);
    $.ajax({
        dataType: "json",
        url: mw.util.wikiScript('api'),
        data: {
            format: 'json',
            action: 'query',
            prop: 'categories',
            clcategories: 'Category:All disambiguation pages|Category:All articles sourced only to NRIS',
            cllimit: 'max',
            titles: TitleList,
            redirects: 'true'
        },
        error: function(ArticlejsonObject,status,errorThrown) {
        	ArticlejsonObject.errorThrown=errorThrown;
        	console.log("QueryProgressStats: error fetching "+TempTitles);
        },
        complete: function(ArticlejsonObject,status) {
        	try {
                ProgressArticleChecked(ArticlejsonObject,status,Rows,Stats,Titles,TempTitles,StartIndex,title,currentTable,currentRow);
        	}
        	catch (e) {
        		console.log("QueryProgressStats: Eception checking article "+TempTitles+": "+e);
		        ProgressFatalError(0,title,currentTable,currentRow);
        	}
        }
    });
    return;
}

// parse API response for article query
function ProgressArticleChecked(ArticlejsonObject,status,Rows,Stats,Titles,TempTitles,StartIndex,title,currentTable,currentRow) {
    if (status!="success") {
        NewWarning("Articles "+ArticlejsonObject.errorThrown);

        setTimeout(function(){ // try again after delay if rate limit reached
            QueryProgressStats(Rows,Stats,Titles,TempTitles,StartIndex,title,currentTable,currentRow);
        }, 250);
        return;
    }
    // won't get here unless successful
    ProgressStructure[currentTable][currentRow].ArticleQueried+=TempTitles.length;

    var responseText=JSON.parse(ArticlejsonObject.responseText);
    var i;
    if (responseText.query.normalized) { // normalize any weird titles
        for (var n in responseText.query.normalized) {
            for (i=0; i<TempTitles.length; i++) {
                if (TempTitles[i]==responseText.query.normalized[n].from) TempTitles[i]=responseText.query.normalized[n].to;
            }
            for (i=0; i<Titles.length; i++) { // also update in Titles array to prepare to query talk pages
                if (Titles[i]==responseText.query.normalized[n].from) Titles[i]=responseText.query.normalized[n].to;
            }
        }
    }
    if (responseText.query.redirects) { // resolve any redirects also
        for (var r in responseText.query.redirects) {
            for (i=0; i<TempTitles.length; i++) {
                if (TempTitles[i]==responseText.query.redirects[r].from) TempTitles[i]=responseText.query.redirects[r].to;
            }
            for (i=0; i<Titles.length; i++) { // also update in Titles array to prepare to query talk pages
                if (Titles[i]==responseText.query.redirects[r].from) Titles[i]=responseText.query.redirects[r].to;
            }
        }
    }

    // now determine the number of bluelinks and NRIS-only articles
    for (var page in responseText.query.pages) {
        var articled=true;   // default to articled, not NRIS-only
        var NRISonly=false;
        var pagetitle=responseText.query.pages[page].title;
        if (typeof responseText.query.pages[page].missing!="undefined") {
            // redlink=unarticled
            articled=false;
        }
        if (responseText.query.pages[page].categories) {
            for (var category in responseText.query.pages[page].categories) {
                if (responseText.query.pages[page].categories[category].title=="Category:All disambiguation pages") {
                    // dab=unarticled
                    articled=false;
                }
                if (responseText.query.pages[page].categories[category].title.indexOf("sourced only to NRIS")!=-1) {
                    // mark as NRIS-only
                    NRISonly=true;
                }
            }
        }
        for (i=0; i<TempTitles.length; i++) { // if page is duplicated, count it multiple times
            if (TempTitles[i]==pagetitle) {
                if (articled) Stats.Articled++;
                if (NRISonly) Stats.NRISonly++;
            }
        }
    }

    if (ProgressStructure[currentTable][currentRow].ArticleQueried==Stats.Total) { // after querying all articles, query talk pages
        for (i=0; i<Titles.length; i++) {
            Titles[i]="Talk:"+Titles[i];
        }
        StartIndex=0;
        LoadNextProgressTalkQuery(Rows,Stats,Titles,StartIndex,title,currentTable,currentRow);
    }
}

// ready next batch of talk pages to query
function LoadNextProgressTalkQuery(Rows,Stats,Titles,StartIndex,title,currentTable,currentRow) {
    if (StartIndex==Stats.Total) { // all queries begun for this list
        return;
    }
    var TempTitles;
    // must have some more rows to query
    if (Stats.Total-StartIndex>50) {
        TempTitles=Titles.slice(StartIndex,StartIndex+50);
    } else {
        TempTitles=Titles.slice(StartIndex);
    }
    StartIndex+=TempTitles.length;

    setTimeout(function(){ // short delay to prevent API overload
        QueryProgressTalkStats(Rows,Stats,Titles,TempTitles,StartIndex,title,currentTable,currentRow);
        LoadNextProgressTalkQuery(Rows,Stats,Titles,StartIndex,title,currentTable,currentRow);
    }, DefaultQueryPause);
    return;
}

// query the next batch of talk pages
function QueryProgressTalkStats(Rows,Stats,Titles,TempTitles,StartIndex,title,currentTable,currentRow) {
    var catlist='Category:FA-Class National Register of Historic Places articles|Category:A-Class National Register of Historic Places ';
    catlist+='articles|Category:GA-Class National Register of Historic Places articles|Category:B-Class National Register of Historic ';
    catlist+='Places articles|Category:C-Class National Register of Historic Places articles|Category:Start-Class National Register of ';
    catlist+='Historic Places articles|Category:Stub-Class National Register of Historic Places articles|Category:Unassessed National ';
    catlist+='Register of Historic Places articles|Category:List-Class National Register of Historic Places articles|Category:Redirect-';
    catlist+='Class National Register of Historic Places articles';

    var TitleList=TempTitles.join("|");
    $.ajax({
        dataType: "json",
        url: mw.util.wikiScript('api'),
        data: {
            format: 'json',
            action: 'query',
            prop: 'categories',
            clcategories: catlist,
            cllimit: 'max',
            titles: TitleList,
            redirects: 'true'
        },
        error: function(ArticlejsonObject,status,errorThrown) {
        	ArticlejsonObject.errorThrown=errorThrown;
        	console.log("QueryProgressTalkStats: Error fetching "+TitleList);
        },
        complete: function(ArticlejsonObject,status) {
        	try {
                ProgressTalkChecked(ArticlejsonObject,status,Rows,Stats,Titles,TempTitles,StartIndex,title,currentTable,currentRow);
            }
            catch (e) {
            	alert("QueryProgressTalkStats: Exception processing "+TitleList+": "+e);
		        ProgressFatalError(0,title,currentTable,currentRow);
            }
        }
    });
    return;
}

// parse API response for talk page query
function ProgressTalkChecked(ArticlejsonObject,status,Rows,Stats,Titles,TempTitles,StartIndex,title,currentTable,currentRow) {
    if (status!="success") {
        NewWarning("Talk "+ArticlejsonObject.errorThrown);

        setTimeout(function(){ // try again after delay if rate limit reached
            QueryProgressTalkStats(Rows,Stats,Titles,TempTitles,StartIndex,title,currentTable,currentRow);
        }, 250);
        return;
    }
    // won't get here unless successful
    ProgressStructure[currentTable][currentRow].TalkQueried+=TempTitles.length;

    // now determine quality statistics
    var responseText=JSON.parse(ArticlejsonObject.responseText);
    for (var page in responseText.query.pages) {
        if (typeof responseText.query.pages[page].missing!="undefined") continue; // skip talk page if not articled

        var untagged=true; // default to untagged
        var articled=true; // assume articled to check for link to other county/MPS lists
        var stub=false;
        var unassessed=false;
        var startPlus=false;
        var pagetitle=responseText.query.pages[page].title;
        if (responseText.query.pages[page].categories) {
            untagged=false; // if cat hit, mark as tagged
            for (var category in responseText.query.pages[page].categories) {
                var CatTitle=responseText.query.pages[page].categories[category].title;
                if (CatTitle.indexOf("Stub")!=-1) {
                    stub=true; // mark as stub
                }
                if (CatTitle.indexOf("Unassessed")!=-1||CatTitle.indexOf("Redirect")!=-1) {
                    unassessed=true; // mark as unassessed
                }
                if  (CatTitle.indexOf("List")!=-1) { // count links to other county/MPS lists as unarticled; other list-class as stubs
                    if (responseText.query.pages[page].title.indexOf("National Register of Historic Places")!=-1){
                        articled=false;
                    } else {
                        stub=true;
                    }
                }
            }
        }
        if (articled&&!untagged&&!unassessed&&!stub) { // if articled, tagged, and assessed, but not stub, must be Start+
            startPlus=true;
        }

        for (var i=0; i<TempTitles.length; i++) {
            if (TempTitles[i]==pagetitle) {
                if (!articled) Stats.Articled--; // reduce the count of articled if links to lists
                if (untagged) Stats.Untagged++;
                if (stub) Stats.Stubs++;
                if (unassessed) Stats.Unassessed++;
                if (startPlus) Stats.StartPlus++;
            }
        }
    }

    if (ProgressStructure[currentTable][currentRow].TalkQueried==Stats.Total) {
        ProgressStructure[currentTable][currentRow].Total=Stats.Total;
        ProgressStructure[currentTable][currentRow].Illustrated=Stats.Illustrated;
        ProgressStructure[currentTable][currentRow].Articled=Stats.Articled;
        ProgressStructure[currentTable][currentRow].Stubs=Stats.Stubs;
        ProgressStructure[currentTable][currentRow].NRISonly=Stats.NRISonly;
        ProgressStructure[currentTable][currentRow].StartPlus=Stats.StartPlus;
        ProgressStructure[currentTable][currentRow].Unassessed=Stats.Unassessed;
        ProgressStructure[currentTable][currentRow].Untagged=Stats.Untagged;
        ProgressStructure[currentTable][currentRow].Link=title;

        TotalQueried++;
        if (TotalQueried==TotalToQuery) CalculateProgressTotals();
    }
}

// keep track of warnings encountered while querying API
function NewWarning(warning) {
    var i, NewWarning=true;
    for (i=0; i<WarningCount.length; i++) { // check if already encountered error
        if (warning==WarningCount[i][0]||WarningCount[i][0]==="") {WarningCount[i][0]=warning; WarningCount[i][1]++; NewWarning=false;}
    }
    if (NewWarning) WarningCount[WarningCount.length]=[warning,1]; // if new warning, make new entry

    var test=0;
    for (i=0; i<WarningCount.length; i++) {
        test+=WarningCount[i][1];
    }
    if (test%50===0) DefaultQueryPause++; // for every 50 errors encountered, increase time between each query to throttle speed
}

var g_fetchErrors = {};

function ProgressFatalError2(code,title, curtable, currow, ex)
{
	console.log('ProgressFatalError2: ',code,title,curtable,currow, ex);
    var errorArray = ['No county section found for ','No list found for ','Incorrectly formatted list for '];
    if (!g_fetchErrors.hasOwnProperty(title)) g_fetchErrors[title] = 0;
    g_fetchErrors[title]++;
    var retry = true;
    if (g_fetchErrors[title] > 5) {
    	retry=confirm(errorArray[code]+title+"!\n\nCancel=Skip                   OK=Retry");
    	if (retry) g_fetchErrors[title] = 0; // reset counter for this title
    }
    if (retry) {
        processTableRow(title,curtable,currow);
    } else {
        TotalQueried++;
        ErrorCount++;
        if (TotalQueried==TotalToQuery) CalculateProgressTotals();
    }
    return;
}
// these errors require user input; can't just be ignored
function ProgressFatalError(code,title,currentTable,currentRow) {
    var errorArray = ['No county section found for ','No list found for ','Incorrectly formatted list for '];
    if (!g_fetchErrors.hasOwnProperty(title)) g_fetchErrors[title] = 0;
    g_fetchErrors[title]++;
    var retry = true;
    if (g_fetchErrors[title] > 5) {
    	retry=confirm(errorArray[code]+title+"!\n\nCancel=Skip                   OK=Retry");
    	if (retry) g_fetchErrors[title] = 0; // reset counter for this title
    }
    if (retry) {
        getProgressListWikitext(title,currentTable,currentRow);
    } else {
        TotalQueried++;
        ErrorCount++;
        if (TotalQueried==TotalToQuery) CalculateProgressTotals();
    }
    return;
}

// update ProgressDiv to let user know what's going on
function UpdateProgressDiv() {
    var ProgressSpan=document.getElementById("ProgressSpan");
    var TimeSpan=document.getElementById("TimeSpan");

    var PercentQueried=Math.round(TotalQueried/TotalToQuery*1000)/10;
    ProgressSpan.innerHTML = "Querying county data... "+TotalQueried+" ("+PercentQueried+"%) of "+TotalToQuery+" lists checked.";

	var TimeRemainingStr="";
    if (TotalQueried>100) {
        var CurrentTime=new Date();
        var SecondsElapsed = (CurrentTime-InitialTime)/1000;
        var Average = SecondsElapsed/TotalQueried;
        SecondsElapsed=Math.round(SecondsElapsed);
        var MinutesElapsed = 0;
        while (SecondsElapsed>=60) {
            SecondsElapsed-=60;
            MinutesElapsed++;
        }
        var SecondsRemaining = Math.round(Average*(TotalToQuery-TotalQueried));
        var MinutesRemaining = 0;
        while (SecondsRemaining>=60) {
            SecondsRemaining-=60;
            MinutesRemaining++;
        }

        TimeRemainingStr = "";
        if (MinutesRemaining!==0) TimeRemainingStr=MinutesRemaining+" min ";
        TimeRemainingStr+=SecondsRemaining+" sec";
        var TimeElapsedStr = "";
        if (MinutesElapsed!==0) TimeElapsedStr=MinutesElapsed+" min ";
        TimeElapsedStr+=SecondsElapsed+" sec";
        TimeRemainingStr+=" ("+TimeElapsedStr+" elapsed)";
    } else {
        TimeRemainingStr="Calculating...";
    }
    TimeSpan.innerHTML="<br>Estimated time remaining: "+TimeRemainingStr;

    if (TotalQueried!=TotalToQuery) { // update ProgressDiv only at regular intervals to prevent CPU overload; stop once done
        ProgressDivTimer=setTimeout(function(){
            UpdateProgressDiv();
        }, 500);
    }
    return;
}

// after all querying complete, calculate totals for states and counties with multiple sublists
function CalculateProgressTotals() {
    clearTimeout(ProgressDivTimer);
    var ProgressSpan=document.getElementById("ProgressSpan");
    var TimeSpan=document.getElementById("TimeSpan");

    ProgressSpan.innerHTML = "Querying county data... Done! "+TotalToQuery+" lists checked.";

    var CurrentTime=new Date();
    var SecondsElapsed = (CurrentTime-InitialTime)/1000;
    SecondsElapsed=Math.round(SecondsElapsed);
    var MinutesElapsed = 0;
    while (SecondsElapsed>=60) {
        SecondsElapsed-=60;
        MinutesElapsed++;
    }
    TimeSpan.innerHTML=" Time elapsed: ";
    if (MinutesElapsed!==0) TimeSpan.innerHTML+=MinutesElapsed+" min ";
    TimeSpan.innerHTML+=SecondsElapsed+" sec";

	var TotalsIndex, NationalTotalsIndex;
    for (var i=1; i<ProgressStructure.length; i++) { // i=table number; skip national table until end
        for (var j=0; j<ProgressStructure[i].length-2; j++) {
            TotalsIndex=ProgressStructure[i].length-1;
            if (!isNaN(parseFloat(ProgressStructure[i][j].ID))) { // if regular county without sublists, add to totals
                ProgressStructure[i][TotalsIndex].Total+=ProgressStructure[i][j].Total;
                ProgressStructure[i][TotalsIndex].Illustrated+=ProgressStructure[i][j].Illustrated;
                ProgressStructure[i][TotalsIndex].Articled+=ProgressStructure[i][j].Articled;
                ProgressStructure[i][TotalsIndex].Stubs+=ProgressStructure[i][j].Stubs;
                ProgressStructure[i][TotalsIndex].NRISonly+=ProgressStructure[i][j].NRISonly;
                ProgressStructure[i][TotalsIndex].StartPlus+=ProgressStructure[i][j].StartPlus;
                ProgressStructure[i][TotalsIndex].Unassessed+=ProgressStructure[i][j].Unassessed;
                ProgressStructure[i][TotalsIndex].Untagged+=ProgressStructure[i][j].Untagged;
            } else if (ProgressStructure[i][j].ID=="-----") { // if county sublist, find total county row and add there
                var CountyTotalsIndex=j+1;
                while (CountyTotalsIndex<ProgressStructure[i].length-2) {
                    if (!isNaN(parseFloat(ProgressStructure[i][CountyTotalsIndex].ID))) break;
                    CountyTotalsIndex++;
                }
                ProgressStructure[i][CountyTotalsIndex].Total+=ProgressStructure[i][j].Total;
                ProgressStructure[i][CountyTotalsIndex].Illustrated+=ProgressStructure[i][j].Illustrated;
                ProgressStructure[i][CountyTotalsIndex].Articled+=ProgressStructure[i][j].Articled;
                ProgressStructure[i][CountyTotalsIndex].Stubs+=ProgressStructure[i][j].Stubs;
                ProgressStructure[i][CountyTotalsIndex].NRISonly+=ProgressStructure[i][j].NRISonly;
                ProgressStructure[i][CountyTotalsIndex].StartPlus+=ProgressStructure[i][j].StartPlus;
                ProgressStructure[i][CountyTotalsIndex].Unassessed+=ProgressStructure[i][j].Unassessed;
                ProgressStructure[i][CountyTotalsIndex].Untagged+=ProgressStructure[i][j].Untagged;
            } else if (ProgressStructure[i][j].ID=="ddddd") { // if county duplicate row, subtract from county total
                ProgressStructure[i][j+1].Total-=ProgressStructure[i][j].Total;
                ProgressStructure[i][j+1].Illustrated-=ProgressStructure[i][j].Illustrated;
                ProgressStructure[i][j+1].Articled-=ProgressStructure[i][j].Articled;
                ProgressStructure[i][j+1].Stubs-=ProgressStructure[i][j].Stubs;
                ProgressStructure[i][j+1].NRISonly-=ProgressStructure[i][j].NRISonly;
                ProgressStructure[i][j+1].StartPlus-=ProgressStructure[i][j].StartPlus;
                ProgressStructure[i][j+1].Unassessed-=ProgressStructure[i][j].Unassessed;
                ProgressStructure[i][j+1].Untagged-=ProgressStructure[i][j].Untagged;
            } else { // unknown ID; skip it
                alert("Error! Unknown ID="+ProgressStructure[i][j].ID+" in Table "+i+", Row "+j+". Skipping this list in totals.");
            }
        }
        // subtract state duplicates
        ProgressStructure[i][TotalsIndex].Total-=ProgressStructure[i][TotalsIndex-1].Total;
        ProgressStructure[i][TotalsIndex].Illustrated-=ProgressStructure[i][TotalsIndex-1].Illustrated;
        ProgressStructure[i][TotalsIndex].Articled-=ProgressStructure[i][TotalsIndex-1].Articled;
        ProgressStructure[i][TotalsIndex].Stubs-=ProgressStructure[i][TotalsIndex-1].Stubs;
        ProgressStructure[i][TotalsIndex].NRISonly-=ProgressStructure[i][TotalsIndex-1].NRISonly;
        ProgressStructure[i][TotalsIndex].StartPlus-=ProgressStructure[i][TotalsIndex-1].StartPlus;
        ProgressStructure[i][TotalsIndex].Unassessed-=ProgressStructure[i][TotalsIndex-1].Unassessed;
        ProgressStructure[i][TotalsIndex].Untagged-=ProgressStructure[i][TotalsIndex-1].Untagged;

        // record state totals in national table
        ProgressStructure[0][i-1].Total=ProgressStructure[i][TotalsIndex].Total;
        ProgressStructure[0][i-1].Illustrated=ProgressStructure[i][TotalsIndex].Illustrated;
        ProgressStructure[0][i-1].Articled=ProgressStructure[i][TotalsIndex].Articled;
        ProgressStructure[0][i-1].Stubs=ProgressStructure[i][TotalsIndex].Stubs;
        ProgressStructure[0][i-1].NRISonly=ProgressStructure[i][TotalsIndex].NRISonly;
        ProgressStructure[0][i-1].StartPlus=ProgressStructure[i][TotalsIndex].StartPlus;
        ProgressStructure[0][i-1].Unassessed=ProgressStructure[i][TotalsIndex].Unassessed;
        ProgressStructure[0][i-1].Untagged=ProgressStructure[i][TotalsIndex].Untagged;

        // add state totals to national totals
        NationalTotalsIndex=ProgressStructure[0].length-1;
        ProgressStructure[0][NationalTotalsIndex].Total+=ProgressStructure[0][i-1].Total;
        ProgressStructure[0][NationalTotalsIndex].Illustrated+=ProgressStructure[0][i-1].Illustrated;
        ProgressStructure[0][NationalTotalsIndex].Articled+=ProgressStructure[0][i-1].Articled;
        ProgressStructure[0][NationalTotalsIndex].Stubs+=ProgressStructure[0][i-1].Stubs;
        ProgressStructure[0][NationalTotalsIndex].NRISonly+=ProgressStructure[0][i-1].NRISonly;
        ProgressStructure[0][NationalTotalsIndex].StartPlus+=ProgressStructure[0][i-1].StartPlus;
        ProgressStructure[0][NationalTotalsIndex].Unassessed+=ProgressStructure[0][i-1].Unassessed;
        ProgressStructure[0][NationalTotalsIndex].Untagged+=ProgressStructure[0][i-1].Untagged;
    }
    // special row for Tangier, Morocco
    ProgressStructure[0][NationalTotalsIndex].Total+=ProgressStructure[0][NationalTotalsIndex-2].Total;
    ProgressStructure[0][NationalTotalsIndex].Illustrated+=ProgressStructure[0][NationalTotalsIndex-2].Illustrated;
    ProgressStructure[0][NationalTotalsIndex].Articled+=ProgressStructure[0][NationalTotalsIndex-2].Articled;
    ProgressStructure[0][NationalTotalsIndex].Stubs+=ProgressStructure[0][NationalTotalsIndex-2].Stubs;
    ProgressStructure[0][NationalTotalsIndex].NRISonly+=ProgressStructure[0][NationalTotalsIndex-2].NRISonly;
    ProgressStructure[0][NationalTotalsIndex].StartPlus+=ProgressStructure[0][NationalTotalsIndex-2].StartPlus;
    ProgressStructure[0][NationalTotalsIndex].Unassessed+=ProgressStructure[0][NationalTotalsIndex-2].Unassessed;
    ProgressStructure[0][NationalTotalsIndex].Untagged+=ProgressStructure[0][NationalTotalsIndex-2].Untagged;

    // subtract national duplicates
    ProgressStructure[0][NationalTotalsIndex].Total-=ProgressStructure[0][NationalTotalsIndex-1].Total;
    ProgressStructure[0][NationalTotalsIndex].Illustrated-=ProgressStructure[0][NationalTotalsIndex-1].Illustrated;
    ProgressStructure[0][NationalTotalsIndex].Articled-=ProgressStructure[0][NationalTotalsIndex-1].Articled;
    ProgressStructure[0][NationalTotalsIndex].Stubs-=ProgressStructure[0][NationalTotalsIndex-1].Stubs;
    ProgressStructure[0][NationalTotalsIndex].NRISonly-=ProgressStructure[0][NationalTotalsIndex-1].NRISonly;
    ProgressStructure[0][NationalTotalsIndex].StartPlus-=ProgressStructure[0][NationalTotalsIndex-1].StartPlus;
    ProgressStructure[0][NationalTotalsIndex].Unassessed-=ProgressStructure[0][NationalTotalsIndex-1].Unassessed;
    ProgressStructure[0][NationalTotalsIndex].Untagged-=ProgressStructure[0][NationalTotalsIndex-1].Untagged;

    setTimeout(function() {ParseProgressSource()},1); // small delay for non-Firefox browsers to update screen
}

// update progress page source with queried totals
async function ParseProgressSource() {
    var pagesourceresponse = await getPageSource(mw.config.get('wgPageName'));
    console.log("Page source query",pagesourceresponse);
    var pagesource = getPageTextFromQueryResponse(pagesourceresponse);
    var newpagesource=pagesource;
    var TableStartIndex=pagesource.indexOf("==State totals");
    var test;
    var RowEndIndex;
    var oldRow;
    var firstColumn;
    var newRow;
    var temp;
    var str;
    for (var i=0; i<ProgressStructure.length; i++) {
        TableStartIndex=pagesource.indexOf("{|",TableStartIndex+1);    // find next table in old page source
        var TableEndIndex=pagesource.indexOf("|}",TableStartIndex)+2;
        var oldTable=pagesource.substr(TableStartIndex,TableEndIndex-TableStartIndex);
        var newTable=oldTable;

        var RowStartIndex=0;
        for (var j=0; j<ProgressStructure[i].length-2; j++) {
            RowStartIndex=oldTable.indexOf("\n|-",RowStartIndex+1); // find next row in old table
            if (ProgressStructure[i][j].ID=="ddddd") continue;   // skip duplicate rows
            RowEndIndex=oldTable.indexOf("\n|-",RowStartIndex+1);
            oldRow=oldTable.substr(RowStartIndex,RowEndIndex-RowStartIndex);
            firstColumn=oldRow.indexOf("\n|");
            var stop=4;        // skip one cell for national table, three for states
            if (i===0) stop=2;
            for (var inc=0; inc<stop; inc++) {
                firstColumn=oldRow.indexOf("\n|",firstColumn+1);  // find next cell
            }

            // build up new row
            newRow=oldRow.substr(0,firstColumn);
            temp=ProgressStructure[i][j];
            // total
            str=temp.Total.toString();
            if (temp.Total>999) str=str.replace(/\B(?=(\d{3})+(?!\d))/g, ","); // add thousands separators
            newRow+="\n| "+str;
            // illustrated
            str=temp.Illustrated.toString();
            if (temp.Illustrated>999) str=str.replace(/\B(?=(\d{3})+(?!\d))/g, ",");
            newRow+="\n| "+str;
            // illustrated percent
            if (temp.Total===0) {
                str="-";
            } else {
                str = Math.round(temp.Illustrated/temp.Total*1000)/10;
                test = str.toString().indexOf(".");
                if (test==-1 && str!=100 && str!==0) str+=".0"; // force decimal
                str+="%";
            }
            newRow+="\n| "+str;
            // articled
            str=temp.Articled.toString();
            if (temp.Articled>999) str=str.replace(/\B(?=(\d{3})+(?!\d))/g, ",");
            newRow+="\n| "+str;
            // articled percent
            if (temp.Total===0) {
                str="-";
            } else {
                str = Math.round(temp.Articled/temp.Total*1000)/10;
                test = str.toString().indexOf(".");
                if (test==-1 && str!=100 && str!==0) str+=".0";
                str+="%";
            }
            newRow+="\n| "+str;
            // stubs
            str=temp.Stubs.toString();
            if (temp.Stubs>999) str=str.replace(/\B(?=(\d{3})+(?!\d))/g, ",");
            newRow+="\n| "+str;
            // NRIS-only
            str=temp.NRISonly.toString();
            if (temp.NRISonly>999) str=str.replace(/\B(?=(\d{3})+(?!\d))/g, ",");
            newRow+="\n| "+str;
            // Start+
            str=temp.StartPlus.toString();
            if (temp.StartPlus>999) str=str.replace(/\B(?=(\d{3})+(?!\d))/g, ",");
            newRow+="\n| "+str;
            // Start+ percent
            if (temp.Total===0) {
                str="-";
            } else {
                str = Math.round(temp.StartPlus/temp.Total*1000)/10;
                test = str.toString().indexOf(".");
                if (test==-1 && str!=100 && str!==0) str+=".0"; // force decimal
                str+="%";
            }
            newRow+="\n| "+str;
            // unassessed
            str=temp.Unassessed.toString();
            if (temp.Unassessed>999) str=str.replace(/\B(?=(\d{3})+(?!\d))/g, ",");
            newRow+="\n| "+str;
            // untagged
            str=temp.Untagged.toString();
            if (temp.Untagged>999) str=str.replace(/\B(?=(\d{3})+(?!\d))/g, ",");
            newRow+="\n| "+str;
            // net quality
            if (temp.Total===0) {
                str="-";
            } else {
                str=temp.StartPlus+0.5*temp.Stubs+0.5*temp.Unassessed-0.5*temp.Untagged-0.75*temp.NRISonly;
                str=Math.round((0.75*str/temp.Total+0.25*temp.Illustrated/temp.Total)*1000)/10;
                if (str<0) str=0;
                test=str.toString().indexOf(".");
                if (test==-1 && str!=100 && str!==0) str+=".0";
                str+="%";
            }
            newRow+="\n| "+str;

            // update new table with new row
            newTable=newTable.replace(oldRow,newRow);
        }
        RowStartIndex=oldTable.indexOf("\n|-",RowStartIndex+1); // skip duplicate row
        RowStartIndex=oldTable.indexOf("\n|-",RowStartIndex+1);
        RowEndIndex=oldTable.indexOf("\n|}",RowStartIndex+1);
        oldRow=oldTable.substr(RowStartIndex,RowEndIndex-RowStartIndex);
        firstColumn=oldRow.indexOf("\n!"); // totals row uses ! instead of |
        firstColumn=oldRow.indexOf("\n!",firstColumn+1); // skip first cell

        // build up new totals row
        newRow=oldRow.substr(0,firstColumn);
        temp=ProgressStructure[i][ProgressStructure[i].length-1];
        // total
        str=temp.Total.toString();
        if (temp.Total>999) str=str.replace(/\B(?=(\d{3})+(?!\d))/g, ","); // add thousands separators
        newRow+="\n! "+str;
        // illustrated
        str=temp.Illustrated.toString();
        if (temp.Illustrated>999) str=str.replace(/\B(?=(\d{3})+(?!\d))/g, ",");
        newRow+="\n! "+str;
        // illustrated percent
        if (temp.Total===0) {
            str="-";
        } else {
            str = Math.round(temp.Illustrated/temp.Total*1000)/10;
            test = str.toString().indexOf(".");
            if (test==-1 && str!=100 && str!==0) str+=".0"; // force decimal
            str+="%";
        }
        newRow+="\n! "+str;
        // articled
        str=temp.Articled.toString();
        if (temp.Articled>999) str=str.replace(/\B(?=(\d{3})+(?!\d))/g, ",");
        newRow+="\n! "+str;
        // articled percent
        if (temp.Total===0) {
            str="-";
        } else {
            str = Math.round(temp.Articled/temp.Total*1000)/10;
            test = str.toString().indexOf(".");
            if (test==-1 && str!=100 && str!==0) str+=".0"; // force decimal
            str+="%";
        }
        newRow+="\n! "+str;
        // stubs
        str=temp.Stubs.toString();
        if (temp.Stubs>999) str=str.replace(/\B(?=(\d{3})+(?!\d))/g, ",");
        newRow+="\n! "+str;
        // NRIS-only
        str=temp.NRISonly.toString();
        if (temp.NRISonly>999) str=str.replace(/\B(?=(\d{3})+(?!\d))/g, ",");
        newRow+="\n! "+str;
        // Start+
        str=temp.StartPlus.toString();
        if (temp.StartPlus>999) str=str.replace(/\B(?=(\d{3})+(?!\d))/g, ",");
        newRow+="\n! "+str;
        // Start+ percent
        if (temp.Total===0) {
            str="-";
        } else {
            str = Math.round(temp.StartPlus/temp.Total*1000)/10;
            test = str.toString().indexOf(".");
            if (test==-1 && str!=100 && str!==0) str+=".0"; // force decimal
            str+="%";
        }
        newRow+="\n! "+str;
        // unassessed
        str=temp.Unassessed.toString();
        if (temp.Unassessed>999) str=str.replace(/\B(?=(\d{3})+(?!\d))/g, ",");
        newRow+="\n! "+str;
        // untagged
        str=temp.Untagged.toString();
        if (temp.Untagged>999) str=str.replace(/\B(?=(\d{3})+(?!\d))/g, ",");
        newRow+="\n! "+str;
        // net quality
        if (temp.Total===0) {
            str="-";
        } else {
            str=temp.StartPlus+0.5*temp.Stubs+0.5*temp.Unassessed-0.5*temp.Untagged-0.75*temp.NRISonly;
            str=Math.round((0.75*str/temp.Total+0.25*temp.Illustrated/temp.Total)*1000)/10;
            if (str<0) str=0;
            test=str.toString().indexOf(".");
            if (test==-1 && str!=100 && str!==0) str+=".0";
            str+="%";
        }
        newRow+="\n! "+str;

        // update new page source with new table
        newTable=newTable.replace(oldRow,newRow);
        newpagesource=newpagesource.replace(oldTable,newTable);
    }

    // now edit page to provide new source
    var ProgressDiv=document.getElementById("ProgressDiv");
    ProgressDiv.innerHTML+="<br>Editing page (this might take up to one minute)... ";
    InitializeEdit(newpagesource);
}

// initialize edit
function InitializeEdit(pagesource) {
    var d=new Date();
    var months=['January','February','March','April','May','June','July','August','September','October','November','December'];
    var year=d.getYear();
    if (year < 1000) year += 1900;
    var DateStr=months[d.getMonth()]+" "+d.getDate()+", "+year;

    regex=/(January|February|March|April|May|June|July|August|September|October|November|December) [0-9]{1,2}, [0-9]{4}/g;
    var tempstring=pagesource.split('==County totals=='); // ignore dates in lead (e.g. date of last map update)
    pagesource=tempstring[0]+'==County totals=='+tempstring[1].replace(regex,DateStr); // update date strings above tables

    var ErrorStr = '';
    if (ErrorCount>0) ErrorStr = " Errors encountered for "+ErrorCount+" counties, which were skipped. Human attention needed.";
    var summary='Updating county data as of '+DateStr+' using [[User:Magicpiano/NRBot/UpdateNRHPProgress.js|script]].'+ErrorStr;
    editPage(pagesource,mw.config.get('wgPageName'),summary);
}

function PageEdited(ajaxResponse,status,pagesource) {
    var ProgressDiv=document.getElementById("ProgressDiv");
    if (status!="success") {
        var retry=confirm("Error: "+ajaxResponse.errorThrown+" while editing page!\n\nCancel=Abort                   OK=Retry");
        if (retry) {
            ProgressDiv.innerHTML+="Retrying... ";
            InitializeEdit(pagesource); // try again
        } else {
            ProgressDiv.innerHTML+="Edit failure! Script aborted!";
        }
        return;
    }
    var responseText=JSON.parse(ajaxResponse.responseText);
    var diff=responseText.edit.newrevid;
    var linkStr="//en.wikipedia.org/w/index.php?diff="+diff;
    ProgressDiv.innerHTML+="Page edited! Click <a href='"+linkStr+"'>here</a> for diff.";
    
    console.log("Number of restricted images found: "+RestrictedImageCount);

    // output technical information to console
    var WarningText="NRHP Progress Warnings: ";
    for (var i=0; i<WarningCount.length; i++) {
        WarningText+=WarningCount[i][0]+" ("+WarningCount[i][1]+"), ";
    }
    if (WarningCount[0][0]!=="") {
        WarningText=WarningText.substr(0,WarningText.length-2);
    } else {
        WarningText="NRHP Progress Warnings: none";
    }
    console.log(WarningText);
}

function editPage(text,title,summary) { // edit page when done
	var edit_token;
	edit_token = mw.user.tokens.get( 'editToken' );
	if (edit_token === null) {
		edit_token = mw.user.tokens.get( 'csrfToken');
	}
    $.ajax({
        dataType: 'json',
        url: mw.util.wikiScript( 'api' ),
        type: 'POST',
        data: {
            format: 'json',
            action: 'edit',
            title: title,
            text: text,
            summary: summary,
            token: edit_token
        },
        success: function( data ) {
            if (data && data.edit && data.edit.result && data.edit.result == 'Success') {
                console.log('Page edit succeeded');
            } else {
            	var msg;
            	if (data && data.error) {
            		msg = "Error editing page: ";
            		msg += " code=" + data.error.code;
            		msg += " info=" + data.error.info;
        		} else {
        			msg = "Error editing page (no data or error)";
        		}
                alert(msg);
            }
        },
        error: function(ajaxResponse,status,errorThrown) {
        	ajaxResponse.errorThrown=errorThrown;
        	console.log("EditPage: Error editing "+title);
        },
        complete: function(ajaxResponse,status) {PageEdited(ajaxResponse,status,text)}
    });
}

function getPageTextFromQueryResponse(response)
{
	var txt;
	var query = response.query;
	console.log('getPageText',response);
    for (var pageid in query.pages) {
    	if (pageid == "-1") return null; // this means 
        txt=query.pages[pageid].revisions[0].slots.main['*'];
    }

	//console.log('Body:');
	//console.log(txt);
	return txt;	
}

async function getPageSource(page) {
    console.log("getPageSource page",page);
    var api = new mw.Api();
    var rsp;

	// nb this will throw an exception if it fails
    rsp = await api.get({
            format: 'json',
            action: 'query',
            prop: 'revisions',
            rvprop: 'content',
            rvslots: '*',
            titles: page,
            indexpageids: true,
            redirects: 'true'
        });

	console.log("getPageSource response",rsp);

	return rsp;
}

function getProgressListWikitext(title,currentTable,currentRow) {   // asynchronous fetch of each list's wikitext
    console.log("GetProgressListWikiText: fetch "+title);
    $.ajax({
        dataType: "json",
        url: mw.util.wikiScript('api'),
        data: {
            format: 'json',
            action: 'query',
            prop: 'revisions',
            rvprop: 'content',
            titles: title,
            indexpageids: true,
            redirects: 'true'
        },
        error: function(ajaxResponse,status,errorThrown) {
        	console.log("GetProgressListWikiText: Error fetching "+title);
        	ajaxResponse.errorThrown=errorThrown;
        },
        complete: function(ajaxResponse,status) {
        	console.log("GetProgressListWikitext: Status fetching "+title+": "+status);
        	try {
	        	WikitextFetched(ajaxResponse,status,title,currentTable,currentRow);
        	}
        	catch (e) {
        		alert("Exception after fetching "+title+": "+e);
		        ProgressFatalError(0,title,currentTable,currentRow);
        	}
        }
    });
}

//jQuery(ProgressButton);
$.when($.ready).then(ProgressButton);