Jump to content

User:WildBot/wildBot.py

From Wikipedia, the free encyclopedia
#!/usr/bin/python
# -*- coding: utf-8 -*-

"""
WildBot: 
The members of Category:All disambiguation pages (169,187 entries) and 
Category:Redirects from incomplete disambiguations (3,215 entries) 
will be considered to be ambiguous links. Articles or redirects 
containing "(disambiguation)" will not be considered ambiguous.

The bot will operate off a cached copy of this list, updated periodically 
via an API call to categorymembers to retrive new additions, and periodic 
checks against its watchlist (containing all known disambiguation pages; 
assuming there's no technical limitation with a watchlist having 172402 
pages) to check for removals. If I'm granted a toolsever account maintaining 
this list might be better done via SQL queries.

Periodically (I propose every minute) queries API for New Pages since the 
last query in namespaces 0 (mainspace), 6 (file), 10 (template) and 14 (category).
    New redirects are excluded.
    New disambiguation pages are excluded.
    Each new page will be checked for any ambiguous links. If a page 
        has ambiguous links, a message will be left on that talk page.
    Affected pages will be monitored, 
        and the template changed or removed as the article changes.
"""

import sys, traceback, threading
import wikipedia
import dab_template_placer, article_queue, watchlist_monitor, category_filter, haltpage_excluder
import codecs

__metaclass__ = type


class ConfigFileProducer( threading.Thread ):
    def __init__(self, shutdown, queue, site=None):
        self.shutdown = shutdown
        self.queue = queue
        if site is None:
            site = wikipedia.getSite()
        self.site = site
        self.source_file = 'disambiguations/sample.txt'
        threading.Thread.__init__(self)

    def run(self):
        try:
            f = codecs.open(self.source_file, 'r', 'utf-8')
            try:
                for logtext in f:
                    print '>>%s<<' % logtext.strip()
                    page = wikipedia.Page(self.site, logtext.strip())
                    if page:
                        self.queue.add_page(page)
            finally:
                f.close()
        except:
            self.shutdown.set()
            self.queue.add_page(None)
            wikipedia.stopme()
            raise


class NewPagesProducer( threading.Thread ):
    def __init__(self, shutdown, queue, namespace=0, site=None):
        self.shutdown = shutdown
        self.queue = queue
        if site is None:
            site = wikipedia.getSite()
        self.site = site
        self.namespace = namespace
        self.number_to_fetch = 50
        threading.Thread.__init__(self)

    def newpages(self, get_redirect = False):
        """
        Yield new articles (as Page objects) from Special:Newpages.

        Starts with the newest article and fetches a number of articles. 
        It fetches again. If there is no new page, it blocks until there 
        is one, sleeping between subsequent fetches of Newpages.

        Page objects are yielded.
        """
        seen = set()
        try:
            d = self.site.apipath()
            del d
        except NotImplementedError:
            self.site.config.use_api = False

        while not self.shutdown.isSet():
            if wikipedia.config.use_api and self.site.versionnumber() >= 10:
                params = {
                    'action': 'query',
                    'list': 'recentchanges',
                    'rctype': 'new',
                    'rcnamespace': self.namespace,
                    'rclimit': int(self.number_to_fetch),
                    'rcprop': ['ids','title', 'ns'],
                    'rcshow': ['!bot','!redirect'],
                    #'': '',
                }
                self.number_to_fetch = 7
                data = wikipedia.query.GetData(params, self.site)['query']['recentchanges']

                for np in data:
                    if np['pageid'] not in seen:
                        seen.add(np['pageid'])
                        page = wikipedia.Page(self.site, np['title'], defaultNamespace=np['ns'])
                        yield page
            else:
                path = self.newpages_address(n=number, namespace=namespace)
                # The throttling is important here, so always enabled.
                get_throttle()
                html = self.site.getUrl(path)

                entryR = re.compile('<li[^>]*>(?P<date>.+?) \S*?<a href=".+?"'
                    ' title="(?P<title>.+?)">.+?</a>.+?[\(\[](?P<length>[\d,.]+)[^\)\]]*[\)\]]'
                    ' .?<a href=".+?" title=".+?:(?P<username>.+?)">')
                for m in entryR.finditer(html):
                    date = m.group('date')
                    title = m.group('title')
                    title = title.replace('&quot;', '"')
                    length = int(re.sub("[,.]", "", m.group('length')))
                    loggedIn = u''
                    username = m.group('username')
                    comment = u''

                    if title not in seen:
                        seen.add(title)
                        page = wikipedia.Page(self.site, title)
                        yield page
            self.shutdown.wait(30)

    def run(self):
        try:
            #Load in all dab pages
            for (page) in self.newpages(get_redirect = False):
                wikipedia.output(u'New Page: %s' % page.title())
                self.queue.add_page(page)
        except:
            self.shutdown.set()
            self.queue.add_page(None)
            wikipedia.stopme()
            raise


# MAIN
if __name__ == "__main__":
    # Shutdown event
    shutdown = threading.Event()
    # communications queues for new articles
    new_pages = article_queue.ArticleQueue()
    nondab_pages = article_queue.ArticleQueue()
    # communications queues for changed pages
    changed_pages_1 = article_queue.ArticleQueue()
    changed_pages_2 = article_queue.ArticleQueue()
    try:
        print str(sys.stdout.encoding)
        for arg in wikipedia.handleArgs():
            wikipedia.output(u'Warning: argument "%s" not understood; ignoring.' % arg)
# WildBot Task 1
        # start message-placing and message-updating threads
        # message placer stores some statistics
        # message updater co-operates with watchlister to ensure only new changes are acted on
        TalkPageDabMsger = dab_template_placer.NewPageChecker(shutdown, nondab_pages)
        TalkPageDabMsger.start()
        #start thread to monitor stop page(s) and stop the world if they change
        halt_pages = []
        halt_pages.append('User:WildBot/Halt')
        halt_checker = haltpage_excluder.HaltpageFilter(
                shutdown, changed_pages_1, changed_pages_2, halt_pages)
        halt_checker.start()
        # start thread to remove dab pages from the new_pages queue
        dab_cats = []
        dab_cats.append('Category:All disambiguation pages')
        dab_page_remover = category_filter.CategoryFilter(
                shutdown, new_pages, nondab_pages, dab_cats)
        dab_page_remover.start()
        # start finding stuff threads, one per namespace
        newpages_watcher = NewPagesProducer(shutdown, new_pages, [0, 6, 10, 14])
        newpages_watcher.start()
        # start checking for changes to clear off the template
        TalkPageCleaner = dab_template_placer.TalkCleaner(shutdown, changed_pages_2)
        TalkPageCleaner.start()
        # start watchlist thread
        changes_watcher = watchlist_monitor.WatchlistProducer(shutdown, changed_pages_1)
        changes_watcher.start()
#        revist = ConfigFileProducer(shutdown, new_pages)
#        revist.start()
    except:
        shutdown.set()
        new_pages.add_page(None)
        changed_pages.add_page(None)
        wikipedia.stopme()
        raise