File:Processor families in TOP500 supercomputers.svg

#!/usr/bin/python
 
import BeautifulSoup
import urllib, re, sys
import numpy as np
from itertools import count
 
def extend_axis(arr, axis=-1, by=1, value=0):
    sh = list(arr.shape)
    sh[axis] = by
    extension = np.empty(sh, dtype=arr.dtype)
    if value is not None: extension.fill(value)
    return np.append(arr, extension, axis)
 
category, output = sys.argv[1:]
 
# stat_ijk = value of column k on date dates[i] for choice choices[k]
# thus: stats.shape = ( len(dates), len(choices), len(columns) )
stats = np.zeros((0,0,4), dtype=int) # n_dates X n_choices X 4-columns
dates = []
choices = []
columns = []
 
for ii in count(1):
    url = 'http://www.top500.org/stats/list/%d/%s' % (ii, category)
    print "Loading %s..." % url
    html = urllib.urlopen(url).read()
    print "Scraping data..."
    soup = BeautifulSoup.BeautifulSoup(html)
 
    # get month and year from title
    title = soup.findChild('h1', {'class':'title'}).string
    m = re.search('(.+) share for (\d+)/(\d+)', title)
    if m:
        catname, month, year = m.groups()
        date = float(year) + float(month)/12.0
        print "Got '%s' data for date %g" % (catname, date)
    else:
        break
 
    # get data table and make sure it's what we expect
    table = soup.findChild('table')
    columns = [ x.string for x in table.findChild('thead').findChildren(('th','td')) ]
    assert columns[0]==catname
 
    # get the data in numeric form
    array = np.array(
        [[x.string for x in row.findChildren('td')]
         for row in table.findChildren('tr')] )
    these_choices = array[:, 0]
    these_stats = array[:, (1,3,4,5)].astype(int)
 
    ############################################################
 
    # figure out how the choices correspond to those already seen
    choice_map = np.zeros_like(these_choices).astype(int)
    for ii, choice in enumerate(these_choices):
        try:
            choice_map[ii] = choices.index(choice)
        except ValueError:
            choices.append(choice)
            choice_map[ii] = len(choices) - 1
 
    # extend axis one of stats to accomodate new choices
    stats = extend_axis(stats, 1, len(choices)-stats.shape[1])
 
    # map our data onto the current ordering of choices
    stats = extend_axis(stats, 0, 1)
    stats[-1, (choice_map,)] = these_stats
 
    # add the current date to the list
    dates.append(date)
 
np.savez(output, dates=dates, choices=choices, stats=stats)

#!/usr/bin/python
 
import sys
from itertools import cycle
import numpy as np
import pylab as pl
pl.rcParams['font.size']*=1.1
pl.rcParams['legend.fontsize']*=1.1
pl.rcParams['xtick.labelsize']*=1.25
pl.rcParams['ytick.labelsize']*=1.25
 
 
colors = cycle( list('bgrcmyw')
                + ['orange', 'pink', 'grey', 'brown', 'lightblue', 'lightgreen', 'turquoise', 'navy', 'purple', 'gold',
                   'aqua', 'silver', 'tan', 'tomato', 'steelblue', 'lightcoral', 'chocolate', 'fuchsia', 'indianred'])
 
input, var, ylabel, title = sys.argv[1:]
 
# get the data
bag = np.load(input)
dates = bag['dates']
choices = bag['choices']
stats = bag['stats'] # n_dates X n_choices X 4-columns
 
if var=='count':
    timeseries = stats[:, :, 0]
    ylabel = "Number of systems"
elif var=='rmaxshare':
    timeseries = stats[:, :, 1] / stats[:, :, 1].sum(axis=1)[:,:,None]
    ylabel = "Share of Rmax performance"
elif var=='rpeakshare':
    timeseries = stats[:, :, 2] / stats[:, :, 2].sum(axis=1)[:,:,None]
    ylabel = "Share of Rpeak performance"
elif var=='procshare':
    timeseries = stats[:, :, 3] / stats[:, :, 3].sum(axis=1)[:,:,None]
    ylabel = "Share of processors"
elif var=='procsum':
    timeseries = stats[:, :, 3]
    ylabel = "Number of processors"
else:
    raise RuntimeError
order = np.argsort(timeseries[-1])
 
# plot it
pl.figure()
base = 0
patches, labels = [], []
for ii in order:
    data = timeseries[:, ii]
    choice = choices[ii]
 
    facecolor = colors.next()
    pl.fill_between(dates, base, base+data, edgecolor='k', facecolor=facecolor, label=choice)
    patches.append( pl.Rectangle((0,0), 1, 1, edgecolor='k', facecolor=facecolor) )
    labels.append( choice )
 
    base += data
 
# show legend and labels
pl.legend(patches, labels, loc='upper right')
pl.xlabel("Year")
pl.ylabel(ylabel)
pl.title(title)
 
pl.show()