Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- #!/usr/bin/python
- # coding: utf-8
- from urllib import urlencode
- from xml.dom import minidom as xml
- import urllib2, json
- from cookielib import CookieJar
- from datetime import datetime
- from threading import Thread
- from sys import exit
- class WikiDataUpdater:
- _CDATA = [{"l":"Albania","n":"Albania","i":"167"},{"l":"Argentina","n":"Argentina","i":"27"},{"l":"Australia","n":"Australia","i":"50"},{"l":"Austria","n":"Austria","i":"33"},{"l":"Belarus","n":"Belarus","i":"83"},{"l":"Belgium","n":"Belgium","i":"32"},{"l":"Bolivia","n":"Bolivia","i":"76"},{"l":"Bosnia-Herzegovina","n":"Bosnia and Herzegovina","i":"69"},{"l":"Brazil","n":"Brazil","i":"9"},{"l":"Bulgaria","n":"Bulgaria","i":"42"},{"l":"Canada","n":"Canada","i":"23"},{"l":"Chile","n":"Chile","i":"64"},{"l":"China","n":"China","i":"14"},{"l":"Colombia","n":"Colombia","i":"78"},{"l":"Croatia","n":"Croatia","i":"63"},{"l":"Cyprus","n":"Cyprus","i":"82"},{"l":"Czech-Republic","n":"Czech Republic","i":"34"},{"l":"Denmark","n":"Denmark","i":"55"},{"l":"Egypt","n":"Egypt","i":"165"},{"l":"Estonia","n":"Estonia","i":"70"},{"l":"Finland","n":"Finland","i":"39"},{"l":"France","n":"France","i":"11"},{"l":"Germany","n":"Germany","i":"12"},{"l":"Greece","n":"Greece","i":"44"},{"l":"Hungary","n":"Hungary","i":"13"},{"l":"India","n":"India","i":"48"},{"l":"Indonesia","n":"Indonesia","i":"49"},{"l":"Iran","n":"Iran","i":"56"},{"l":"Ireland","n":"Ireland","i":"54"},{"l":"Israel","n":"Israel","i":"58"},{"l":"Italy","n":"Italy","i":"10"},{"l":"Japan","n":"Japan","i":"45"},{"l":"Latvia","n":"Latvia","i":"71"},{"l":"Lithuania","n":"Lithuania","i":"72"},{"l":"Malaysia","n":"Malaysia","i":"66"},{"l":"Mexico","n":"Mexico","i":"26"},{"l":"Montenegro","n":"Montenegro","i":"80"},{"l":"Netherlands","n":"Netherlands","i":"31"},{"l":"New-Zealand","n":"New Zealand","i":"84"},{"l":"North-Korea","n":"North Korea","i":"73"},{"l":"Norway","n":"Norway","i":"37"},{"l":"Pakistan","n":"Pakistan","i":"57"},{"l":"Paraguay","n":"Paraguay","i":"75"},{"l":"Peru","n":"Peru","i":"77"},{"l":"Philippines","n":"Philippines","i":"67"},{"l":"Poland","n":"Poland","i":"35"},{"l":"Portugal","n":"Portugal","i":"53"},{"l":"Republic-of-China-Taiwan","n":"Republic of China (Taiwan)","i":"81"},{"l":"Republic-of-Macedonia-FYROM","n":"Republic of Macedonia (FYROM)","i":"79"},{"l":"Republic-of-Moldova","n":"Republic of Moldova","i":"52"},{"l":"Romania","n":"Romania","i":"1"},{"l":"Russia","n":"Russia","i":"41"},{"l":"Saudi-Arabia","n":"Saudi Arabia","i":"164"},{"l":"Serbia","n":"Serbia","i":"65"},{"l":"Singapore","n":"Singapore","i":"68"},{"l":"Slovakia","n":"Slovakia","i":"36"},{"l":"Slovenia","n":"Slovenia","i":"61"},{"l":"South-Africa","n":"South Africa","i":"51"},{"l":"South-Korea","n":"South Korea","i":"47"},{"l":"Spain","n":"Spain","i":"15"},{"l":"Sweden","n":"Sweden","i":"38"},{"l":"Switzerland","n":"Switzerland","i":"30"},{"l":"Thailand","n":"Thailand","i":"59"},{"l":"Turkey","n":"Turkey","i":"43"},{"l":"Ukraine","n":"Ukraine","i":"40"},{"l":"United-Arab-Emirates","n":"United Arab Emirates","i":"166"},{"l":"United-Kingdom","n":"UK","i":"29"},{"l":"Uruguay","n":"Uruguay","i":"74"},{"l":"USA","n":"USA","i":"24"},{"l":"Venezuela","n":"Venezuela","i":"28"}]
- def __init__(self):
- w = Wiki()
- w.start()
- self.edit_data = []
- print 'Fetching region, mppa and embargo data'
- self.mapXML = self.fetch_map_xml()
- self.parse_data_from_map_xml()
- print 'Done. Fetching resource bonuses.'
- self.bonus_fetcher()
- print 'Done. Fetching tax data.'
- self.fetch_tax_data()
- print 'Done. Fetching latest events.'
- self.fetch_events()
- print 'Done. Editing pages'
- w.params = self.edit_data
- w.doEdit()
- def parse_data_from_map_xml(self):
- country_list_code = "<noinclude>This is part of [[Template:Regions]]. When country name is passed to this template it returns a formatted list of regions owned by the country in question. <br /><br /> '''This page is updated automatically updated by a bot. The bot is administrated by [[Code-Y]]. Please refrain from editing this page by hand and if it's out-dated just write on his User Talk.'''</noinclude><includeonly>{{#switch: {{{1}}}|\n"
- regionstable_code = "<noinclude>This is part of [[Template:RegionsTable]]. When country name is passed to this template it returns a formatted list of regions owned by the country in question. <br /><br /> '''This page is updated automatically updated by a bot. The bot is administrated by [[Code-Y]]. Please refrain from editing this page by hand and if it's out-dated just write on his User Talk.'''</noinclude><includeonly>{{#switch: {{{1}}}|\n"
- region_list_code = "<noinclude>This is part of [[Template:Regions]]. When a region name is passed to this template, it returns the current owner country of that region. <br /><br /> '''This page is updated automatically updated by a bot. The bot is administrated by [[Code-Y]]. Please refrain from editing this page by hand and if it's out-dated just write on his User Talk.'''\n\nNotes:\n*If you want to get data for [[Ha'il]] you must pass <code>Ha&#39;il</code> to the template.\n*If you want to get data for [[Svalbard & Jan Mayen]] or [[Yorkshire & Humberside]], you need to replace '&' with '&amp;'</noinclude><includeonly>{{#switch: {{{1}}}|\n"
- mpp_code="<noinclude>When a country name is passed to this template, it returns the current list of MPP's that country has with expire date. <br /><br /> '''This page is updated automatically updated by a bot. The bot is administrated by [[Code-Y]]. Please refrain from editing this page by hand and if it's out-dated just write on his User Talk.'''</noinclude><includeonly>{{#switch: {{{1|}}}|\n"
- embargo_code = "<noinclude>When a country name is passed to this template, it returns the current list of embargoes that country has with expire date. <br /><br /> '''This page is updated automatically updated by a bot. The bot is administrated by [[Code-Y]]. Please refrain from editing this page by hand and if it's out-dated just write on his User Talk.'''</noinclude><includeonly>{{#switch: {{{1|}}}|\n"
- countries = self.mapXML.getElementsByTagName('country')
- for c in countries:
- capitalcity = ''
- regions = list()
- country_link = c.getAttribute("link")
- country_name = self.cNameFromLink(country_link)
- country_list_code += ' %s =' % country_name
- region_elements = c.getElementsByTagName('region')
- names = []
- for reg in region_elements:
- region_name = reg.getAttribute('name')
- wikiname = self.wikiregion(region_name)
- region_list_code += ' %s=%s|\n' % (wikiname.replace('&','&').replace("'",'''), country_name)
- capital = reg.getAttribute('capitalcity')
- if capital == '1':
- capitalcity = wikiname.replace('&','&').replace("'",''')
- wikiname = u' · [[%s]] [[File:Icon-capital.gif|link=Capital (political)]]' % wikiname
- else:
- names.append(wikiname.replace('&','&').replace("'",'''))
- wikiname = u' · [[%s]]' % wikiname
- regions.append(wikiname)
- regions.sort()
- names.sort()
- regionstable_code += ' %s = {{RegionsTable/Table|%s|showprod={{{showprod}}}|showmap={{{showmap}}}|capital=%s}}|\n' % (country_name,'|'.join(names),capitalcity)
- country_list_code += ''.join(regions)
- country_list_code += '|\n'
- mpp_code += ' %s={{MPPList/List' % country_name
- index = 0
- c_mpps = []
- for mpp in c.getElementsByTagName('mpp'):
- mppc = self.cNameFromId(mpp.getAttribute('c_id'))
- expires = mpp.getAttribute('expires')
- c_mpps.append('|cx=%s|ex=%s' % (mppc, expires))
- c_mpps.sort()
- for mpp in c_mpps:
- mpp_code = '%s%s' % (mpp_code, mpp.replace('cx=','c%s=' % str(index)).replace('ex=','e%s=' % str(index)))
- index += 1
- mpp_code = '%s}}|\n' % mpp_code
- embargo_code = '%s %s={{MPPList/List|type=embargo'% (embargo_code, country_name)
- index = 0
- c_embargoes = []
- for embargo in c.getElementsByTagName('embargo'):
- embargoc = self.cNameFromId(embargo.getAttribute('c_id'))
- expires = embargo.getAttribute('expires')
- c_embargoes.append('|cx=%s|ex=%s' % (embargoc, expires))
- c_embargoes.sort()
- for embargo in c_embargoes:
- embargo_code = '%s%s' % (embargo_code, embargo.replace('cx=','c%s=' % str(index)).replace('ex=','e%s=' % str(index)))
- index += 1
- embargo_code = '%s}}|\n' % embargo_code
- mpp_list = '%s #default=error}}</includeonly>' % mpp_code
- embargo_list = '%s #default=error}}</includeonly>' % embargo_code
- m_data = {'bot':'','format':'txt','action':'edit','token':'','title':'Template:MPPList/Data','summary':'Cron update.','text':mpp_list}
- e_data = {'bot':'','format':'txt','action':'edit','token':'','title':'Template:EmbargoList/Data','summary':'Cron update.','text':embargo_list}
- country_list_code += '#default=FAIL\n}}</includeonly>'
- regionstable_code += '#default=FAIL\n}}</includeonly>'
- region_list_code += ' #default=FAIL\n}}</includeonly>'
- country_list_code = country_list_code.replace(u'= · ','=')
- temp = {'bot':'','format':'txt','action':'edit','token':'','title':'Template:Regions/Regions','summary':'Cron update.','text':country_list_code.encode('UTF-8')}
- temp2 = {'bot':'','format':'txt','action':'edit','token':'','title':'Template:RegionsTable/OwnerData','summary':'Cron update','text':regionstable_code.encode('UTF-8')}
- temp3 = {'bot':'','format':'txt','action':'edit','token':'','title':'Template:Regions/Owners','summary':'Cron update.','text':region_list_code.encode('UTF-8')}
- self.edit_data.append(temp)
- self.edit_data.append(temp2)
- self.edit_data.append(temp3)
- self.edit_data.append(m_data)
- self.edit_data.append(e_data)
- def bonus_fetcher(self):
- _DATA_URL = 'http://api.1way.it/erep/regions/'
- _WPN_RES = 'Iron Oil Saltpeter Aluminum Rubber'
- _FOD_RES = 'Grain Fruits Fish Deer Cattle'
- bonusdata = []
- code = "<noinclude>This template can be used to get the amount of resource bonuses provided in different countries.\n\n==Usage==\n*For food and food raw material use: <code><nowiki>{{ResourceBonus|</nowiki>''country_name''_f}}</code>\n*For weapons and weapon raw material use: <code><nowiki>{{ResourceBonus|</nowiki>''country_name''_w}}</code>\n\n=== Example ===\n*<code><nowiki>{{ResourceBonus|Finland_f}}</nowiki></code> --> {{ResourceBonus|Finland_f}} (the percentage of food bonus in Finland)\n\n'''Note''': The output is plain number without '%' sign!\n\n<small>''Data is provided by the [http://api.1way.it/erep/ heapy (Hate Erepublik API)] made by Martin Sileno.''</small></noinclude><includeonly>{{#switch: {{{1|}}}|\n"
- for i in self._CDATA:
- url = '%s%s' % (_DATA_URL, i["i"])
- response = urllib2.urlopen(url)
- json_data = json.loads(response.read())
- country = json_data["country"]["name"]
- regions = json_data["regions"]
- weapon_raws = []
- food_raws = []
- for region in regions:
- route = region["is_connected"]
- if route:
- res = region["resource"]["name"]
- if res in _WPN_RES:
- weapon_raws.append(res)
- elif res in _FOD_RES:
- food_raws.append(res)
- else:
- print 'Unknown resource: %s' % res
- food_raws = list(set(food_raws))
- weapon_raws = list(set(weapon_raws))
- food_bonus = len(food_raws) * 20
- weapon_bonus = len(weapon_raws) * 20
- bonusdata.append(' %s_f=%s' % (country, food_bonus))
- bonusdata.append(' %s_w=%s' % (country, weapon_bonus))
- code += '|'.join(bonusdata)
- code = '%s|\n #default=err\n}}</includeonly>' % code
- self.edit_data.append({'bot':'','format':'txt','action':'edit','token':'','title':'Template:ResourceBonus','summary':'Cron update.','text':code})
- def fetch_tax_data(self):
- url = 'http://api.1way.it/erep/tax/0'
- try:
- data = urllib2.urlopen(url)
- except:
- print 'Error fetching tax data'
- taxes = json.loads(data.read())
- code = '<noinclude>{{Documentation}}</noinclude><includeonly>{{#switch: {{{1|}}}|'
- ind_codes = {"Food":"food","Iron":"wrm","Grain":"frm","Weapons":"weapons","Hospital":"hosp","Defense Systems":"ds","Housing":"house","Moving Tickets":"mt"}
- for c in taxes:
- name = c["country"]["name"]
- timestamp = c["last_updated_ts"]
- tax_data = c["taxes"]
- code += '%s_last=%i|' % (name, timestamp)
- for inds in tax_data:
- try:
- ind = ind_codes[inds["industry"]["name"]]
- except NameError:
- print "Unknown industry: %s" % inds["industry"]["name"]
- continue
- vat = inds["vat"]
- imp = inds["import"]
- inc = inds["income"]
- if ind == 'wrm' or ind == 'frm':
- code += '%s_%s_income=%i|%s_%s_import=%i|' % (name, ind, inc, name, ind, imp)
- elif ind == 'food' or ind == 'weapons':
- code += '%s_%s_income=%i|%s_%s_import=%i|%s_%s_vat=%i|' % (name, ind, inc, name, ind, imp, name, ind, vat)
- else:
- code += '%s_%s_import=%i|%s_%s_vat=%i|' % (name, ind, imp, name, ind, vat)
- code += '#default=<span title="Error: Nothing matched your query. Either something is not working or there\'s no such tax for the industry.">0</span>}}</includeonly>'
- self.edit_data.append({'bot':'','format':'txt','action':'edit','token':'','title':'Template:Taxes/Data','summary':'Cron update.','text':code})
- def fetch_events(self):
- url = 'http://www.erepublik.com/rss/allMilitaryEvents'
- try:
- rss = urllib2.urlopen(url)
- except Exception, e:
- print 'Error fetching latest events: %s' % e
- events = xml.parseString(rss.read()).getElementsByTagName('entry')
- event_code = ''
- for i in range(10):
- event = self.format_event(events[i].getElementsByTagName('title')[0].firstChild.data)
- event_code += '* %s\n' % event
- self.edit_data.append({'bot':'','format':'txt','action':'edit','token':'','title':'Portal:Main Page/Latest events','summary':'Cron update.','text':event_code.encode('UTF-8')})
- def format_event(self, string):
- if 'was secured' in string:
- alue = self.wikiregion(string.split(' was secured')[0])
- voittaja = string.split('secured by ')[1].split(' in the war')[0]
- if 'Resistance force' in string:
- haviaja = string.split('force of ')[1]
- koodi = '[[%s]] was secured by [[%s]] in the war versus [[%s|Resistance force of %s]]' % (alue, voittaja, haviaja, haviaja)
- else:
- haviaja = string.split('war versus ')[1]
- koodi = '[[%s]] was secured by [[%s]] in the war versus [[%s]]' % (alue, voittaja, haviaja)
- return koodi
- if 'was conquered' in string:
- alue = self.wikiregion(string.split(' was conquered')[0])
- voittaja = string.split('conquered by ')[1].split(' in the war')[0]
- tappio = string.split('war versus ')[1]
- if 'Resistance' in voittaja:
- voittaja = voittaja.split('force of ')[1]
- koodi = '[[%s]] was conquered by [[%s|Resistance force of %s]] in the war versus [[%s]]' % (alue, voittaja, voittaja, tappio)
- else:
- koodi = '[[%s]] was conquered by [[%s]] in the war versus [[%s]]' % (alue, voittaja, tappio)
- return koodi
- if 'resistance has started' in string:
- alue = self.wikiregion(string.split('started in ')[1])
- string = 'A [[resistance]] has started in [[%s]], [[{{Regions/Owners|%s}}]]' % (alue, alue.replace('&','&').replace("'","'"))
- return string
- if 'attacked' in string:
- att = string.split(' attacked')[0]
- alue = self.wikiregion(string.split('attacked ')[1].split(',')[0])
- defe = string.split(', ')[1]
- string = '[[%s]] attacked [[%s]], [[%s]]' % (att, alue, defe)
- return string
- if 'signed an alliance' in string:
- maa1 = string.split(' signed')[0]
- maa2 = string.split(' with ')[1]
- string = '[[%s]] signed an [[alliance]] with [[%s]]' % (maa1, maa2)
- return string
- if ' as a Natural Enemy' in string:
- maa1 = string.split(' has declared')[0]
- maa2 = string.split('has declared ')[1].split(' as a Natu')[0]
- string = '[[%s]] has declared [[%s]] as a [[Natural enemy]]' % (maa1, maa2)
- return string
- if 'is no longer' in string:
- maa1 = string.split(' is no longer')[0]
- maa2 = string.split('Enemy for ')[1]
- string = '[[%s]] is no longer a [[natural enemy]] for [[%s]]' % (maa1, maa2)
- return string
- if 'no longer has a Natural Enemy' in string:
- maa = string.split(' no longer has a')[0]
- string = '[[%s]] no longer has a [[Natural enemy]]' % maa
- return string
- if 'was impeached' in string:
- maa = string.split('President of ')[1].split(',')[0]
- nimi = string.split(', was impeached')[0].split(', ')[1]
- string = 'The [[President]] of [[%s]], [[%s]], was [[Impeachment|impeached]]' % (maa, nimi)
- return string
- if 'signed a peace treaty' in string:
- maa = string.split(' signed a')[0]
- maai = string.split('treaty with ')[1]
- string = '[[%s]] signed a [[peace treaty]] with [[%s]]' % (maa, maai)
- return string
- return string
- def fetch_map_xml(self):
- try:
- api_xml = xml.parseString(urllib2.urlopen(self.maplink()).read())
- if len(api_xml.getElementsByTagName('countries')) == 0:
- exit(1)
- return api_xml
- except IOError:
- exit(1)
- def wikiregion(self, r):
- if 'ko District' in r:
- return unicode('Brčko District','UTF-8')
- else:
- return r.replace("Friuli-","Friuli ").replace("Haifa d","Haifa D").replace("Maluku i","Maluku I").replace("Trentino-","Trentino ")
- def maplink(self):
- time = datetime.now()
- y = time.year
- m = time.month
- d = time.day
- if m < 10:
- m = '0%i' % m
- if d < 10:
- d = '0%i' % d
- timestring = "%s%s%s" % (y, m, d)
- apiurl = 'http://api.erepublik.com/map/data/%s' % timestring
- return apiurl
- def cNameFromLink(self, link):
- for c in self._CDATA:
- if c["l"] == link:
- return c["n"]
- print "No country found with link %s" % link
- def cNameFromId(self, cid):
- for c in self._CDATA:
- if c["i"] == cid:
- return c["n"]
- print "No country found with ID %s" % cid
- class Wiki(Thread):
- def __init__(self):
- Thread.__init__(self)
- def run(self):
- self.wiki_api_url = 'http://wiki.erepublik.com/api.php'
- self.params = None
- self.opener = None
- self.login()
- def login(self):
- name = 'SampoBot'
- pw = 'bottaillaan'
- logindata = {'action':'login','format':'xml','lgname':name,'lgpassword':pw}
- request = urllib2.urlopen(self.wiki_api_url, urlencode(logindata))
- logindata["lgtoken"] = xml.parseString(request.read()).getElementsByTagName('login')[0].getAttribute('token')
- confirm = urllib2.urlopen(self.wiki_api_url,urlencode(logindata))
- result = xml.parseString(confirm.read()).getElementsByTagName('login')[0].getAttribute('result')
- if result == 'Success':
- self.opener = opener
- if self.params is not None:
- self.doEdit()
- else:
- print 'Error: %s' % result
- exit(1)
- def doEdit(self, mdata=None):
- if mdata is not None:
- self.params = mdata
- if self.opener is None: return
- pages = [x["title"] for x in self.params]
- self.fetch_edit_tokens(pages)
- self.do_edit()
- def fetch_edit_tokens(self, pages):
- names = "|".join(pages)
- query_paramns = {"action":"query","prop":"info","intoken":"edit","format":"xml","titles":names}
- query = urllib2.urlopen(self.wiki_api_url, urlencode(query_paramns))
- response = query.read()
- response_xml = xml.parseString(response)
- tokens = response_xml.getElementsByTagName('page')
- for t in tokens:
- token = t.getAttribute('edittoken')
- page = t.getAttribute('title')
- self.add_token(token, page)
- def add_token(self, token, page):
- for item in self.params:
- if item["title"] == page:
- item["token"] = token
- break
- def do_edit(self):
- for page in self.params:
- edit = urllib2.urlopen(self.wiki_api_url, urlencode(page))
- result = edit.read()
- if 'nochange' in result:
- print '%s: No changes after latest update.' % page["title"]
- elif 'Success' in result:
- print '%s: Success' % page["title"]
- else:
- print '%s: Error - Response was %s' % (page["title"], result)
- if __name__ == '__main__':
- opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(CookieJar()))
- opener.add_headers = [('User-Agent','WikiBot/Python-urllib/2.7')]
- urllib2.install_opener(opener)
- w = WikiDataUpdater()
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement