Guest User

Untitled

a guest
Oct 9th, 2020
91
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 7.36 KB | None | 0 0
  1. python3
  2. import requests, html, math
  3.  
  4. geni=lambda x:requests.get("https://query.wikidata.org/sparql?query=SELECT%20%3Fid%7B%3Fid%20wdt%3AP2600%20%22"+str(x)+"%22%7D%20LIMIT%201",headers={"Accept":"text/csv","User-Agent":"Bugreporter"}).text[35:].strip()
  5.  
  6. def gettarget(q,p):
  7. q=str(q)
  8. if q[0]!="Q": q="Q"+q
  9. p=str(p)
  10. if p[0]!="P": p="P"+p
  11. data=requests.get("https://www.wikidata.org/w/api.php?action=wbgetclaims&format=json&entity="+q+"&property="+p+"&props=",headers={"User-Agent":"Bugreporter"}).json()
  12. if "claims" not in data:
  13. print(q,p)
  14. data=data["claims"]
  15. if p not in data: return None
  16. data=data[p]
  17. data=[i for i in data if i["rank"]!="deprecated"]
  18. data1=[i for i in data if i["rank"]=="preferred"]
  19. if len(data1)>0: data=data1
  20. if len(data)==0: return None
  21. data=data[0]["mainsnak"]
  22. if data["snaktype"]!="value": return None
  23. data=data["datavalue"]
  24. if data["type"]=="wikibase-entityid": return data["value"]["id"]
  25. return data["value"]
  26.  
  27. def resolve_redirect(q):
  28. if not q:
  29. return ""
  30. data=requests.get("https://www.wikidata.org/w/api.php?action=wbgetentities&format=json&ids="+q+"&props=info",headers={"User-Agent":"Bugreporter"}).json()["entities"]
  31. if q not in data:
  32. return ""
  33. else:
  34. return data[q]["id"]
  35.  
  36. def getlabel(q):
  37. q=str(q)
  38. if q[0]!="Q": q="Q"+q
  39. data=requests.get("https://www.wikidata.org/w/api.php?action=wbgetentities&format=json&ids="+q+"&props=labels&languages=en",headers={"User-Agent":"Bugreporter"}).json()["entities"][q]["labels"]
  40. if "en" in data:
  41. return data["en"]["value"]
  42. else:
  43. return ""
  44.  
  45. import pywikibot
  46.  
  47. pywikibot.config.put_throttle=0
  48. pywikibot.config.minthrottle=0
  49. pywikibot.config.maxthrottle=0
  50. pywikibot.config.maxlag=0
  51.  
  52. geni_items={}
  53. def geni_get(id):
  54. if id in geni_items:
  55. return geni_items[id]
  56. geni_id=geni(id)
  57. if geni_id!="":
  58. geni_items[id]=geni_id
  59. return geni_id
  60. return ""
  61.  
  62.  
  63. def geni_get_or_create(id,name,gender,desc):
  64. if geni_get(id)!="":
  65. return geni_get(id)
  66. data={"labels":{"en":name},"descriptions":{"en":desc},"claims":{}}
  67. data['claims']['P31']=[{"mainsnak": {"snaktype": "value","property": "P31","datatype": "wikibase-item",
  68. "datavalue": {"value": {"entity-type": "item","numeric-id": 5},"type": "wikibase-entityid"}
  69. },"type": "statement","rank": "normal","references": []}]
  70. data['claims']['P21']=[{"mainsnak": {"snaktype": "value","property": "P21","datatype": "wikibase-item",
  71. "datavalue": {"value": {"entity-type": "item","numeric-id": 6581072 if gender=="F" else 6581097},"type": "wikibase-entityid"}
  72. },"type": "statement","rank": "normal","references": []}]
  73. data['claims']['P2600']=[{"mainsnak": {"snaktype": "value","property": "P2600","datatype": "external-id",
  74. "datavalue": {"value": id,"type": "string"}
  75. },"type": "statement","rank": "normal","references": []}]
  76. ip=pywikibot.ItemPage(pywikibot.Site('wikidata', 'wikidata'))
  77. ip.editEntity(data, summary="semi-automatic Geni import")
  78. geni_items[id]=ip.id
  79. return ip.id
  80.  
  81. def showq(id):
  82. gender=gettarget(id,"P21")
  83. dob=gettarget(id,"P569")
  84. dob=dob["time"][1:].replace("T00:00:00Z","") if dob else ""
  85. dod=gettarget(id,"P570")
  86. dod=dod["time"][1:].replace("T00:00:00Z","") if dod else ""
  87. if gender=="Q6581072":
  88. return getlabel(id)+" ["+id+" "+dob+" - "+dod+"]"
  89. else:
  90. return getlabel(id)+" ("+id+" "+dob+" - "+dod+")"
  91.  
  92. repo = pywikibot.Site("wikidata", "wikidata")
  93. genip22={}
  94. genip25={}
  95. p40=[]
  96. def dogeni(id,anf):
  97. if id=="" or anf==1:
  98. return
  99. if id[0]=="Q":
  100. old_id=id
  101. id=gettarget(id,"P2600")
  102. if not id:
  103. return
  104. geni_items[id]=old_id
  105. old_id=geni_get(id)
  106. layer=1<<int(math.log(anf,2)-1)
  107. nextanf=anf-layer-(anf&layer)
  108. nextgender="M" if anf&layer==0 else "F"
  109. nextid=(genip22 if anf&layer==0 else genip25).get(id,"-")
  110. if nextid!="-":
  111. dogeni(nextid,nextanf)
  112. return
  113. data=requests.get("https://www.geni.com/api/profile-g"+id+"/immediate-family?access_token="+access_token).json()
  114. profile_id=data["focus"]["id"]
  115. unions=data["nodes"][profile_id]["edges"]
  116. union_id=[i for i in unions.keys() if unions[i]["rel"]=="child"]
  117. if len(union_id)==0:
  118. return
  119. union_id=union_id[0]
  120. fmid=[i for i in data["nodes"].keys() if union_id in data["nodes"][i]["edges"] and data["nodes"][i]["edges"][union_id]["rel"]=="partner" and data["nodes"][i]["gender"][0].upper()==nextgender]
  121. if len(fmid)==0:
  122. return
  123. fmid=fmid[0]
  124. fmdata=requests.get("https://www.geni.com/api/"+fmid+"?access_token="+access_token).json()
  125. cfmid=fmdata["guid"]
  126. curitem=geni_get(cfmid)
  127. fmname=fmdata["name"]
  128. fm=resolve_redirect(gettarget(old_id,("P22" if nextgender=="M" else "P25")))
  129. input(showq(old_id)+"/"+id+"/"+nextgender+str(anf)+" -- "+fmname+"/"+fm+'/'+cfmid)
  130. if fm:
  131. nfmid=gettarget(fm,"P2600")
  132. if nfmid!=cfmid:
  133. fmlabel=getlabel(fm)
  134. fmlabel=fmlabel.replace("(?)","").strip()
  135. if (fmlabel not in fmname) or curitem or nfmid:
  136. x=input(showq(fm)+(("/"+nfmid) if nfmid else "")+" vs "+fmname+(" (" if nextgender else " [")+(")" if nextgender else "]")+"/"+cfmid+("/"+curitem if curitem else "")).strip()
  137. if x!="":
  138. return
  139. item = pywikibot.ItemPage(repo, fm)
  140. claim = pywikibot.Claim(repo, 'P2600')
  141. claim.setTarget(cfmid)
  142. item.addClaim(claim, summary="semi-automatic Geni import")
  143. geni_items[cfmid]=fm
  144. else:
  145. if not curitem:
  146. curitem=geni_get_or_create(cfmid,fmname,nextgender,"")
  147. if (curitem,old_id) not in p40:
  148. if int(old_id[1:])<96000000 and int(curitem[1:])<96000000:
  149. x=input(showq(old_id)+(" P22 " if nextgender=="M" else " P25 ")+showq(curitem))
  150. if x!="":
  151. return
  152. item1 = pywikibot.ItemPage(repo, curitem)
  153. item2 = pywikibot.ItemPage(repo, old_id)
  154. claim1 = pywikibot.Claim(repo, ("P22" if nextgender=="M" else "P25"))
  155. claim1.setTarget(item1)
  156. statedin = pywikibot.Claim(repo, u'P248')
  157. statedin.setTarget(pywikibot.ItemPage(repo, "Q2621214"))
  158. claim1.addSources([statedin])
  159. item2.addClaim(claim1, summary="semi-automatic Geni import")
  160. claim2 = pywikibot.Claim(repo, "P40")
  161. claim2.setTarget(item2)
  162. statedin = pywikibot.Claim(repo, u'P248')
  163. statedin.setTarget(pywikibot.ItemPage(repo, "Q2621214"))
  164. claim2.addSources([statedin])
  165. item1.addClaim(claim2, summary="semi-automatic Geni import")
  166. p40.append((curitem,old_id))
  167. (genip22 if anf&layer==0 else genip25)[id]=cfmid
  168. dogeni(cfmid,nextanf)
  169.  
  170.  
Add Comment
Please, Sign In to add comment