Guest User

Untitled

a guest
May 23rd, 2018
96
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 3.49 KB | None | 0 0
  1. import json
  2. import os
  3. import sqlite3
  4.  
  5.  
  6. def init_db(filename):
  7. if os.path.exists(filename):
  8. return
  9.  
  10. conn = sqlite3.connect(filename)
  11. conn.executescript(
  12. """
  13. CREATE TABLE [commit] (
  14. [commit] VARCHAR(40),
  15. tree TEXT,
  16. author_name TEXT,
  17. author_email TEXT,
  18. author_date INTEGER,
  19. author_timezone TEXT,
  20. committer_name TEXT,
  21. committer_email TEXT,
  22. committer_date INTEGER,
  23. committer_timezone TEXT,
  24. message TEXT,
  25. PRIMARY KEY ([commit])
  26. );
  27. CREATE TABLE [change] (
  28. [commit] VARCHAR(40),
  29. added INTEGER,
  30. deleted INTEGER,
  31. filepath TEXT,
  32. FOREIGN KEY ([commit]) REFERENCES [commit]("commit")
  33. );
  34. CREATE TABLE parent (
  35. [commit] VARCHAR(40),
  36. parent VARCHAR(40),
  37. FOREIGN KEY ("parent") REFERENCES [commit]("commit")
  38. );
  39. CREATE INDEX parent_commit ON parent("commit");
  40. CREATE INDEX parent_parent ON parent("parent");
  41. """
  42. )
  43. conn.close()
  44.  
  45.  
  46. def create_and_populate_fts(conn):
  47. create_sql = """
  48. CREATE VIRTUAL TABLE "commit_fts"
  49. USING FTS4 (author_name, committer_name, message, content="commit")
  50. """
  51. conn.executescript(create_sql)
  52. conn.executescript(
  53. """
  54. INSERT INTO "commit_fts" (rowid, author_name, committer_name, message)
  55. SELECT rowid, author_name, committer_name, message
  56. FROM [commit];
  57. """)
  58.  
  59.  
  60. def insert_or_replace(conn, table, record):
  61. pairs = record.items()
  62. columns = [p[0] for p in pairs]
  63. params = [p[1] for p in pairs]
  64. sql = """
  65. INSERT OR REPLACE INTO [{table}] ({column_list})
  66. VALUES ({value_list});
  67. """.format(
  68. table=table,
  69. column_list=", ".join('[{}]'.format(c) for c in columns),
  70. value_list=", ".join(["?" for p in params]),
  71. )
  72. print(sql)
  73. conn.execute(sql, params)
  74.  
  75.  
  76. def parse_and_load(data, db):
  77. for commit in data:
  78. insert_or_replace(
  79. db,
  80. "commit",
  81. {
  82. "commit": commit["commit"],
  83. "tree": commit["tree"],
  84. "author_name": commit["author"]["name"],
  85. "author_email": commit["author"]["email"],
  86. "author_date": commit["author"]["date"],
  87. "author_timezone": commit["author"]["timezone"],
  88. "committer_name": commit["committer"]["name"],
  89. "committer_email": commit["committer"]["email"],
  90. "committer_date": commit["committer"]["date"],
  91. "committer_timezone": commit["committer"]["timezone"],
  92. "message": commit["message"],
  93. },
  94. )
  95. for parent in commit["parents"]:
  96. insert_or_replace(
  97. db,
  98. "parent",
  99. {
  100. "commit": commit["commit"],
  101. "parent": parent,
  102. },
  103. )
  104. for change in commit["changes"]:
  105. insert_or_replace(
  106. db,
  107. "change",
  108. {
  109. "commit": commit["commit"],
  110. "added": change[0],
  111. "deleted": change[1],
  112. "filepath": change[2],
  113. },
  114. )
  115.  
  116.  
  117. if __name__ == "__main__":
  118. import sys
  119.  
  120. dbfile = sys.argv[-1]
  121. assert dbfile.endswith(".db")
  122. init_db(dbfile)
  123. db = sqlite3.connect(dbfile)
  124. # Read from stdin
  125. parse_and_load(json.load(sys.stdin), db)
  126. create_and_populate_fts(db)
  127. db.close()
Add Comment
Please, Sign In to add comment