Advertisement
Guest User

Untitled

a guest
Apr 29th, 2019
120
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. import * as fs from "fs-extra";
  2. import * as Octokit from "@octokit/rest";
  3. import * as parse from "parse-link-header";
  4. import { sleep } from "../../utils/sleep";
  5. import { fetchPackageJson, fetchLanguagesRepository } from "../../utils/github";
  6. import { isReactNativeProject } from "../../utils/validation";
  7. import { GithubRepository, GithubCommit } from "../../models/github";
  8.  
  9. (async function main() {
  10.   const rawSearchResultsContent = fs.readFileSync("../../crawler/rawSearchResults.json", "utf-8");
  11.   const repositories: GithubRepository[] = JSON.parse(rawSearchResultsContent);
  12.  
  13.   const octokit = new Octokit({
  14.     headers: {
  15.       accept: "application/vnd.github.mercy-preview+json",
  16.       "user-agent": "octokit/rest.js v1.2.3", // v1.2.3 will be current version
  17.     },
  18.  
  19.     // custom GitHub Enterprise URL
  20.     baseUrl: "https://api.github.com",
  21.  
  22.     // Node only: advanced request options can be passed as http(s) agent
  23.     agent: undefined
  24.   });
  25.  
  26.   octokit.authenticate({
  27.     type: "basic",
  28.     username: "albertininmourato",
  29.     password: "albertininTCC10"
  30.   });
  31.  
  32.   octokit.authenticate({
  33.     type: "basic",
  34.     username: "DLRibeiro",
  35.     password: "Bl@d3150396"
  36.   });
  37.   console.log("Getting unique repositories now")
  38.   const uniqueRepositories = repositories.filter((repository, index) => {
  39.     return repositories.findIndex(repo => repo.html_url === repository.html_url) === index;
  40.   });
  41.   console.log("Filterring by main languange");
  42.  
  43.   await filterLanguage(octokit, uniqueRepositories);
  44.  
  45. })();
  46.  
  47. async function filterLanguage(octokit: Octokit, repositories: GithubRepository[]) {
  48.   console.log("Filterring by main languange");
  49.  
  50.   const failedRepositories = [];
  51.  
  52.   const androidJavaKotlinRepositories: GithubRepository[] = [];
  53.   //LANGUAGE FILTER
  54.   for (let i = 0; i < 5; i++) {
  55.     const repository = repositories[i];
  56.     try {
  57.       console.log(`Getting language list from repository ${repository.html_url} - ${i}`)
  58.       const response = await octokit.repos.listLanguages({ "owner": repository.owner.login, "repo": repository.name });
  59.       const languages = response.data;
  60.       const languageList = Object.keys(languages);
  61.  
  62.       console.log('Repositories remaining:', repositories.length - i + 1);
  63.       console.log('Languages: ', languageList);
  64.       const containsKotlinAndJava = languageList.includes('Kotlin') && languageList.includes('Java');
  65.       const isJavaWithKotlin = languageList[0] == 'Java' && languageList[1] == 'Kotlin';
  66.       const isKotlinWithJava = languageList[0] == 'Kotlin' && languageList[1] == 'Java';
  67.  
  68.       if (isJavaWithKotlin) {
  69.         androidJavaKotlinRepositories.push(repository);
  70.         console.log("Added repository based on language");
  71.       }
  72.  
  73.       await sleep(700);
  74.     } catch (e) {
  75.       console.log(`Repository ${repository.html_url} - ${i} Failed`);
  76.       console.log(e);
  77.       failedRepositories.push(repository);
  78.       await sleep(1000);
  79.     }
  80.   }
  81.  
  82.  
  83.   fs.writeFileSync(
  84.     "./filteredJavaKotlinSearchResults.json",
  85.     JSON.stringify(androidJavaKotlinRepositories),
  86.     "utf-8"
  87.   );
  88.  
  89.  
  90.   fs.writeFileSync(
  91.     "./failedJavaKotlinSearchResults.json",
  92.     JSON.stringify(failedRepositories),
  93.     "utf-8"
  94.   );
  95. }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement