Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- import * as fs from "fs-extra";
- import * as Octokit from "@octokit/rest";
- import * as parse from "parse-link-header";
- import { sleep } from "../../utils/sleep";
- import { fetchPackageJson, fetchLanguagesRepository } from "../../utils/github";
- import { isReactNativeProject } from "../../utils/validation";
- import { GithubRepository, GithubCommit } from "../../models/github";
- (async function main() {
- const rawSearchResultsContent = fs.readFileSync("../../crawler/rawSearchResults.json", "utf-8");
- const repositories: GithubRepository[] = JSON.parse(rawSearchResultsContent);
- const octokit = new Octokit({
- headers: {
- accept: "application/vnd.github.mercy-preview+json",
- "user-agent": "octokit/rest.js v1.2.3", // v1.2.3 will be current version
- },
- // custom GitHub Enterprise URL
- baseUrl: "https://api.github.com",
- // Node only: advanced request options can be passed as http(s) agent
- agent: undefined
- });
- octokit.authenticate({
- type: "basic",
- username: "albertininmourato",
- password: "albertininTCC10"
- });
- octokit.authenticate({
- type: "basic",
- username: "DLRibeiro",
- password: "Bl@d3150396"
- });
- console.log("Getting unique repositories now")
- const uniqueRepositories = repositories.filter((repository, index) => {
- return repositories.findIndex(repo => repo.html_url === repository.html_url) === index;
- });
- console.log("Filterring by main languange");
- await filterLanguage(octokit, uniqueRepositories);
- })();
- async function filterLanguage(octokit: Octokit, repositories: GithubRepository[]) {
- console.log("Filterring by main languange");
- const failedRepositories = [];
- const androidJavaKotlinRepositories: GithubRepository[] = [];
- //LANGUAGE FILTER
- for (let i = 0; i < 5; i++) {
- const repository = repositories[i];
- try {
- console.log(`Getting language list from repository ${repository.html_url} - ${i}`)
- const response = await octokit.repos.listLanguages({ "owner": repository.owner.login, "repo": repository.name });
- const languages = response.data;
- const languageList = Object.keys(languages);
- console.log('Repositories remaining:', repositories.length - i + 1);
- console.log('Languages: ', languageList);
- const containsKotlinAndJava = languageList.includes('Kotlin') && languageList.includes('Java');
- const isJavaWithKotlin = languageList[0] == 'Java' && languageList[1] == 'Kotlin';
- const isKotlinWithJava = languageList[0] == 'Kotlin' && languageList[1] == 'Java';
- if (isJavaWithKotlin) {
- androidJavaKotlinRepositories.push(repository);
- console.log("Added repository based on language");
- }
- await sleep(700);
- } catch (e) {
- console.log(`Repository ${repository.html_url} - ${i} Failed`);
- console.log(e);
- failedRepositories.push(repository);
- await sleep(1000);
- }
- }
- fs.writeFileSync(
- "./filteredJavaKotlinSearchResults.json",
- JSON.stringify(androidJavaKotlinRepositories),
- "utf-8"
- );
- fs.writeFileSync(
- "./failedJavaKotlinSearchResults.json",
- JSON.stringify(failedRepositories),
- "utf-8"
- );
- }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement