Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- Python:
- import os
- import csv
- import sys
- import time
- import mmap
- import asyncio
- loop = asyncio.get_event_loop()
- def get_files(count, base):
- files = []
- with open('chunks.csv') as csvfile:
- spamreader = csv.reader(csvfile, delimiter=',')
- for row in spamreader:
- if os.path.exists(os.path.join(base, 'cache', row[0])):
- files.append(os.path.join(base, 'cache', row[0]))
- elif os.path.exists(os.path.join(base, 'cache_small', row[0])):
- files.append(os.path.join(base, 'cache_small', row[0]))
- if count > len(files) or count < 0:
- return files
- return files[:count]
- def read_file(file_name, devnull):
- with open(file_name, 'rb') as f:
- data = f.read(1024*1024*10)
- while data:
- devnull.write(data)
- data = f.read(1024*1024*10)
- def read_file_os(file_name, devnull):
- f = os.open(file_name, os.O_RDONLY)
- data = True
- while data:
- data = os.read(f, 1024*1024*10)
- devnull.write(data)
- os.close(f)
- @asyncio.coroutine
- def read_file_coro(file_name, devnull):
- with open(file_name, 'rb') as f:
- devnull.write(f.read())
- def test_case(base, file_list, devnull):
- start_time = time.time()
- for file in file_list:
- read_file_os(file, devnull)
- end_time = time.time()
- return end_time - start_time
- def test_case_coro(base, file_list, devnull):
- start_time = time.time()
- to_do = [read_file_coro(file, devnull) for file in file_list]
- wait_coro = asyncio.wait(to_do)
- res, _ = loop.run_until_complete(wait_coro)
- end_time = time.time()
- return end_time - start_time
- def main(base_path):
- if len(sys.argv) < 3:
- print('Usage: %s file_count base_path <base_path>...' % sys.argv[0])
- sys.exit(1)
- if not os.path.exists(base_path):
- print('Path not exists %s' % base_path)
- sys.exit(2)
- file_count = int(sys.argv[1])
- file_list = get_files(file_count, base_path)
- etime = []
- with open(os.devnull, 'wb') as devnull:
- print("Searching file in %s" % base_path)
- for i in range(10):
- os.system('sync ; echo 3 > /proc/sys/vm/drop_caches')
- etime.append(test_case(base_path, file_list, devnull))
- print("Process time %d: %f seconds for %d files in %s" % (i, etime[i], file_count, base_path))
- return etime
- if __name__ == '__main__':
- for base in sys.argv[2:]:
- main(base)
- loop.close()
- C++
- #include <iostream>
- #include <string>
- #include <vector>
- #include <fstream>
- #include <chrono>
- #include <memory>
- #include <stdlib.h>
- #include <stdio.h>
- #include <cstring>
- #define OUT
- #define BUF_SIZE 1024*1024*10
- float test_case(std::string base, std::vector<std::string> files, std::ostream& devnull) {
- std::string line;
- char* buff = new char[BUF_SIZE];
- int length = 0;
- auto begin = std::chrono::high_resolution_clock::now();
- for(std::string file : files) {
- std::ifstream infile(file);
- while(infile) {
- std::memset(buff, 0, BUF_SIZE);
- infile.read(buff, BUF_SIZE-1);
- devnull << buff;
- }
- infile.close();
- }
- auto end = std::chrono::high_resolution_clock::now();
- delete[] buff;
- return std::chrono::duration_cast<std::chrono::nanoseconds>(end-begin).count() / 1000000000.0;
- }
- void get_files(int count, std::string base, OUT std::vector<std::string>& files) {
- std::cout << "Searching file in " << base << std::endl;
- std::ifstream infile("chunks.csv");
- std::string line;
- for(int i = 0; i < count && infile >> line; ++i) {
- files.push_back(base + line);
- }
- }
- void show_usage(char* program_name) {
- std::cout << "Usage: " << program_name << " file_count base_path <base_path>..." << std::endl;
- }
- int main(int argc, char** argv) {
- if(argc < 3) {
- show_usage(argv[0]);
- return -1;
- }
- std::vector<std::string> files;
- int count = std::stoi(argv[1], nullptr);
- std::vector<std::string> bases;
- for(int i = 2; i < argc; ++i) {
- bases.push_back(argv[i]);
- }
- std::ostream devnull(0);
- for(std::string base : bases) {
- std::vector<float> times;
- get_files(count, base, files);
- for(int i = 0; i < 10; ++i) {
- system("sync ; echo 3 > /proc/sys/vm/drop_caches");
- times.push_back(test_case(base, files, devnull));
- }
- for(float time : times) {
- std::cout << "Process time: " << time << " seconds for " << count << " files in " << base << std::endl;
- }
- }
- return 0;
- }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement