Advertisement
Guest User

Untitled

a guest
Mar 24th, 2017
96
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 4.75 KB | None | 0 0
  1. Python:
  2. import os
  3. import csv
  4. import sys
  5. import time
  6. import mmap
  7. import asyncio
  8.  
  9. loop = asyncio.get_event_loop()
  10.  
  11.  
  12. def get_files(count, base):
  13.     files = []
  14.     with open('chunks.csv') as csvfile:
  15.         spamreader = csv.reader(csvfile, delimiter=',')
  16.         for row in spamreader:
  17.             if os.path.exists(os.path.join(base, 'cache', row[0])):
  18.                 files.append(os.path.join(base, 'cache', row[0]))
  19.             elif os.path.exists(os.path.join(base, 'cache_small', row[0])):
  20.                 files.append(os.path.join(base, 'cache_small', row[0]))
  21.     if count > len(files) or count < 0:
  22.         return files
  23.     return files[:count]
  24.  
  25.  
  26. def read_file(file_name, devnull):
  27.     with open(file_name, 'rb') as f:
  28.         data = f.read(1024*1024*10)
  29.         while data:
  30.             devnull.write(data)
  31.             data = f.read(1024*1024*10)
  32.  
  33.  
  34. def read_file_os(file_name, devnull):
  35.     f = os.open(file_name, os.O_RDONLY)
  36.     data = True
  37.     while data:
  38.         data = os.read(f, 1024*1024*10)
  39.         devnull.write(data)
  40.     os.close(f)
  41.  
  42.  
  43. @asyncio.coroutine
  44. def read_file_coro(file_name, devnull):
  45.     with open(file_name, 'rb') as f:
  46.         devnull.write(f.read())
  47.  
  48.  
  49. def test_case(base, file_list, devnull):
  50.     start_time = time.time()
  51.     for file in file_list:
  52.         read_file_os(file, devnull)
  53.     end_time = time.time()
  54.     return end_time - start_time
  55.  
  56.  
  57. def test_case_coro(base, file_list, devnull):
  58.     start_time = time.time()
  59.     to_do = [read_file_coro(file, devnull) for file in file_list]
  60.     wait_coro = asyncio.wait(to_do)
  61.     res, _ = loop.run_until_complete(wait_coro)
  62.     end_time = time.time()
  63.     return end_time - start_time
  64.  
  65.  
  66. def main(base_path):
  67.     if len(sys.argv) < 3:
  68.         print('Usage: %s file_count base_path <base_path>...' % sys.argv[0])
  69.         sys.exit(1)
  70.     if not os.path.exists(base_path):
  71.         print('Path not exists %s' % base_path)
  72.         sys.exit(2)
  73.     file_count = int(sys.argv[1])
  74.     file_list = get_files(file_count, base_path)
  75.     etime = []
  76.  
  77.     with open(os.devnull, 'wb') as devnull:
  78.         print("Searching file in %s" % base_path)
  79.         for i in range(10):
  80.             os.system('sync ; echo 3 > /proc/sys/vm/drop_caches')
  81.             etime.append(test_case(base_path, file_list, devnull))
  82.             print("Process time %d: %f seconds for %d files in %s" % (i, etime[i], file_count, base_path))
  83.     return etime
  84.  
  85.  
  86. if __name__ == '__main__':
  87.     for base in sys.argv[2:]:
  88.         main(base)
  89.     loop.close()
  90.  
  91.  
  92. C++
  93. #include <iostream>
  94. #include <string>
  95. #include <vector>
  96. #include <fstream>
  97. #include <chrono>
  98. #include <memory>
  99. #include <stdlib.h>
  100. #include <stdio.h>
  101. #include <cstring>
  102.  
  103. #define OUT
  104. #define BUF_SIZE 1024*1024*10
  105.  
  106. float test_case(std::string base, std::vector<std::string> files, std::ostream& devnull) {
  107.     std::string line;
  108.     char* buff = new char[BUF_SIZE];
  109.     int length = 0;
  110.     auto begin = std::chrono::high_resolution_clock::now();
  111.     for(std::string file : files) {
  112.            std::ifstream infile(file);
  113.            while(infile) {
  114.                 std::memset(buff, 0, BUF_SIZE);
  115.                 infile.read(buff, BUF_SIZE-1);
  116.                 devnull << buff;
  117.            }
  118.            infile.close();
  119.     }
  120.     auto end = std::chrono::high_resolution_clock::now();
  121.     delete[] buff;
  122.     return std::chrono::duration_cast<std::chrono::nanoseconds>(end-begin).count() / 1000000000.0;
  123. }
  124.  
  125. void get_files(int count, std::string base, OUT std::vector<std::string>& files) {
  126.     std::cout << "Searching file in " << base << std::endl;
  127.     std::ifstream infile("chunks.csv");
  128.     std::string line;
  129.     for(int i = 0; i < count && infile >> line; ++i) {
  130.         files.push_back(base + line);
  131.     }
  132. }
  133.  
  134.  
  135. void show_usage(char* program_name) {
  136.     std::cout << "Usage: " << program_name << " file_count base_path <base_path>..." << std::endl;
  137. }
  138.  
  139.  
  140. int main(int argc, char** argv) {
  141.     if(argc < 3) {
  142.            show_usage(argv[0]);
  143.            return -1;
  144.     }
  145.     std::vector<std::string> files;
  146.     int count = std::stoi(argv[1], nullptr);
  147.     std::vector<std::string> bases;
  148.  
  149.     for(int i = 2; i < argc; ++i) {
  150.            bases.push_back(argv[i]);
  151.     }
  152.     std::ostream devnull(0);
  153.  
  154.     for(std::string base : bases) {
  155.            std::vector<float> times;
  156.            get_files(count, base, files);
  157.         for(int i = 0; i < 10; ++i) {
  158.             system("sync ; echo 3 > /proc/sys/vm/drop_caches");
  159.                   times.push_back(test_case(base, files, devnull));
  160.            }
  161.         for(float time : times) {
  162.                   std::cout << "Process time: " << time << " seconds for " << count << " files in " << base << std::endl;
  163.            }
  164.     }
  165.  
  166.     return 0;
  167. }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement