daily pastebin goal
40%
SHARE
TWEET

Untitled

a guest Sep 14th, 2018 52 Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. reading and writing in chunks on linux using c
  2. Record-1:15 characters
  3. Record-2:200 characters
  4. Record-3:500 characters
  5. ...
  6. ...
  7. Record-n: X characters
  8.    
  9. FILE *stream;
  10. char buffer[104857600]; //100 MB char array
  11. fread(buffer, sizeof(buffer), 104857600, stream);
  12.    
  13. fwrite(buffer, sizeof(buffer), 104857600, stream);
  14.    
  15. char linebuf[1000];
  16. FILE *file = ...;
  17. while (fgets(linebuf, sizeof(linebuf), file) {
  18.     // decode one line
  19. }
  20.    
  21. char buffer[104857600]; // too big
  22.    
  23. int r, fdes;
  24. struct stat st;
  25. void *ptr;
  26. size_t sz;
  27.  
  28. fdes = open(filename, O_RDONLY);
  29. if (fdes < 0) abort();
  30. r = fstat(fdes, &st);
  31. if (r) abort();
  32. if (st.st_size > (size_t) -1) abort(); // too big to map
  33. sz = st.st_size;
  34. ptr = mmap(NULL, sz, PROT_READ, MAP_SHARED, fdes, 0);
  35. if (ptr == MAP_FAILED) abort();
  36. close(fdes); // file no longer needed
  37.  
  38. // now, ptr has the data, sz has the data length
  39. // you can use ordinary string functions
  40.    
  41. #include <stdio.h>
  42. #include <sys/stat.h>
  43. #include <sys/types.h>
  44. #include <unistd.h>
  45. #include <sys/mman.h>
  46.  
  47.  
  48. /* ... */
  49.  
  50. struct stat stat_buf;
  51. long pagesz = sysconf(_SC_PAGESIZE);
  52. int fd = fileno(stream);
  53. off_t line_start = 0;
  54. char *file_chunk = NULL;
  55. char *input_line;
  56. off_t cur_off = 0;
  57. off_t map_offset = 0;
  58. /* map 16M plus pagesize to ensure any record <= 16M will always fit in the mapped area */
  59. size_t map_size = 16*1024*1024+pagesz;
  60. if (map_offset + map_size > stat_buf.st_size) {
  61.   map_size = stat_buf.st_size - map_offset;
  62. }
  63. fstat(fd, &stat_buf);
  64. /* map the first chunk of the file */
  65. file_chunk = mmap(NULL, map_size, PROT_READ, MAP_SHARED, fd, map_offset);
  66. // until we reach the end of the file
  67. while (cur_off < stat_buf.st_size) {
  68.   /* check if we're about to read outside the current chunk */
  69.   if (!(cur_off-map_offset < map_size)) {
  70.     // destroy the previous mapping
  71.     munmap(file_chunk, map_size);
  72.     // round down to the page before line_start
  73.     map_offset = (line_start/pagesz)*pagesz;
  74.     // limit mapped region to size of file
  75.     if (map_offset + map_size > stat_buf.st_size) {
  76.       map_size = stat_buf.st_size - map_offset;
  77.     }
  78.     // map the next chunk
  79.     file_chunk = mmap(NULL, map_size, PROT_READ, MAP_SHARED, fd, map_offset);
  80.     // adjust the line start for the new mapping
  81.     input_line = &file_chunk[line_start-map_offset];
  82.   }
  83.   if (file_chunk[cur_off-map_offset] == 'n') {
  84.     // found a new line, process the current line
  85.     process_line(input_line, cur_off-line_start);
  86.     // set up for the next one
  87.     line_start = cur_off+1;
  88.     input_line = &file_chunk[line_start-map_offset];
  89.   }
  90.   cur_off++;
  91. }
  92.    
  93. char *file_data = mmap(NULL, stat_buf.st_size, PROT_READ, MAP_SHARED, fd, 0);
  94.    
  95. if( (total+strlen(buff)) > 104857600 )
RAW Paste Data
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy. OK, I Understand
 
Top