Guest User

Untitled

a guest
Sep 14th, 2018
111
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 2.62 KB | None | 0 0
  1. reading and writing in chunks on linux using c
  2. Record-1:15 characters
  3. Record-2:200 characters
  4. Record-3:500 characters
  5. ...
  6. ...
  7. Record-n: X characters
  8.  
  9. FILE *stream;
  10. char buffer[104857600]; //100 MB char array
  11. fread(buffer, sizeof(buffer), 104857600, stream);
  12.  
  13. fwrite(buffer, sizeof(buffer), 104857600, stream);
  14.  
  15. char linebuf[1000];
  16. FILE *file = ...;
  17. while (fgets(linebuf, sizeof(linebuf), file) {
  18. // decode one line
  19. }
  20.  
  21. char buffer[104857600]; // too big
  22.  
  23. int r, fdes;
  24. struct stat st;
  25. void *ptr;
  26. size_t sz;
  27.  
  28. fdes = open(filename, O_RDONLY);
  29. if (fdes < 0) abort();
  30. r = fstat(fdes, &st);
  31. if (r) abort();
  32. if (st.st_size > (size_t) -1) abort(); // too big to map
  33. sz = st.st_size;
  34. ptr = mmap(NULL, sz, PROT_READ, MAP_SHARED, fdes, 0);
  35. if (ptr == MAP_FAILED) abort();
  36. close(fdes); // file no longer needed
  37.  
  38. // now, ptr has the data, sz has the data length
  39. // you can use ordinary string functions
  40.  
  41. #include <stdio.h>
  42. #include <sys/stat.h>
  43. #include <sys/types.h>
  44. #include <unistd.h>
  45. #include <sys/mman.h>
  46.  
  47.  
  48. /* ... */
  49.  
  50. struct stat stat_buf;
  51. long pagesz = sysconf(_SC_PAGESIZE);
  52. int fd = fileno(stream);
  53. off_t line_start = 0;
  54. char *file_chunk = NULL;
  55. char *input_line;
  56. off_t cur_off = 0;
  57. off_t map_offset = 0;
  58. /* map 16M plus pagesize to ensure any record <= 16M will always fit in the mapped area */
  59. size_t map_size = 16*1024*1024+pagesz;
  60. if (map_offset + map_size > stat_buf.st_size) {
  61. map_size = stat_buf.st_size - map_offset;
  62. }
  63. fstat(fd, &stat_buf);
  64. /* map the first chunk of the file */
  65. file_chunk = mmap(NULL, map_size, PROT_READ, MAP_SHARED, fd, map_offset);
  66. // until we reach the end of the file
  67. while (cur_off < stat_buf.st_size) {
  68. /* check if we're about to read outside the current chunk */
  69. if (!(cur_off-map_offset < map_size)) {
  70. // destroy the previous mapping
  71. munmap(file_chunk, map_size);
  72. // round down to the page before line_start
  73. map_offset = (line_start/pagesz)*pagesz;
  74. // limit mapped region to size of file
  75. if (map_offset + map_size > stat_buf.st_size) {
  76. map_size = stat_buf.st_size - map_offset;
  77. }
  78. // map the next chunk
  79. file_chunk = mmap(NULL, map_size, PROT_READ, MAP_SHARED, fd, map_offset);
  80. // adjust the line start for the new mapping
  81. input_line = &file_chunk[line_start-map_offset];
  82. }
  83. if (file_chunk[cur_off-map_offset] == 'n') {
  84. // found a new line, process the current line
  85. process_line(input_line, cur_off-line_start);
  86. // set up for the next one
  87. line_start = cur_off+1;
  88. input_line = &file_chunk[line_start-map_offset];
  89. }
  90. cur_off++;
  91. }
  92.  
  93. char *file_data = mmap(NULL, stat_buf.st_size, PROT_READ, MAP_SHARED, fd, 0);
  94.  
  95. if( (total+strlen(buff)) > 104857600 )
Add Comment
Please, Sign In to add comment