Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- reading and writing in chunks on linux using c
- Record-1:15 characters
- Record-2:200 characters
- Record-3:500 characters
- ...
- ...
- Record-n: X characters
- FILE *stream;
- char buffer[104857600]; //100 MB char array
- fread(buffer, sizeof(buffer), 104857600, stream);
- fwrite(buffer, sizeof(buffer), 104857600, stream);
- char linebuf[1000];
- FILE *file = ...;
- while (fgets(linebuf, sizeof(linebuf), file) {
- // decode one line
- }
- char buffer[104857600]; // too big
- int r, fdes;
- struct stat st;
- void *ptr;
- size_t sz;
- fdes = open(filename, O_RDONLY);
- if (fdes < 0) abort();
- r = fstat(fdes, &st);
- if (r) abort();
- if (st.st_size > (size_t) -1) abort(); // too big to map
- sz = st.st_size;
- ptr = mmap(NULL, sz, PROT_READ, MAP_SHARED, fdes, 0);
- if (ptr == MAP_FAILED) abort();
- close(fdes); // file no longer needed
- // now, ptr has the data, sz has the data length
- // you can use ordinary string functions
- #include <stdio.h>
- #include <sys/stat.h>
- #include <sys/types.h>
- #include <unistd.h>
- #include <sys/mman.h>
- /* ... */
- struct stat stat_buf;
- long pagesz = sysconf(_SC_PAGESIZE);
- int fd = fileno(stream);
- off_t line_start = 0;
- char *file_chunk = NULL;
- char *input_line;
- off_t cur_off = 0;
- off_t map_offset = 0;
- /* map 16M plus pagesize to ensure any record <= 16M will always fit in the mapped area */
- size_t map_size = 16*1024*1024+pagesz;
- if (map_offset + map_size > stat_buf.st_size) {
- map_size = stat_buf.st_size - map_offset;
- }
- fstat(fd, &stat_buf);
- /* map the first chunk of the file */
- file_chunk = mmap(NULL, map_size, PROT_READ, MAP_SHARED, fd, map_offset);
- // until we reach the end of the file
- while (cur_off < stat_buf.st_size) {
- /* check if we're about to read outside the current chunk */
- if (!(cur_off-map_offset < map_size)) {
- // destroy the previous mapping
- munmap(file_chunk, map_size);
- // round down to the page before line_start
- map_offset = (line_start/pagesz)*pagesz;
- // limit mapped region to size of file
- if (map_offset + map_size > stat_buf.st_size) {
- map_size = stat_buf.st_size - map_offset;
- }
- // map the next chunk
- file_chunk = mmap(NULL, map_size, PROT_READ, MAP_SHARED, fd, map_offset);
- // adjust the line start for the new mapping
- input_line = &file_chunk[line_start-map_offset];
- }
- if (file_chunk[cur_off-map_offset] == 'n') {
- // found a new line, process the current line
- process_line(input_line, cur_off-line_start);
- // set up for the next one
- line_start = cur_off+1;
- input_line = &file_chunk[line_start-map_offset];
- }
- cur_off++;
- }
- char *file_data = mmap(NULL, stat_buf.st_size, PROT_READ, MAP_SHARED, fd, 0);
- if( (total+strlen(buff)) > 104857600 )
Add Comment
Please, Sign In to add comment