lamiastella

process huge csv files

Jun 21st, 2020
255
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 3.24 KB | None | 0 0
  1. ---------------------------------------------------------------------------
  2. ParserError Traceback (most recent call last)
  3. <ipython-input-6-14cf239fd04e> in <module>
  4. 3 import dask.dataframe as dd
  5. 4
  6. ----> 5 df= dd.read_csv("tweets_withheader.csv", quoting=csv.QUOTE_NONE)
  7. 6
  8. 7 df = df.compute()
  9.  
  10. ~/anaconda3/lib/python3.7/site-packages/dask/dataframe/io/csv.py in read(urlpath, blocksize, collection, lineterminator, compression, sample, enforce, assume_missing, storage_options, include_path_column, **kwargs)
  11. 576 storage_options=storage_options,
  12. 577 include_path_column=include_path_column,
  13. --> 578 **kwargs
  14. 579 )
  15. 580
  16.  
  17. ~/anaconda3/lib/python3.7/site-packages/dask/dataframe/io/csv.py in read_pandas(reader, urlpath, blocksize, collection, lineterminator, compression, sample, enforce, assume_missing, storage_options, include_path_column, **kwargs)
  18. 442
  19. 443 # Use sample to infer dtypes and check for presence of include_path_column
  20. --> 444 head = reader(BytesIO(b_sample), **kwargs)
  21. 445 if include_path_column and (include_path_column in head.columns):
  22. 446 raise ValueError(
  23.  
  24. ~/anaconda3/lib/python3.7/site-packages/pandas/io/parsers.py in parser_f(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, dialect, error_bad_lines, warn_bad_lines, delim_whitespace, low_memory, memory_map, float_precision)
  25. 674 )
  26. 675
  27. --> 676 return _read(filepath_or_buffer, kwds)
  28. 677
  29. 678 parser_f.__name__ = name
  30.  
  31. ~/anaconda3/lib/python3.7/site-packages/pandas/io/parsers.py in _read(filepath_or_buffer, kwds)
  32. 452
  33. 453 try:
  34. --> 454 data = parser.read(nrows)
  35. 455 finally:
  36. 456 parser.close()
  37.  
  38. ~/anaconda3/lib/python3.7/site-packages/pandas/io/parsers.py in read(self, nrows)
  39. 1131 def read(self, nrows=None):
  40. 1132 nrows = _validate_integer("nrows", nrows)
  41. -> 1133 ret = self._engine.read(nrows)
  42. 1134
  43. 1135 # May alter columns / col_dict
  44.  
  45. ~/anaconda3/lib/python3.7/site-packages/pandas/io/parsers.py in read(self, nrows)
  46. 2035 def read(self, nrows=None):
  47. 2036 try:
  48. -> 2037 data = self._reader.read(nrows)
  49. 2038 except StopIteration:
  50. 2039 if self._first_chunk:
  51.  
  52. pandas/_libs/parsers.pyx in pandas._libs.parsers.TextReader.read()
  53.  
  54. pandas/_libs/parsers.pyx in pandas._libs.parsers.TextReader._read_low_memory()
  55.  
  56. pandas/_libs/parsers.pyx in pandas._libs.parsers.TextReader._read_rows()
  57.  
  58. pandas/_libs/parsers.pyx in pandas._libs.parsers.TextReader._tokenize_rows()
  59.  
  60. pandas/_libs/parsers.pyx in pandas._libs.parsers.raise_parser_error()
  61.  
  62. ParserError: Error tokenizing data. C error: Expected 25 fields in line 5, saw 26
Add Comment
Please, Sign In to add comment