df = sqlc.read.format('com.databricks.spark.csv').
    options(header='true', inferSchema='true', delimiter=',').load(my_file)
assembler = VectorAssembler(inputCols=input_cols, outputCol="features")
df = assembler.transform(df)
rm_cols = [c for c in df.columns if c not in ['features', 'label']]
df = df.drop(*rm_cols)

lr = LogisticRegression(maxIter=10, regParam=0.3, elasticNetParam=0.8)
lr.fit(sdf)

---------------------------------------------------------------------------
Py4JJavaError                             Traceback (most recent call last)
<ipython-input-63-1e9f885a2ebc> in <module>()
      1 lr = LogisticRegression(maxIter=10, regParam=0.3, elasticNetParam=0.8)
----> 2 lr.fit(sdf)

/opt/spark/python/pyspark/ml/base.py in fit(self, dataset, params)
     62                 return self.copy(params)._fit(dataset)
     63             else:
---> 64                 return self._fit(dataset)
     65         else:
     66             raise ValueError("Params must be either a param map or a list/tuple of param maps, "

/opt/spark/python/pyspark/ml/wrapper.py in _fit(self, dataset)
    263
    264     def _fit(self, dataset):
--> 265         java_model = self._fit_java(dataset)
    266         return self._create_model(java_model)
    267

/opt/spark/python/pyspark/ml/wrapper.py in _fit_java(self, dataset)
    260         """
    261         self._transfer_params_to_java()
--> 262         return self._java_obj.fit(dataset._jdf)
    263
    264     def _fit(self, dataset):

/opt/spark/python/lib/py4j-0.10.4-src.zip/py4j/java_gateway.py in __call__(self, *args)
   1131         answer = self.gateway_client.send_command(command)
   1132         return_value = get_return_value(
-> 1133             answer, self.gateway_client, self.target_id, self.name)
   1134
   1135         for temp_arg in temp_args:

/opt/spark/python/pyspark/sql/utils.py in deco(*a, **kw)
     61     def deco(*a, **kw):
     62         try:
---> 63             return f(*a, **kw)
     64         except py4j.protocol.Py4JJavaError as e:
     65             s = e.java_exception.toString()

/opt/spark/python/lib/py4j-0.10.4-src.zip/py4j/protocol.py in get_return_value(answer, gateway_client, target_id, name)
    317                 raise Py4JJavaError(
    318                     "An error occurred while calling {0}{1}{2}.n".
--> 319                     format(target_id, ".", name), value)
    320             else:
    321                 raise Py4JError(

Py4JJavaError: An error occurred while calling o1789.fit.

  [1]: https://i.stack.imgur.com/owwCX.png