import luigi
from luigi.contrib.spark import PySparkTask

class SinglePySparkTask(PySparkTask):
    task_namespace = 'spark'

    def main(self, sc, *args):
    """

    :type sc: pyspark.context.SparkContext
    """
    rdd = sc.parallelize([1, 2, 3])
    print(rdd.collect())

if name == "main":
   luigi.build([SinglePySparkTask(), ], local_scheduler=True)