# Template for a Spark Job Server configuration file
# When deployed these settings are loaded when job server starts
#
# Spark Cluster / Job Server configuration
spark {

  master = "yarn"

  # Default # of CPUs for jobs to use for Spark standalone cluster
  job-number-cpus = 2

  jobserver {
    port = 8090

    context-per-jvm = true

    # Default client mode will start up a new JobManager int local machine
    # You can use mesos-cluster mode with REMOTE_JOBSERVER_DIR and MESOS_SPARK_DISPATCHER
    # environment value set in xxxx.sh file to launch JobManager in remote node
    # Mesos will take responsibility to offer resource to the JobManager process
    #driver-mode = client

    # Note: JobFileDAO is deprecated from v0.7.0 because of issues in
    # production and will be removed in future, now defaults to H2 file.
    jobdao = spark.jobserver.io.JobSqlDAO

    filedao {
      rootdir = /tmp/spark-jobserver/filedao/data
    }

    datadao {
      # storage directory for files that are uploaded to the server
      # via POST/data commands
      rootdir = /tmp/spark-jobserver/upload
    }

    sqldao {
      # Slick database driver, full classpath
      # slick-driver = slick.driver.H2Driver
      slick-driver = slick.driver.MySQLDriver
      # JDBC driver, full classpath
      #jdbc-driver = org.h2.Driver
      jdbc-driver = com.mysql.jdbc.Driver
      # Directory where default H2 driver stores its data. Only needed for H2.
      #rootdir = /tmp/spark-jobserver/sqldao/data

      # Full JDBC URL / init string, along with username and password.  Sorry, needs to match above.
      # Substitutions may be used to launch job-server, but leave it out here in the default or tests won't pass
      #jdbc {
      #  url = "jdbc:h2:file:/tmp/spark-jobserver/sqldao/data/h2-db"
      #  user = ""
      #  password = ""
      #}
      jdbc {
        url = "jdbc:mysql://localhost:3666/spark_jobserver?useSSL=false"
        user = "jobserver"
        password = ""
      }

      # DB connection pool settings
      dbcp {
        #enabled = false
        maxactive = 20
        maxidle = 10
        initialsize = 10
      }
    }
    # When using chunked transfer encoding with scala Stream job results, this is the size of each chunk
    result-chunk-size = 1m
  }

  # Predefined Spark contexts
  # contexts {
  #   my-low-latency-context {
  #     num-cpu-cores = 1           # Number of cores to allocate.  Required.
  #     memory-per-node = 512m         # Executor memory per node, -Xmx style eg 512m, 1G, etc.
  #   }
  #   # define additional contexts here
  # }

  # Universal context configuration.  These settings can be overridden, see README.md
  context-settings {
    #num-cpu-cores = 2           # Number of cores to allocate.  Required.
    #memory-per-node = 2g         # Executor memory per node, -Xmx style eg 512m, #1G, etc.
    spark.driver.port = 32456
    spark.yarn.jar = "/opt/mapreducelab/spark/jars/*"
    spark.dynamicAllocation.enabled = true
    spark.executor.memory = 6632m
    spark.yarn.executor.memoryOverhead = 664

    # In case spark distribution should be accessed from HDFS (as opposed to being installed on every Mesos slave)
    # spark.executor.uri = "hdfs://namenode:8020/apps/spark/spark.tgz"

    # URIs of Jars to be loaded into the classpath for this context.
    # Uris is a string list, or a string separated by commas ','
    # dependent-jar-uris = ["file:///some/path/present/in/each/mesos/slave/somepackage.jar"]

    # Add settings you wish to pass directly to the sparkConf as-is such as Hadoop connection
    # settings that don't use the "spark." prefix
    passthrough {
      spark.dynamicAllocation.enabled = true
      spark.executor.memory = 6632m
      spark.yarn.executor.memoryOverhead = 664
      spark.cassandra.driver = "org.apache.spark.sql.cassandra"
      spark.cassandra.connection.host = "host1, host2, host3"
      spark.cassandra.connection.port = "9042"
      spark.cassandra.auth.username   = "noetl"
      spark.cassandra.auth.password   = "noetl"
      spark.driver.allowMultipleContexts = true
    }
  }

  # This needs to match SPARK_HOME for cluster SparkContexts to be created successfully
  home = "/opt/mapreducelab/spark"
}
spray.can.server.parsing.max-content-length = 250m
# Note that you can use this file to define settings not only for job server,
# but for your Spark jobs as well.  Spark job configuration merges with this configuration file as defaults.

akka {
  remote.netty.tcp {
    # This controls the maximum message size, including job results, that can be sent
    maximum-frame-size = 40 MiB
  }
}
#client {
#connecting-timeout = infinite
#}
spray.can.server {
  idle-timeout =70 s
  request-timeout = 60 s
  parsing.max-content-length = 300m
  verbose-error-logging = "on"
  verbose-error-messages = "on"
}
flyway.locations="db/mysql/migration"