spark

Type Members

class Accumulable[R, T] extends Serializable

A data type that can be accumulated, ie has an commutative and associative "add" operation, but where the result type, R, may be different from the element type being added, T.
trait AccumulableParam[R, T] extends Serializable

Helper object defining how to accumulate values of a particular type.
class Accumulator[T] extends Accumulable[T, T]

A simpler value of Accumulable where the result type being accumulated is the same as the types of elements being merged, i.
trait AccumulatorParam[T] extends AccumulableParam[T, T]

A simpler version of org.apache.spark.AccumulableParam where the only data type you can add in is the same type as the accumulated value.
case class Aggregator[K, V, C](createCombiner: (V) ⇒ C, mergeValue: (C, V) ⇒ C, mergeCombiners: (C, C) ⇒ C) extends Product with Serializable

:: DeveloperApi :: A set of functions used to aggregate data.
class ComplexFutureAction[T] extends FutureAction[T]

A FutureAction for actions that could trigger multiple Spark jobs.
abstract class Dependency[T] extends Serializable

:: DeveloperApi :: Base class for dependencies.
case class ExceptionFailure(className: String, description: String, stackTrace: Array[StackTraceElement], fullStackTrace: String, metrics: Option[TaskMetrics]) extends TaskFailedReason with Product with Serializable

:: DeveloperApi :: Task failed due to a runtime exception.
case class ExecutorLostFailure(execId: String) extends TaskFailedReason with Product with Serializable

:: DeveloperApi :: The task failed because the executor that it was running on was lost.
case class FetchFailed(bmAddress: BlockManagerId, shuffleId: Int, mapId: Int, reduceId: Int, message: String) extends TaskFailedReason with Product with Serializable

:: DeveloperApi :: Task failed to fetch shuffle data from a remote node.
trait FutureAction[T] extends Future[T]

A future for the result of an action to support cancellation.
class HashPartitioner extends Partitioner

A org.apache.spark.Partitioner that implements hash-based partitioning using Java's Object.hashCode.
class InterruptibleIterator[+T] extends Iterator[T]

:: DeveloperApi :: An iterator that wraps around an existing iterator to provide task killing functionality.
class JavaSparkListener extends SparkListener
class JobExecutionStatus extends Enum[JobExecutionStatus]
trait Logging extends AnyRef

:: DeveloperApi :: Utility trait for classes that want to log data.
abstract class NarrowDependency[T] extends Dependency[T]

:: DeveloperApi :: Base class for dependencies where each partition of the child RDD depends on a small number of partitions of the parent RDD.
class OneToOneDependency[T] extends NarrowDependency[T]

:: DeveloperApi :: Represents a one-to-one dependency between partitions of the parent and child RDDs.
trait Partition extends Serializable

An identifier for a partition in an RDD.
abstract class Partitioner extends Serializable

An object that defines how the elements in a key-value pair RDD are partitioned by key.
class RangeDependency[T] extends NarrowDependency[T]

:: DeveloperApi :: Represents a one-to-one dependency between ranges of partitions in the parent and child RDDs.
class RangePartitioner[K, V] extends Partitioner

A org.apache.spark.Partitioner that partitions sortable records by range into roughly equal ranges.
class SerializableWritable[T <: Writable] extends Serializable

Annotations
@DeveloperApi()
class ShuffleDependency[K, V, C] extends Dependency[Product2[K, V]]

:: DeveloperApi :: Represents a dependency on the output of a shuffle stage.
class SimpleFutureAction[T] extends FutureAction[T]

A FutureAction holding the result of an action that triggers a single job.
class SparkConf extends Cloneable with Logging

Configuration for a Spark application.
class SparkContext extends Logging with ExecutorAllocationClient

Main entry point for Spark functionality.
class SparkEnv extends Logging

:: DeveloperApi :: Holds all the runtime environment objects for a running Spark instance (either master or worker), including the serializer, Akka actor system, block manager, map output tracker, etc.
class SparkException extends Exception
class SparkFirehoseListener extends SparkListener
trait SparkJobInfo extends Serializable
trait SparkStageInfo extends Serializable
class SparkStatusTracker extends AnyRef

Low-level status reporting APIs for monitoring job and stage progress.
case class TaskCommitDenied(jobID: Int, partitionID: Int, attemptID: Int) extends TaskFailedReason with Product with Serializable

:: DeveloperApi :: Task requested the driver to commit, but was denied.
abstract class TaskContext extends Serializable

Contextual information about a task which can be read or mutated during execution.
sealed trait TaskEndReason extends AnyRef

:: DeveloperApi :: Various possible reasons why a task ended.
sealed trait TaskFailedReason extends TaskEndReason

:: DeveloperApi :: Various possible reasons why a task failed.
class TaskKilledException extends RuntimeException

:: DeveloperApi :: Exception thrown when a task is explicitly killed (i.

Value Members

object AccumulatorParam extends Serializable
object HeartbeatReceiver
object Partitioner extends Serializable
object Resubmitted extends TaskFailedReason with Product with Serializable

:: DeveloperApi :: A org.apache.spark.scheduler.ShuffleMapTask that completed successfully earlier, but we lost the executor before the stage completed.
val SPARK_VERSION: String
object SparkContext extends Logging

The SparkContext object contains a number of implicit conversions and parameters for use with various Spark features.
object SparkEnv extends Logging
object SparkFiles

Resolves paths to files added through SparkContext.addFile().
object Success extends TaskEndReason with Product with Serializable

:: DeveloperApi :: Task succeeded.
object TaskContext extends Serializable
object TaskKilled extends TaskFailedReason with Product with Serializable

:: DeveloperApi :: Task was killed intentionally and needs to be rescheduled.
object TaskResultLost extends TaskFailedReason with Product with Serializable

:: DeveloperApi :: The task finished successfully, but the result was lost from the executor's block manager before it was fetched.
object UnknownReason extends TaskFailedReason with Product with Serializable

:: DeveloperApi :: We don't know why the task ended -- for example, because of a ClassNotFound exception when deserializing the task result.
object WritableConverter extends Serializable
object WritableFactory extends Serializable
package annotation

Spark annotations to mark an API experimental or intended only for advanced usages by developers.
package api
package bagel

Bagel: An implementation of Pregel in Spark.
package broadcast

Spark's broadcast variables, used to broadcast immutable datasets to all nodes.
package examples
package graphx

ALPHA COMPONENT GraphX is a graph processing framework built on top of Spark.
package input
package io

IO codecs used for compression.
package launcher
package mapred
package ml

Spark ML is a BETA component that adds a new set of machine learning APIs to let users quickly assemble and configure practical machine learning pipelines.
package mllib

Spark's machine learning library.
package partial

:: Experimental ::
package rdd

Provides several RDD implementations.
package scheduler

Spark's scheduling components.
package serializer

Pluggable serializers for RDD and shuffle data.
package sql

Allows the execution of relational queries, including those expressed in SQL using Spark.
package status
package storage
package streaming

Spark Streaming functionality.
package ui
package util

Spark utilities.

package spark

Type Members

class Accumulable[R, T] extends Serializable

trait AccumulableParam[R, T] extends Serializable

class Accumulator[T] extends Accumulable[T, T]

trait AccumulatorParam[T] extends AccumulableParam[T, T]

case class Aggregator[K, V, C](createCombiner: (V) ⇒ C, mergeValue: (C, V) ⇒ C, mergeCombiners: (C, C) ⇒ C) extends Product with Serializable

class ComplexFutureAction[T] extends FutureAction[T]

abstract class Dependency[T] extends Serializable

case class ExceptionFailure(className: String, description: String, stackTrace: Array[StackTraceElement], fullStackTrace: String, metrics: Option[TaskMetrics]) extends TaskFailedReason with Product with Serializable

case class ExecutorLostFailure(execId: String) extends TaskFailedReason with Product with Serializable

case class FetchFailed(bmAddress: BlockManagerId, shuffleId: Int, mapId: Int, reduceId: Int, message: String) extends TaskFailedReason with Product with Serializable

trait FutureAction[T] extends Future[T]

class HashPartitioner extends Partitioner

class InterruptibleIterator[+T] extends Iterator[T]

class JavaSparkListener extends SparkListener

class JobExecutionStatus extends Enum[JobExecutionStatus]

trait Logging extends AnyRef

abstract class NarrowDependency[T] extends Dependency[T]

class OneToOneDependency[T] extends NarrowDependency[T]

trait Partition extends Serializable

abstract class Partitioner extends Serializable

class RangeDependency[T] extends NarrowDependency[T]

class RangePartitioner[K, V] extends Partitioner

class SerializableWritable[T <: Writable] extends Serializable

class ShuffleDependency[K, V, C] extends Dependency[Product2[K, V]]

class SimpleFutureAction[T] extends FutureAction[T]

class SparkConf extends Cloneable with Logging

class SparkContext extends Logging with ExecutorAllocationClient

class SparkEnv extends Logging

class SparkException extends Exception

class SparkFirehoseListener extends SparkListener

trait SparkJobInfo extends Serializable

trait SparkStageInfo extends Serializable

class SparkStatusTracker extends AnyRef

case class TaskCommitDenied(jobID: Int, partitionID: Int, attemptID: Int) extends TaskFailedReason with Product with Serializable

abstract class TaskContext extends Serializable

sealed trait TaskEndReason extends AnyRef

sealed trait TaskFailedReason extends TaskEndReason

class TaskKilledException extends RuntimeException

Value Members

object AccumulatorParam extends Serializable

object HeartbeatReceiver

object Partitioner extends Serializable

object Resubmitted extends TaskFailedReason with Product with Serializable

val SPARK_VERSION: String

object SparkContext extends Logging

object SparkEnv extends Logging

object SparkFiles

object Success extends TaskEndReason with Product with Serializable

object TaskContext extends Serializable

object TaskKilled extends TaskFailedReason with Product with Serializable

object TaskResultLost extends TaskFailedReason with Product with Serializable

object UnknownReason extends TaskFailedReason with Product with Serializable

object WritableConverter extends Serializable

object WritableFactory extends Serializable

package annotation

package api

package bagel

package broadcast

package examples

package graphx

package input

package io

package launcher

package mapred

package ml

package mllib

package partial

package rdd

package scheduler

package serializer

package sql

package status

package storage

package streaming

package ui

package util

Inherited from AnyRef