Breaking News: Grepper is joining You.com. Read the official announcement!

pyspark sparse data

Innocent Iguana answered on February 23, 2024 Popularity 3/10 Helpfulness 7/10

answer pyspark sparse data

pyspark sparse data

Comment

from pyspark.ml.linalg import DenseVector, SparseVector, Vectors
dense_vector = DenseVector([1.0, 0.0, 3.0, 0.0, 5.0]) # [1.0,0.0,3.0,0.0,5.0]
sparse_vector = SparseVector(5, [0, 2, 4], [1.0, 3.0, 5.0]) # (size, index, values) representation. Others are 0s

from pyspark.sql.functions import udf
# Given you have "spark" session and "dense_array" column
########### DENSE TO SPARSE #############
def dense_to_sparse(dense_array):
    return Vectors.dense(dense_array).toSparse()

from pyspark.sql.types import ArrayType, DoubleType
dense_to_sparse_udf = udf(dense_to_sparse, ArrayType(DoubleType()))
sparse_df = dense_df.withColumn("sparse_array", dense_to_sparse_udf("dense_array"))

########### SPARSE TO DENSE  #############
def sparse_to_dense(sparse_array):
    return Vectors.sparse(len(sparse_array), [i for i, v in enumerate(sparse_array) if v != 0], 
                          [v for v in sparse_array if v != 0]).toArray()

from pyspark.sql.types import ArrayType, DoubleType
sparse_to_dense_udf = udf(sparse_to_dense, ArrayType(DoubleType()))
dense_df = sparse_df.withColumn("dense_array", sparse_to_dense_udf("sparse_array"))

############## FOR TEXT DATA ###################

from pyspark.ml.feature import CountVectorizer, VectorAssembler # Efficient sparse representation for text
vectorizer = CountVectorizer(inputCol="text_col", outputCol="features") 
vectorizer_model = vectorizer.fit(text_df)
sparse_df = vectorizer_model.transform(text_df)

# Convert sparse vectors to dense vectors
vector_assembler = VectorAssembler(inputCols=["features"], outputCol="dense_features")
dense_df = vector_assembler.transform(sparse_df).select("dense_features")

# Convert sparse vector column to array
def sparse_to_array(sparse_vector):
    return sparse_vector.toArray().tolist()

from pyspark.sql.functions import udf
from pyspark.sql.types import ArrayType, DoubleType
sparse_to_array_udf = udf(sparse_to_array, ArrayType(DoubleType()))
array_df = sparse_df.withColumn("array_features", sparse_to_array_udf("features"))

### Checking if sparse array is empty
def is_sparse_array_empty(sparse_array):
    return sparse_array.numNonzeros() == 0

is_sparse_array_empty_udf = udf(is_sparse_array_empty, BooleanType())
df_with_boolean = df.withColumn("is_empty", is_sparse_array_empty_udf("sparse_col"))

### Using machine learning
train_data, test_data = df.randomSplit([0.7, 0.3], seed=123)
lr = LogisticRegression(featuresCol="features", labelCol="label")
lr_model = lr.fit(train_data)
predictions = lr_model.transform(test_data)
evaluator = BinaryClassificationEvaluator(labelCol="label", metricName="areaUnderROC")
roc_auc = evaluator.evaluate(predictions) # lr_model.evaluate(test_data)

Popularity 3/10 Helpfulness 7/10 Language python

Source: Grepper

Tags: pyspark python

Link to this answer
Share Copy Link

Contributed on Feb 23 2024

Innocent Iguana

0 Answers Avg Quality 2/10

pyspark sparse data

Contents

More Related Answers

pyspark sparse data

Grepper

Documentation

Social

Legal

Contact

Oops, You will need to install Grepper and log-in to perform this action.