Breaking News: Grepper is joining You.com. Read the official announcement!

pyspark collaborative filtering

Innocent Iguana answered on February 18, 2024 Popularity 6/10 Helpfulness 5/10

answer pyspark collaborative filtering

pyspark collaborative filtering

Comment

from pyspark.sql import SparkSession

# Initialize SparkSession
spark = SparkSession.builder \
    .appName("ALS Example") \
    .getOrCreate()

# Sample data (User ID, Item ID, Rating, Additional Column1, Additional Column2)
data = [
    (1, 1, 5, "A", "X"),
# .............................
    (3, 2, 2, "F", "U")
]
df = spark.createDataFrame(data, ["userId", "itemId", "rating", "user_cat", "item_cat"])
# If user ID and/or Item ID do not exist and rather only categories exist, make unique identifier in separate dataframes for each distinct category and then join
users = df.select("user_cat").distinct().coalesce(1) # coalesce to put them in a single partition for consistent increase of id
users = users.withColumn("userId", monotonically_increasing_id()).persist() # caching to make sure the values do not change
items = df.select("item_cat").distinct().coalesce(1) # coalesce to put them in a single partition for consistent increase of id
items = items.withColumn("itemId", monotonically_increasing_id()).persist() # caching to make sure the values do not change
new_rating_df = df.join(users, "user_cat", "left").join(items, "item_cat", "left") # If rating dataframe already exists
## NOTE: If rating dataframe does not exist, then create one:
users = df1.select("userId").distinct()
items = df2.select("itemId").distinct()
custom_rating_df = users.crossJoin(items).join(df3, ["userId", "itemId"], "left").fillna(0)

# Sparsity = (Number of empty entries) / (Total number of entries)
numerator = ratings.count()
num_users = ratings.select("userId").distinct().count()
num_items = ratings.select("itemId").distinct().count()
denominator = num_users * num_items
sparsity = (1.0 - (numerator *1.0)/denominator)*100

(training_data, test_data) = df.randomSplit([0.8, 0.2]) # Split data into training and test sets

# k flod Cross-validation
from pyspark.ml.recommendation import ALS
from pyspark.ml.tuning import ParamGridBuilder, CrossValidator
from pyspark.ml.evaluation import RegressionEvaluator

als = ALS(userCol="userId", itemCol="itemId", ratingCol="rating",
        coldStartStrategy="drop" , nonnegative =True , implicitPrefs = False)
evaluator = RegressionEvaluator(metricName="rmse", labelCol="rating", predictionCol="prediction")
param_grid = ParamGridBuilder()
                .addGrid(als.rank, [5, 40, 80, 120])
                .addGrid(als.maxIter, [5, 100, 250, 500])
                .addGrid(als.regParam, [.05, .1, 1.5])
                .build()
cv = CrossValidator(estimator = als,
                estimatorParamMaps = param_grid,
                evaluator = evaluator,
                numFolds = 5)
len(param_grid) # Total number of models in the grid
model = cv.fit(training_data) # Train with ALS model
best_model = model.bestModel # Best Model
predictions = best_model.transform(test_data) # Make predictions on test data
rmse = evaluator.evaluate(predictions) # Evaluate predictions using RMSE
best_model.getRank()
best_model.getMaxIter()
best_model.getRegParam()
recommendations_df = best_model.recommendForAllUsers(5) # Generate all recommendations for users up to userId 5
from pyspark.sql.functions import explode
# Explode recommendations column to separate itemId and prediction
df = recommendations_df.withColumn("itemId_rating", explode("recommendations")) # Put each recommendation in new row
df = df.withColumn("itemId", col("itemId_rating").getField("itemId"))\
                .withColumn("rating", col("itemId_rating").getField("rating")) # Generate separate columns from the pair
df.join(ratings, ["userId", "itemId"], "left").filter(ratings['rating'].isNull()).show() # Only show recommendation for items not rated yet

### NOTE: For implicit rating columns (eg: No of clicks / binary rating etc) use custom evaluation metric (ROEM) instead of RMSE
### Or create weighted rating column by feature engineering. Also, make sure to put "implicitPrefs = True" in ALS model param
from pyspark.ml.evaluation import Evaluator
class ROEMEvaluator(Evaluator):
    def __init__(self, userCol="userId", itemCol="itemId", ratingCol="num_clicks"):
        self.userCol = userCol
        self.itemCol = itemCol
        self.ratingCol = ratingCol

    def _evaluate(self, predictions):
        predictions.createOrReplaceTempView("predictions")
        denominator = predictions.groupBy().sum(self.ratingCol).collect()[0][0]
        rankings = spark.sql("SELECT " + self.userCol + ", " + self.ratingCol + ", PERCENT_RANK() OVER (PARTITION BY " + self.  userCol + " ORDER BY prediction DESC) AS rank FROM predictions")
        rankings.createOrReplaceTempView("rankings")
        numerator = spark.sql("SELECT SUM(" + self.ratingCol + " * rank) FROM rankings").collect()[0][0]
        performance = numerator / denominator
        return performance
evaluator = ROEMEvaluator( ratingCol="colName")

Popularity 6/10 Helpfulness 5/10 Language python

Source: Grepper

Tags: collaborative-filtering pyspark python

Link to this answer
Share Copy Link

Contributed on Feb 18 2024

Innocent Iguana

0 Answers Avg Quality 2/10

pyspark collaborative filtering

Contents

More Related Answers

pyspark collaborative filtering

Grepper

Documentation

Social

Legal

Contact

Oops, You will need to install Grepper and log-in to perform this action.