Spark构建推荐系统引擎--来源于Spark机器学习

import org.apache.spark.mllib.evaluation.RegressionMetrics import org.apache.spark.mllib.recommendation.{ALS, Rating} import org.apache.spark.{SparkConf, SparkContext} import org.jblas.DoubleMatrix /** * Created by zgr on 2017/3/14. */ //spark进行推荐系统 object RecommandSystem { def main(args: Array[String]) { val sparkConf = new SparkConf().setAppName("RecommandSystem").setMaster("spark://10.149.252.106:7077"); val sc = new SparkContext(sparkConf); val rawData = sc.textFile("hdfs://10.149.252.106:9000/input/ml-100k/u.data"); rawData.first();//返回第一个元素 //取出u.data中除了时间戳的数据 val rawRatings = rawData.map(_.split("\t").take(3)) /* Construct the RDD of Rating objects */ val ratings = rawRatings.map { case Array(user, movie, rating) => Rating(user.toInt, movie.toInt, rating.toDouble) } //Train the ALS model with rank=50, iterations=10, lambda=0.01 //训练ALS模型，rank：对应ALS模型中的因子个数，也就是在低阶近似矩阵中的隐含特征个 //iterations：对应运行时的迭代次数 //lambda：该参数控制模型的正则化过程，从而控制模型的过拟合情况。其值越高，正则化越严厉 val model = ALS.train(ratings,50,10,0.01); /* Inspect the user factors */ model.userFeatures; /* Count user factors and force computation */ model.userFeatures.count(); model.productFeatures.count(); /* Make a prediction for a single user and movie pair */ val predictRating = model.predict(789,123); /* Make predictions for a single user across all movies */ val userId = 789 val K = 10 val topKRecs = model.recommendProducts(userId,K);//为了789推荐前10产品 println(topKRecs.mkString("\n"));//转成字符串 println("========================到此为为用户推荐电影====================================="); //进行推荐内容的检验 //要直观地检验推荐的效果，可以简单比对下用户所评级过的电影的标题和被推荐的那些电影的电影 val movies = sc.textFile("hdfs://10.149.252.106:9000/input/ml-100k/u.item"); val titles = movies.map(line => line.split("\\|").take(2)).map(array => (array(0).toInt,array(1))).collectAsMap(); val moviesForUser = ratings.keyBy(_.user).lookup(789);//在上面的评级中找789用户 println(moviesForUser.size)//该用户对多少电影评级过 // moviesForUser.sortBy(-_.rating).take(10).map(rating => (titles(rating.product),rating.rating)).foreach(println);//实际中789用户已经看的 topKRecs.map(rating => (titles(rating.product),rating.rating)).foreach(println);//推荐系统中推荐的 通过上下对比 println("===========================推荐检验结束================================================"); //进行物品推荐 //利用余弦相似度来对指定物品的因子向量与其他物品的做比 val itemId = 567; val itemFactor = model.productFeatures.lookup(itemId).head; val itemVector = new DoubleMatrix(itemFactor); //现在求各个物品的余弦相似度： val sims = model.productFeatures.map{case (id,factor) => val factorVector = new DoubleMatrix(factor); val sim = cosineSimilarit(itemVector,factorVector); (id,sim);//返回 } val sortedSims = sims.top(K)(Ordering.by[(Int, Double), Double]{case (id,similiarity) => similiarity});//通过相似度 println(sortedSims.mkString("\n")); //检查推荐的相似物品 println(titles(itemId)); //这一次我们取前11部最相似电影，以排除给定的那部。所以，可以选取列表中的第1到11项 val sortedSims2 = sims.top(K+1)(Ordering.by[(Int, Double), Double]{case (id,similiarity) => similiarity}); println(sortedSims.slice(1,11).map{case (id,sim) => (titles(id),sim)}.mkString("\n")); println("=======================物品推荐完成====================================================") //推荐模型效果的评估 /* Compute squared error between a predicted and actual rating */ //We'll take the first rating for our example user 789 val actualRating = moviesForUser.take(1)(0) println(actualRating.rating); //然后，求模型的预计评级 val predictedRating = model.predict(789,actualRating.product); println(predictedRating); //最后，我们计算实际评级和预计评级的平方误差 val squaredError = math.pow(actualRating.rating-predictedRating,2.0); //整个数据集上的MSE，需要对每一条(user, movie, actual rating, predictedrating)记录都计算该平均误差，然后求和，再除以总的评级次数 val usersProducts = ratings.map{case Rating(user, product, rating) => (user, product)}; val predictions = model.predict(usersProducts).map{case Rating(user, product, rating) => ((user, product), rating)} //这个RDD的主键为“用户物品”对，键值为相应的实际评级和预计评级。 val ratingsAndPredictions = ratings.map{ case Rating(user, product, rating) => ((user, product), rating) }.join(predictions) //最后，求上述MSE var MSE = ratingsAndPredictions.map{case((user,product),(actual,predicted)) => math.pow(actual-predicted,2.0)}.reduce(_+_)/ratingsAndPredictions.count(); println("Mean Squared Error = "+MSE); //均跟方误差 val RMSE = math.sqrt(MSE) println("Root Mean Squared Error = " + RMSE) println("=====================s使用MLlib下的RegressionMetrics和RankingMetrics+======================") //实际中MLlib下的RegressionMetrics和RankingMetrics类也提供了相应的函数 //RegressionMetrics来求解MSE和RMSE得分 val predictedAndTrue = ratingsAndPredictions.map{case((user,product),(actual,predicted)) => (actual,predicted)}; val regressionMetrics = new RegressionMetrics(predictedAndTrue); //之后就可以查看各种指标的情况，包括MSE和RMSE。 println(regressionMetrics.meanSquaredError); println(regressionMetrics.rootMeanSquaredError); } //求两个向量的余弦度，。1表示完全相似，0表示两者互不相关（即无相似性） def cosineSimilarit(vec1: DoubleMatrix, vec2: DoubleMatrix):Double={ vec1.dot(vec2)/(vec1.norm2() * vec2.norm2()); } }