-
Notifications
You must be signed in to change notification settings - Fork 0
/
KMeansMapper.java
113 lines (105 loc) · 3.55 KB
/
KMeansMapper.java
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
/*=============================================================================
| Assignment: Final Project - Multiple Document Summarization
| Author: Group7 - (Sampath, Ajay, Visesh)
| Grader: Walid Shalaby
|
| Course: ITCS 6190
| Instructor: Srinivas Akella
|
| Language: Java
| Version : 1.8.0_101
|
| Deficiencies: No logical errors.
*===========================================================================*/
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.io.Text;
/*
* Mapper class to emit cluster center as key and vectors as values
* */
public class KMeansMapper extends Mapper<LongWritable, Text, ClusterCenter, Text> {
private final List<ClusterCenter> centers = new ArrayList<>();
private DistanceMeasurer distanceMeasurer;
/*
* first iteration, k-random centers, in every follow-up iteration we have
* new calculated centers
*/
@SuppressWarnings("deprecation")
@Override
protected void setup(Context context) throws IOException, InterruptedException {
super.setup(context);
Configuration conf = context.getConfiguration();
Path centroids = new Path(conf.get("centroid.path"));
FileSystem fs = FileSystem.get(conf);
try (SequenceFile.Reader reader = new SequenceFile.Reader(fs, centroids, conf)) {
ClusterCenter key = new ClusterCenter();
IntWritable value = new IntWritable();
int index = 0;
while (reader.next(key, value)) {
ClusterCenter clusterCenter = new ClusterCenter(key);
clusterCenter.setClusterIndex(index++);
centers.add(clusterCenter);
}
}
distanceMeasurer = new ManhattanDistance();
}
@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
ClusterCenter nearest = null;
double nearestDistance = Double.MAX_VALUE;
String line = value.toString();
String fileName, vectorValues;
String[] fileNameAndValues;
String[] sec = line.split("\t");
if (sec.length == 1) {
fileNameAndValues = line.split("=");
fileName = fileNameAndValues[0];
vectorValues = fileNameAndValues[1];
} else {
fileNameAndValues = sec[1].split("=");
fileName = fileNameAndValues[0];
vectorValues = fileNameAndValues[1];
}
Pattern p = Pattern.compile("\\[(.*?)\\]");
Matcher m = p.matcher(vectorValues);
String v = null;
while (m.find()) {
v = m.group(1);
}
String[] vec = v.split(",");
double[] vecArray = new double[vec.length];
for (int i = 0; i < vec.length; i++) {
String trim = vec[i].replaceAll("\\s+", "");
vecArray[i] = Double.parseDouble(trim);
}
VectorWritable vw = new VectorWritable(vecArray);
for (ClusterCenter c : centers) {
// calculating manhattan distance between cluster center and vector.
double dist = distanceMeasurer.measureDistance(c.getCenterVector(), vw.getVector());
// assign vector to the nearest center.
if (nearest == null) {
nearest = c;
nearestDistance = dist;
} else {
if (nearestDistance > dist) {
nearest = c;
nearestDistance = dist;
}
}
}
String finalValue = fileName + "=" + vw;
context.write(nearest, new Text(finalValue));
}
}