I'm making a distributed sorting system, where each machine (client) has a specific range of values (partition) to store. I don't have any problem sending and receiving data, but the synchronization is my problem. Before retrieving the wanted partition from the server, it must first arrive to the server from one of the other clients.
I've tried with delay
and various other RPC calls to try to try to synchronize the clients and server, but the clients run out of synchronization, and the program breaks. I think I have to incorporate future
or promise
, but I'm not sure how.
Every client has a unique ID (from 0 and up), and will store partitions according to that number. One line of data looks like this: AsfAGHM5om 00000000000000000000000000000000 0000222200002222000022220000222200002222000000001111
EDIT: updated code for example
Client
val numberOfClients = 3
var sentPartitionsCounter = 0
while(sentPartitionsCounter < numberOfClients) {
if(sentPartitionsCounter != id){
sendUnwantedPartition("src/main/scala/testFile."+sentPartitionsCounter.toString) //"partition."+client.sentPartitionsCounter
sentPartitionsCounter += 1
}
client.getWantedPartition()
}
def sendUnwantedPartition(filename: String): Unit = {
val dataList = Source.fromFile(filename).getLines.toList
val dataSeq = for {
dataLine <- dataList
dataValues = dataLine.split(" ", 2)
} yield (Data(key = dataValues(0), value = dataValues(1)))
val partitionID = filename takeRight 1
val request = Dataset(data = dataSeq, partitionID = partitionID.toInt)
val response = blockingStub.getUnwantedPartitions(request)
print("sent partitions")
}
def getWantedPartition(): Unit = {
val request = ID(id = id)
val response = blockingStub.sendWantedPartitions(request)
val wantedPartitions = new File("clientFiles/wantedPartitions"+id.toString+".txt")
val printWriter: PrintWriter = new PrintWriter(new FileWriter(wantedPartitions, true));
if(!wantedPartitions.exists()){
wantedPartitions.createNewFile()
}
for {
data <- response.data
} yield (printWriter.append(data.key + " " + data.value + "\n"))
printWriter.close();
}
Server
override def getUnwantedPartitions(req: Dataset) = {
val filename = "serverPartitions/partition"+req.partitionID+".txt"
val partition = new File(filename)
val printWriter: PrintWriter = new PrintWriter(new FileWriter(partition, true));
if(!partition.exists()){
partition.createNewFile()
}
for {
data <- req.data
} yield (printWriter.append(data.key + " " + data.value + "\n"))
printWriter.close();
Future.successful(Text("Got unwanted data"))
}
override def sendWantedPartitions(req: ID) = {
val filename = "serverPartitions/partition"+req.id+".txt"
try {
val dataList = Source.fromFile(filename).getLines.toList
val dataSeq = for {
dataLine <- dataList
dataValues = dataLine.split(" ", 2)
} yield (Data(key = dataValues(0), value = dataValues(1)))
val reply = Dataset(data = dataSeq, partitionID = req.id)
new File(filename).delete()
Future.successful(reply)
} catch {
// Partition hasn't been received yet
case e: FileNotFoundException => Future.successful(Dataset())
}
}
Proto
syntax = "proto3";
package protoPackage;
service DistrSorting {
rpc getUnwantedPartitions(Dataset) returns (Text) {}
rpc sendWantedPartitions(ID) returns (Dataset) {}
}
message ID {
int32 id = 1;
}
message Text {
string text = 1;
}
message Dataset {
repeated Data data = 1;
int32 partitionID = 2;
}
message Data {
string key = 1;
string value = 2;
}