0

I'm making a distributed sorting system, where each machine (client) has a specific range of values (partition) to store. I don't have any problem sending and receiving data, but the synchronization is my problem. Before retrieving the wanted partition from the server, it must first arrive to the server from one of the other clients.

I've tried with delay and various other RPC calls to try to try to synchronize the clients and server, but the clients run out of synchronization, and the program breaks. I think I have to incorporate future or promise, but I'm not sure how.

Every client has a unique ID (from 0 and up), and will store partitions according to that number. One line of data looks like this: AsfAGHM5om 00000000000000000000000000000000 0000222200002222000022220000222200002222000000001111

EDIT: updated code for example

Client

  val numberOfClients = 3
  var sentPartitionsCounter = 0
  while(sentPartitionsCounter < numberOfClients) {
    if(sentPartitionsCounter != id){
      sendUnwantedPartition("src/main/scala/testFile."+sentPartitionsCounter.toString) //"partition."+client.sentPartitionsCounter
      sentPartitionsCounter += 1
    }
    client.getWantedPartition()
  }

  def sendUnwantedPartition(filename: String): Unit = {
    val dataList = Source.fromFile(filename).getLines.toList
    val dataSeq = for {
                    dataLine <- dataList
                    dataValues = dataLine.split(" ", 2)
                  } yield (Data(key = dataValues(0), value = dataValues(1)))
    val partitionID = filename takeRight 1 
    val request = Dataset(data = dataSeq, partitionID = partitionID.toInt)
    val response = blockingStub.getUnwantedPartitions(request)
    print("sent partitions")
  }

  def getWantedPartition(): Unit = {
    val request = ID(id = id)
    val response = blockingStub.sendWantedPartitions(request)
    val wantedPartitions = new File("clientFiles/wantedPartitions"+id.toString+".txt")
    val printWriter: PrintWriter = new PrintWriter(new FileWriter(wantedPartitions, true));

    if(!wantedPartitions.exists()){
      wantedPartitions.createNewFile()
    } 

    for {
      data <- response.data
    } yield (printWriter.append(data.key + " " + data.value + "\n"))
    
    printWriter.close();
  }

Server

override def getUnwantedPartitions(req: Dataset) = {
  val filename = "serverPartitions/partition"+req.partitionID+".txt" 
  val partition = new File(filename)
  val printWriter: PrintWriter = new PrintWriter(new FileWriter(partition, true));

  if(!partition.exists()){ 
    partition.createNewFile()
  } 

  for {
    data <- req.data
  } yield (printWriter.append(data.key + " " + data.value + "\n"))

  printWriter.close();
  Future.successful(Text("Got unwanted data"))
}

override def sendWantedPartitions(req: ID) = {
  val filename = "serverPartitions/partition"+req.id+".txt"
  try {
    val dataList = Source.fromFile(filename).getLines.toList
    val dataSeq = for {
                    dataLine <- dataList
                    dataValues = dataLine.split(" ", 2)
                  } yield (Data(key = dataValues(0), value = dataValues(1)))
    val reply = Dataset(data = dataSeq, partitionID = req.id)
    new File(filename).delete()
    Future.successful(reply)
  } catch {
    // Partition hasn't been received yet
    case e: FileNotFoundException => Future.successful(Dataset())
  }
}

Proto

syntax = "proto3";

package protoPackage;

service DistrSorting {
  rpc getUnwantedPartitions(Dataset) returns (Text) {}
  rpc sendWantedPartitions(ID) returns (Dataset) {}
}

message ID {
  int32 id = 1;
}

message Text {
  string text = 1;
}

message Dataset {
  repeated Data data = 1;
  int32 partitionID = 2;
}

message Data {
  string key = 1;
  string value = 2;
}
Bassusour
  • 175
  • 6
  • 14
  • Can you simplify the question and the given example? It's unclear what is wanted and wanted, how the partitions make it to server, and why the client send partition ids that it *does not* want. – thesamet Dec 09 '21 at 17:22
  • @thesamet I updated the code now – Bassusour Dec 10 '21 at 01:41
  • 2
    What do you want the server to do if the partition isn't available yet? To keep things simple you can just return an error, and let the client retry. – thesamet Dec 10 '21 at 16:58
  • That's a good idea @thesamet. I think I'll try to do it that way – Bassusour Dec 11 '21 at 05:37

0 Answers0