Home | About | Sematext search-lucene.com search-hadoop.com
 Search Hadoop and all its subprojects:

Switch to Threaded View
HBase >> mail # user >> Coprocessor concurrency


Copy link to this message
-
Coprocessor concurrency
See below a null endpoint which takes 60-70msec on my 4 node ec2 cluster for a table with 45 regions.
When I run 64 concurrent clients for this the latency jumps to 3000-3700 msec.

(zookeeper maxClientCnxs is set to 0 (unlimited) and hbase regionserver handler count is 800). I hope I am not missing any configs for concurrency)

----------------------------------------------------------------------------------------------------------------------------------------------------------

package com.serendio.hbase.coprocessor;

import java.io.IOException;
import java.util.Map;

import org.apache.hadoop.hbase.coprocessor.BaseEndpointCoprocessor;

//Aggregation implementation at a region.
public class SearchEndpoint extends BaseEndpointCoprocessor implements
SearchProtocol {
@Override
public Map<String, Double> foo(Map<String, Double> input, int topN)
throws IOException {
// Implement your logic here
// Map<String, Double> ret = new HashMap<String, Double>();
// ret.put("foo", 1.0);
return null;
}
}
==================================================================
package com.serendio.hbase.coprocessor;

import java.io.IOException;
import java.util.Map;

import org.apache.hadoop.hbase.ipc.CoprocessorProtocol;

public interface SearchProtocol extends CoprocessorProtocol {

public Map<String, Double> foo(Map<String, Double> input,
int topN) throws IOException;
}

=========================================================================package com.serendio.hbase.coprocessor;

import java.io.IOException;
import java.util.HashMap;
import java.util.Map;

import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.coprocessor.Batch.Call;

public class Search {
public static final String TEST_TABLE = "c4";
public static final String TEST_FAMILY = "info";
public static final String TEST_QUALIFIER = "c1";

/**
 * ``
 * 
 * @param args
 */
public static void main(String[] args) {

org.apache.hadoop.conf.Configuration conf = HBaseConfiguration.create();
HTableInterface table = null;
try {
table = new HTable(conf, TEST_TABLE);

} catch (IOException e2) {
// TODO Auto-generated catch block
e2.printStackTrace();
}

Map<byte[], Map<String, Double>> results = null;
final Map<String, Double> input = new HashMap<String, Double>();
input.put("test", 1.0);
final int topN = 10;
long start = System.currentTimeMillis();
try {
results = table.coprocessorExec(SearchProtocol.class, null, null,
new Call<SearchProtocol, Map<String, Double>>() {

@Override
public Map<String, Double> call(SearchProtocol instance)
throws IOException {
// TODO Auto-generated method stub
return instance.foo(input, topN);
}
});
} catch (IOException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
} catch (Throwable e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
}
long end = System.currentTimeMillis();
System.out.println("Total search elapsed time: "
+ Long.toString(end - start));
}

}
 
Regards,
- kiru