Home | About | Sematext search-lucene.com search-hadoop.com
NEW: Monitor These Apps!
elasticsearch, apache solr, apache hbase, hadoop, redis, casssandra, amazon cloudwatch, mysql, memcached, apache kafka, apache zookeeper, apache storm, ubuntu, centOS, red hat, debian, puppet labs, java, senseiDB
 Search Hadoop and all its subprojects:

Switch to Threaded View
Hadoop >> mail # user >> Problems with block compression using native codecs (Snappy, LZO) and MapFile.Reader.get()


Copy link to this message
-
Re: Problems with block compression using native codecs (Snappy, LZO) and MapFile.Reader.get()
if You are getting a SIGSEG it never hurts to try a more recent JVM.
21 has many bug fixes at this point.

On Tue, May 22, 2012 at 11:45 AM, Jason B <[EMAIL PROTECTED]> wrote:
> JIRA entry created:
>
> https://issues.apache.org/jira/browse/HADOOP-8423
>
>
> On 5/21/12, Jason B <[EMAIL PROTECTED]> wrote:
>> Sorry about using attachment. The code is below for the reference.
>> (I will also file a jira as you suggesting)
>>
>> package codectest;
>>
>> import com.hadoop.compression.lzo.LzoCodec;
>> import java.io.IOException;
>> import java.util.Formatter;
>> import org.apache.hadoop.conf.Configuration;
>> import org.apache.hadoop.fs.FileSystem;
>> import org.apache.hadoop.io.MapFile;
>> import org.apache.hadoop.io.SequenceFile.CompressionType;
>> import org.apache.hadoop.io.Text;
>> import org.apache.hadoop.io.compress.CompressionCodec;
>> import org.apache.hadoop.io.compress.DefaultCodec;
>> import org.apache.hadoop.io.compress.SnappyCodec;
>> import org.apache.hadoop.util.Tool;
>> import org.apache.hadoop.util.ToolRunner;
>>
>> public class MapFileCodecTest implements Tool {
>>     private Configuration conf = new Configuration();
>>
>>     private void createMapFile(Configuration conf, FileSystem fs, String
>> path,
>>             CompressionCodec codec, CompressionType type, int records)
>> throws IOException {
>>         MapFile.Writer writer = new MapFile.Writer(conf, fs, path,
>> Text.class, Text.class,
>>                 type, codec, null);
>>         Text key = new Text();
>>         for (int j = 0; j < records; j++) {
>>             StringBuilder sb = new StringBuilder();
>>             Formatter formatter = new Formatter(sb);
>>             formatter.format("%03d", j);
>>             key.set(sb.toString());
>>             writer.append(key, key);
>>         }
>>         writer.close();
>>     }
>>
>>     private void testCodec(Configuration conf, Class<? extends
>> CompressionCodec> clazz,
>>             CompressionType type, int records) throws IOException {
>>         FileSystem fs = FileSystem.getLocal(conf);
>>         try {
>>             System.out.println("Creating MapFiles with " + records  +
>>                     " records using codec " + clazz.getSimpleName());
>>             String path = clazz.getSimpleName() + records;
>>             createMapFile(conf, fs, path, clazz.newInstance(), type,
>> records);
>>             MapFile.Reader reader = new MapFile.Reader(fs, path, conf);
>>             Text key1 = new Text("002");
>>             if (reader.get(key1, new Text()) != null) {
>>                 System.out.println("1st key found");
>>             }
>>             Text key2 = new Text("004");
>>             if (reader.get(key2, new Text()) != null) {
>>                 System.out.println("2nd key found");
>>             }
>>         } catch (Throwable ex) {
>>             ex.printStackTrace();
>>         }
>>     }
>>
>>     @Override
>>     public int run(String[] strings) throws Exception {
>>         System.out.println("Using native library " +
>> System.getProperty("java.library.path"));
>>
>>         testCodec(conf, DefaultCodec.class, CompressionType.RECORD, 100);
>>         testCodec(conf, SnappyCodec.class, CompressionType.RECORD, 100);
>>         testCodec(conf, LzoCodec.class, CompressionType.RECORD, 100);
>>
>>         testCodec(conf, DefaultCodec.class, CompressionType.RECORD, 10);
>>         testCodec(conf, SnappyCodec.class, CompressionType.RECORD, 10);
>>         testCodec(conf, LzoCodec.class, CompressionType.RECORD, 10);
>>
>>         testCodec(conf, DefaultCodec.class, CompressionType.BLOCK, 100);
>>         testCodec(conf, SnappyCodec.class, CompressionType.BLOCK, 100);
>>         testCodec(conf, LzoCodec.class, CompressionType.BLOCK, 100);
>>
>>         testCodec(conf, DefaultCodec.class, CompressionType.BLOCK, 10);
>>         testCodec(conf, SnappyCodec.class, CompressionType.BLOCK, 10);
>>         testCodec(conf, LzoCodec.class, CompressionType.BLOCK, 10);
>>         return 0;
>>     }
NEW: Monitor These Apps!
elasticsearch, apache solr, apache hbase, hadoop, redis, casssandra, amazon cloudwatch, mysql, memcached, apache kafka, apache zookeeper, apache storm, ubuntu, centOS, red hat, debian, puppet labs, java, senseiDB