Home | About | Sematext search-lucene.com search-hadoop.com
 Search Hadoop and all its subprojects:

Switch to Threaded View
Hadoop, mail # user - Problems with block compression using native codecs (Snappy, LZO) and MapFile.Reader.get()


Copy link to this message
-
Re: Problems with block compression using native codecs (Snappy, LZO) and MapFile.Reader.get()
Edward Capriolo 2012-05-22, 15:59
if You are getting a SIGSEG it never hurts to try a more recent JVM.
21 has many bug fixes at this point.

On Tue, May 22, 2012 at 11:45 AM, Jason B <[EMAIL PROTECTED]> wrote:
> JIRA entry created:
>
> https://issues.apache.org/jira/browse/HADOOP-8423
>
>
> On 5/21/12, Jason B <[EMAIL PROTECTED]> wrote:
>> Sorry about using attachment. The code is below for the reference.
>> (I will also file a jira as you suggesting)
>>
>> package codectest;
>>
>> import com.hadoop.compression.lzo.LzoCodec;
>> import java.io.IOException;
>> import java.util.Formatter;
>> import org.apache.hadoop.conf.Configuration;
>> import org.apache.hadoop.fs.FileSystem;
>> import org.apache.hadoop.io.MapFile;
>> import org.apache.hadoop.io.SequenceFile.CompressionType;
>> import org.apache.hadoop.io.Text;
>> import org.apache.hadoop.io.compress.CompressionCodec;
>> import org.apache.hadoop.io.compress.DefaultCodec;
>> import org.apache.hadoop.io.compress.SnappyCodec;
>> import org.apache.hadoop.util.Tool;
>> import org.apache.hadoop.util.ToolRunner;
>>
>> public class MapFileCodecTest implements Tool {
>>     private Configuration conf = new Configuration();
>>
>>     private void createMapFile(Configuration conf, FileSystem fs, String
>> path,
>>             CompressionCodec codec, CompressionType type, int records)
>> throws IOException {
>>         MapFile.Writer writer = new MapFile.Writer(conf, fs, path,
>> Text.class, Text.class,
>>                 type, codec, null);
>>         Text key = new Text();
>>         for (int j = 0; j < records; j++) {
>>             StringBuilder sb = new StringBuilder();
>>             Formatter formatter = new Formatter(sb);
>>             formatter.format("%03d", j);
>>             key.set(sb.toString());
>>             writer.append(key, key);
>>         }
>>         writer.close();
>>     }
>>
>>     private void testCodec(Configuration conf, Class<? extends
>> CompressionCodec> clazz,
>>             CompressionType type, int records) throws IOException {
>>         FileSystem fs = FileSystem.getLocal(conf);
>>         try {
>>             System.out.println("Creating MapFiles with " + records  +
>>                     " records using codec " + clazz.getSimpleName());
>>             String path = clazz.getSimpleName() + records;
>>             createMapFile(conf, fs, path, clazz.newInstance(), type,
>> records);
>>             MapFile.Reader reader = new MapFile.Reader(fs, path, conf);
>>             Text key1 = new Text("002");
>>             if (reader.get(key1, new Text()) != null) {
>>                 System.out.println("1st key found");
>>             }
>>             Text key2 = new Text("004");
>>             if (reader.get(key2, new Text()) != null) {
>>                 System.out.println("2nd key found");
>>             }
>>         } catch (Throwable ex) {
>>             ex.printStackTrace();
>>         }
>>     }
>>
>>     @Override
>>     public int run(String[] strings) throws Exception {
>>         System.out.println("Using native library " +
>> System.getProperty("java.library.path"));
>>
>>         testCodec(conf, DefaultCodec.class, CompressionType.RECORD, 100);
>>         testCodec(conf, SnappyCodec.class, CompressionType.RECORD, 100);
>>         testCodec(conf, LzoCodec.class, CompressionType.RECORD, 100);
>>
>>         testCodec(conf, DefaultCodec.class, CompressionType.RECORD, 10);
>>         testCodec(conf, SnappyCodec.class, CompressionType.RECORD, 10);
>>         testCodec(conf, LzoCodec.class, CompressionType.RECORD, 10);
>>
>>         testCodec(conf, DefaultCodec.class, CompressionType.BLOCK, 100);
>>         testCodec(conf, SnappyCodec.class, CompressionType.BLOCK, 100);
>>         testCodec(conf, LzoCodec.class, CompressionType.BLOCK, 100);
>>
>>         testCodec(conf, DefaultCodec.class, CompressionType.BLOCK, 10);
>>         testCodec(conf, SnappyCodec.class, CompressionType.BLOCK, 10);
>>         testCodec(conf, LzoCodec.class, CompressionType.BLOCK, 10);
>>         return 0;
>>     }