Home | About | Sematext search-lucene.com search-hadoop.com
NEW: Monitor These Apps!
elasticsearch, apache solr, apache hbase, hadoop, redis, casssandra, amazon cloudwatch, mysql, memcached, apache kafka, apache zookeeper, apache storm, ubuntu, centOS, red hat, debian, puppet labs, java, senseiDB
 Search Hadoop and all its subprojects:

Switch to Threaded View
Hive >> mail # user >> Textfile compression using Gzip codec


Copy link to this message
-
Re: Textfile compression using Gzip codec
Hi Stephen,

*hive> show create table facts520_normal_text;*
*OK*
*CREATE  TABLE facts520_normal_text(*
*  fact_key bigint,*
*  products_key int,*
*  retailers_key int,*
*  suppliers_key int,*
*  time_key int,*
*  units int)*
*ROW FORMAT DELIMITED*
*  FIELDS TERMINATED BY ','*
*  LINES TERMINATED BY '\n'*
*STORED AS INPUTFORMAT*
*  'org.apache.hadoop.mapred.TextInputFormat'*
*OUTPUTFORMAT*
*  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'*
*LOCATION*
*  'hdfs://
aana1.ird.com/user/hive/warehouse/facts_520.db/facts520_normal_text'*
*TBLPROPERTIES (*
*  'numPartitions'='0',*
*  'numFiles'='1',*
*  'transient_lastDdlTime'='1369395430',*
*  'numRows'='0',*
*  'totalSize'='545216508',*
*  'rawDataSize'='0')*
*Time taken: 0.353 seconds*
The syserror log shows this:

*java.lang.IllegalArgumentException: Compression codec
org.apache.hadoop.io.compress.GZipCodec was not found.*
* at
org.apache.hadoop.mapred.FileOutputFormat.getOutputCompressorClass(FileOutputFormat.java:85)
*
* at
org.apache.hadoop.hive.ql.exec.Utilities.getFileExtension(Utilities.java:934)
*
* at
org.apache.hadoop.hive.ql.exec.FileSinkOperator.createBucketFiles(FileSinkOperator.java:469)
*
* at
org.apache.hadoop.hive.ql.exec.FileSinkOperator.processOp(FileSinkOperator.java:543)
*
* at org.apache.hadoop.hive.ql.exec.Operator.process(Operator.java:474)*
* at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:800)*
* at
org.apache.hadoop.hive.ql.exec.SelectOperator.processOp(SelectOperator.java:84)
*
* at org.apache.hadoop.hive.ql.exec.Operator.process(Operator.java:474)*
* at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:800)*
* at
org.apache.hadoop.hive.ql.exec.TableScanOperator.processOp(TableScanOperator.java:83)
*
* at org.apache.hadoop.hive.ql.exec.Operator.process(Operator.java:474)*
* at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:800)*
* at
org.apache.hadoop.hive.ql.exec.MapOperator.process(MapOperator.java:546)*
* at org.apache.hadoop.hive.ql.exec.ExecMapper.map(ExecMapper.java:143)*
* at org.apache.hadoop.mapred.MapRunner.run(MapRunner.java:50)*
* at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:418)*
* at org.apache.hadoop.mapred.MapTask.run(MapTask.java:333)*
* at org.apache.hadoop.mapred.Child$4.run(Child.java:268)*
* at java.security.AccessController.doPrivileged(Native Method)*
* at javax.security.auth.Subject.doAs(Subject.java:415)*
* at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1408)
*
* at org.apache.hadoop.mapred.Child.main(Child.java:262)*
*Caused by: java.lang.ClassNotFoundException: Class
org.apache.hadoop.io.compress.GZipCodec not found*
* at
org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:1493)
*
* at
org.apache.hadoop.mapred.FileOutputFormat.getOutputCompressorClass(FileOutputFormat.java:82)
*
* ... 21 more*
*java.lang.IllegalArgumentException: Compression codec
org.apache.hadoop.io.compress.GZipCodec was not found.*
* at
org.apache.hadoop.mapred.FileOutputFormat.getOutputCompressorClass(FileOutputFormat.java:85)
*
* at
org.apache.hadoop.hive.ql.exec.Utilities.getFileExtension(Utilities.java:934)
*
* at
org.apache.hadoop.hive.ql.exec.FileSinkOperator.createBucketFiles(FileSinkOperator.java:469)
*
* at
org.apache.hadoop.hive.ql.exec.FileSinkOperator.closeOp(FileSinkOperator.java:739)
*
* at org.apache.hadoop.hive.ql.exec.Operator.close(Operator.java:558)*
* at org.apache.hadoop.hive.ql.exec.Operator.close(Operator.java:567)*
* at org.apache.hadoop.hive.ql.exec.Operator.close(Operator.java:567)*
* at org.apache.hadoop.hive.ql.exec.Operator.close(Operator.java:567)*
* at org.apache.hadoop.hive.ql.exec.ExecMapper.close(ExecMapper.java:193)*
* at org.apache.hadoop.mapred.MapRunner.run(MapRunner.java:57)*
* at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:418)*
* at org.apache.hadoop.mapred.MapTask.run(MapTask.java:333)*
* at org.apache.hadoop.mapred.Child$4.run(Child.java:268)*
* at java.security.AccessController.doPrivileged(Native Method)*
* at javax.security.auth.Subject.doAs(Subject.java:415)*
* at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1408)
*
* at org.apache.hadoop.mapred.Child.main(Child.java:262)*
*Caused by: java.lang.ClassNotFoundException: Class
org.apache.hadoop.io.compress.GZipCodec not found*
* at
org.apache.hadoop.conf.Configuration.getClassByName(Configuration.java:1493)
*
* at
org.apache.hadoop.mapred.FileOutputFormat.getOutputCompressorClass(FileOutputFormat.java:82)
*
* ... 16 more*
*org.apache.hadoop.hive.ql.metadata.HiveException:
java.lang.IllegalArgumentException: Compression codec
org.apache.hadoop.io.compress.GZipCodec was not found.*
* at
org.apache.hadoop.hive.ql.exec.FileSinkOperator.createBucketFiles(FileSinkOperator.java:479)
*
* at
org.apache.hadoop.hive.ql.exec.FileSinkOperator.closeOp(FileSinkOperator.java:739)
*
* at org.apache.hadoop.hive.ql.exec.Operator.close(Operator.java:558)*
* at org.apache.hadoop.hive.ql.exec.Operator.close(Operator.java:567)*
* at org.apache.hadoop.hive.ql.exec.Operator.close(Operator.java:567)*
* at org.apache.hadoop.hive.ql.exec.Operator.close(Operator.java:567)*
* at org.apache.hadoop.hive.ql.exec.ExecMapper.close(ExecMapper.java:193)*
* at org.apache.hadoop.mapred.MapRunner.run(MapRunner.java:57)*
* at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:418)*
* at org.apache.hadoop.mapred.MapTask.run(MapTask.java:333)*
* at org.apache.hadoop.mapred.Child$4.run(Child.java:268)*
* at java.security.AccessController.doPrivileged(Native Method)*
* at javax.security.auth.Subject.doAs(Subject.java:415)*
* at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1408)
*
* at org.apache.hadoop.mapred.Child.main(Child.java:262)*
*Caused by: java.lang.IllegalArgumentException: Compression codec
org.apache.hadoop.io.compress.GZipCodec was not found.*
* at
org.apache.hadoop.mapred.FileOutputFormat.getOutputCompressorClass(FileOutputFormat.java:85
NEW: Monitor These Apps!
elasticsearch, apache solr, apache hbase, hadoop, redis, casssandra, amazon cloudwatch, mysql, memcached, apache kafka, apache zookeeper, apache storm, ubuntu, centOS, red hat, debian, puppet labs, java, senseiDB