Home | About | Sematext search-lucene.com search-hadoop.com
NEW: Monitor These Apps!
elasticsearch, apache solr, apache hbase, hadoop, redis, casssandra, amazon cloudwatch, mysql, memcached, apache kafka, apache zookeeper, apache storm, ubuntu, centOS, red hat, debian, puppet labs, java, senseiDB
 Search Hadoop and all its subprojects:

Switch to Plain View
Hadoop >> mail # user >> Writing Reducer output to database


+
Adeel Qureshi 2011-02-03, 22:45
+
Ted Yu 2011-02-04, 03:29
+
Adeel Qureshi 2011-02-04, 20:00
+
Adeel Qureshi 2011-02-06, 16:12
Copy link to this message
-
Re: Writing Reducer output to database
I think you have looked at
src/examples/org/apache/hadoop/examples/DBCountPageView.java
where:
    job.setMapOutputKeyClass(Text.class);

and:
public class Text extends BinaryComparable
    implements WritableComparable<BinaryComparable> {
...
  static {
    // register this comparator
    WritableComparator.define(Text.class, new Comparator());
  }

Modify your LogRecord accordingly.

On Fri, Feb 4, 2011 at 12:00 PM, Adeel Qureshi <[EMAIL PROTECTED]>wrote:

> Thanks - I switched to using the mapreduce version of dboutputformat and
> things look a little better but I am getting a ClassCastException ..
>
> here is my writable class
> public class LogRecord implements Writable, DBWritable {
>    private long timestamp;
>    private String userId;
>    private String action;
>
>    public LogRecord() {
>    }
>
>    public LogRecord(long timestamp, String userId, String action,
>            String pageType, String pageName, String attrPath, String
> attrName,
>            String forEntity, String forEntityInfo, long rendTime) {
>        this.timestamp = timestamp;
>        this.userId = userId;
>        this.action = action;
>    }
>
>    public void clearFields(){
>        this.timestamp = 0;
>        this.userId = "";
>        this.action = "";
>    }
>
>    @Override
>    public int hashCode() {
>        final int prime = 31;
>        int result = 1;
>        result = prime * result + ((action == null) ? 0 :
> action.hashCode());
>        result = prime * result + (int) (timestamp ^ (timestamp >>> 32));
>        result = prime * result + ((userId == null) ? 0 :
> userId.hashCode());
>        return result;
>    }
>
>    @Override
>    public boolean equals(Object obj) {
>        if (this == obj)
>            return true;
>        if (obj == null)
>            return false;
>        if (getClass() != obj.getClass())
>            return false;
>        LogRecord other = (LogRecord) obj;
>        if (action == null) {
>            if (other.action != null)
>                return false;
>        } else if (!action.equals(other.action))
>            return false;
>        if (timestamp != other.timestamp)
>            return false;
>        if (userId == null) {
>            if (other.userId != null)
>                return false;
>        } else if (!userId.equals(other.userId))
>            return false;
>        return true;
>    }
>
>    @Override
>    public void readFields(DataInput in) throws IOException {
>        this.timestamp = in.readLong();
>        this.userId = Text.readString(in);
>        this.action = Text.readString(in);
>    }
>
>    @Override
>    public void write(DataOutput out) throws IOException {
>        out.writeLong(this.timestamp);
>        Text.writeString(out, this.userId);
>        Text.writeString(out, this.action);
>    }
>
>    @Override
>    public void readFields(ResultSet rs) throws SQLException {
>        this.timestamp = rs.getLong(1);
>        this.userId = rs.getString(2);
>        this.action = rs.getString(3);
>    }
>
>    @Override
>    public void write(PreparedStatement stmt) throws SQLException {
>        stmt.setLong(1, this.timestamp);
>        stmt.setString(2, this.userId);
>        stmt.setString(3, this.action);
>    }
>
>    public void setTimestamp(long timestamp) {
>        this.timestamp = timestamp;
>    }
>    public void setUserId(String userId) {
>        this.userId = userId;
>    }
>    public void setAction(String action) {
>        this.action = action;
>    }
> }
> **************************************
>
> here is my job runner/configuration code
>
> //configuration
>        Configuration conf = new Configuration();
>        Job job = new Job(conf, "Log Parser Job");
>
>        //configure database output
>        job.setOutputFormatClass(DBOutputFormat.class);
>        DBConfiguration.configureDB(conf,
> "com.microsoft.sqlserver.jdbc.SQLServerDriver",
>                    "jdbc:sqlserver://..........",
>                    "...", "...");
>        String[] fields = {"timestamp", "userId", "action"};
+
Adeel Qureshi 2011-02-06, 23:28
+
Ted Yu 2011-02-07, 04:55
+
Adeel Qureshi 2011-02-07, 14:16
+
adeelmahmood 2011-02-04, 00:34
NEW: Monitor These Apps!
elasticsearch, apache solr, apache hbase, hadoop, redis, casssandra, amazon cloudwatch, mysql, memcached, apache kafka, apache zookeeper, apache storm, ubuntu, centOS, red hat, debian, puppet labs, java, senseiDB