Home | About | Sematext search-lucene.com search-hadoop.com
 Search Hadoop and all its subprojects:

Switch to Threaded View
Hadoop >> mail # user >> Writing Reducer output to database


Copy link to this message
-
Re: Writing Reducer output to database
I think you have looked at
src/examples/org/apache/hadoop/examples/DBCountPageView.java
where:
    job.setMapOutputKeyClass(Text.class);

and:
public class Text extends BinaryComparable
    implements WritableComparable<BinaryComparable> {
...
  static {
    // register this comparator
    WritableComparator.define(Text.class, new Comparator());
  }

Modify your LogRecord accordingly.

On Fri, Feb 4, 2011 at 12:00 PM, Adeel Qureshi <[EMAIL PROTECTED]>wrote:

> Thanks - I switched to using the mapreduce version of dboutputformat and
> things look a little better but I am getting a ClassCastException ..
>
> here is my writable class
> public class LogRecord implements Writable, DBWritable {
>    private long timestamp;
>    private String userId;
>    private String action;
>
>    public LogRecord() {
>    }
>
>    public LogRecord(long timestamp, String userId, String action,
>            String pageType, String pageName, String attrPath, String
> attrName,
>            String forEntity, String forEntityInfo, long rendTime) {
>        this.timestamp = timestamp;
>        this.userId = userId;
>        this.action = action;
>    }
>
>    public void clearFields(){
>        this.timestamp = 0;
>        this.userId = "";
>        this.action = "";
>    }
>
>    @Override
>    public int hashCode() {
>        final int prime = 31;
>        int result = 1;
>        result = prime * result + ((action == null) ? 0 :
> action.hashCode());
>        result = prime * result + (int) (timestamp ^ (timestamp >>> 32));
>        result = prime * result + ((userId == null) ? 0 :
> userId.hashCode());
>        return result;
>    }
>
>    @Override
>    public boolean equals(Object obj) {
>        if (this == obj)
>            return true;
>        if (obj == null)
>            return false;
>        if (getClass() != obj.getClass())
>            return false;
>        LogRecord other = (LogRecord) obj;
>        if (action == null) {
>            if (other.action != null)
>                return false;
>        } else if (!action.equals(other.action))
>            return false;
>        if (timestamp != other.timestamp)
>            return false;
>        if (userId == null) {
>            if (other.userId != null)
>                return false;
>        } else if (!userId.equals(other.userId))
>            return false;
>        return true;
>    }
>
>    @Override
>    public void readFields(DataInput in) throws IOException {
>        this.timestamp = in.readLong();
>        this.userId = Text.readString(in);
>        this.action = Text.readString(in);
>    }
>
>    @Override
>    public void write(DataOutput out) throws IOException {
>        out.writeLong(this.timestamp);
>        Text.writeString(out, this.userId);
>        Text.writeString(out, this.action);
>    }
>
>    @Override
>    public void readFields(ResultSet rs) throws SQLException {
>        this.timestamp = rs.getLong(1);
>        this.userId = rs.getString(2);
>        this.action = rs.getString(3);
>    }
>
>    @Override
>    public void write(PreparedStatement stmt) throws SQLException {
>        stmt.setLong(1, this.timestamp);
>        stmt.setString(2, this.userId);
>        stmt.setString(3, this.action);
>    }
>
>    public void setTimestamp(long timestamp) {
>        this.timestamp = timestamp;
>    }
>    public void setUserId(String userId) {
>        this.userId = userId;
>    }
>    public void setAction(String action) {
>        this.action = action;
>    }
> }
> **************************************
>
> here is my job runner/configuration code
>
> //configuration
>        Configuration conf = new Configuration();
>        Job job = new Job(conf, "Log Parser Job");
>
>        //configure database output
>        job.setOutputFormatClass(DBOutputFormat.class);
>        DBConfiguration.configureDB(conf,
> "com.microsoft.sqlserver.jdbc.SQLServerDriver",
>                    "jdbc:sqlserver://..........",
>                    "...", "...");
>        String[] fields = {"timestamp", "userId", "action"};