PA1a

Training

set mapred.reduce.tasks=64;

drop table e2006tfidf_pa1a_model ;
create table e2006tfidf_pa1a_model as
select 
 feature,
 avg(weight) as weight
from 
 (select 
     train_pa1a_regr(add_bias(features),target) as (feature,weight)
  from 
     e2006tfidf_train_x3
 ) t 
group by feature;

-- reset to the default setting
set mapred.reduce.tasks=-1;

prediction

create or replace view e2006tfidf_pa1a_predict
as
select
  t.rowid, 
  sum(m.weight * t.value) as predicted
from 
  e2006tfidf_test_exploded t LEFT OUTER JOIN
  e2006tfidf_pa1a_model m ON (t.feature = m.feature)
group by
  t.rowid;

evaluation

WITH submit as (
  select 
    t.target as actual, 
    p.predicted as predicted
  from 
    e2006tfidf_test t
    JOIN e2006tfidf_pa1a_predict p 
      on (t.rowid = p.rowid)
)
select 
   rmse(predicted, actual) as RMSE,
   mse(predicted, actual) as MSE, 
   mae(predicted, actual) as MAE,
   r2(predicted, actual) as R2
from 
   submit;

PA2a

Training

set mapred.reduce.tasks=64;
drop table e2006tfidf_pa2a_model;
create table e2006tfidf_pa2a_model as
select 
 feature,
 avg(weight) as weight
from 
 (select 
     train_pa2a_regr(add_bias(features),target) as (feature,weight)
  from 
     e2006tfidf_train_x3
 ) t 
group by feature;
set mapred.reduce.tasks=-1;

prediction

create or replace view e2006tfidf_pa2a_predict
as
select
  t.rowid, 
  sum(m.weight * t.value) as predicted
from 
  e2006tfidf_test_exploded t LEFT OUTER JOIN
  e2006tfidf_pa2a_model m ON (t.feature = m.feature)
group by
  t.rowid;

evaluation

WITH submit as (
  select 
    t.target as actual, 
    p.predicted as predicted
  from 
    e2006tfidf_test t
    JOIN e2006tfidf_pa2a_predict p 
      on (t.rowid = p.rowid)
)
select 
   rmse(predicted, actual) as RMSE,
   mse(predicted, actual) as MSE, 
   mae(predicted, actual) as MAE,
   r2(predicted, actual) as R2
from 
   submit;

AROW

Training

set mapred.reduce.tasks=64;
drop table e2006tfidf_arow_model ;
create table e2006tfidf_arow_model as
select 
 feature,
 -- avg(weight) as weight -- [hivemall v0.1]
 argmin_kld(weight, covar) as weight -- [hivemall v0.2 or later]
from 
 (select 
     train_arow_regr(add_bias(features),target) as (feature,weight,covar)
  from 
     e2006tfidf_train_x3
 ) t 
group by feature;
set mapred.reduce.tasks=-1;

prediction

create or replace view e2006tfidf_arow_predict
as
select
  t.rowid, 
  sum(m.weight * t.value) as predicted
from 
  e2006tfidf_test_exploded t LEFT OUTER JOIN
  e2006tfidf_arow_model m ON (t.feature = m.feature)
group by
  t.rowid;

evaluation

WITH submit as (
  select 
    t.target as actual, 
    p.predicted as predicted
  from 
    e2006tfidf_test t
    JOIN e2006tfidf_arow_predict p 
      on (t.rowid = p.rowid)
)
select 
   rmse(predicted, actual) as RMSE,
   mse(predicted, actual) as MSE, 
   mae(predicted, actual) as MAE,
   r2(predicted, actual) as R2
from 
   submit;

AROWe

AROWe is a modified version of AROW that uses Hinge loss (epsilion = 0.1)

Training

-- set mapred.reduce.tasks=64;

drop table e2006tfidf_arowe_model ;
create table e2006tfidf_arowe_model as
select 
 feature,
 argmin_kld(weight, covar) as weight 
from 
 (select 
     train_arowe_regr(add_bias(features),target) as (feature,weight,covar)
  from 
     e2006tfidf_train_x3
 ) t 
group by feature;
set mapred.reduce.tasks=-1;

prediction

create or replace view e2006tfidf_arowe_predict
as
select
  t.rowid, 
  sum(m.weight * t.value) as predicted
from 
  e2006tfidf_test_exploded t LEFT OUTER JOIN
  e2006tfidf_arowe_model m ON (t.feature = m.feature)
group by
  t.rowid;

evaluation

WITH submit as (
  select 
    t.target as actual, 
    p.predicted as predicted
  from 
    e2006tfidf_test t
    JOIN e2006tfidf_arowe_predict p 
      on (t.rowid = p.rowid)
)
select 
   rmse(predicted, actual) as RMSE,
   mse(predicted, actual) as MSE, 
   mae(predicted, actual) as MAE,
   r2(predicted, actual) as R2
from 
   submit;