Extract clusters and assign N cluster IDs to each article

create or replace view news20_cluster
as
select 
  minhash(rowid, features) as (clusterId, rowid)
from 
  news20mc_train;

create table news20_with_clusterid
as
select 
  t1.clusterid, 
  t1.rowid, 
  o1.features
from 
  news20_cluster t1
  JOIN news20mc_train o1 ON (t1.rowid = o1.rowid);

Query expression with cluster id

set hivevar:noWeight=false;

create table extract_target_cluster
as
select 
  features,
  clusterid
from (
  select 
     features,
     minhashes(features,${noWeight}) as clusters
  from 
     news20mc_test 
  where 
     rowid = 1
) t1
LATERAL VIEW explode(clusters) t2 AS clusterid;

kNN search using minhashing

set hivevar:topn=10;

select 
  t1.rowid, 
  cosine_similarity(t1.features, q1.features, false) as similarity
from
  news20_with_clusterid t1
  JOIN extract_target_cluster q1 ON (t1.clusterid = q1.clusterid)
order by
  similarity DESC
limit ${topn};

Time taken: 22.161 seconds

Brute force kNN search (based on cosine similarity)

select
  t1.rowid,
  cosine_similarity(t1.features, q1.features) as similarity -- hive v0.3.2 or later
  -- cosine_similarity(t1.features, q1.features, false) as similarity -- hive v0.3.1 or before
from 
  news20mc_train t1
  CROSS JOIN
  (select features from news20mc_test where rowid = 1) q1
ORDER BY
  similarity DESC
limit ${topn};

Time taken: 24.335 seconds

Refer this page for efficient top-k kNN computation.