kudu 虚拟机环境使用

 

安装

curl -s https://raw.githubusercontent.com/cloudera/kudu-examples/master/demo-vm-setup/bootstrap.sh | bash

备注： 会比较慢，可以使用其他的下载工具，提前下载放到目录即可，同时默认会自动启动服务

基本使用

a. 链接

ssh demo@quickstart.cloudera

b. 查看服务

$ ps aux | grep kudu
$ ps aux | grep impalad

c. 加载数据

wget http://kudu-sample-data.s3.amazonaws.com/sfmtaAVLRawData01012013.csv.gz

hdfs dfs -mkdir /sfmta

zcat sfmtaAVLRawData01012013.csv.gz | tr -d '\r' | hadoop fs -put - /sfmta/data.csv


d. 使用impala 进行数据处理

ssh demo@quickstart.cloudera -t impala-shell

CREATE EXTERNAL TABLE sfmta_raw (
  revision int,
  report_time string,
  vehicle_tag int,
  longitude float,
  latitude float,
  speed float,
  heading float
)
ROW FORMAT DELIMITED
FIELDS TERMINATED BY ','
LOCATION '/sfmta/'
TBLPROPERTIES ('skip.header.line.count'='1');


SELECT count(*) FROM sfmta_raw;

+----------+
| count(*) |
+----------+
| 859086 |
+----------+

CREATE TABLE sfmta
PRIMARY KEY (report_time, vehicle_tag)
PARTITION BY HASH(report_time) PARTITIONS 8
STORED AS KUDU
AS SELECT
  UNIX_TIMESTAMP(report_time, 'MM/dd/yyyy HH:mm:ss') AS report_time,
  vehicle_tag,
  longitude,
  latitude,
  speed,
  heading
FROM sfmta_raw;

SELECT * FROM sfmta ORDER BY speed DESC LIMIT 1;

参考资料

http://kudu.apache.org/docs/quickstart.html