E-MapReduce HadoopHive 开发手册是什么?

在 Hive 中使用 OSS




要在 Hive 中读写 OSS，需要在使用 OSS 的 URI 时加上 AccessKeyId、AccessKeySecret 以及 endpoint。如下示例介绍了如何创建一个 external 的表：

1. [backcolor=transparent]CREATE EXTERNAL TABLE eusers [backcolor=transparent](
2. [backcolor=transparent]  userid INT[backcolor=transparent])[backcolor=transparent]
3. [backcolor=transparent] LOCATION [backcolor=transparent]'oss://emr/users'[backcolor=transparent];

1. [backcolor=transparent]CREATE EXTERNAL TABLE eusers [backcolor=transparent](
2. [backcolor=transparent]  userid INT[backcolor=transparent])[backcolor=transparent]
3. [backcolor=transparent] LOCATION [backcolor=transparent]'oss://${AccessKeyId}:${AccessKeySecret}@${bucket}.${endpoint}/users'[backcolor=transparent];

1. [backcolor=transparent]set[backcolor=transparent] hive[backcolor=transparent].[backcolor=transparent]execution[backcolor=transparent].[backcolor=transparent]engine[backcolor=transparent]=[backcolor=transparent]tez

1. 
   编写如下脚本，保存为 hiveSample1.sql，并上传到 OSS 上。[backcolor=transparent] USE DEFAULT[backcolor=transparent];
2. [backcolor=transparent] [backcolor=transparent]set[backcolor=transparent] hive[backcolor=transparent].[backcolor=transparent]input[backcolor=transparent].[backcolor=transparent]format[backcolor=transparent]=[backcolor=transparent]org[backcolor=transparent].[backcolor=transparent]apache[backcolor=transparent].[backcolor=transparent]hadoop[backcolor=transparent].[backcolor=transparent]hive[backcolor=transparent].[backcolor=transparent]ql[backcolor=transparent].[backcolor=transparent]io[backcolor=transparent].[backcolor=transparent]HiveInputFormat[backcolor=transparent];
3. [backcolor=transparent] [backcolor=transparent]set[backcolor=transparent] hive[backcolor=transparent].[backcolor=transparent]stats[backcolor=transparent].[backcolor=transparent]autogather[backcolor=transparent]=[backcolor=transparent]false[backcolor=transparent];
5. [backcolor=transparent] DROP TABLE emrusers[backcolor=transparent];
6. [backcolor=transparent] CREATE EXTERNAL TABLE emrusers [backcolor=transparent](
7. [backcolor=transparent]   userid INT[backcolor=transparent],
8. [backcolor=transparent]   movieid INT[backcolor=transparent],
9. [backcolor=transparent]   rating INT[backcolor=transparent],
10. [backcolor=transparent]   unixtime STRING [backcolor=transparent])[backcolor=transparent]
11. [backcolor=transparent]  ROW FORMAT DELIMITED
12. [backcolor=transparent]  FIELDS TERMINATED BY [backcolor=transparent]'\t'[backcolor=transparent]
13. [backcolor=transparent]  STORED AS TEXTFILE
14. [backcolor=transparent]  LOCATION [backcolor=transparent]'oss://${AccessKeyId}:${AccessKeySecret}@${bucket}.${endpoint}/yourpath'[backcolor=transparent];
16. [backcolor=transparent] SELECT COUNT[backcolor=transparent](*)[backcolor=transparent] FROM emrusers[backcolor=transparent];
18. [backcolor=transparent] SELECT [backcolor=transparent]*[backcolor=transparent] [backcolor=transparent]from[backcolor=transparent] emrusers limit [backcolor=transparent]100[backcolor=transparent];
20. [backcolor=transparent] SELECT movieid[backcolor=transparent],[backcolor=transparent]count[backcolor=transparent]([backcolor=transparent]userid[backcolor=transparent])[backcolor=transparent] [backcolor=transparent]as[backcolor=transparent] usercount [backcolor=transparent]from[backcolor=transparent] emrusers [backcolor=transparent]group[backcolor=transparent] [backcolor=transparent]by[backcolor=transparent] movieid order [backcolor=transparent]by[backcolor=transparent] usercount desc limit [backcolor=transparent]50[backcolor=transparent];

1. [backcolor=transparent] [backcolor=transparent]-[backcolor=transparent]f ossref[backcolor=transparent]:[backcolor=transparent]//${bucket}/yourpath/hiveSample1.sql

1. 
   编写如下脚本。[backcolor=transparent] USE DEFAULT[backcolor=transparent];
2. [backcolor=transparent] [backcolor=transparent]set[backcolor=transparent] hive[backcolor=transparent].[backcolor=transparent]input[backcolor=transparent].[backcolor=transparent]format[backcolor=transparent]=[backcolor=transparent]org[backcolor=transparent].[backcolor=transparent]apache[backcolor=transparent].[backcolor=transparent]hadoop[backcolor=transparent].[backcolor=transparent]hive[backcolor=transparent].[backcolor=transparent]ql[backcolor=transparent].[backcolor=transparent]io[backcolor=transparent].[backcolor=transparent]HiveInputFormat[backcolor=transparent];
3. [backcolor=transparent] [backcolor=transparent]set[backcolor=transparent] mapreduce[backcolor=transparent].[backcolor=transparent]job[backcolor=transparent].[backcolor=transparent]maps[backcolor=transparent]=[backcolor=transparent]12[backcolor=transparent];
4. [backcolor=transparent] [backcolor=transparent]set[backcolor=transparent] mapreduce[backcolor=transparent].[backcolor=transparent]job[backcolor=transparent].[backcolor=transparent]reduces[backcolor=transparent]=[backcolor=transparent]6[backcolor=transparent];
5. [backcolor=transparent] [backcolor=transparent]set[backcolor=transparent] hive[backcolor=transparent].[backcolor=transparent]stats[backcolor=transparent].[backcolor=transparent]autogather[backcolor=transparent]=[backcolor=transparent]false[backcolor=transparent];
7. [backcolor=transparent] DROP TABLE uservisits[backcolor=transparent];
8. [backcolor=transparent] CREATE EXTERNAL TABLE uservisits [backcolor=transparent]([backcolor=transparent]sourceIP STRING[backcolor=transparent],[backcolor=transparent]destURL STRING[backcolor=transparent],[backcolor=transparent]visitDate STRING[backcolor=transparent],[backcolor=transparent]adRevenue DOUBLE[backcolor=transparent],[backcolor=transparent]userAgent STRING[backcolor=transparent],[backcolor=transparent]countryCode STRING[backcolor=transparent],[backcolor=transparent]languageCode STRING[backcolor=transparent],[backcolor=transparent]searchWord STRING[backcolor=transparent],[backcolor=transparent]duration INT [backcolor=transparent])[backcolor=transparent] ROW FORMAT DELIMITED FIELDS TERMINATED BY [backcolor=transparent]','[backcolor=transparent] STORED AS SEQUENCEFILE LOCATION [backcolor=transparent]'oss://${AccessKeyId}:${AccessKeySecret}@${bucket}.${endpoint}/sample-data/hive/Scan/Input/uservisits'[backcolor=transparent];

准备测试数据。您可通过下面的地址下载作业需要的资源，然后将其放到您 OSS 对应的目录下。
资源下载：uservisits
创建作业。将步骤 1 中编写的脚本存储到 OSS 中，假设存储路径为oss://emr/jars/scan.hive，在 E-MapReduce 中创建如下作业：

创建执行计划并运行。创建一个执行计划，您可以关联一个已有的集群，也可以自动按需创建一个，然后关联上这个作业。请将作为保存为“手动运行”，回到界面上就可以点击“立即运行”来运行作业了。