Hbase 建表及属性修改-FreeOA

Hbase 建表及属性修改

2017-07-29 22:37:47

阿炯

本文是通过Rest接口来建立表及修改表属性，采用Perl脚本实现。

建表是批量操作的，用于在从集群上创建表。

shell中创建表

create 'zw_img', {NAME => 'content', DATA_BLOCK_ENCODING => 'NONE', BLOOMFILTER => 'ROW', REPLICATION_SCOPE => '0', VERSIONS => '1', COMPRESSION => 'NONE', MIN_VERSIONS => '0', TTL => 'FOREVER', KEEP_DELETED_CELLS => 'FALSE', BLOCKSIZE => '65536', IN_MEMORY => 'false', BLOCKCACHE => 'true'}

create 'zw_img', {
BLOCKCACHE => 'true',
BLOCKSIZE => 65536,
BLOOMFILTER => 'ROW',
COMPRESSION => 'GZ',
DATA_BLOCK_ENCODING => 'NONE',
IN_MEMORY => 'false',
KEEP_DELETED_CELLS => 'FALSE',
MIN_VERSIONS => 0,
NAME => 'content',
REPLICATION_SCOPE => 0,
TTL => 259200,
VERSIONS => 1,
}

查看表
describe 'zw_img'

删除表
disable 'zw'
drop 'zw'

以json格式取得表结构
curl -X GET -H "Accept: application/json" "http://192.168.20.87:8000/z/schema"

{"name":"z","ColumnSchema":[{"name":"facontent","DATA_BLOCK_ENCODING":"NONE","BLOOMFILTER":"ROW","REPLICATION_SCOPE":"0","VERSIONS":"1","COMPRESSION":"NONE","MIN_VERSIONS":"0","TTL":"2147483647","KEEP_DELETED_CELLS":"FALSE","BLOCKSIZE":"65536","IN_MEMORY":"false","BLOCKCACHE":"true"}],"IS_META":"false"}

以json格式提交创建语句却不被支持，仅支持xml格式的提交。
curl -X POST -H "Accept: application/json" -d '
create 'zw_content', {
BLOCKCACHE => "true",
BLOCKSIZE => 65536,
BLOOMFILTER => "ROW",
COMPRESSION => "GZ",
DATA_BLOCK_ENCODING => "NONE",
IN_MEMORY => "false",
KEEP_DELETED_CELLS => "FALSE",
MIN_VERSIONS => 0,
NAME => "content",
REPLICATION_SCOPE => 0,
TTL => 259200,
VERSIONS => 1,
}' "http://192.168.20.87:8000/zw_content/schema"

curl -v -X POST -H "Accept: text/xml" -H "Content-Type: text/xml" -d '<?xml version="1.0" encoding="UTF-8" standalone="yes"?><TableSchema name="zw" IS_META="false"><ColumnSchema name="content" DATA_BLOCK_ENCODING="NONE" BLOOMFILTER="ROW" REPLICATION_SCOPE="0" VERSIONS="1" COMPRESSION="NONE" MIN_VERSIONS="0" TTL="2147483647" KEEP_DELETED_CELLS="FALSE" BLOCKSIZE="65536" IN_MEMORY="false" BLOCKCACHE="true"/></TableSchema>' "http://192.168.20.87:8000/zw/schema"

HBase REST to return json instead of xml

You can request plain text (the default), XML , or JSON output by adding no header for plain text, or the header "Accept: text/xml" for XML or "Accept: application/json" for JSON.

Post with Mojo::UserAgent

Try to post your $xml_body like so:

my $tx = $ua->post($ebay_api_url => form => $xml_body);

my $tx = $ua->post($ebay_api_url => {Accept => '*/*'} => $xml_body);

取得表的json格式的信息
hb_tab_schema.json.pl

use v5.12;
use utf8;
use Encode;
use JSON::XS;
use Mojo::Log;
use Time::Piece;
use Data::Dumper;
use Time::Seconds;
use Mojo::UserAgent;
use Data::Dump "pp";
use Cwd qw(abs_path realpath);
use File::Basename qw(dirname);

binmode(STDIN, ":encoding(utf8)");
binmode(STDOUT, ":encoding(utf8)");

my $mydir=dirname(abs_path($0));
chdir($mydir);

my $cdt=localtime;
my $log = Mojo::Log->new(path => 'log/hbase.tab.log');
my $ua = Mojo::UserAgent->new;

$log = $log->format(sub {
my ($time, $level, @lines) = @_;
#my $idt=strftime("%Y-%m-%d %H:%M:%S",localtime);
my $idt=localtime->datetime;
return qq{[$idt] [$level] @lines \n};
});

#Hbase hp for hostname & port
my ($hp,@hbtabs)=('192.168.1.120:8080');
#取得所有表的名称
my $json=$ua->get($hp=>{'Accept'=>'application/json'})->res->json;
push @hbtabs,$_->{name} foreach (@{$json->{table}});
#say Dumper(encode_json $json);

foreach my $tab (@hbtabs){
   my $tabsch=$ua->get("$hp/$tab/schema/"=>{'Accept'=>'application/json'})->res->json;
   my $creatabsch=qq[create '$tabsch->{name}', ];
   my $rs=getab_meta($tabsch->{ColumnSchema}[0]);
   $creatabsch.=$rs;
   say $creatabsch;
}

sub getab_meta{
   my $tm=shift;
   while(my ($k, $v) = each(%$tm)) {
       $tm->{uc($k)} = $v,delete $tm->{$k} if($k=~/[a-z]+/);
   }
   return pp($tm);
}

取得表的xml信息及表在从集群上创建(含表移除，请慎用)
hb_tab_schema.pl

use v5.12;
use utf8;
use Encode;
use Mojo::Log;
use Time::Piece;
use XML::Simple;
use Data::Dumper;
use Time::Seconds;
use Mojo::UserAgent;
use Data::Dump "pp";
use Cwd qw(abs_path realpath);
use File::Basename qw(dirname);

binmode(STDIN, ":encoding(utf8)");
binmode(STDOUT, ":encoding(utf8)");

my $mydir=dirname(abs_path($0));
chdir($mydir);

my $cdt=localtime;
my $log = Mojo::Log->new(path => 'log/hbase.tab.log');
my $ua = Mojo::UserAgent->new;
my $xs = XML::Simple->new();

$log = $log->format(sub {
my ($time, $level, @lines) = @_;
#my $idt=strftime("%Y-%m-%d %H:%M:%S",localtime);
my $idt=localtime->datetime;
return qq{[$idt] [$level] @lines \n};
});

#Hbase shp for sourec hostname & port
my ($shp,$dhp,@hbtabs)=('192.168.0.9:8080','192.168.0.26:8080');
#######################################
#取得所有表的名称
my $hrs=$ua->get($shp=>{'Accept'=>'text/xml'})->res->body;
#push @hbtabs,$_->{name} foreach (@{$json->{table}});
push @hbtabs,$_ foreach (keys %{$xs->XMLin($hrs)->{table}});
#my $ref = $xs->XMLin($hrs);
#say Dumper($hrs,$ref);

=begin comment
foreach my $tab (@hbtabs){
   my $tabsch=$ua->get("$shp/$tab/schema/"=>{'Accept'=>'text/xml'})->res->body;
   my $ts=$xs->XMLin($tabsch);
   my $creatabsch=qq[create '$ts->{name}', ];
   my $rs=getab_meta($ts->{ColumnSchema});
   $creatabsch.=$rs;
   say $creatabsch;
}
=cut

foreach my $tab (@hbtabs){
   my $tabsch=$ua->get("$shp/$tab/schema/"=>{'Accept'=>'text/xml'})->res->body;
   my $crs=postab($tab,$tabsch);
   #my $crs=deletab($tab);
   say "handle table $tab:$crs";
}

sub getab_meta{
   my $tm=shift;
   while(my ($k, $v) = each(%$tm)) {
       $tm->{uc($k)} = $v,delete $tm->{$k} if($k=~/[a-z]+/);
   }
   return pp($tm);
}

sub postab{
    my ($tab,$cr)=@_;
   my $tx = $ua->post("$dhp/$tab/schema" => {'Content-Type' => 'text/xml'} => $cr);
   return $tx->res->{message};
}

sub deletab{
   my ($tab)=@_;
   my $tx = $ua->delete("$dhp/$tab/schema" => {'Content-Type' => 'text/xml'});
    return $tx->res->{message};
}

批量修改表的属性
hb_tab_schema_modify.pl

use v5.12;
use utf8;
use Encode;
use Mojo::Log;
use Time::Piece;
use XML::Simple;
use Data::Dumper;
use Time::Seconds;
use Mojo::UserAgent;
use Data::Dump "pp";
use Cwd qw(abs_path realpath);
use File::Basename qw(dirname);

binmode(STDIN, ":encoding(utf8)");
binmode(STDOUT, ":encoding(utf8)");

my $mydir=dirname(abs_path($0));
chdir($mydir);

my $cdt=localtime;
my $log = Mojo::Log->new(path => 'log/hbase.tab.log');
my $ua = Mojo::UserAgent->new;
my $xs = XML::Simple->new();

$log = $log->format(sub {
my ($time, $level, @lines) = @_;
#my $idt=strftime("%Y-%m-%d %H:%M:%S",localtime);
my $idt=localtime->datetime;
return qq{[$idt] [$level] @lines \n};
});

#Hbase hp for sourec hostname & port
my ($shp,$dhp,$hbdir,@hbtabs)=('192.168.0.9:8080','192.168.0.21:8080','/usr/local/hbase');
#######################################
#取得所有表的名称
my $hrs=$ua->get($shp=>{'Accept'=>'text/xml'})->res->body;
#push @hbtabs,$_->{name} foreach (@{$json->{table}});
push @hbtabs,$_ foreach (keys %{$xs->XMLin($hrs)->{table}});
#my $ref = $xs->XMLin($hrs);
#say Dumper($hrs,$ref);

=begin comment
foreach my $tab (@hbtabs){
   my $tabsch=$ua->get("$shp/$tab/schema/"=>{'Accept'=>'text/xml'})->res->body;
   my $ts=$xs->XMLin($tabsch);
   my $creatabsch=qq[create '$ts->{name}', ];
   my $rs=getab_meta($ts->{ColumnSchema});
   $creatabsch.=$rs;
   say $creatabsch;
}
=cut

foreach my $tab (@hbtabs){
   my $tabsch=$ua->get("$shp/$tab/schema/"=>{'Accept'=>'text/xml'})->res->body;
   #my $crs=postab($tab,$tabsch);
   #my $crs=deletab($tab);
   my $crs=modtab($tab);
   say "handle table $tab:$crs";
}

sub getab_meta{
   my $tm=shift;
   while(my ($k, $v) = each(%$tm)) {
       $tm->{uc($k)} = $v,delete $tm->{$k} if($k=~/[a-z]+/);
   }
   return pp($tm);
}

sub modtab{
   my ($tab)=@_;
   my $cmdis=qq[echo "disable '$tab'" | $hbdir/bin/hbase shell];
   my $rs=system($cmdis);
   my $cmdif=qq[echo "alter '$tab', {NAME => 'cf',REPLICATION_SCOPE => '1'}" | $hbdir/bin/hbase shell];
   $rs.=system($cmdif);
   my $cmden=qq[echo "enable '$tab'" | $hbdir/bin/hbase shell];
   $rs.=system($cmden);
   #$henv.="$_:$ENV{$_}\n" foreach (keys %ENV);
   say $rs;
   $log->info("Alter on $tab,return code is:$rs.");
}

sub postab{
    my ($tab,$cr)=@_;
   my $tx = $ua->post("$dhp/$tab/schema" => {'Content-Type' => 'text/xml'} => $cr);
   return $tx->res->{message};
}

sub deletab{
   my ($tab)=@_;
   my $tx = $ua->delete("$dhp/$tab/schema" => {'Content-Type' => 'text/xml'});
    return $tx->res->{message};
}