pinpoint1.7.3开启kerberos认证

/ pinpoint / 5 条评论 / 351浏览

官方不支持Hbase开启kerberos认证,下面介绍如何修改官方的代码及配置,使之支持kerberos认证。

补充:这种方式大概1天以后还是会出现无法认证的问题:Not attempting to re-login since the last re-login was attempted less than 600 seconds before.。给官方提了issue,暂未收到回复。

Java连接

我们先用Java写个开启kerberos认证的Hbase的例子,然后再修改官方源码。

例子

在看了n多博客,尝试修改了很多版本,最终以下代码是可以正确运行的(HDP版本为HDP-2.6.0.3)

package com.example.keberos;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.security.UserGroupInformation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.util.Enumeration;
import java.util.Properties;

/**
 * @author xuan
 * @since 1.0.0
 */
public class App {

    private static final Logger LOG = LoggerFactory.getLogger(App.class);

    public static void main(String[] args) throws IOException {
        BufferedReader in = new BufferedReader(new FileReader("config.properties"));
        Properties prop = new Properties();
        prop.load(in);

        System.setProperty("java.security.krb5.conf", "/etc/krb5.conf");
        Configuration conf = HBaseConfiguration.create();
        conf.addResource(new Path(prop.getProperty("core.site")));
        conf.addResource(new Path(prop.getProperty("hbase.site")));
        setConf(conf, prop);

        login(conf, prop);

        put(conf);
        list(conf);
    }

    private static void list(Configuration conf) throws IOException {
        Connection conn = ConnectionFactory.createConnection(conf);
        Admin admin = conn.getAdmin();
        TableName[] tableNames = admin.listTableNames();
        for (TableName tableName : tableNames) {
            LOG.info("tableName: {}", tableName);
        }
        admin.close();
        conn.close();
    }

    private static void put(Configuration conf) throws IOException {
        Connection conn = ConnectionFactory.createConnection(conf);
        Table table = conn.getTable(TableName.valueOf("User"));
        Put put = new Put(Bytes.toBytes("1"));
        put.addColumn(Bytes.toBytes("info"), Bytes.toBytes("name"), Bytes.toBytes("张三"));
        table.put(put);
        table.close();
        conn.close();
    }

    private static void login(Configuration conf, Properties prop) throws IOException {
        UserGroupInformation.setConfiguration(conf);
        UserGroupInformation.loginUserFromKeytab(prop.getProperty("kerberos.principal"), prop.getProperty("kerberos.keytab"));
    }

    private static void setConf(Configuration conf, Properties properties) {
        Enumeration<?> props = properties.propertyNames();
        while (props.hasMoreElements()) {
            String key = props.nextElement().toString();
            conf.set(key, properties.getProperty(key));
        }
    }

}

其中config.properties配置如下

core.site=/etc/hadoop/2.6.0.3-8/0/core-site.xml
hbase.site=/etc/hbase/2.6.0.3-8/0/hbase-site.xml

hadoop.security.authentication=kerberos
hbase.security.authentication=kerberos

hbase.zookeeper.quorum=10.24.15.43,10.24.15.44,10.24.15.45
hbase.zookeeper.property.clientPort=2181
zookeeper.znode.parent=/hbase-secure

hbase.master.kerberos.principal=hbase/_HOST@EXAMPLE.COM
# hbase.regionserver.kerberos.principal=hbase/_HOST@EXAMPLE.COM

kerberos.principal=omm@EXAMPLE.COM
kerberos.keytab=/cmcc/kettle/source/rpone/user.keytab

主要代码配置:

// krb5
System.setProperty("java.security.krb5.conf", "/etc/krb5.conf");
// hbase配置
Configuration conf = HBaseConfiguration.create();
conf.set("hbase.zookeeper.quorum", "10.24.15.43,10.24.15.44,10.24.15.45");
conf.set("hbase.zookeeper.property.clientPort", "2181");
conf.set("zookeeper.znode.parent", "/hbase-secure");
// 最开始没有加这两个配置,发现一直报错:Not attempting to re-login since the last re-login was attempted less than 600 seconds before
conf.addResource(new Path("/etc/hadoop/2.6.0.3-8/0/core-site.xml"));
conf.addResource(new Path("/etc/hadoop/2.6.0.3-8/0/hbase-site.xml"));
// kerberos认证
conf.set("hadoop.security.authentication", "kerberos");
conf.set("hbase.security.authentication", "kerberos");
conf.set("hbase.master.kerberos.principal", "hbase/_HOST@EXAMPLE.COM");
// 登录
UserGroupInformation.setConfiguration(conf);
UserGroupInformation.loginUserFromKeytab("omm@EXAMPLE.COM", "/cmcc/kettle/source/rpone/user.keytab");

// conn做些什么
Connection conn = ConnectionFactory.createConnection(conf);

修改pinpoint源码及配置

其实,我们只需要能加入一点kerberos的配置信息即可,很容易发现源码中HbaseConfigurationFactoryBean是用来配置Configuration

很容易修改为:

/*
 * Copyright 2011 the original author or authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.navercorp.pinpoint.common.hbase;

import org.apache.commons.lang.BooleanUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.security.UserGroupInformation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.FactoryBean;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.util.Assert;

import java.io.IOException;
import java.util.Enumeration;
import java.util.Properties;

/**
 * Factory for creating HBase specific configuration. By default cleans up any connection associated with the current configuration.
 *
 * @author Costin Leau
 */
public class HbaseConfigurationFactoryBean implements InitializingBean, FactoryBean<Configuration> {

    private static final Logger LOG = LoggerFactory.getLogger(HbaseConfigurationFactoryBean.class);

    private static final String KEBEROS_ENABLE = "kerberos.enable";
    private static final String CORE_SITE = "core.site";
    private static final String HBASE_SITE = "hbase.site";
    private static final String KRB5_CONF = "java.security.krb5.conf";
    private static final String KEBEROS_PRINCIPAL = "kerberos.principal";
    private static final String KEBEROS_KEYTAB = "kerberos.keytab";

    private Configuration configuration;
    private Configuration hadoopConfig;
    private Properties properties;

    /**
     * Sets the Hadoop configuration to use.
     *
     * @param configuration The configuration to set.
     */
    public void setConfiguration(Configuration configuration) {
        this.hadoopConfig = configuration;
    }

    /**
     * Sets the configuration properties.
     *
     * @param properties The properties to set.
     */
    public void setProperties(Properties properties) {
        this.properties = properties;
    }

    @Override
    public void afterPropertiesSet() throws IOException {
        configuration = (hadoopConfig != null ? HBaseConfiguration.create(hadoopConfig) : HBaseConfiguration.create());
        addProperties(configuration, properties);
        // 是否开启kerberos认证
        if (BooleanUtils.toBoolean(properties.getProperty(KEBEROS_ENABLE))) {
            LOG.info("==================kerberos config=======================");
            configuration.addResource(new Path(properties.getProperty(CORE_SITE)));
            configuration.addResource(new Path(properties.getProperty(HBASE_SITE)));
            // 设置 auth
            System.setProperty(KRB5_CONF, properties.getProperty(KRB5_CONF));
            // 登录
            UserGroupInformation.setConfiguration(configuration);
            UserGroupInformation.loginUserFromKeytab(properties.getProperty(KEBEROS_PRINCIPAL), properties.getProperty(KEBEROS_KEYTAB));
            // 测试连接
            Connection conn = ConnectionFactory.createConnection(configuration);
            Admin admin = conn.getAdmin();
            TableName[] tableNames = admin.listTableNames();
            LOG.info("==================kerberos testing=======================");
            for (TableName tableName : tableNames) {
                LOG.info("tableName: {}", tableName);
            }
            LOG.info("==================kerberos testing=======================");
            conn.close();
        }
    }

    /**
     * Adds the specified properties to the given {@link Configuration} object.  
     *
     * @param configuration configuration to manipulate. Should not be null.
     * @param properties properties to add to the configuration. May be null.
     */
    private void addProperties(Configuration configuration, Properties properties) {
        Assert.notNull(configuration, "A non-null configuration is required");
        if (properties != null) {
            Enumeration<?> props = properties.propertyNames();
            while (props.hasMoreElements()) {
                String key = props.nextElement().toString();
                configuration.set(key, properties.getProperty(key));
            }
        }
    }

    @Override
    public Configuration getObject() {
        return configuration;
    }

    @Override
    public Class<? extends Configuration> getObjectType() {
        return (configuration != null ? configuration.getClass() : Configuration.class);
    }

    @Override
    public boolean isSingleton() {
        return true;
    }

}

主要修改afterPropertiesSet方法即可,如果kerberos配置开启(kerberos.enable=true)了,那么进行配置。

我们还需要修改collectorweb模块下applicationContext-hbase.xmlhbase.properties

applicationContext-hbase.xml中找到hbaseConfiguration

<bean id="hbaseConfiguration" class="com.navercorp.pinpoint.common.hbase.HbaseConfigurationFactoryBean">
    <property name="properties">
        <props>
            <prop key="hbase.zookeeper.quorum">${hbase.client.host}</prop>
            <prop key="hbase.zookeeper.property.clientPort">${hbase.client.port}</prop>

            <!--Root ZNode for HBase in ZooKeeper.-->
            <prop key="zookeeper.znode.parent">${hbase.zookeeper.znode.parent:/hbase}</prop>

            <!-- hbase default:true -->
            <prop key="hbase.ipc.client.tcpnodelay">${hbase.ipc.client.tcpnodelay}</prop>
            <!-- hbase default:60000 -->
            <prop key="hbase.rpc.timeout">${hbase.rpc.timeout}</prop>
            <!-- hbase default:Integer.MAX_VALUE -->
            <prop key="hbase.client.operation.timeout">${hbase.client.operation.timeout}</prop>

            <!-- hbase socket read timeout. default: 200000-->
            <prop key="hbase.ipc.client.socket.timeout.read">${hbase.ipc.client.socket.timeout.read}</prop>
            <!-- socket write timeout. hbase default: 600000-->
            <prop key="hbase.ipc.client.socket.timeout.write">${hbase.ipc.client.socket.timeout.write}</prop>

            <!-- hbase async put operation. -->
            <prop key="hbase.client.async.enable">${hbase.client.async.enable:false}</prop>
            <prop key="hbase.client.async.in.queuesize">${hbase.client.async.in.queuesize:10000}</prop>
            <prop key="hbase.tablemultiplexer.flush.period.ms">${hbase.client.async.flush.period.ms:100}</prop>
            <prop key="hbase.client.max.retries.in.queue">${hbase.client.async.max.retries.in.queue:10}</prop>
            <!-- kerberos -->
            <prop key="kerberos.enable">${kerberos.enable:false}</prop>
            <prop key="core.site">${core.site}</prop>
            <prop key="hbase.site">${hbase.site}</prop>
            <prop key="java.security.krb5.conf">${java.security.krb5.conf}</prop>
            <prop key="hbase.master.kerberos.principal">${hbase.master.kerberos.principal}</prop>
            <prop key="kerberos.principal">${kerberos.principal}</prop>
            <prop key="kerberos.keytab">${kerberos.keytab}</prop>
            <prop key="hadoop.security.authentication">${hadoop.security.authentication}</prop>
            <prop key="hbase.security.authentication">${hbase.security.authentication}</prop>
        </props>
    </property>
</bean>

hbase.properties最后面添加

# keberos auth
kerberos.enable=true
core.site=/etc/hadoop/2.6.0.3-8/0/core-site.xml
hbase.site=/etc/hbase/2.6.0.3-8/0/hbase-site.xml
java.security.krb5.conf=/etc/krb5.conf
hadoop.security.authentication=kerberos
hbase.security.authentication=kerberos
hbase.master.kerberos.principal=hbase/_HOST@EXAMPLE.COM
# hbase.regionserver.kerberos.principal=hbase/_HOST@EXAMPLE.COM
kerberos.principal=omm@EXAMPLE.COM
kerberos.keytab=/cmcc/kettle/source/rpone/user.keytab

提示:如果直接下载的war包,在$ROOT/WEB-INF/classes/下直接修改配置即可,对于java源码,则需要自己编译pinpoint-commons-hbase-1.7.3.jar。当然,也可以写HbaseConfigurationFactoryBean,编译成class,替换进去即可。 替换

对于,偷懒的道友,也可以下载pinpoint-commons-hbase-1.7.3.jar,替换war包中的jar即可。

参考

补充

kerberos过期问题

程序运行一段时间后,发现就无法连接到hbase了。。由于kerberos认证会过期,过期后要么renew,要么relogin。

先看看spark如何处理这个问题:

spark renew

因此,pinpoint中HbaseConfigurationFactoryBean得加点刷新ticket得逻辑。

/*
 * Copyright 2011 the original author or authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.navercorp.pinpoint.common.hbase;

import org.apache.commons.lang.BooleanUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.security.UserGroupInformation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.FactoryBean;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.util.Assert;

import java.io.IOException;
import java.util.Enumeration;
import java.util.Properties;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;

/**
 * Factory for creating HBase specific configuration. By default cleans up any connection associated with the current configuration.
 *
 * @author Costin Leau
 */
public class HbaseConfigurationFactoryBean implements InitializingBean, FactoryBean<Configuration> {

    private static final Logger LOG = LoggerFactory.getLogger(HbaseConfigurationFactoryBean.class);

    private static final String KEBEROS_ENABLE = "kerberos.enable";
    private static final String CORE_SITE = "core.site";
    private static final String HBASE_SITE = "hbase.site";
    private static final String KRB5_CONF = "java.security.krb5.conf";
    private static final String KEBEROS_PRINCIPAL = "kerberos.principal";
    private static final String KEBEROS_KEYTAB = "kerberos.keytab";

    private ScheduledThreadPoolExecutor renewExecutor;

    private Configuration configuration;
    private Configuration hadoopConfig;
    private Properties properties;

    /**
     * Sets the Hadoop configuration to use.
     *
     * @param configuration The configuration to set.
     */
    public void setConfiguration(Configuration configuration) {
        this.hadoopConfig = configuration;
    }

    /**
     * Sets the configuration properties.
     *
     * @param properties The properties to set.
     */
    public void setProperties(Properties properties) {
        this.properties = properties;
    }

    @Override
    public void afterPropertiesSet() throws IOException {
        configuration = (hadoopConfig != null ? HBaseConfiguration.create(hadoopConfig) : HBaseConfiguration.create());
        addProperties(configuration, properties);
        // 是否开启kerberos认证
        if (BooleanUtils.toBoolean(properties.getProperty(KEBEROS_ENABLE))) {
            configuration.addResource(new Path(properties.getProperty(CORE_SITE)));
            configuration.addResource(new Path(properties.getProperty(HBASE_SITE)));
            // 设置 auth
            System.setProperty(KRB5_CONF, properties.getProperty(KRB5_CONF));

            doLogin();

            renewExecutor = new ScheduledThreadPoolExecutor(1);
            renewExecutor.scheduleWithFixedDelay(() -> {
                try {
                    LOG.info("==================kerberos renew=======================");
                    UserGroupInformation.getLoginUser().checkTGTAndReloginFromKeytab();
                } catch (IOException e) {
                    LOG.error("kerberos renew error", e);
                }
            }, 5, 5, TimeUnit.MINUTES);
        }
    }

    private void doLogin() throws IOException {
        LOG.info("==================kerberos config begin=======================");
        // 登录
        UserGroupInformation.setConfiguration(configuration);
        UserGroupInformation.loginUserFromKeytab(properties.getProperty(KEBEROS_PRINCIPAL), properties.getProperty(KEBEROS_KEYTAB));
        // 测试连接
        Connection conn = ConnectionFactory.createConnection(configuration);
        Admin admin = conn.getAdmin();
        TableName[] tableNames = admin.listTableNames();
        LOG.info("==================kerberos testing=======================");
        for (TableName tableName : tableNames) {
            LOG.info("tableName: {}", tableName);
        }
        LOG.info("==================kerberos testing=======================");
        conn.close();
        LOG.info("==================kerberos config end=======================");
    }

    /**
     * Adds the specified properties to the given {@link Configuration} object.  
     *
     * @param configuration configuration to manipulate. Should not be null.
     * @param properties properties to add to the configuration. May be null.
     */
    private void addProperties(Configuration configuration, Properties properties) {
        Assert.notNull(configuration, "A non-null configuration is required");
        if (properties != null) {
            Enumeration<?> props = properties.propertyNames();
            while (props.hasMoreElements()) {
                String key = props.nextElement().toString();
                configuration.set(key, properties.getProperty(key));
            }
        }
    }

    @Override
    public Configuration getObject() {
        return configuration;
    }

    @Override
    public Class<? extends Configuration> getObjectType() {
        return (configuration != null ? configuration.getClass() : Configuration.class);
    }

    @Override
    public boolean isSingleton() {
        return true;
    }

}

点击下载最新的pinpoint-commons-hbase-1.7.3.jar

  1. 你好,我按照你最后的办法放入了pingpoint中一天之后还是认证失败,报错内容,而且我看日志,一直都在renew呀 Couldn't setup connection for XXX@XXX.COM to hbase/XXX@XXX.COM 2019-07-18 17:57:03 [INFO ](p.c.h.HbaseConfigurationFactoryBean) ==================kerberos renew======================= 2019-07-18 18:02:03 [INFO ](p.c.h.HbaseConfigurationFactoryBean) ==================kerberos renew======================= 2019-07-18 18:07:03 [INFO ](p.c.h.HbaseConfigurationFactoryBean) ==================kerberos renew======================= 2019-07-18 18:09:49 [WARN ](o.a.h.s.UserGroupInformation ) Not attempting to re-login since the last re-login was attempted less than 600 seconds before. 2019-07-18 18:09:53 [WARN ](o.a.h.s.UserGroupInformation ) Not attempting to re-login since the last re-login was attempted less than 600 seconds before. 2019-07-18 18:09:58 [WARN ](o.a.h.s.UserGroupInformation ) Not attempting to re-login since the last re-login was attempted less than 600 seconds before. 2019-07-18 18:10:01 [WARN ](o.a.h.s.UserGroupInformation ) Not attempting to re-login since the last re-login was attempted less than 600 seconds before. 2019-07-18 18:10:03 [WARN ](o.a.h.h.i.RpcClientImpl ) Couldn't setup connection for XXX@XXX.COM to hbase/XXX@XXX.COM javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)]

    回复
    1. @墨渊

      我之前也发现了大概2~3天的样子就认证失败了。报错:Not attempting to re-login since the last re-login was attempted less than 600 seconds before。后面无奈之下放弃了,给官方提了issue,暂未有回复

      回复
  2. xxx

    嘿嘿 波哥,是不是在划水

    回复
  3. mk

    你写的博客,越来越看不懂了

    回复
    1. @mk

      别搞我啊:)

      回复