With the new release of HDP 2.3 comes Ambari 2.1 that brings among other improvements the provisioning and management of Apache Ranger. Ranger together with new agents for a centralized authorization management brings a new KMS key storage for HDFS encryption. HDP components in Ambari can be installed and configured through blueprints that are described in a JSON notation.
Ranger Overview
Architectural overview of Apach Ranger:
Common configurations via blueprint:
Ranger KMS Blueprint
Settings for the Ramger Key Management Server:
{ ... "ranger-kms-security" : { "properties_attributes" : { }, "properties" : { "ranger.plugin.kms.policy.cache.dir" : "/etc/ranger/{{repo_name}}/policycache", "ranger.plugin.kms.policy.pollIntervalMs" : "30000", "ranger.plugin.kms.policy.rest.ssl.config.file" : "/etc/kms/conf/ranger-policymgr-ssl.xml", "ranger.plugin.kms.policy.rest.url" : "{{policymgr_mgr_url}}", "ranger.plugin.kms.policy.source.impl" : "org.apache.ranger.admin.client.RangerAdminRESTClient", "ranger.plugin.kms.service.name" : "{{repo_name}}" } } }, { "kms-site" : { "properties_attributes" : { }, "properties" : { "hadoop.kms.audit.aggregation.window.ms" : "10000", "hadoop.kms.authentication.kerberos.keytab" : "${user.home}/kms.keytab", "hadoop.kms.authentication.kerberos.name.rules" : "DEFAULT", "hadoop.kms.authentication.kerberos.principal" : "HTTP/localhost", "hadoop.kms.authentication.signer.secret.provider" : "random", "hadoop.kms.authentication.signer.secret.provider.zookeeper.auth.type" : "kerberos", "hadoop.kms.authentication.signer.secret.provider.zookeeper.connection.string" : "#HOSTNAME#:#PORT#,...", "hadoop.kms.authentication.signer.secret.provider.zookeeper.kerberos.keytab" : "/etc/hadoop/conf/kms.keytab", "hadoop.kms.authentication.signer.secret.provider.zookeeper.kerberos.principal" : "kms/#HOSTNAME#", "hadoop.kms.authentication.signer.secret.provider.zookeeper.path" : "/hadoop-kms/hadoop-auth-signature-secret", "hadoop.kms.authentication.type" : "simple", "hadoop.kms.cache.enable" : "true", "hadoop.kms.cache.timeout.ms" : "600000", "hadoop.kms.current.key.cache.timeout.ms" : "30000", "hadoop.kms.key.provider.uri" : "dbks://http@localhost:9292/kms", "hadoop.kms.security.authorization.manager" : "org.apache.ranger.authorization.kms.authorizer.RangerKmsAuthorizer" } } }, { "dbks-site" : { "properties_attributes" : { }, "properties" : { "hadoop.kms.blacklist.DECRYPT_EEK" : "hdfs", "ranger.ks.jdbc.sqlconnectorjar" : "{{driver_curl_target}}", "ranger.ks.jpa.jdbc.credential.alias" : "ranger.ks.jdbc.password", "ranger.ks.jpa.jdbc.credential.provider.path" : "/etc/ranger/kms/rangerkms.jceks", "ranger.ks.jpa.jdbc.dialect" : "{{jdbc_dialect}}", "ranger.ks.jpa.jdbc.driver" : "{{db_jdbc_driver}}", "ranger.ks.jpa.jdbc.url" : "{{db_jdbc_url}}", "ranger.ks.jpa.jdbc.user" : "{{db_user}}", "ranger.ks.masterkey.credential.alias" : "ranger.ks.masterkey.password" } } }, { "ranger-kms-audit" : { "properties_attributes" : { }, "properties" : { "xasecure.audit.credential.provider.file" : "jceks://file{{credential_file}}", "xasecure.audit.destination.db" : "false", "xasecure.audit.destination.db.jdbc.driver" : "{{jdbc_driver}}", "xasecure.audit.destination.db.jdbc.url" : "{{audit_jdbc_url}}", "xasecure.audit.destination.db.user" : "{{xa_audit_db_user}}", "xasecure.audit.destination.hdfs" : "true", "xasecure.audit.destination.hdfs.batch.filespool.dir" : "/var/log/kms/audit/hdfs/spool", "xasecure.audit.destination.hdfs.dir" : "hdfs://NAMENODE_HOSTNAME:8020/ranger/audit", "xasecure.audit.destination.solr" : "true", "xasecure.audit.destination.solr.batch.filespool.dir" : "/var/log/kms/audit/solr/spool", "xasecure.audit.destination.solr.urls" : "{{ranger_audit_solr_urls}}", "xasecure.audit.destination.solr.zookeepers" : "none", "xasecure.audit.is.enabled" : "true", "xasecure.audit.provider.summary.enabled" : "false" } } }, { "kms-env" : { "properties_attributes" : { }, "properties" : { "kms_group" : "kms", "kms_log_dir" : "/var/log/ranger/kms", "kms_port" : "9292", "kms_user" : "kms" } } }, { "kms-properties" : { "properties_attributes" : { }, "properties" : { "DB_FLAVOR" : "MYSQL", "KMS_MASTER_KEY_PASSWD" : "123", "REPOSITORY_CONFIG_USERNAME" : "keyadmin", "SQL_CONNECTOR_JAR" : "/usr/share/java/mysql-connector-java.jar", "db_host" : "sandbox.hortonworks.com", "db_name" : "rangerkms", "db_root_user" : "root", "db_user" : "rangerkms" } } }, { "kms-log4j" : { "properties_attributes" : { }, "properties" : { "content" : "n#n# Licensed under the Apache License, Version 2.0 (the "License");n# you may not use this file except in compliance with the License.n# You may obtain a copy of the License atn#n# http://www.apache.org/licenses/LICENSE-2.0n#n# Unless required by applicable law or agreed to in writing, softwaren# distributed under the License is distributed on an "AS IS" BASIS,n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.n# See the License for the specific language governing permissions andn# limitations under the License. See accompanying LICENSE file.n#nn# If the Java System property 'kms.log.dir' is not defined at KMS start up timen# Setup sets its value to '${kms.home}/logs'nnlog4j.appender.kms=org.apache.log4j.DailyRollingFileAppendernlog4j.appender.kms.DatePattern='.'yyyy-MM-ddnlog4j.appender.kms.File=${kms.log.dir}/kms.lognlog4j.appender.kms.Append=truenlog4j.appender.kms.layout=org.apache.log4j.PatternLayoutnlog4j.appender.kms.layout.ConversionPattern=%d{ISO8601} %-5p %c{1} - %m%nnnlog4j.appender.kms-audit=org.apache.log4j.DailyRollingFileAppendernlog4j.appender.kms-audit.DatePattern='.'yyyy-MM-ddnlog4j.appender.kms-audit.File=${kms.log.dir}/kms-audit.lognlog4j.appender.kms-audit.Append=truenlog4j.appender.kms-audit.layout=org.apache.log4j.PatternLayoutnlog4j.appender.kms-audit.layout.ConversionPattern=%d{ISO8601} %m%nnnlog4j.logger.kms-audit=INFO, kms-auditnlog4j.additivity.kms-audit=falsennlog4j.rootLogger=ALL, kmsnlog4j.logger.org.apache.hadoop.conf=ERRORnlog4j.logger.org.apache.hadoop=INFOnlog4j.logger.com.sun.jersey.server.wadl.generators.WadlGeneratorJAXBGrammarGenerator=OFF" } } }, { "ranger-kms-site" : { "properties_attributes" : { }, "properties" : { "ranger.contextName" : "/kms", "ranger.service.host" : "{{ranger_admin_hosts}}", "ranger.service.http.port" : "9292", "ranger.service.shutdown.port" : "7085", "xa.webapp.dir" : "./webapp" } } } ... }
Ranger UGSYNC Blueprint
User synchronization settings:
{ ... "ranger-ugsync-site" : { "properties_attributes" : { }, "properties" : { "ranger.usersync.credstore.filename" : "/etc/ranger/usersync/ugsync.jceks", "ranger.usersync.enabled" : "true", "ranger.usersync.filesource.file" : "/tmp/usergroup.txt", "ranger.usersync.filesource.text.delimiter" : ",", "ranger.usersync.group.memberattributename" : "member", "ranger.usersync.group.nameattribute" : "cn", "ranger.usersync.group.objectclass" : "groupofnames", "ranger.usersync.group.searchbase" : "ou=groups,dc=hadoop,dc=apache,dc=org", "ranger.usersync.group.searchenabled" : "false", "ranger.usersync.group.searchfilter" : "empty", "ranger.usersync.group.searchscope" : "sub", "ranger.usersync.group.usermapsyncenabled" : "false", "ranger.usersync.keystore.file" : "./conf/cert/unixauthservice.jks", "ranger.usersync.ldap.bindalias" : "testldapalias", "ranger.usersync.ldap.binddn" : "cn=admin,dc=xasecure,dc=net", "ranger.usersync.ldap.bindkeystore" : "", "ranger.usersync.ldap.groupname.caseconversion" : "lower", "ranger.usersync.ldap.searchBase" : "dc=hadoop,dc=apache,dc=org", "ranger.usersync.ldap.url" : "ldap://localhost:389", "ranger.usersync.ldap.user.groupnameattribute" : "memberof, ismemberof", "ranger.usersync.ldap.user.nameattribute" : "cn", "ranger.usersync.ldap.user.objectclass" : "person", "ranger.usersync.ldap.user.searchbase" : "ou=users,dc=xasecure,dc=net", "ranger.usersync.ldap.user.searchfilter" : "empty", "ranger.usersync.ldap.user.searchscope" : "sub", "ranger.usersync.ldap.username.caseconversion" : "lower", "ranger.usersync.logdir" : "/var/log/ranger/usersync", "ranger.usersync.pagedresultsenabled" : "true", "ranger.usersync.pagedresultssize" : "500", "ranger.usersync.passwordvalidator.path" : "./native/credValidator.uexe", "ranger.usersync.policymanager.baseURL" : "{{ranger_external_url}}", "ranger.usersync.policymanager.maxrecordsperapicall" : "1000", "ranger.usersync.policymanager.mockrun" : "false", "ranger.usersync.port" : "5151", "ranger.usersync.sink.impl.class" : "org.apache.ranger.unixusersync.process.PolicyMgrUserGroupBuilder", "ranger.usersync.sleeptimeinmillisbetweensynccycle" : "5", "ranger.usersync.source.impl.class" : "org.apache.ranger.unixusersync.process.UnixUserGroupBuilder", "ranger.usersync.ssl" : "true", "ranger.usersync.truststore.file" : "./conf/cert/mytruststore.jks", "ranger.usersync.unix.minUserId" : "500" } } }, ... }
Complete Blueprint incl. Spark
{ "Blueprints" : { "stack_name" : "HDP", "stack_version" : "2.3" }, "host_groups" : [ { "name" : "host_group_1", "configurations" : [ ], "components" : [ { "name" : "RANGER_USERSYNC" }, { "name" : "RANGER_KMS_SERVER" }, { "name" : "RANGER_ADMIN" } ], "cardinality" : "1" } ] "configurations" : [ { "ranger-kms-security" : { "properties_attributes" : { }, "properties" : { "ranger.plugin.kms.policy.cache.dir" : "/etc/ranger/{{repo_name}}/policycache", "ranger.plugin.kms.policy.pollIntervalMs" : "30000", "ranger.plugin.kms.policy.rest.ssl.config.file" : "/etc/kms/conf/ranger-policymgr-ssl.xml", "ranger.plugin.kms.policy.rest.url" : "{{policymgr_mgr_url}}", "ranger.plugin.kms.policy.source.impl" : "org.apache.ranger.admin.client.RangerAdminRESTClient", "ranger.plugin.kms.service.name" : "{{repo_name}}" } } }, { "kms-site" : { "properties_attributes" : { }, "properties" : { "hadoop.kms.audit.aggregation.window.ms" : "10000", "hadoop.kms.authentication.kerberos.keytab" : "${user.home}/kms.keytab", "hadoop.kms.authentication.kerberos.name.rules" : "DEFAULT", "hadoop.kms.authentication.kerberos.principal" : "HTTP/localhost", "hadoop.kms.authentication.signer.secret.provider" : "random", "hadoop.kms.authentication.signer.secret.provider.zookeeper.auth.type" : "kerberos", "hadoop.kms.authentication.signer.secret.provider.zookeeper.connection.string" : "#HOSTNAME#:#PORT#,...", "hadoop.kms.authentication.signer.secret.provider.zookeeper.kerberos.keytab" : "/etc/hadoop/conf/kms.keytab", "hadoop.kms.authentication.signer.secret.provider.zookeeper.kerberos.principal" : "kms/#HOSTNAME#", "hadoop.kms.authentication.signer.secret.provider.zookeeper.path" : "/hadoop-kms/hadoop-auth-signature-secret", "hadoop.kms.authentication.type" : "simple", "hadoop.kms.cache.enable" : "true", "hadoop.kms.cache.timeout.ms" : "600000", "hadoop.kms.current.key.cache.timeout.ms" : "30000", "hadoop.kms.key.provider.uri" : "dbks://http@localhost:9292/kms", "hadoop.kms.security.authorization.manager" : "org.apache.ranger.authorization.kms.authorizer.RangerKmsAuthorizer" } } }, { "dbks-site" : { "properties_attributes" : { }, "properties" : { "hadoop.kms.blacklist.DECRYPT_EEK" : "hdfs", "ranger.ks.jdbc.sqlconnectorjar" : "{{driver_curl_target}}", "ranger.ks.jpa.jdbc.credential.alias" : "ranger.ks.jdbc.password", "ranger.ks.jpa.jdbc.credential.provider.path" : "/etc/ranger/kms/rangerkms.jceks", "ranger.ks.jpa.jdbc.dialect" : "{{jdbc_dialect}}", "ranger.ks.jpa.jdbc.driver" : "{{db_jdbc_driver}}", "ranger.ks.jpa.jdbc.url" : "{{db_jdbc_url}}", "ranger.ks.jpa.jdbc.user" : "{{db_user}}", "ranger.ks.masterkey.credential.alias" : "ranger.ks.masterkey.password" } } }, { "ranger-ugsync-site" : { "properties_attributes" : { }, "properties" : { "ranger.usersync.credstore.filename" : "/etc/ranger/usersync/ugsync.jceks", "ranger.usersync.enabled" : "true", "ranger.usersync.filesource.file" : "/tmp/usergroup.txt", "ranger.usersync.filesource.text.delimiter" : ",", "ranger.usersync.group.memberattributename" : "member", "ranger.usersync.group.nameattribute" : "cn", "ranger.usersync.group.objectclass" : "groupofnames", "ranger.usersync.group.searchbase" : "ou=groups,dc=hadoop,dc=apache,dc=org", "ranger.usersync.group.searchenabled" : "false", "ranger.usersync.group.searchfilter" : "empty", "ranger.usersync.group.searchscope" : "sub", "ranger.usersync.group.usermapsyncenabled" : "false", "ranger.usersync.keystore.file" : "./conf/cert/unixauthservice.jks", "ranger.usersync.ldap.bindalias" : "testldapalias", "ranger.usersync.ldap.binddn" : "cn=admin,dc=xasecure,dc=net", "ranger.usersync.ldap.bindkeystore" : "", "ranger.usersync.ldap.groupname.caseconversion" : "lower", "ranger.usersync.ldap.searchBase" : "dc=hadoop,dc=apache,dc=org", "ranger.usersync.ldap.url" : "ldap://localhost:389", "ranger.usersync.ldap.user.groupnameattribute" : "memberof, ismemberof", "ranger.usersync.ldap.user.nameattribute" : "cn", "ranger.usersync.ldap.user.objectclass" : "person", "ranger.usersync.ldap.user.searchbase" : "ou=users,dc=xasecure,dc=net", "ranger.usersync.ldap.user.searchfilter" : "empty", "ranger.usersync.ldap.user.searchscope" : "sub", "ranger.usersync.ldap.username.caseconversion" : "lower", "ranger.usersync.logdir" : "/var/log/ranger/usersync", "ranger.usersync.pagedresultsenabled" : "true", "ranger.usersync.pagedresultssize" : "500", "ranger.usersync.passwordvalidator.path" : "./native/credValidator.uexe", "ranger.usersync.policymanager.baseURL" : "{{ranger_external_url}}", "ranger.usersync.policymanager.maxrecordsperapicall" : "1000", "ranger.usersync.policymanager.mockrun" : "false", "ranger.usersync.port" : "5151", "ranger.usersync.sink.impl.class" : "org.apache.ranger.unixusersync.process.PolicyMgrUserGroupBuilder", "ranger.usersync.sleeptimeinmillisbetweensynccycle" : "5", "ranger.usersync.source.impl.class" : "org.apache.ranger.unixusersync.process.UnixUserGroupBuilder", "ranger.usersync.ssl" : "true", "ranger.usersync.truststore.file" : "./conf/cert/mytruststore.jks", "ranger.usersync.unix.minUserId" : "500" } } }, { "ranger-kafka-audit" : { "properties_attributes" : { }, "properties" : { "xasecure.audit.credential.provider.file" : "jceks://file{{credential_file}}", "xasecure.audit.destination.db" : "false", "xasecure.audit.destination.db.jdbc.driver" : "{{jdbc_driver}}", "xasecure.audit.destination.db.jdbc.url" : "{{audit_jdbc_url}}", "xasecure.audit.destination.db.user" : "{{xa_audit_db_user}}", "xasecure.audit.destination.hdfs" : "true", "xasecure.audit.destination.hdfs.batch.filespool.dir" : "/var/log/kafka/audit/hdfs/spool", "xasecure.audit.destination.hdfs.dir" : "hdfs://NAMENODE_HOSTNAME:8020/ranger/audit", "xasecure.audit.destination.solr" : "true", "xasecure.audit.destination.solr.batch.filespool.dir" : "/var/log/kafka/audit/solr/spool", "xasecure.audit.destination.solr.urls" : "{{ranger_audit_solr_urls}}", "xasecure.audit.destination.solr.zookeepers" : "none", "xasecure.audit.is.enabled" : "true", "xasecure.audit.provider.summary.enabled" : "true" } } }, { "ranger-yarn-plugin-properties" : { "properties_attributes" : { }, "properties" : { "REPOSITORY_CONFIG_USERNAME" : "yarn", "common.name.for.certificate" : "", "hadoop.rpc.protection" : "-", "policy_user" : "ambari-qa", "ranger-yarn-plugin-enabled" : "No" } } }, { "ssl-server" : { "properties_attributes" : { }, "properties" : { "ssl.server.keystore.location" : "/etc/security/serverKeys/keystore.jks", "ssl.server.keystore.type" : "jks", "ssl.server.truststore.location" : "/etc/security/serverKeys/all.jks", "ssl.server.truststore.reload.interval" : "10000", "ssl.server.truststore.type" : "jks" } } }, { "ranger-hdfs-audit" : { "properties_attributes" : { }, "properties" : { "xasecure.audit.credential.provider.file" : "jceks://file{{credential_file}}", "xasecure.audit.destination.db" : "false", "xasecure.audit.destination.db.jdbc.driver" : "{{jdbc_driver}}", "xasecure.audit.destination.db.jdbc.url" : "{{audit_jdbc_url}}", "xasecure.audit.destination.db.user" : "{{xa_audit_db_user}}", "xasecure.audit.destination.hdfs" : "true", "xasecure.audit.destination.hdfs.batch.filespool.dir" : "/var/log/hadoop/audit/hdfs/spool", "xasecure.audit.destination.hdfs.dir" : "hdfs://NAMENODE_HOSTNAME:8020/ranger/audit", "xasecure.audit.destination.solr" : "false", "xasecure.audit.destination.solr.batch.filespool.dir" : "/var/log/hadoop/audit/solr/spool", "xasecure.audit.destination.solr.urls" : "{{ranger_audit_solr_urls}}", "xasecure.audit.destination.solr.zookeepers" : "none", "xasecure.audit.is.enabled" : "true", "xasecure.audit.provider.summary.enabled" : "false" } } }, { "spark-defaults" : { "properties_attributes" : { }, "properties" : { "spark.driver.extraJavaOptions" : "-Dhdp.version={{hdp_full_version}}", "spark.history.kerberos.keytab" : "none", "spark.history.kerberos.principal" : "none", "spark.history.provider" : "org.apache.spark.deploy.yarn.history.YarnHistoryProvider", "spark.history.ui.port" : "18080", "spark.yarn.am.extraJavaOptions" : "-Dhdp.version={{hdp_full_version}}", "spark.yarn.applicationMaster.waitTries" : "10", "spark.yarn.containerLauncherMaxThreads" : "25", "spark.yarn.driver.memoryOverhead" : "384", "spark.yarn.executor.memoryOverhead" : "384", "spark.yarn.historyServer.address" : "{{spark_history_server_host}}:{{spark_history_ui_port}}", "spark.yarn.max.executor.failures" : "3", "spark.yarn.preserve.staging.files" : "false", "spark.yarn.queue" : "default", "spark.yarn.scheduler.heartbeat.interval-ms" : "5000", "spark.yarn.services" : "org.apache.spark.deploy.yarn.history.YarnHistoryService", "spark.yarn.submit.file.replication" : "3" } } }, { "ranger-hdfs-plugin-properties" : { "properties_attributes" : { }, "properties" : { "REPOSITORY_CONFIG_USERNAME" : "hadoop", "common.name.for.certificate" : "", "hadoop.rpc.protection" : "-", "policy_user" : "ambari-qa", "ranger-hdfs-plugin-enabled" : "No" } } }, { "ranger-yarn-audit" : { "properties_attributes" : { }, "properties" : { "xasecure.audit.credential.provider.file" : "jceks://file{{credential_file}}", "xasecure.audit.destination.db" : "false", "xasecure.audit.destination.db.jdbc.driver" : "{{jdbc_driver}}", "xasecure.audit.destination.db.jdbc.url" : "{{audit_jdbc_url}}", "xasecure.audit.destination.db.user" : "{{xa_audit_db_user}}", "xasecure.audit.destination.hdfs" : "true", "xasecure.audit.destination.hdfs.batch.filespool.dir" : "/var/log/yarn/audit/hdfs/spool", "xasecure.audit.destination.hdfs.dir" : "hdfs://NAMENODE_HOSTNAME:8020/ranger/audit", "xasecure.audit.destination.solr" : "false", "xasecure.audit.destination.solr.batch.filespool.dir" : "/var/log/yarn/audit/solr/spool", "xasecure.audit.destination.solr.urls" : "{{ranger_audit_solr_urls}}", "xasecure.audit.destination.solr.zookeepers" : "none", "xasecure.audit.is.enabled" : "true", "xasecure.audit.provider.summary.enabled" : "false" } } }, { "ranger-storm-security" : { "properties_attributes" : { }, "properties" : { "ranger.plugin.storm.policy.cache.dir" : "/etc/ranger/{{repo_name}}/policycache", "ranger.plugin.storm.policy.pollIntervalMs" : "30000", "ranger.plugin.storm.policy.rest.ssl.config.file" : "/usr/hdp/current/storm-client/conf/ranger-policymgr-ssl.xml", "ranger.plugin.storm.policy.rest.url" : "{{policymgr_mgr_url}}", "ranger.plugin.storm.policy.source.impl" : "org.apache.ranger.admin.client.RangerAdminRESTClient", "ranger.plugin.storm.service.name" : "{{repo_name}}" } } }, { "ranger-kafka-plugin-properties" : { "properties_attributes" : { }, "properties" : { "REPOSITORY_CONFIG_USERNAME" : "kafka", "common.name.for.certificate" : "-", "hadoop.rpc.protection" : "-", "policy_user" : "ambari-qa", "ranger-kafka-plugin-enabled" : "No", "zookeeper.connect" : "localhost:2181" } } }, { "ranger-hbase-audit" : { "properties_attributes" : { }, "properties" : { "xasecure.audit.credential.provider.file" : "jceks://file{{credential_file}}", "xasecure.audit.destination.db" : "false", "xasecure.audit.destination.db.jdbc.driver" : "{{jdbc_driver}}", "xasecure.audit.destination.db.jdbc.url" : "{{audit_jdbc_url}}", "xasecure.audit.destination.db.user" : "{{xa_audit_db_user}}", "xasecure.audit.destination.hdfs" : "true", "xasecure.audit.destination.hdfs.batch.filespool.dir" : "/var/log/hbase/audit/hdfs/spool", "xasecure.audit.destination.hdfs.dir" : "hdfs://NAMENODE_HOSTNAME:8020/ranger/audit", "xasecure.audit.destination.solr" : "false", "xasecure.audit.destination.solr.batch.filespool.dir" : "/var/log/hbase/audit/solr/spool", "xasecure.audit.destination.solr.urls" : "{{ranger_audit_solr_urls}}", "xasecure.audit.destination.solr.zookeepers" : "none", "xasecure.audit.is.enabled" : "true", "xasecure.audit.provider.summary.enabled" : "true" } } }, { "ranger-hdfs-policymgr-ssl" : { "properties_attributes" : { }, "properties" : { "xasecure.policymgr.clientssl.keystore" : "/etc/hadoop/conf/ranger-plugin-keystore.jks", "xasecure.policymgr.clientssl.keystore.credential.file" : "jceks://file{{credential_file}}", "xasecure.policymgr.clientssl.truststore" : "/etc/hadoop/conf/ranger-plugin-truststore.jks", "xasecure.policymgr.clientssl.truststore.credential.file" : "jceks://file{{credential_file}}" } } }, { "ranger-kafka-security" : { "properties_attributes" : { }, "properties" : { "ranger.plugin.kafka.policy.cache.dir" : "/etc/ranger/{{repo_name}}/policycache", "ranger.plugin.kafka.policy.pollIntervalMs" : "30000", "ranger.plugin.kafka.policy.rest.ssl.config.file" : "/etc/kafka/conf/ranger-policymgr-ssl.xml", "ranger.plugin.kafka.policy.rest.url" : "{{policymgr_mgr_url}}", "ranger.plugin.kafka.policy.source.impl" : "org.apache.ranger.admin.client.RangerAdminRESTClient", "ranger.plugin.kafka.service.name" : "{{repo_name}}" } } }, { "ranger-hbase-plugin-properties" : { "properties_attributes" : { }, "properties" : { "REPOSITORY_CONFIG_USERNAME" : "hbase", "common.name.for.certificate" : "", "policy_user" : "ambari-qa", "ranger-hbase-plugin-enabled" : "No" } } }, { "ranger-storm-plugin-properties" : { "properties_attributes" : { }, "properties" : { "REPOSITORY_CONFIG_USERNAME" : "stormtestuser@EXAMPLE.COM", "common.name.for.certificate" : "", "policy_user" : "storm", "ranger-storm-plugin-enabled" : "No" } } }, { "ranger-admin-site" : { "properties_attributes" : { }, "properties" : { "ranger.audit.solr.urls" : "http://solr_host:6083/solr/ranger_audits", "ranger.audit.solr.username" : "ranger_solr", "ranger.audit.solr.zookeepers" : "NONE", "ranger.audit.source.type" : "db", "ranger.authentication.method" : "UNIX", "ranger.credential.provider.path" : "/etc/ranger/admin/rangeradmin.jceks", "ranger.externalurl" : "{{ranger_external_url}}", "ranger.https.attrib.keystore.file" : "/etc/ranger/admin/keys/server.jks", "ranger.jpa.audit.jdbc.credential.alias" : "rangeraudit", "ranger.jpa.audit.jdbc.dialect" : "{{jdbc_dialect}}", "ranger.jpa.audit.jdbc.driver" : "{{ranger_jdbc_driver}}", "ranger.jpa.audit.jdbc.url" : "{{audit_jdbc_url}}", "ranger.jpa.audit.jdbc.user" : "{{ranger_audit_db_user}}", "ranger.jpa.jdbc.credential.alias" : "rangeradmin", "ranger.jpa.jdbc.dialect" : "{{jdbc_dialect}}", "ranger.jpa.jdbc.driver" : "com.mysql.jdbc.Driver", "ranger.jpa.jdbc.url" : "jdbc:mysql://sandbox.hortonworks.com/ranger", "ranger.jpa.jdbc.user" : "{{ranger_db_user}}", "ranger.ldap.ad.domain" : "localhost", "ranger.ldap.ad.url" : "ldap://ad.xasecure.net:389", "ranger.ldap.group.roleattribute" : "cn", "ranger.ldap.group.searchbase" : "ou=groups,dc=xasecure,dc=net", "ranger.ldap.group.searchfilter" : "(member=uid={0},ou=users,dc=xasecure,dc=net)", "ranger.ldap.url" : "ldap://71.127.43.33:389", "ranger.ldap.user.dnpattern" : "uid={0},ou=users,dc=xasecure,dc=net", "ranger.service.host" : "{{ranger_host}}", "ranger.service.http.enabled" : "true", "ranger.service.http.port" : "6080", "ranger.service.https.attrib.clientAuth" : "false", "ranger.service.https.attrib.keystore.keyalias" : "mkey", "ranger.service.https.attrib.keystore.pass" : "ranger", "ranger.service.https.attrib.ssl.enabled" : "false", "ranger.service.https.port" : "6182", "ranger.unixauth.remote.login.enabled" : "true", "ranger.unixauth.service.hostname" : "localhost", "ranger.unixauth.service.port" : "5151" } } }, { "ranger-hdfs-security" : { "properties_attributes" : { }, "properties" : { "ranger.plugin.hdfs.policy.cache.dir" : "/etc/ranger/{{repo_name}}/policycache", "ranger.plugin.hdfs.policy.pollIntervalMs" : "30000", "ranger.plugin.hdfs.policy.rest.ssl.config.file" : "/etc/hadoop/conf/ranger-policymgr-ssl.xml", "ranger.plugin.hdfs.policy.rest.url" : "{{policymgr_mgr_url}}", "ranger.plugin.hdfs.policy.source.impl" : "org.apache.ranger.admin.client.RangerAdminRESTClient", "ranger.plugin.hdfs.service.name" : "{{repo_name}}", "xasecure.add-hadoop-authorization" : "true" } } }, { "ranger-hive-security" : { "properties_attributes" : { }, "properties" : { "ranger.plugin.hive.policy.cache.dir" : "/etc/ranger/{{repo_name}}/policycache", "ranger.plugin.hive.policy.pollIntervalMs" : "30000", "ranger.plugin.hive.policy.rest.ssl.config.file" : "/usr/hdp/current/hive-server2/conf/ranger-policymgr-ssl.xml", "ranger.plugin.hive.policy.rest.url" : "{{policymgr_mgr_url}}", "ranger.plugin.hive.policy.source.impl" : "org.apache.ranger.admin.client.RangerAdminRESTClient", "ranger.plugin.hive.service.name" : "{{repo_name}}", "xasecure.hive.update.xapolicies.on.grant.revoke" : "true" } } }, { "ranger-hbase-security" : { "properties_attributes" : { }, "properties" : { "ranger.plugin.hbase.policy.cache.dir" : "/etc/ranger/{{repo_name}}/policycache", "ranger.plugin.hbase.policy.pollIntervalMs" : "30000", "ranger.plugin.hbase.policy.rest.ssl.config.file" : "/etc/hbase/conf/ranger-policymgr-ssl.xml", "ranger.plugin.hbase.policy.rest.url" : "{{policymgr_mgr_url}}", "ranger.plugin.hbase.policy.source.impl" : "org.apache.ranger.admin.client.RangerAdminRESTClient", "ranger.plugin.hbase.service.name" : "{{repo_name}}", "xasecure.hbase.update.xapolicies.on.grant.revoke" : "true" } } }, { "ranger-storm-policymgr-ssl" : { "properties_attributes" : { }, "properties" : { "xasecure.policymgr.clientssl.keystore" : "/usr/hdp/current/storm-client/conf/ranger-plugin-keystore.jks", "xasecure.policymgr.clientssl.keystore.credential.file" : "jceks://file{{credential_file}}", "xasecure.policymgr.clientssl.truststore" : "/usr/hdp/current/storm-client/conf/ranger-plugin-truststore.jks", "xasecure.policymgr.clientssl.truststore.credential.file" : "jceks://file{{credential_file}}" } } }, { "ranger-kms-policymgr-ssl" : { "properties_attributes" : { }, "properties" : { "xasecure.policymgr.clientssl.keystore" : "/etc/ranger/kms/conf/ranger-plugin-keystore.jks", "xasecure.policymgr.clientssl.keystore.credential.file" : "jceks://file{{credential_file}}", "xasecure.policymgr.clientssl.truststore" : "/etc/ranger/kms/conf/ranger-plugin-truststore.jks", "xasecure.policymgr.clientssl.truststore.credential.file" : "jceks://file{{credential_file}}" } } }, { "ranger-site" : { "properties_attributes" : { }, "properties" : { } } }, { "ranger-kms-audit" : { "properties_attributes" : { }, "properties" : { "xasecure.audit.credential.provider.file" : "jceks://file{{credential_file}}", "xasecure.audit.destination.db" : "false", "xasecure.audit.destination.db.jdbc.driver" : "{{jdbc_driver}}", "xasecure.audit.destination.db.jdbc.url" : "{{audit_jdbc_url}}", "xasecure.audit.destination.db.user" : "{{xa_audit_db_user}}", "xasecure.audit.destination.hdfs" : "true", "xasecure.audit.destination.hdfs.batch.filespool.dir" : "/var/log/kms/audit/hdfs/spool", "xasecure.audit.destination.hdfs.dir" : "hdfs://NAMENODE_HOSTNAME:8020/ranger/audit", "xasecure.audit.destination.solr" : "true", "xasecure.audit.destination.solr.batch.filespool.dir" : "/var/log/kms/audit/solr/spool", "xasecure.audit.destination.solr.urls" : "{{ranger_audit_solr_urls}}", "xasecure.audit.destination.solr.zookeepers" : "none", "xasecure.audit.is.enabled" : "true", "xasecure.audit.provider.summary.enabled" : "false" } } }, { "kms-log4j" : { "properties_attributes" : { }, "properties" : { "content" : "n#n# Licensed under the Apache License, Version 2.0 (the "License");n# you may not use this file except in compliance with the License.n# You may obtain a copy of the License atn#n# http://www.apache.org/licenses/LICENSE-2.0n#n# Unless required by applicable law or agreed to in writing, softwaren# distributed under the License is distributed on an "AS IS" BASIS,n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.n# See the License for the specific language governing permissions andn# limitations under the License. See accompanying LICENSE file.n#nn# If the Java System property 'kms.log.dir' is not defined at KMS start up timen# Setup sets its value to '${kms.home}/logs'nnlog4j.appender.kms=org.apache.log4j.DailyRollingFileAppendernlog4j.appender.kms.DatePattern='.'yyyy-MM-ddnlog4j.appender.kms.File=${kms.log.dir}/kms.lognlog4j.appender.kms.Append=truenlog4j.appender.kms.layout=org.apache.log4j.PatternLayoutnlog4j.appender.kms.layout.ConversionPattern=%d{ISO8601} %-5p %c{1} - %m%nnnlog4j.appender.kms-audit=org.apache.log4j.DailyRollingFileAppendernlog4j.appender.kms-audit.DatePattern='.'yyyy-MM-ddnlog4j.appender.kms-audit.File=${kms.log.dir}/kms-audit.lognlog4j.appender.kms-audit.Append=truenlog4j.appender.kms-audit.layout=org.apache.log4j.PatternLayoutnlog4j.appender.kms-audit.layout.ConversionPattern=%d{ISO8601} %m%nnnlog4j.logger.kms-audit=INFO, kms-auditnlog4j.additivity.kms-audit=falsennlog4j.rootLogger=ALL, kmsnlog4j.logger.org.apache.hadoop.conf=ERRORnlog4j.logger.org.apache.hadoop=INFOnlog4j.logger.com.sun.jersey.server.wadl.generators.WadlGeneratorJAXBGrammarGenerator=OFF" } } }, { "ranger-kms-site" : { "properties_attributes" : { }, "properties" : { "ranger.contextName" : "/kms", "ranger.service.host" : "{{ranger_admin_hosts}}", "ranger.service.http.port" : "9292", "ranger.service.shutdown.port" : "7085", "xa.webapp.dir" : "./webapp" } } }, { "ranger-hive-policymgr-ssl" : { "properties_attributes" : { }, "properties" : { "xasecure.policymgr.clientssl.keystore" : "/etc/hive/conf/ranger-plugin-keystore.jks", "xasecure.policymgr.clientssl.keystore.credential.file" : "jceks://file{{credential_file}}", "xasecure.policymgr.clientssl.truststore" : "/etc/hive/conf/ranger-plugin-truststore.jks", "xasecure.policymgr.clientssl.truststore.credential.file" : "jceks://file{{credential_file}}" } } }, { "kms-env" : { "properties_attributes" : { }, "properties" : { "kms_group" : "kms", "kms_log_dir" : "/var/log/ranger/kms", "kms_port" : "9292", "kms_user" : "kms" } } }, { "ranger-env" : { "properties_attributes" : { }, "properties" : { "admin_username" : "admin", "create_db_dbuser" : "true", "ranger_admin_log_dir" : "/var/log/ranger/admin", "ranger_admin_username" : "amb_ranger_admin", "ranger_group" : "ranger", "ranger_pid_dir" : "/var/run/ranger", "ranger_user" : "ranger", "ranger_usersync_log_dir" : "/var/log/ranger/usersync", "xml_configurations_supported" : "true" } } }, { "kms-properties" : { "properties_attributes" : { }, "properties" : { "DB_FLAVOR" : "MYSQL", "KMS_MASTER_KEY_PASSWD" : "123", "REPOSITORY_CONFIG_USERNAME" : "keyadmin", "SQL_CONNECTOR_JAR" : "/usr/share/java/mysql-connector-java.jar", "db_host" : "sandbox.hortonworks.com", "db_name" : "rangerkms", "db_root_user" : "root", "db_user" : "rangerkms" } } }, { "ranger-hive-plugin-properties" : { "properties_attributes" : { }, "properties" : { "REPOSITORY_CONFIG_USERNAME" : "hive", "common.name.for.certificate" : "", "jdbc.driverClassName" : "org.apache.hive.jdbc.HiveDriver", "policy_user" : "ambari-qa", "ranger-hive-plugin-enabled" : "No" } } }, { "ranger-yarn-security" : { "properties_attributes" : { }, "properties" : { "ranger.plugin.yarn.policy.cache.dir" : "/etc/ranger/{{repo_name}}/policycache", "ranger.plugin.yarn.policy.pollIntervalMs" : "30000", "ranger.plugin.yarn.policy.rest.ssl.config.file" : "/etc/yarn/conf/ranger-policymgr-ssl.xml", "ranger.plugin.yarn.policy.rest.url" : "{{policymgr_mgr_url}}", "ranger.plugin.yarn.policy.source.impl" : "org.apache.ranger.admin.client.RangerAdminRESTClient", "ranger.plugin.yarn.service.name" : "{{repo_name}}" } } }, { "ranger-yarn-policymgr-ssl" : { "properties_attributes" : { }, "properties" : { "xasecure.policymgr.clientssl.keystore" : "/etc/hadoop/conf/ranger-plugin-keystore.jks", "xasecure.policymgr.clientssl.keystore.credential.file" : "jceks://file{{credential_file}}", "xasecure.policymgr.clientssl.truststore" : "/etc/hadoop/conf/ranger-plugin-truststore.jks", "xasecure.policymgr.clientssl.truststore.credential.file" : "jceks://file{{credential_file}}" } } }, { "ranger-storm-audit" : { "properties_attributes" : { }, "properties" : { "xasecure.audit.credential.provider.file" : "jceks://file{{credential_file}}", "xasecure.audit.destination.db" : "false", "xasecure.audit.destination.db.jdbc.driver" : "{{jdbc_driver}}", "xasecure.audit.destination.db.jdbc.url" : "{{audit_jdbc_url}}", "xasecure.audit.destination.db.user" : "{{xa_audit_db_user}}", "xasecure.audit.destination.hdfs" : "true", "xasecure.audit.destination.hdfs.batch.filespool.dir" : "/var/log/storm/audit/hdfs/spool", "xasecure.audit.destination.hdfs.dir" : "hdfs://NAMENODE_HOSTNAME:8020/ranger/audit", "xasecure.audit.destination.solr" : "false", "xasecure.audit.destination.solr.batch.filespool.dir" : "/var/log/storm/audit/solr/spool", "xasecure.audit.destination.solr.urls" : "{{ranger_audit_solr_urls}}", "xasecure.audit.destination.solr.zookeepers" : "none", "xasecure.audit.is.enabled" : "true", "xasecure.audit.provider.summary.enabled" : "false" } } }, { "ranger-kafka-policymgr-ssl" : { "properties_attributes" : { }, "properties" : { "xasecure.policymgr.clientssl.keystore" : "/etc/kafka/conf/ranger-plugin-keystore.jks", "xasecure.policymgr.clientssl.keystore.credential.file" : "jceks://file/{{credential_file}}", "xasecure.policymgr.clientssl.truststore" : "/etc/kafka/conf/ranger-plugin-truststore.jks", "xasecure.policymgr.clientssl.truststore.credential.file" : "jceks://file/{{credential_file}}" } } }, { "ranger-hbase-policymgr-ssl" : { "properties_attributes" : { }, "properties" : { "xasecure.policymgr.clientssl.keystore" : "/etc/hbase/conf/ranger-plugin-keystore.jks", "xasecure.policymgr.clientssl.keystore.credential.file" : "jceks://file{{credential_file}}", "xasecure.policymgr.clientssl.truststore" : "/etc/hbase/conf/ranger-plugin-truststore.jks", "xasecure.policymgr.clientssl.truststore.credential.file" : "jceks://file{{credential_file}}" } } }, { "ranger-hive-audit" : { "properties_attributes" : { }, "properties" : { "xasecure.audit.credential.provider.file" : "jceks://file{{credential_file}}", "xasecure.audit.destination.db" : "false", "xasecure.audit.destination.db.jdbc.driver" : "{{jdbc_driver}}", "xasecure.audit.destination.db.jdbc.url" : "{{audit_jdbc_url}}", "xasecure.audit.destination.db.user" : "{{xa_audit_db_user}}", "xasecure.audit.destination.hdfs" : "true", "xasecure.audit.destination.hdfs.batch.filespool.dir" : "/var/log/hive/audit/hdfs/spool", "xasecure.audit.destination.hdfs.dir" : "hdfs://NAMENODE_HOSTNAME:8020/ranger/audit", "xasecure.audit.destination.solr" : "false", "xasecure.audit.destination.solr.batch.filespool.dir" : "/var/log/hive/audit/solr/spool", "xasecure.audit.destination.solr.urls" : "{{ranger_audit_solr_urls}}", "xasecure.audit.destination.solr.zookeepers" : "none", "xasecure.audit.is.enabled" : "true", "xasecure.audit.provider.summary.enabled" : "false" } } } ] }
https://issues.apache.org/jira/browse/AMBARI-12413
LikeLike
@David
Wasn’t an issue for me if I specified ranger.jpa.jdbc.url and ranger.jpa.jdbc.url and ranger.jpa.audit.jdbc.url in ranger-admin-site:
Ambari 2.2.1
{
“ranger-admin-site” : {
“properties_attributes” : { },
“properties” : {
“ranger.jpa.jdbc.url” : “jdbc:mysql://mn03.vagrant:3306/ranger”,
“ranger.jpa.audit.jdbc.url” : “jdbc:mysql://mn03.vagrant:3306/rangeradmin”
}
}
},
LikeLike