diff --git a/Bigdata/livy/0.8.0/24.03-lts-sp2/Dockerfile b/Bigdata/livy/0.8.0/24.03-lts-sp2/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..8273b10495bb2aaa0b48c26dd62dfd4c81c1407e --- /dev/null +++ b/Bigdata/livy/0.8.0/24.03-lts-sp2/Dockerfile @@ -0,0 +1,48 @@ +ARG BASE=openeuler/openeuler:24.03-lts-sp2 +FROM ${BASE} as builder +ARG VERSION=0.8.0 + +ENV SCALA_VERSION=2.12 \ + SPARK_HOME=/usr/lib/spark \ + SPARK_VERSION=3.4.2 \ + LIVY_HOME=/opt/livy \ + HADOOP_CONF_DIR=/etc/hadoop/conf + +RUN dnf install -y wget unzip && \ + dnf clean all + +# Download livy +RUN wget https://dlcdn.apache.org/incubator/livy/${VERSION}-incubating/apache-livy-${VERSION}-incubating_${SCALA_VERSION}-bin.zip \ + && unzip apache-livy-${VERSION}-incubating_${SCALA_VERSION}-bin.zip \ + && mv apache-livy-${VERSION}-incubating_${SCALA_VERSION}-bin ${LIVY_HOME} \ + && rm -f apache-livy-${VERSION}-incubating_${SCALA_VERSION}-bin.zip + +# Download Spark (as mentioned in quick start for Spark engine) +RUN wget https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop3.tgz \ + && tar -xzf spark-${SPARK_VERSION}-bin-hadoop3.tgz \ + && mv spark-${SPARK_VERSION}-bin-hadoop3 ${SPARK_HOME} \ + && rm -f spark-${SPARK_VERSION}-bin-hadoop3.tgz + + +FROM ${BASE} + +ENV LIVY_HOME=/opt/livy \ + SPARK_HOME=/usr/lib/spark \ + HADOOP_CONF_DIR=/etc/hadoop/conf + +COPY --from=builder ${LIVY_HOME} ${LIVY_HOME} +COPY --from=builder ${SPARK_HOME} ${SPARK_HOME} + +COPY entrypoint.sh /opt/entrypoint.sh +COPY log4j.properties ${LIVY_HOME}/conf/log4j.properties + +RUN dnf install -y java-1.8.0-openjdk \ + && chmod +x /opt/entrypoint.sh \ + && dnf clean all \ + && rm -rf /var/cache/dnf + +EXPOSE 8080 + +WORKDIR ${LIVY_HOME} + +ENTRYPOINT ["/opt/entrypoint.sh"] diff --git a/Bigdata/livy/0.8.0/24.03-lts-sp2/entrypoint.sh b/Bigdata/livy/0.8.0/24.03-lts-sp2/entrypoint.sh new file mode 100644 index 0000000000000000000000000000000000000000..590fa3c8f9fcbfb6ccf7af7626961c1d674a4dea --- /dev/null +++ b/Bigdata/livy/0.8.0/24.03-lts-sp2/entrypoint.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +/opt/livy/bin/livy-server start +tail -f /opt/livy/logs/livy--server.out \ No newline at end of file diff --git a/Bigdata/livy/0.8.0/24.03-lts-sp2/livy.conf b/Bigdata/livy/0.8.0/24.03-lts-sp2/livy.conf new file mode 100644 index 0000000000000000000000000000000000000000..aedc63293c88955dbf746c484a4eba2463d95ce3 --- /dev/null +++ b/Bigdata/livy/0.8.0/24.03-lts-sp2/livy.conf @@ -0,0 +1,194 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Use this keystore for the SSL certificate and key. +# livy.keystore = + +# Specify the keystore password. +# livy.keystore.password = +# +# Specify the key password. +# livy.key-password = + +# Hadoop Credential Provider Path to get "livy.keystore.password" and "livy.key-password". +# Credential Provider can be created using command as follow: +# hadoop credential create "livy.keystore.password" -value "secret" -provider jceks://hdfs/path/to/livy.jceks +# livy.hadoop.security.credential.provider.path = + +# What host address to start the server on. By default, Livy will bind to all network interfaces. +# livy.server.host = 0.0.0.0 + +# What port to start the server on. +# livy.server.port = 8998 + +# What base path ui should work on. By default UI is mounted on "/". +# E.g.: livy.ui.basePath = /my_livy - result in mounting UI on /my_livy/ +# livy.ui.basePath = "" + +# What spark master Livy sessions should use. +# livy.spark.master = local + +# What spark deploy mode Livy sessions should use. +# livy.spark.deploy-mode = + +# Configure Livy server http request and response header size. +# livy.server.request-header.size = 131072 +# livy.server.response-header.size = 131072 + +# Whether or not to send server version in http response. +# livy.server.send-server-version = false + +# Enabled to check whether timeout Livy sessions should be stopped. +# livy.server.session.timeout-check = true +# +# Whether or not to skip timeout check for a busy session +# livy.server.session.timeout-check.skip-busy = false + +# Time in milliseconds on how long Livy will wait before timing out an inactive session. +# Note that the inactive session could be busy running jobs. +# livy.server.session.timeout = 1h +# +# How long a finished session state should be kept in LivyServer for query. +# livy.server.session.state-retain.sec = 600s + +# If livy should impersonate the requesting users when creating a new session. +# livy.impersonation.enabled = false + +# Logs size livy can cache for each session/batch. 0 means don't cache the logs. +# livy.cache-log.size = 200 + +# Comma-separated list of Livy RSC jars. By default Livy will upload jars from its installation +# directory every time a session is started. By caching these files in HDFS, for example, startup +# time of sessions on YARN can be reduced. +# livy.rsc.jars = + +# Comma-separated list of Livy REPL jars. By default Livy will upload jars from its installation +# directory every time a session is started. By caching these files in HDFS, for example, startup +# time of sessions on YARN can be reduced. Please list all the repl dependencies including +# Scala version-specific livy-repl jars, Livy will automatically pick the right dependencies +# during session creation. +# livy.repl.jars = + +# Location of PySpark archives. By default Livy will upload the file from SPARK_HOME, but +# by caching the file in HDFS, startup time of PySpark sessions on YARN can be reduced. +# livy.pyspark.archives = + +# Location of the SparkR package. By default Livy will upload the file from SPARK_HOME, but +# by caching the file in HDFS, startup time of R sessions on YARN can be reduced. +# livy.sparkr.package = + +# List of local directories from where files are allowed to be added to user sessions. By +# default it's empty, meaning users can only reference remote URIs when starting their +# sessions. +# livy.file.local-dir-whitelist = + +# Whether to enable csrf protection, by default it is false. If it is enabled, client should add +# http-header "X-Requested-By" in request if the http method is POST/DELETE/PUT/PATCH. +# livy.server.csrf-protection.enabled = + +# Whether to enable HiveContext in livy interpreter, if it is true hive-site.xml will be detected +# on user request and then livy server classpath automatically. +# livy.repl.enable-hive-context = + +# Recovery mode of Livy. Possible values: +# off: Default. Turn off recovery. Every time Livy shuts down, it stops and forgets all sessions. +# recovery: Livy persists session info to the state store. When Livy restarts, it recovers +# previous sessions from the state store. +# Must set livy.server.recovery.state-store and livy.server.recovery.state-store.url to +# configure the state store. +# livy.server.recovery.mode = off +# Zookeeper address used for HA and state store. e.g. host1:port1, host2:port2 +# livy.server.zookeeper.url = + +# Where Livy should store state to for recovery. Possible values: +# : Default. State store disabled. +# filesystem: Store state on a file system. +# zookeeper: Store state in a Zookeeper instance. +# livy.server.recovery.state-store = + +# For filesystem state store, the path of the state store directory. Please don't use a filesystem +# that doesn't support atomic rename (e.g. S3). e.g. file:///tmp/livy or hdfs:///. +# For zookeeper, the address to the Zookeeper servers. e.g. host1:port1,host2:port2 +# If livy.server.recovery.state-store is zookeeper, this config is for back-compatibility, +# so if both this config and livy.server.zookeeper.url exist, +# livy uses livy.server.zookeeper.url first. +# livy.server.recovery.state-store.url = + +# The policy of curator connecting to zookeeper. +# For example, m, n means retry m times and the interval of retry is n milliseconds. +# Please use the new config: livy.server.zk.retry-policy. +# Keep this config for back-compatibility. +# If both this config and livy.server.zk.retry-policy exist, +# livy uses livy.server.zk.retry-policy first. +# livy.server.recovery.zk-state-store.retry-policy = 5,100 + +# The policy of curator connecting to zookeeper. +# For example, m, n means retry m times and the interval of retry is n milliseconds +# livy.server.zk.retry-policy = + +# The dir in zk to store the data about session. +# livy.server.recovery.zk-state-store.key-prefix = livy + +# If Livy can't find the yarn app within this time, consider it lost. +# livy.server.yarn.app-lookup-timeout = 120s +# When the cluster is busy, we may fail to launch yarn app in app-lookup-timeout, then it would +# cause session leakage, so we need to check session leakage. +# How long to check livy session leakage +# livy.server.yarn.app-leakage.check-timeout = 600s +# how often to check livy session leakage +# livy.server.yarn.app-leakage.check-interval = 60s + +# How often Livy polls YARN to refresh YARN app state. +# livy.server.yarn.poll-interval = 5s +# +# Days to keep Livy server request logs. +# livy.server.request-log-retain.days = 5 + +# If the Livy Web UI should be included in the Livy Server. Enabled by default. +# livy.ui.enabled = true + +# Whether to enable Livy server access control, if it is true then all the income requests will +# be checked if the requested user has permission. +# livy.server.access-control.enabled = false + +# Allowed users to access Livy, by default any user is allowed to access Livy. If user want to +# limit who could access Livy, user should list all the permitted users with comma separated. +# livy.server.access-control.allowed-users = * + +# A list of users with comma separated has the permission to change other user's submitted +# session, like submitting statements, deleting session. +# livy.server.access-control.modify-users = + +# A list of users with comma separated has the permission to view other user's infomation, like +# submitted session state, statement results. +# livy.server.access-control.view-users = +# +# Authentication support for Livy server +# Livy has a built-in SPnego authentication support for HTTP requests with below configurations. +# livy.server.auth.type = kerberos +# livy.server.auth.kerberos.principal = +# livy.server.auth.kerberos.keytab = +# livy.server.auth.kerberos.name-rules = DEFAULT +# +# If user wants to use custom authentication filter, configurations are: +# livy.server.auth.type = +# livy.server.auth..class = +# livy.server.auth..param. = +# livy.server.auth..param. = + +# Enable to allow custom classpath by proxy user in cluster mode +# The below configuration parameter is disabled by default. +# livy.server.session.allow-custom-classpath = true diff --git a/Bigdata/livy/0.8.0/24.03-lts-sp2/log4j.properties b/Bigdata/livy/0.8.0/24.03-lts-sp2/log4j.properties new file mode 100644 index 0000000000000000000000000000000000000000..70b67a6d50d0a18e2ef97b2cffe78201e4ee6deb --- /dev/null +++ b/Bigdata/livy/0.8.0/24.03-lts-sp2/log4j.properties @@ -0,0 +1,24 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# The default Livy logging configuration. +log4j.rootCategory=INFO, console +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n + +log4j.logger.org.eclipse.jetty=WARN diff --git a/Bigdata/livy/README.md b/Bigdata/livy/README.md index 98e5a334688480a923c3c50b1b4857e1a5bde4e6..491fa66b79ae2f3afdbc5627fc360110c98e7794 100644 --- a/Bigdata/livy/README.md +++ b/Bigdata/livy/README.md @@ -18,6 +18,7 @@ The tag of each livy docker image is consist of the version of livy and the vers | Tags | Currently | Architectures| |--------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------|--| +|[0.8.0-oe2403sp2](https://gitee.com/openeuler/openeuler-docker-images/blob/master/Bigdata/livy/0.8.0/24.03-lts-sp2/Dockerfile) | livy 0.8.0 on openEuler 24.03-LTS-SP2 | amd64, arm64 | | [0.8.0-oe2403sp1](https://gitee.com/openeuler/openeuler-docker-images/blob/master/Bigdata/livy/0.8.0/24.03-lts-sp1/Dockerfile) | Livy 0.8.0 on openEuler 24.03-LTS-SP1 | amd64, arm64 | # Usage diff --git a/Bigdata/livy/doc/image-info.yml b/Bigdata/livy/doc/image-info.yml index 7c4abfc3a2be5c38d9cdad09da6ecefe83b5aa0a..6dbd07f41eebffb366e7c430e8b0baab424f305d 100644 --- a/Bigdata/livy/doc/image-info.yml +++ b/Bigdata/livy/doc/image-info.yml @@ -11,6 +11,7 @@ tags: | | Tag | Currently | Architectures | |----------|-------------|------------------| + |[0.8.0-oe2403sp2](https://gitee.com/openeuler/openeuler-docker-images/blob/master/Bigdata/livy/0.8.0/24.03-lts-sp2/Dockerfile) | livy 0.8.0 on openEuler 24.03-LTS-SP2 | amd64, arm64 | |[0.8.0-oe2403sp1](https://gitee.com/openeuler/openeuler-docker-images/blob/master/Bigdata/livy/0.8.0/24.03-lts-sp1/Dockerfile)| Livy 0.8.0 on openEuler 24.03-LTS-SP1 | amd64, arm64 | download: | diff --git a/Bigdata/livy/meta.yml b/Bigdata/livy/meta.yml index c1053ef3264b2c74a825ee0b4407ffab73274a47..6e452ae4fcdcebf482fc86835166d41dd687316b 100644 --- a/Bigdata/livy/meta.yml +++ b/Bigdata/livy/meta.yml @@ -1,2 +1,5 @@ 0.8.0-oe2403sp1: path: 0.8.0/24.03-lts-sp1/Dockerfile + +0.8.0-oe2403sp2: + path: 0.8.0/24.03-lts-sp2/Dockerfile \ No newline at end of file