Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 23 additions & 14 deletions Containerfile
Original file line number Diff line number Diff line change
Expand Up @@ -10,42 +10,51 @@ LABEL name='streams-e2e' \
ENV STREAMS_HOME=/opt/streams-e2e
ENV KUBECONFIG=/opt/kubeconfig/config
ENV OPERATOR_SDK_VERSION=1.41.1

COPY . /opt/streams-e2e
ENV HELM_VERSION=3.17.3

USER root
RUN microdnf --setopt=install_weak_deps=0 --setopt=tsflags=nodocs install -y unzip git && microdnf clean all
RUN microdnf --setopt=install_weak_deps=0 --setopt=tsflags=nodocs install -y unzip git bsdtar && microdnf clean all

# Install kubectl, oc, operator-sdk and helm3 clients
# Install kubectl, oc, operator-sdk and helm clients
RUN export ARCH=$(case $(uname -m) in x86_64) echo -n amd64 ;; aarch64) echo -n arm64 ;; *) echo -n $(uname -m) ;; esac) && \
export OS=$(uname | awk '{print tolower($0)}') && \
export OPERATOR_SDK_DL_URL=https://github.com/operator-framework/operator-sdk/releases/download/v${OPERATOR_SDK_VERSION} && \
curl -L "https://mirror.openshift.com/pub/openshift-v4/clients/ocp/stable/openshift-client-linux-${ARCH}-rhel9.tar.gz" -o openshift-client-linux.tar.gz && \
tar -xzf openshift-client-linux.tar.gz && \
bsdtar -xzf openshift-client-linux.tar.gz && \
chmod +x oc kubectl && \
mv oc /usr/local/bin/ && \
mv kubectl /usr/local/bin/ && \
rm -f openshift-client-linux.tar.gz README.md && \
curl -LO ${OPERATOR_SDK_DL_URL}/operator-sdk_${OS}_${ARCH} && \
chmod +x operator-sdk_${OS}_${ARCH} && \
mv operator-sdk_${OS}_${ARCH} /usr/local/bin/operator-sdk && \
curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 && \
chmod 700 get_helm.sh && \
./get_helm.sh

mv operator-sdk_${OS}_${ARCH} /usr/local/bin/operator-sdk && \
curl -L "https://get.helm.sh/helm-v${HELM_VERSION}-${OS}-${ARCH}.tar.gz" -o helm.tar.gz && \
bsdtar -xzf helm.tar.gz && \
mv ${OS}-${ARCH}/helm /usr/local/bin/helm && \
chmod +x /usr/local/bin/helm && \
rm -rf helm.tar.gz ${OS}-${ARCH}

RUN mkdir -p /opt/kubeconfig && chown 185:0 /opt/kubeconfig && \
chown -R 185:0 /opt/streams-e2e && chmod +x /opt/streams-e2e/mvnw
mkdir -p /opt/streams-e2e && chown -R 185:0 /opt/streams-e2e

# Copy only build definition files first to cache dependency resolution
COPY --chown=185:0 pom.xml mvnw /opt/streams-e2e/
COPY --chown=185:0 .mvn /opt/streams-e2e/.mvn

USER 185

WORKDIR $STREAMS_HOME

# Cache dependencies - only re-runs when pom.xml or wrapper changes
RUN ./mvnw dependency:go-offline -B -q

# Copy full source
COPY --chown=185:0 . /opt/streams-e2e

VOLUME ["/opt/kubeconfig"]
VOLUME ["${STREAMS_HOME}/operator-install-files"]

RUN ./mvnw dependency:go-offline -B -q \
&& ./mvnw install -Pget-operator-files \
&& ./mvnw compile test-compile -B -q -Dcheckstyle.skip=true
# Download operator files (generate-sources) + compile main and test in one pass
RUN ./mvnw test-compile -Pget-operator-files -B -q -Dcheckstyle.skip=true

CMD ["./mvnw", "verify", "-Ptest"]
8 changes: 4 additions & 4 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
<maven.compiler.release>21</maven.compiler.release>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

<kubetest4j.version>1.0.0</kubetest4j.version>
<kubetest4j.version>1.1.0</kubetest4j.version>
<skodjob-doc.version>0.6.0</skodjob-doc.version>
<fabric8.version>7.6.1</fabric8.version>
<log4j.version>2.25.3</log4j.version>
Expand Down Expand Up @@ -57,15 +57,15 @@
<debezium.operator.files.destination>debezium</debezium.operator.files.destination>
<keycloak.operator.files.destination>keycloak</keycloak.operator.files.destination>
<!-- Operators versions -->
<strimzi.version>0.51.0</strimzi.version>
<strimzi.version>1.0.0</strimzi.version>
<flink.version>1.12.1</flink.version>
<cert.manager.version>1.18.2</cert.manager.version>
<apicurio.registry.version>1.1.3-v2.6.4.final</apicurio.registry.version>
<apicurio.api.version>1.1.2</apicurio.api.version>
<apicurio.api.version>1.1.3</apicurio.api.version>
<apicurio.lib.version>2.6.13.Final</apicurio.lib.version>
<debezium.operator.version>3.1.1</debezium.operator.version>
<kafka.version>4.2.0</kafka.version>
<keycloak.version>26.3.3</keycloak.version>
<keycloak.version>26.6.1</keycloak.version>
</properties>

<dependencyManagement>
Expand Down
4 changes: 2 additions & 2 deletions src/test/java/io/streams/e2e/flink/sql/SqlExampleST.java
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@
@Label(value = FLINK),
}
)
public class SqlExampleST extends Abstract {
class SqlExampleST extends Abstract {

String namespace = "flink";
Path exampleFiles = TestConstants.YAML_MANIFEST_PATH.resolve("examples").resolve("sql-example");
Expand Down Expand Up @@ -99,7 +99,7 @@ void prepareOperators() {
}
)
@Test
void testRecommendationApp() throws IOException {
void testRecommendationApp() {
Allure.step("Prepare " + namespace + " namespace", () -> {
// Create namespace
KubeResourceManager.get().createOrUpdateResourceWithWait(
Expand Down
2 changes: 1 addition & 1 deletion src/test/java/io/streams/e2e/flink/sql/SqlJobRunnerST.java
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@
@Label(value = FLINK),
}
)
public class SqlJobRunnerST extends Abstract {
class SqlJobRunnerST extends Abstract {
final String kafkaClusterName = "my-cluster";

@BeforeAll
Expand Down
49 changes: 37 additions & 12 deletions src/test/java/io/streams/e2e/flink/sql/SqlSecurityST.java
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@
import io.fabric8.kubernetes.api.model.NamespaceBuilder;
import io.fabric8.kubernetes.api.model.Secret;
import io.fabric8.kubernetes.api.model.SecretBuilder;
import io.fabric8.kubernetes.api.model.SecretVolumeSourceBuilder;
import io.fabric8.kubernetes.api.model.VolumeMountBuilder;
import io.qameta.allure.Allure;
import io.skodjob.annotations.Desc;
import io.skodjob.annotations.Label;
Expand Down Expand Up @@ -36,9 +38,8 @@
import io.streams.sql.TestStatements;
import io.streams.utils.StrimziClientUtils;
import io.streams.utils.TestUtils;
import io.strimzi.api.kafka.model.common.CertSecretSourceBuilder;
import io.strimzi.api.kafka.model.kafka.listener.GenericKafkaListenerBuilder;
import io.strimzi.api.kafka.model.kafka.listener.KafkaListenerAuthenticationOAuthBuilder;
import io.strimzi.api.kafka.model.kafka.listener.KafkaListenerAuthenticationCustomBuilder;
import io.strimzi.api.kafka.model.kafka.listener.KafkaListenerAuthenticationScramSha512;
import io.strimzi.api.kafka.model.kafka.listener.KafkaListenerAuthenticationTls;
import io.strimzi.api.kafka.model.kafka.listener.KafkaListenerType;
Expand Down Expand Up @@ -79,7 +80,7 @@
@Label(value = FLINK),
}
)
public class SqlSecurityST extends Abstract {
class SqlSecurityST extends Abstract {
final String kafkaClusterName = "my-cluster";

@BeforeAll
Expand Down Expand Up @@ -222,18 +223,42 @@ void testOauthWithTls() {
.withTls(true)
.withType(KafkaListenerType.INTERNAL)
.withPort((9093))
.withAuth(new KafkaListenerAuthenticationOAuthBuilder()
.withValidIssuerUri(keycloakUrl + "/realms/streams-e2e")
.withJwksEndpointUri(keycloakUrl + "/realms/streams-e2e/protocol/openid-connect/certs")
.withUserNameClaim("preferred_username")
.withTlsTrustedCertificates(new CertSecretSourceBuilder()
.withSecretName("keycloak-tls-secret")
.withCertificate("tls.crt")
.build()
)
.withAuth(new KafkaListenerAuthenticationCustomBuilder()
.withSasl(true)
.addToListenerConfig("sasl.enabled.mechanisms", "OAUTHBEARER")
.addToListenerConfig("oauthbearer.sasl.server.callback.handler.class",
"io.strimzi.kafka.oauth.server.JaasServerOauthValidatorCallbackHandler")
.addToListenerConfig("oauthbearer.sasl.jaas.config",
"org.apache.kafka.common.security.oauthbearer.OAuthBearerLoginModule required"
+ " unsecuredLoginStringClaim_sub=\"thePrincipalName\""
+ " oauth.valid.issuer.uri=\"" + keycloakUrl + "/realms/streams-e2e\""
+ " oauth.jwks.endpoint.uri=\""
+ keycloakUrl + "/realms/streams-e2e/protocol/openid-connect/certs\""
+ " oauth.username.claim=\"preferred_username\""
+ " oauth.ssl.truststore.location=\"/mnt/oauth-certs/tls.crt\""
+ " oauth.ssl.truststore.type=\"PEM\";")
.addToListenerConfig("connections.max.reauth.ms", 3600000)
.build())
.build()
)
.addToConfig("principal.builder.class",
"io.strimzi.kafka.oauth.server.OAuthKafkaPrincipalBuilder")
.editOrNewTemplate()
.editOrNewPod()
.addNewVolume()
.withName("oauth-certs")
.withSecret(new SecretVolumeSourceBuilder()
.withSecretName("keycloak-tls-secret")
.build())
.endVolume()
.endPod()
.editOrNewKafkaContainer()
.addToVolumeMounts(new VolumeMountBuilder()
.withName("oauth-certs")
.withMountPath("/mnt/oauth-certs")
.build())
.endKafkaContainer()
.endTemplate()
.endKafka()
.endSpec()
.build());
Expand Down
2 changes: 1 addition & 1 deletion src/test/java/io/streams/unit/SqlWithTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@

import static org.junit.jupiter.api.Assertions.assertEquals;

public class SqlWithTest {
class SqlWithTest {
@Test
void testCreateTableWithFilesystemConnector() {
String expectedSql = "CREATE TABLE ProductInventoryTable ( product_id STRING, category STRING, stock STRING, rating STRING ) " +
Expand Down
Loading