Skip to content

Commit e62366c

Browse files
sbernauersbernauer
committed
chore: Remodel URLs to hostname, port, path and TLS client details
1 parent e9aedb9 commit e62366c

10 files changed

Lines changed: 465 additions & 135 deletions

File tree

‎docs/modules/trino/pages/usage-guide/catalogs/iceberg.adoc‎

Lines changed: 30 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -23,11 +23,24 @@ spec:
2323
iceberg:
2424
catalog:
2525
rest:
26-
uri: https://iceberg-rest-catalog.example.com # <2>
27-
authentication: # <3>
26+
hostname: iceberg-rest-catalog.example.com # <2>
27+
port: 9001 # <3>
28+
path: /catalog # <4>
29+
tls: # <5>
30+
verification:
31+
server:
32+
caCert:
33+
secretClass: tls
34+
authentication: # <6>
2835
oAuth2:
29-
serverUri: https://idp.example.com/oauth2/token # <4>
30-
clientCredentialsSecretName: trino-iceberg-client # <5>
36+
hostname: idp.example.com # <7>
37+
path: /oauth2/token
38+
tls:
39+
verification:
40+
server:
41+
caCert:
42+
webPki: {}
43+
clientCredentialsSecretName: trino-iceberg-client # <8>
3144
s3:
3245
inline:
3346
host: test-minio
@@ -45,19 +58,23 @@ stringData:
4558
clientSecret: my-client-secret
4659
----
4760
<1> The name of the catalog as it will appear in Trino
48-
<2> The URI of the REST catalog server
49-
<3> Optional, defaults to no authentication
50-
<4> The token endpoint of the OAuth2 server, see <<oauth2-server-uri>>
51-
<5> Name of a Secret containing the `clientId` and `clientSecret` keys, which are exchanged for a token using the OAuth2 client credentials flow.
61+
<2> The hostname of the REST catalog server
62+
<3> Optional, defaults to 443 if TLS is used and to 80 otherwise
63+
<4> Optional, defaults to `/`
64+
<5> Optional, uses the same TLS configuration as xref:usage-guide/s3.adoc#tls[S3 connections].
65+
If not specified, no TLS is used. Disabling the TLS verification (`verification.none`) is not supported.
66+
<6> Optional, defaults to no authentication
67+
<7> `hostname`, `port`, `path` and `tls` of the token endpoint of the OAuth2 server, see <<oauth2-server>>
68+
<8> Name of a Secret containing the `clientId` and `clientSecret` keys, which are exchanged for a token using the OAuth2 client credentials flow.
5269
Alternatively, use `tokenSecretName` to pass a Secret containing a static bearer token in the `token` key.
5370

5471
The operator only supports a subset of the https://trino.io/docs/current/object-storage/metastores.html#rest-catalog[REST catalog properties].
5572
Use `configOverrides` to set additional ones.
5673

57-
[#oauth2-server-uri]
58-
=== OAuth2 server URI
74+
[#oauth2-server]
75+
=== OAuth2 server
5976

60-
`serverUri` is the token endpoint of the OAuth2 server.
77+
`hostname`, `port`, `path` and `tls` within `oAuth2` point to the token endpoint of the OAuth2 server.
6178
What it is used for depends on the credential:
6279

6380
* `clientCredentialsSecretName`: Trino fetches its token from this endpoint on startup and refreshes it there before it expires.
@@ -66,8 +83,8 @@ What it is used for depends on the credential:
6683
If the refresh fails, Trino logs a warning and keeps using the current token.
6784
A token that isn't a JWT is never refreshed, so the endpoint is never called.
6885

69-
`serverUri` is required even for tokens that are never refreshed.
70-
Without it, the Iceberg client used by Trino would fall back to the deprecated `<uri>/v1/oauth/tokens` endpoint of the REST catalog, which is planned to be removed (see https://github.com/apache/iceberg/issues/10537[apache/iceberg#10537]).
86+
The OAuth2 server is required even for tokens that are never refreshed.
87+
Without it, the Iceberg client used by Trino would fall back to the deprecated `/v1/oauth/tokens` endpoint of the REST catalog, which is planned to be removed (see https://github.com/apache/iceberg/issues/10537[apache/iceberg#10537]).
7188

7289
== Example Iceberg Hive metastore configuration
7390

‎extra/crds.yaml‎

Lines changed: 136 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -4105,6 +4105,17 @@ spec:
41054105
description: |-
41064106
Use OAuth2 to authenticate against the REST catalog.
41074107
4108+
`hostname`, `port`, `path` and `tls` point to the token endpoint of the OAuth2 server.
4109+
It is used to
4110+
4111+
* fetch a token using the client credentials (`clientCredentialsSecretName`) and
4112+
* refresh the token before it expires. A static token (`tokenSecretName`) is only
4113+
refreshed if it is a JWT with an `exp` claim, otherwise the endpoint is never called.
4114+
4115+
It is required nevertheless, as without it the Iceberg client would fall back to the
4116+
deprecated `/v1/oauth/tokens` endpoint of the REST catalog, which is planned to be removed
4117+
(see <https://github.com/apache/iceberg/issues/10537>).
4118+
41084119
Note that we only support configuring a subset of Trino's properties, you might need to use
41094120
`configOverrides` to be able to set all [available properties](https://trino.io/docs/current/object-storage/metastores.html#iceberg-specific-metastores).
41104121
oneOf:
@@ -4120,19 +4131,69 @@ spec:
41204131
41214132
The Secret needs to contain the `clientId` and `clientSecret` keys.
41224133
type: string
4123-
serverUri:
4134+
hostname:
41244135
description: |-
4125-
The token endpoint of the OAuth2 server. It is used to
4126-
4127-
* fetch a token using the client credentials (`clientCredentialsSecretName`) and
4128-
* refresh the token before it expires. A static token (`tokenSecretName`) is only
4129-
refreshed if it is a JWT with an `exp` claim, otherwise the endpoint is never called.
4130-
4131-
It is required nevertheless, as without it the Iceberg client would fall back to the
4132-
deprecated `<uri>/v1/oauth/tokens` endpoint of the REST catalog, which is planned to be
4133-
removed (see <https://github.com/apache/iceberg/issues/10537>).
4134-
format: uri
4136+
Hostname of the server without any protocol, port or path.
4137+
For example: `iceberg-rest-catalog.example.com`.
41354138
type: string
4139+
path:
4140+
default: /
4141+
description: Path of the endpoint on the server, e.g. `/catalog`. Defaults to `/`.
4142+
type: string
4143+
port:
4144+
description: |-
4145+
Port the server listens on.
4146+
Defaults to 443 if TLS is used and to 80 otherwise.
4147+
format: uint16
4148+
maximum: 65535.0
4149+
minimum: 0.0
4150+
nullable: true
4151+
type: integer
4152+
tls:
4153+
description: Use a TLS connection. If not specified no TLS will be used.
4154+
nullable: true
4155+
properties:
4156+
verification:
4157+
description: The verification method used to verify the certificates of the server and/or the client.
4158+
oneOf:
4159+
- required:
4160+
- none
4161+
- required:
4162+
- server
4163+
properties:
4164+
none:
4165+
description: Use TLS but don't verify certificates.
4166+
type: object
4167+
server:
4168+
description: Use TLS and a CA certificate to verify the server.
4169+
properties:
4170+
caCert:
4171+
description: CA cert to verify the server.
4172+
oneOf:
4173+
- required:
4174+
- webPki
4175+
- required:
4176+
- secretClass
4177+
properties:
4178+
secretClass:
4179+
description: |-
4180+
Name of the [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) which will provide the CA certificate.
4181+
Note that a SecretClass does not need to have a key but can also work with just a CA certificate,
4182+
so if you got provided with a CA cert but don't have access to the key you can still use this method.
4183+
type: string
4184+
webPki:
4185+
description: |-
4186+
Use TLS and the CA certificates trusted by the common web browsers to verify the server.
4187+
This can be useful when you e.g. use public AWS S3 or other public available services.
4188+
type: object
4189+
type: object
4190+
required:
4191+
- caCert
4192+
type: object
4193+
type: object
4194+
required:
4195+
- verification
4196+
type: object
41364197
tokenSecretName:
41374198
description: |-
41384199
Authenticate using a bearer token.
@@ -4141,15 +4202,74 @@ spec:
41414202
The Secret needs to contain the `token` key.
41424203
type: string
41434204
required:
4144-
- serverUri
4205+
- hostname
41454206
type: object
41464207
type: object
4147-
uri:
4148-
description: URL of the rest catalog server.
4149-
format: uri
4208+
hostname:
4209+
description: |-
4210+
Hostname of the server without any protocol, port or path.
4211+
For example: `iceberg-rest-catalog.example.com`.
4212+
type: string
4213+
path:
4214+
default: /
4215+
description: Path of the endpoint on the server, e.g. `/catalog`. Defaults to `/`.
41504216
type: string
4217+
port:
4218+
description: |-
4219+
Port the server listens on.
4220+
Defaults to 443 if TLS is used and to 80 otherwise.
4221+
format: uint16
4222+
maximum: 65535.0
4223+
minimum: 0.0
4224+
nullable: true
4225+
type: integer
4226+
tls:
4227+
description: Use a TLS connection. If not specified no TLS will be used.
4228+
nullable: true
4229+
properties:
4230+
verification:
4231+
description: The verification method used to verify the certificates of the server and/or the client.
4232+
oneOf:
4233+
- required:
4234+
- none
4235+
- required:
4236+
- server
4237+
properties:
4238+
none:
4239+
description: Use TLS but don't verify certificates.
4240+
type: object
4241+
server:
4242+
description: Use TLS and a CA certificate to verify the server.
4243+
properties:
4244+
caCert:
4245+
description: CA cert to verify the server.
4246+
oneOf:
4247+
- required:
4248+
- webPki
4249+
- required:
4250+
- secretClass
4251+
properties:
4252+
secretClass:
4253+
description: |-
4254+
Name of the [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) which will provide the CA certificate.
4255+
Note that a SecretClass does not need to have a key but can also work with just a CA certificate,
4256+
so if you got provided with a CA cert but don't have access to the key you can still use this method.
4257+
type: string
4258+
webPki:
4259+
description: |-
4260+
Use TLS and the CA certificates trusted by the common web browsers to verify the server.
4261+
This can be useful when you e.g. use public AWS S3 or other public available services.
4262+
type: object
4263+
type: object
4264+
required:
4265+
- caCert
4266+
type: object
4267+
type: object
4268+
required:
4269+
- verification
4270+
type: object
41514271
required:
4152-
- uri
4272+
- hostname
41534273
type: object
41544274
userProvided:
41554275
description: |-

‎rust/operator-binary/src/catalog/commons.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,7 @@ impl ExtendCatalogConfig for s3::v1alpha1::InlineConnectionOrReference {
102102
}
103103

104104
catalog_config.init_container_extra_start_commands.extend(
105-
config::s3::s3_tls_truststore_commands(&s3.tls)
105+
config::tls::tls_truststore_commands(&s3.tls)
106106
.map_err(|_| S3TlsNoVerificationNotSupportedSnafu.build())?,
107107
);
108108

‎rust/operator-binary/src/catalog/iceberg.rs‎

Lines changed: 35 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
use async_trait::async_trait;
2+
use snafu::ResultExt;
23
use stackable_operator::{
34
client::Client, k8s_openapi::api::core::v1::SecretKeySelector,
45
v2::types::kubernetes::NamespaceName,
@@ -8,11 +9,16 @@ use crate::{
89
catalog::{
910
ExtendCatalogConfig, FromTrinoCatalogError, ToCatalogConfig,
1011
config::{CatalogConfig, calculate_env_name},
12+
from_trino_catalog_error::{
13+
BuildIcebergRestUrlSnafu, ConfigureIcebergRestTlsSnafu,
14+
IcebergRestTlsNoVerificationNotSupportedSnafu,
15+
},
1116
},
17+
config,
1218
crd::catalog::{
1319
TrinoCatalogName,
1420
iceberg::{
15-
IcebergCatalogConnection, IcebergRestCatalogAuthentication,
21+
HttpEndpoint, IcebergCatalogConnection, IcebergRestCatalogAuthentication,
1622
IcebergRestCatalogConnection, IcebergRestCatalogOAuthCredential,
1723
},
1824
v1alpha2::IcebergConnector,
@@ -78,22 +84,24 @@ impl ExtendCatalogConfig for IcebergRestCatalogConnection {
7884
_client: &Client,
7985
) -> Result<(), FromTrinoCatalogError> {
8086
catalog_config.add_property("iceberg.catalog.type", "rest");
81-
catalog_config.add_property("iceberg.rest-catalog.uri", self.uri.as_str());
87+
catalog_config.add_property(
88+
"iceberg.rest-catalog.uri",
89+
self.endpoint.url().context(BuildIcebergRestUrlSnafu)?,
90+
);
91+
add_endpoint_tls(catalog_config, &self.endpoint)?;
8292

8393
// We explicitly use a match here to catch further additions
8494
match &self.authentication {
8595
IcebergRestCatalogAuthentication::None {} => {
8696
catalog_config.add_property("iceberg.rest-catalog.security", "NONE");
8797
}
88-
IcebergRestCatalogAuthentication::OAuth2 {
89-
server_uri,
90-
credential,
91-
} => {
98+
IcebergRestCatalogAuthentication::OAuth2 { server, credential } => {
9299
catalog_config.add_property("iceberg.rest-catalog.security", "OAUTH2");
93100
catalog_config.add_property(
94101
"iceberg.rest-catalog.oauth2.server-uri",
95-
server_uri.as_str(),
102+
server.url().context(BuildIcebergRestUrlSnafu)?,
96103
);
104+
add_endpoint_tls(catalog_config, server)?;
97105
match credential {
98106
IcebergRestCatalogOAuthCredential::ClientCredentialsSecretName(secret_name) => {
99107
// We can't use `add_env_property_from_secret`, as we need to concatenate
@@ -144,3 +152,23 @@ impl ExtendCatalogConfig for IcebergRestCatalogConnection {
144152
Ok(())
145153
}
146154
}
155+
156+
/// Adds the CA certificate of the given endpoint (if any) to the truststore of Trino.
157+
fn add_endpoint_tls(
158+
catalog_config: &mut CatalogConfig,
159+
endpoint: &HttpEndpoint,
160+
) -> Result<(), FromTrinoCatalogError> {
161+
let (volumes, mounts) = endpoint
162+
.tls
163+
.volumes_and_mounts()
164+
.context(ConfigureIcebergRestTlsSnafu)?;
165+
catalog_config.volumes.extend(volumes);
166+
catalog_config.volume_mounts.extend(mounts);
167+
168+
catalog_config.init_container_extra_start_commands.extend(
169+
config::tls::tls_truststore_commands(&endpoint.tls)
170+
.map_err(|_| IcebergRestTlsNoVerificationNotSupportedSnafu.build())?,
171+
);
172+
173+
Ok(())
174+
}

0 commit comments

Comments
 (0)