From 6fa8a01766f89d8e0c00e0de967c6b82c6411277 Mon Sep 17 00:00:00 2001 From: venkat1701 Date: Sun, 25 Oct 2026 21:45:55 +0530 Subject: [PATCH 1/5] feat(security): sign in with SCRAM-SHA-256 on the wire protocol The wire protocol now signs in with SCRAM-SHA-256 (RFC 5802, RFC 7677), with tls-server-end-point channel binding (SCRAM-SHA-256-PLUS) over TLS. The password never crosses the connection, the client checks that the server knows the user's verifier, and a relayed exchange fails. - Users are stored as SCRAM verifiers in the PostgreSQL format. The first open converts the PBKDF2 hashes of older versions in place, without the passwords. - Unknown users get a salt derived from the name and a per-database secret and fail only at the last step, like a wrong password. - AUTHENTICATE ... PASSWORD over the wire protocol is refused with a message to upgrade the client; it still works in embedded sessions and in Studio. hstore connect turns a typed AUTHENTICATE into SCRAM. - CREATE USER and ALTER USER accept a verifier in place of a password, and hstore verifier prints one. - The sign-in throttle is split into reserve and complete so a two-step exchange holds its slot. - Version 0.2.0, because 0.1.0 clients cannot sign in any more. --- benchmarks/pom.xml | 2 +- .../java/io/hstore/bench/HStoreStore.java | 2 +- database/pom.xml | 2 +- .../java/io/hstore/db/HypergraphDatabase.java | 27 ++ .../main/java/io/hstore/db/query/Session.java | 73 ++- .../java/io/hstore/db/security/Scram.java | 417 ++++++++++++++++++ .../java/io/hstore/db/security/Security.java | 91 +++- .../java/io/hstore/db/security/SignIns.java | 54 ++- .../test/java/io/hstore/db/SecurityTest.java | 79 ++++ .../java/io/hstore/db/security/ScramTest.java | 170 +++++++ docs/database/hql.md | 6 + docs/database/security.md | 34 +- docs/operations/cli.md | 23 +- docs/operations/logging.md | 6 +- docs/operations/wire-protocol.md | 131 +++++- engine/pom.xml | 2 +- pom.xml | 2 +- server/pom.xml | 2 +- .../src/main/java/io/hstore/server/Main.java | 11 +- .../java/io/hstore/server/RemoteEndpoint.java | 88 +++- .../main/java/io/hstore/server/Server.java | 96 +++- .../java/io/hstore/server/ServerTest.java | 48 +- 22 files changed, 1266 insertions(+), 100 deletions(-) create mode 100644 database/src/main/java/io/hstore/db/security/Scram.java create mode 100644 database/src/test/java/io/hstore/db/security/ScramTest.java diff --git a/benchmarks/pom.xml b/benchmarks/pom.xml index 5c274f24..934496e5 100644 --- a/benchmarks/pom.xml +++ b/benchmarks/pom.xml @@ -7,7 +7,7 @@ io.hstore hstore-parent - 0.1.0 + 0.2.0 hstore-benchmarks diff --git a/benchmarks/src/main/java/io/hstore/bench/HStoreStore.java b/benchmarks/src/main/java/io/hstore/bench/HStoreStore.java index 2f752aab..64c0ae9f 100644 --- a/benchmarks/src/main/java/io/hstore/bench/HStoreStore.java +++ b/benchmarks/src/main/java/io/hstore/bench/HStoreStore.java @@ -67,7 +67,7 @@ public String name() { @Override public String version() { - return "0.1.0"; + return "0.2.0"; } @Override diff --git a/database/pom.xml b/database/pom.xml index 40cb67a1..9b4eab27 100644 --- a/database/pom.xml +++ b/database/pom.xml @@ -7,7 +7,7 @@ io.hstore hstore-parent - 0.1.0 + 0.2.0 hstore-db diff --git a/database/src/main/java/io/hstore/db/HypergraphDatabase.java b/database/src/main/java/io/hstore/db/HypergraphDatabase.java index e54ce574..3b855f96 100644 --- a/database/src/main/java/io/hstore/db/HypergraphDatabase.java +++ b/database/src/main/java/io/hstore/db/HypergraphDatabase.java @@ -12,6 +12,7 @@ import io.hstore.db.security.Principal; import io.hstore.db.security.Quota.Usage; import io.hstore.db.security.Security; +import io.hstore.db.security.Scram; import io.hstore.db.security.SignIns; import io.hstore.db.semantic.Embedding; import io.hstore.db.semantic.SemanticPlane; @@ -49,6 +50,7 @@ public final class HypergraphDatabase implements AutoCloseable { private final Statistics statistics; private final MaterializedViews views; private final SignIns signIns = new SignIns(); + private final byte[] scramSecret; private HypergraphDatabase(Path directory, DatabaseOptions options) { this.options = options; @@ -62,6 +64,31 @@ private HypergraphDatabase(Path directory, DatabaseOptions options) { engine.onCheckpoint(semantic::persist); this.statistics = new Statistics(engine); this.views = new MaterializedViews(this); + this.scramSecret = prepareSignIns(); + } + + private byte[] prepareSignIns() { + boolean ready = read(reader -> Security.scramSecret(reader.view()).isPresent() + && Security.users(reader.view()).stream().noneMatch(Security::legacy)); + if (!ready) { + write(writer -> { + if (Security.scramSecret(writer.transaction()).isEmpty()) { + Security.createScramSecret(writer.transaction()); + } + Security.upgradeLegacyUsers(writer.transaction()); + return null; + }); + } + return read(reader -> Security.scramSecret(reader.view())).orElseThrow(); + } + + public Scram.Server scram(Optional channelBinding) { + return new Scram.Server(name -> read(reader -> Security.user(reader.view(), name)).map(Security::verifier), scramSecret, + Security.ITERATIONS, channelBinding); + } + + public SignIns signIns() { + return signIns; } public static List> extensionSlots() { diff --git a/database/src/main/java/io/hstore/db/query/Session.java b/database/src/main/java/io/hstore/db/query/Session.java index cb94b99a..85ef2bda 100644 --- a/database/src/main/java/io/hstore/db/query/Session.java +++ b/database/src/main/java/io/hstore/db/query/Session.java @@ -8,7 +8,9 @@ import io.hstore.db.Writer; import io.hstore.db.query.Ast.Statement; import io.hstore.db.security.Principal; +import io.hstore.db.security.Scram; import io.hstore.db.security.Security; +import io.hstore.db.security.SignIns; import io.hstore.engine.HStoreException; import io.hstore.engine.catalog.Branch; import io.hstore.engine.txn.Isolation; @@ -38,6 +40,11 @@ public final class Session implements AutoCloseable { private boolean renewing; private Optional origin = Optional.empty(); private int failedAuthentications; + private Optional scram = Optional.empty(); + private Optional scramUser = Optional.empty(); + + private record ScramAttempt(Scram.Server server, SignIns.Reservation reservation) { + } private Format format = Format.TABLE; public enum Format { TABLE, JSON } @@ -70,24 +77,72 @@ public String render(QueryResult result) { } public boolean authenticate(String user, String password) { + requireCanAuthenticate(); + Optional found = database.authenticate(user, password, origin); + if (found.isEmpty()) { + failedAuthentications++; + return false; + } + signIn(found.get()); + return true; + } + + public String scramFirst(String mechanism, String clientFirst, Optional channelBinding) { + requireCanAuthenticate(); + scram.ifPresent(pending -> pending.reservation().failed()); + scram = Optional.empty(); + Scram.Server server = database.scram(channelBinding); + String serverFirst = server.first(mechanism, clientFirst); + scramUser = Optional.of(server.user()); + SignIns.Reservation reservation = database.signIns().reserve(server.user(), origin); + scram = Optional.of(new ScramAttempt(server, reservation)); + return serverFirst; + } + + public Optional scramFinal(String clientFinal) { + ScramAttempt attempt = scram.orElseThrow(() -> HStoreException.invalid("malformed SCRAM message: no exchange in progress")); + scram = Optional.empty(); + Scram.Outcome outcome; + try { + outcome = attempt.server().last(clientFinal); + } catch (RuntimeException malformed) { + attempt.reservation().failed(); + failedAuthentications++; + throw malformed; + } + Optional found = outcome instanceof Scram.Outcome.Accepted(String user, String _) + ? database.read(reader -> Security.user(reader.view(), user)).map(Security::principal) + : Optional.empty(); + if (found.isEmpty()) { + attempt.reservation().failed(); + failedAuthentications++; + return Optional.empty(); + } + attempt.reservation().succeeded(); + signIn(found.get()); + return Optional.of(((Scram.Outcome.Accepted) outcome).serverFinal()); + } + + public Optional scramUser() { + return scramUser; + } + + private void requireCanAuthenticate() { if (transaction.isPresent()) { throw HStoreException.invalid("cannot authenticate inside a transaction"); } if (failedAuthentications >= MAX_FAILED_AUTHENTICATIONS) { throw HStoreException.invalid("too many failed authentication attempts in this session"); } - Optional found = database.authenticate(user, password, origin); - if (found.isEmpty()) { - failedAuthentications++; - return false; - } - principal = found.get(); + } + + private void signIn(Principal found) { + principal = found; authenticated = true; - credentials = database.read(reader -> Security.user(reader.view(), user)); + credentials = database.read(reader -> Security.user(reader.view(), found.user())); renewing = false; variables.clear(); branch = Branch.MAIN; - return true; } public boolean authenticated() { @@ -289,5 +344,7 @@ public boolean inTransaction() { public void close() { transaction.ifPresent(Transaction::abort); transaction = Optional.empty(); + scram.ifPresent(pending -> pending.reservation().failed()); + scram = Optional.empty(); } } diff --git a/database/src/main/java/io/hstore/db/security/Scram.java b/database/src/main/java/io/hstore/db/security/Scram.java new file mode 100644 index 00000000..09ba43a9 --- /dev/null +++ b/database/src/main/java/io/hstore/db/security/Scram.java @@ -0,0 +1,417 @@ +// SPDX-FileCopyrightText: MetaCognition India Pvt Limited +// SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0 + +package io.hstore.db.security; + +import io.hstore.engine.HStoreException; + +import javax.crypto.Mac; +import javax.crypto.SecretKeyFactory; +import javax.crypto.spec.PBEKeySpec; +import javax.crypto.spec.SecretKeySpec; +import java.nio.charset.StandardCharsets; +import java.security.GeneralSecurityException; +import java.security.MessageDigest; +import java.security.SecureRandom; +import java.security.cert.Certificate; +import java.security.cert.CertificateEncodingException; +import java.security.cert.X509Certificate; +import java.util.Arrays; +import java.util.Base64; +import java.util.LinkedHashMap; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.function.Function; + +public final class Scram { + + public static final String MECHANISM = "SCRAM-SHA-256"; + public static final String MECHANISM_PLUS = "SCRAM-SHA-256-PLUS"; + public static final String CHANNEL_BINDING = "tls-server-end-point"; + private static final String PREFIX = "SCRAM-SHA-256$"; + private static final int NONCE_BYTES = 18; + private static final int MAX_MESSAGE = 4096; + private static final SecureRandom RANDOM = new SecureRandom(); + private static final Base64.Encoder ENCODE = Base64.getEncoder(); + private static final Base64.Decoder DECODE = Base64.getDecoder(); + + private Scram() { + } + + public record Verifier(int iterations, byte[] salt, byte[] storedKey, byte[] serverKey) { + + public static Verifier of(String password, byte[] salt, int iterations) { + return fromSaltedPassword(saltedPassword(password, salt, iterations), salt, iterations); + } + + public static Verifier fromSaltedPassword(byte[] saltedPassword, byte[] salt, int iterations) { + return new Verifier(iterations, salt.clone(), sha256(hmac(saltedPassword, "Client Key")), hmac(saltedPassword, "Server Key")); + } + + public static Optional parse(String text) { + if (!text.startsWith(PREFIX)) { + return Optional.empty(); + } + String[] parts = text.substring(PREFIX.length()).split("\\$", -1); + if (parts.length != 2) { + return Optional.empty(); + } + String[] head = parts[0].split(":", -1); + String[] keys = parts[1].split(":", -1); + if (head.length != 2 || keys.length != 2) { + return Optional.empty(); + } + try { + int iterations = Integer.parseInt(head[0]); + byte[] salt = DECODE.decode(head[1]); + byte[] stored = DECODE.decode(keys[0]); + byte[] server = DECODE.decode(keys[1]); + if (iterations < 1 || salt.length == 0 || stored.length != 32 || server.length != 32) { + return Optional.empty(); + } + return Optional.of(new Verifier(iterations, salt, stored, server)); + } catch (IllegalArgumentException malformed) { + return Optional.empty(); + } + } + + public String encode() { + return PREFIX + iterations + ":" + ENCODE.encodeToString(salt) + "$" + ENCODE.encodeToString(storedKey) + ":" + + ENCODE.encodeToString(serverKey); + } + + public boolean matches(String password) { + byte[] salted = saltedPassword(password, salt, iterations); + return MessageDigest.isEqual(sha256(hmac(salted, "Client Key")), storedKey); + } + + @Override + public boolean equals(Object other) { + return other instanceof Verifier(int i, byte[] s, byte[] stored, byte[] server) && i == iterations + && Arrays.equals(s, salt) && Arrays.equals(stored, storedKey) && Arrays.equals(server, serverKey); + } + + @Override + public int hashCode() { + return 31 * Arrays.hashCode(storedKey) + iterations; + } + + @Override + public String toString() { + return "Verifier[iterations=" + iterations + "]"; + } + } + + public static byte[] saltedPassword(String password, byte[] salt, int iterations) { + try { + PBEKeySpec spec = new PBEKeySpec(password.toCharArray(), salt, iterations, 256); + return SecretKeyFactory.getInstance("PBKDF2WithHmacSHA256").generateSecret(spec).getEncoded(); + } catch (GeneralSecurityException e) { + throw new IllegalStateException("PBKDF2WithHmacSHA256 is unavailable", e); + } + } + + public static byte[] endPoint(X509Certificate certificate) { + String algorithm = certificate.getSigAlgName().toUpperCase(Locale.ROOT); + String digest = algorithm.contains("SHA512") ? "SHA-512" : algorithm.contains("SHA384") ? "SHA-384" : "SHA-256"; + try { + return MessageDigest.getInstance(digest).digest(certificate.getEncoded()); + } catch (GeneralSecurityException | RuntimeException e) { + throw new IllegalStateException("cannot hash the server certificate", e); + } + } + + static byte[] hmac(byte[] key, String message) { + return hmac(key, message.getBytes(StandardCharsets.UTF_8)); + } + + static byte[] hmac(byte[] key, byte[] message) { + try { + Mac mac = Mac.getInstance("HmacSHA256"); + mac.init(new SecretKeySpec(key, "HmacSHA256")); + return mac.doFinal(message); + } catch (GeneralSecurityException e) { + throw new IllegalStateException("HmacSHA256 is unavailable", e); + } + } + + static byte[] sha256(byte[] data) { + try { + return MessageDigest.getInstance("SHA-256").digest(data); + } catch (GeneralSecurityException e) { + throw new IllegalStateException("SHA-256 is unavailable", e); + } + } + + private static byte[] xor(byte[] left, byte[] right) { + byte[] out = new byte[left.length]; + for (int i = 0; i < left.length; i++) { + out[i] = (byte) (left[i] ^ right[i]); + } + return out; + } + + static String nonce() { + byte[] bytes = new byte[NONCE_BYTES]; + RANDOM.nextBytes(bytes); + return ENCODE.encodeToString(bytes); + } + + private static String saslName(String user) { + return user.replace("=", "=3D").replace(",", "=2C"); + } + + private static String unsaslName(String name) { + if (name.replaceAll("=3D|=2C", "").contains("=")) { + throw malformed("invalid user name encoding"); + } + return name.replace("=2C", ",").replace("=3D", "="); + } + + private static Map attributes(String message, int skip) { + Map found = new LinkedHashMap<>(); + String[] parts = message.split(",", -1); + for (int i = skip; i < parts.length; i++) { + String part = parts[i]; + if (part.length() < 2 || part.charAt(1) != '=') { + throw malformed("expected an attribute, found '" + part + "'"); + } + found.putIfAbsent(part.substring(0, 1), part.substring(2)); + } + return found; + } + + private static HStoreException malformed(String detail) { + return HStoreException.invalid("malformed SCRAM message: " + detail); + } + + private static String required(Map attributes, String name) { + String value = attributes.get(name); + if (value == null) { + throw malformed("missing " + name + "="); + } + return value; + } + + private static byte[] decode(String text, String name) { + try { + return DECODE.decode(text); + } catch (IllegalArgumentException e) { + throw malformed(name + " is not base64"); + } + } + + public static final class Client { + private final String user; + private final String password; + private final Optional channelBinding; + private final String nonce; + private String header; + private String firstBare; + private byte[] serverSignature; + + public Client(String user, String password, Optional channelBinding) { + this(user, password, channelBinding, nonce()); + } + + Client(String user, String password, Optional channelBinding, String nonce) { + this.user = user; + this.password = password; + this.channelBinding = channelBinding.map(byte[]::clone); + this.nonce = nonce; + } + + public String mechanism() { + return channelBinding.isPresent() ? MECHANISM_PLUS : MECHANISM; + } + + public String first() { + header = channelBinding.isPresent() ? "p=" + CHANNEL_BINDING + ",," : "n,,"; + firstBare = "n=" + saslName(user) + ",r=" + nonce; + return header + firstBare; + } + + public String last(String serverFirst) { + Map fields = attributes(serverFirst, 0); + if (fields.containsKey("m")) { + throw malformed("unsupported extension"); + } + String combined = required(fields, "r"); + if (!combined.startsWith(nonce) || combined.length() <= nonce.length()) { + throw malformed("the server nonce does not extend the client nonce"); + } + byte[] salt = decode(required(fields, "s"), "salt"); + int iterations; + try { + iterations = Integer.parseInt(required(fields, "i")); + } catch (NumberFormatException e) { + throw malformed("iteration count is not a number"); + } + if (iterations < 4096 || iterations > 10_000_000) { + throw malformed("iteration count " + iterations + " is outside 4096 to 10000000"); + } + byte[] bindData = concat(header.getBytes(StandardCharsets.UTF_8), channelBinding.orElse(new byte[0])); + String withoutProof = "c=" + ENCODE.encodeToString(bindData) + ",r=" + combined; + String authMessage = firstBare + "," + serverFirst + "," + withoutProof; + byte[] salted = saltedPassword(password, salt, iterations); + byte[] clientKey = hmac(salted, "Client Key"); + byte[] proof = xor(clientKey, hmac(sha256(clientKey), authMessage)); + serverSignature = hmac(hmac(salted, "Server Key"), authMessage); + return withoutProof + ",p=" + ENCODE.encodeToString(proof); + } + + public void verify(String serverFinal) { + Map fields = attributes(serverFinal, 0); + if (fields.containsKey("e")) { + throw HStoreException.invalid("the server rejected the sign-in: " + fields.get("e")); + } + if (serverSignature == null || !MessageDigest.isEqual(decode(required(fields, "v"), "verifier"), serverSignature)) { + throw HStoreException.invalid("the server could not prove that it knows the password; it may not be the server you meant to reach"); + } + } + } + + public sealed interface Outcome { + record Accepted(String user, String serverFinal) implements Outcome { + } + + record Rejected(String user) implements Outcome { + } + } + + public static final class Server { + private final Function> verifiers; + private final byte[] mockKey; + private final int mockIterations; + private final Optional channelBinding; + private final String serverNonce; + private String user; + private String header; + private String firstBare; + private String serverFirst; + private String combinedNonce; + private Optional verifier = Optional.empty(); + private boolean finished; + + public Server(Function> verifiers, byte[] mockKey, int mockIterations, Optional channelBinding) { + this(verifiers, mockKey, mockIterations, channelBinding, nonce()); + } + + Server(Function> verifiers, byte[] mockKey, int mockIterations, Optional channelBinding, String serverNonce) { + this.verifiers = verifiers; + this.mockKey = mockKey.clone(); + this.mockIterations = mockIterations; + this.channelBinding = channelBinding.map(byte[]::clone); + this.serverNonce = serverNonce; + } + + public String user() { + if (user == null) { + throw new IllegalStateException("no client-first message yet"); + } + return user; + } + + public String first(String mechanism, String clientFirst) { + if (user != null) { + throw malformed("the exchange already started"); + } + if (clientFirst.length() > MAX_MESSAGE) { + throw malformed("message too long"); + } + boolean plus = switch (mechanism) { + case MECHANISM -> false; + case MECHANISM_PLUS -> true; + default -> throw HStoreException.invalid("unsupported SASL mechanism '" + mechanism + "'; this server offers " + + MECHANISM + (channelBinding.isPresent() ? " and " + MECHANISM_PLUS : "")); + }; + String[] parts = clientFirst.split(",", 3); + if (parts.length != 3) { + throw malformed("expected gs2 header and client-first-message-bare"); + } + String flag = parts[0]; + if (!parts[1].isEmpty()) { + throw malformed("authorization identities are not supported"); + } + if (plus) { + if (channelBinding.isEmpty()) { + throw HStoreException.invalid(MECHANISM_PLUS + " needs a TLS connection"); + } + if (!flag.equals("p=" + CHANNEL_BINDING)) { + throw malformed("only " + CHANNEL_BINDING + " channel binding is supported"); + } + } else if (flag.equals("y")) { + if (channelBinding.isPresent()) { + throw HStoreException.invalid("the client did not use channel binding although this server supports it; " + + "the connection may have been tampered with"); + } + } else if (!flag.equals("n")) { + throw malformed(flag.startsWith("p=") ? "channel binding needs " + MECHANISM_PLUS : "invalid channel binding flag"); + } + header = flag + ",,"; + firstBare = parts[2]; + Map fields = attributes(firstBare, 0); + if (fields.containsKey("m")) { + throw malformed("unsupported extension"); + } + user = unsaslName(required(fields, "n")); + String clientNonce = required(fields, "r"); + if (clientNonce.isEmpty() || clientNonce.contains(",")) { + throw malformed("invalid client nonce"); + } + combinedNonce = clientNonce + serverNonce; + verifier = verifiers.apply(user); + int iterations = verifier.map(Verifier::iterations).orElse(mockIterations); + byte[] salt = verifier.map(Verifier::salt).orElseGet(() -> Arrays.copyOf(hmac(mockKey, "salt:" + user), 16)); + serverFirst = "r=" + combinedNonce + ",s=" + ENCODE.encodeToString(salt) + ",i=" + iterations; + return serverFirst; + } + + public Outcome last(String clientFinal) { + if (serverFirst == null || finished) { + throw malformed("unexpected client-final message"); + } + finished = true; + if (clientFinal.length() > MAX_MESSAGE) { + throw malformed("message too long"); + } + int proofAt = clientFinal.lastIndexOf(",p="); + if (proofAt < 0) { + throw malformed("missing p="); + } + String withoutProof = clientFinal.substring(0, proofAt); + Map fields = attributes(withoutProof, 0); + byte[] expectedBinding = concat(header.getBytes(StandardCharsets.UTF_8), header.startsWith("p=") ? channelBinding.orElseThrow() : new byte[0]); + boolean bound = MessageDigest.isEqual(decode(required(fields, "c"), "channel binding"), expectedBinding); + boolean nonceMatches = required(fields, "r").equals(combinedNonce); + byte[] proof = decode(clientFinal.substring(proofAt + 3), "proof"); + String authMessage = firstBare + "," + serverFirst + "," + withoutProof; + Verifier known = verifier.orElseGet(() -> new Verifier(mockIterations, new byte[16], hmac(mockKey, "stored:" + user), hmac(mockKey, "server:" + user))); + byte[] signature = hmac(known.storedKey(), authMessage); + boolean proven = proof.length == 32 && MessageDigest.isEqual(sha256(xor(proof, signature)), known.storedKey()); + if (verifier.isEmpty() || !bound || !nonceMatches || !proven) { + return new Outcome.Rejected(user); + } + return new Outcome.Accepted(user, "v=" + ENCODE.encodeToString(hmac(known.serverKey(), authMessage))); + } + } + + private static byte[] concat(byte[] left, byte[] right) { + byte[] out = Arrays.copyOf(left, left.length + right.length); + System.arraycopy(right, 0, out, left.length, right.length); + return out; + } + + public static Optional endPointOf(Certificate[] certificates) { + if (certificates == null || certificates.length == 0 || !(certificates[0] instanceof X509Certificate leaf)) { + return Optional.empty(); + } + try { + leaf.getEncoded(); + } catch (CertificateEncodingException e) { + return Optional.empty(); + } + return Optional.of(endPoint(leaf)); + } +} diff --git a/database/src/main/java/io/hstore/db/security/Security.java b/database/src/main/java/io/hstore/db/security/Security.java index 802da2cb..d78007ed 100644 --- a/database/src/main/java/io/hstore/db/security/Security.java +++ b/database/src/main/java/io/hstore/db/security/Security.java @@ -5,6 +5,7 @@ import io.hstore.db.security.Quota.Usage; import io.hstore.engine.HStoreException; +import io.hstore.engine.catalog.EngineSlots; import io.hstore.engine.catalog.Slot; import io.hstore.engine.page.ByteCursor; import io.hstore.engine.tree.EntryMeasure; @@ -15,15 +16,11 @@ import io.hstore.engine.txn.Transaction; import io.hstore.engine.txn.View; -import java.security.GeneralSecurityException; -import java.security.MessageDigest; import java.security.SecureRandom; import java.util.HexFormat; import java.util.List; import java.util.Optional; import java.util.stream.Stream; -import javax.crypto.SecretKeyFactory; -import javax.crypto.spec.PBEKeySpec; public final class Security { @@ -33,7 +30,10 @@ public record Tenant(int id, String name, Quota quota) { public record User(String name, String salt, String hash, int iterations, int tenant, Role role) { } - private static final int ITERATIONS = 600_000; + public static final int ITERATIONS = 600_000; + private static final int MIN_VERIFIER_ITERATIONS = 4096; + private static final long SCRAM_SECRET = 0x5343_5241_4D00_0000L; + private static final int SCRAM_SECRET_WORDS = 4; public static final int MIN_PASSWORD_LENGTH = 8; public static final int MAX_PASSWORD_LENGTH = 1024; private static final byte[] UNKNOWN_USER_SALT = new byte[16]; @@ -140,20 +140,78 @@ public static void createUser(Transaction txn, String name, String password, int if (!name.matches("[A-Za-z_][A-Za-z0-9_.@-]*")) { throw HStoreException.invalid("invalid user name '" + name + "'"); } - requirePassword(password); - byte[] salt = new byte[16]; - RANDOM.nextBytes(salt); - User user = new User(name, HEX.formatHex(salt), HEX.formatHex(derive(password, salt, ITERATIONS)), ITERATIONS, tenant, role); + Scram.Verifier verifier = secret(password); + User user = new User(name, HEX.formatHex(verifier.salt()), verifier.encode(), verifier.iterations(), tenant, role); txn.merge(USERS, userKey(name), Optional::isEmpty, _ -> Optional.of(user)); } public static void setPassword(Transaction txn, String name, String password) { User current = user(txn, name).orElseThrow(() -> HStoreException.invalid("unknown user '" + name + "'")); + Scram.Verifier verifier = secret(password); + txn.put(USERS, userKey(name), new User(name, HEX.formatHex(verifier.salt()), verifier.encode(), verifier.iterations(), + current.tenant(), current.role())); + } + + static Scram.Verifier secret(String password) { + Optional given = Scram.Verifier.parse(password); + if (given.isPresent()) { + if (given.get().iterations() < MIN_VERIFIER_ITERATIONS || given.get().salt().length < 8) { + throw HStoreException.invalid("a SCRAM-SHA-256 verifier needs at least " + MIN_VERIFIER_ITERATIONS + + " iterations and an 8-byte salt"); + } + return given.get(); + } + return newVerifier(password); + } + + public static Scram.Verifier newVerifier(String password) { requirePassword(password); byte[] salt = new byte[16]; RANDOM.nextBytes(salt); - txn.put(USERS, userKey(name), new User(name, HEX.formatHex(salt), HEX.formatHex(derive(password, salt, ITERATIONS)), - ITERATIONS, current.tenant(), current.role())); + return Scram.Verifier.of(password, salt, ITERATIONS); + } + + public static Scram.Verifier verifier(User user) { + return Scram.Verifier.parse(user.hash()).orElseGet(() -> Scram.Verifier.fromSaltedPassword(HEX.parseHex(user.hash()), + HEX.parseHex(user.salt()), user.iterations())); + } + + public static boolean legacy(User user) { + return Scram.Verifier.parse(user.hash()).isEmpty(); + } + + public static int upgradeLegacyUsers(Transaction txn) { + List legacy = users(txn).stream().filter(Security::legacy).toList(); + for (User user : legacy) { + txn.put(USERS, userKey(user.name()), new User(user.name(), user.salt(), verifier(user).encode(), user.iterations(), + user.tenant(), user.role())); + } + return legacy.size(); + } + + public static Optional scramSecret(View view) { + byte[] secret = new byte[SCRAM_SECRET_WORDS * Long.BYTES]; + for (int word = 0; word < SCRAM_SECRET_WORDS; word++) { + Optional value = view.get(EngineSlots.SETTINGS, SCRAM_SECRET + word); + if (value.isEmpty()) { + return Optional.empty(); + } + for (int i = 0; i < Long.BYTES; i++) { + secret[word * Long.BYTES + i] = (byte) (value.get() >>> (8 * i)); + } + } + return Optional.of(secret); + } + + public static void createScramSecret(Transaction txn) { + for (int word = 0; word < SCRAM_SECRET_WORDS; word++) { + long value = RANDOM.nextLong(); + txn.merge(EngineSlots.SETTINGS, SCRAM_SECRET + word, Optional::isEmpty, _ -> Optional.of(value)); + } + } + + public static Principal principal(User user) { + return new Principal(user.name(), user.tenant(), user.role()); } public static void requirePassword(String password) { @@ -181,9 +239,7 @@ public static Optional authenticate(View view, String name, String pa if (stored.isEmpty()) { derive(password, UNKNOWN_USER_SALT, ITERATIONS); } - return stored.filter(user -> MessageDigest.isEqual( - derive(password, HEX.parseHex(user.salt()), user.iterations()), HEX.parseHex(user.hash()))) - .map(user -> new Principal(user.name(), user.tenant(), user.role())); + return stored.filter(user -> verifier(user).matches(password)).map(Security::principal); } public static boolean hasUsers(View view) { @@ -195,11 +251,6 @@ private static long userKey(String name) { } private static byte[] derive(String password, byte[] salt, int iterations) { - try { - PBEKeySpec spec = new PBEKeySpec(password.toCharArray(), salt, iterations, 256); - return SecretKeyFactory.getInstance("PBKDF2WithHmacSHA256").generateSecret(spec).getEncoded(); - } catch (GeneralSecurityException e) { - throw new IllegalStateException("PBKDF2WithHmacSHA256 is unavailable", e); - } + return Scram.saltedPassword(password, salt, iterations); } } diff --git a/database/src/main/java/io/hstore/db/security/SignIns.java b/database/src/main/java/io/hstore/db/security/SignIns.java index 56b2bbc6..59d9a98c 100644 --- a/database/src/main/java/io/hstore/db/security/SignIns.java +++ b/database/src/main/java/io/hstore/db/security/SignIns.java @@ -47,7 +47,41 @@ public SignIns(int accountLimit, int originLimit, Duration window, LongSupplier this.clock = clock; } - public Optional attempt(String user, Optional origin, Supplier> check) { + public final class Reservation { + private final String user; + private final Key account; + private final List keys; + private final long now; + private boolean settled; + + private Reservation(String user, Key account, List keys, long now) { + this.user = user; + this.account = account; + this.keys = keys; + this.now = now; + } + + public String user() { + return user; + } + + public void succeeded() { + if (!settled) { + settled = true; + windows.remove(account.name()); + keys.stream().filter(key -> key != account).forEach(SignIns.this::release); + } + } + + public void failed() { + if (!settled) { + settled = true; + SignIns.this.failed(user, now); + } + } + } + + public Reservation reserve(String user, Optional origin) { Key account = new Key(origin.map(address -> "account:" + user + "\n" + address).orElse("account:" + user), accountLimit); List keys = Stream.concat(Stream.of(account), origin.map(address -> new Key("origin:" + address, originLimit)).stream()).toList(); long now = clock.getAsLong(); @@ -59,12 +93,22 @@ public Optional attempt(String user, Optional origin, Supplier result = check.get(); + return new Reservation(user, account, keys, now); + } + + public Optional attempt(String user, Optional origin, Supplier> check) { + Reservation reservation = reserve(user, origin); + Optional result; + try { + result = check.get(); + } catch (RuntimeException failure) { + reservation.failed(); + throw failure; + } if (result.isPresent()) { - windows.remove(account.name()); - keys.stream().filter(key -> key != account).forEach(this::release); + reservation.succeeded(); } else { - failed(user, now); + reservation.failed(); } return result; } diff --git a/database/src/test/java/io/hstore/db/SecurityTest.java b/database/src/test/java/io/hstore/db/SecurityTest.java index 883bdff2..20f90549 100644 --- a/database/src/test/java/io/hstore/db/SecurityTest.java +++ b/database/src/test/java/io/hstore/db/SecurityTest.java @@ -6,6 +6,9 @@ import io.hstore.db.query.QueryResult; import io.hstore.db.query.Session; import io.hstore.db.security.Principal; +import io.hstore.db.security.Role; +import io.hstore.db.security.Scram; +import io.hstore.db.security.Security; import io.hstore.db.semantic.Embedding; import io.hstore.db.value.Value; import io.hstore.db.view.MaterializedViews; @@ -18,6 +21,7 @@ import io.hstore.db.security.SignIns; import io.hstore.engine.EngineOptions; import io.hstore.engine.HStoreException; +import io.hstore.engine.tree.Hashing; import io.hstore.engine.txn.TxnOptions; import io.hstore.engine.catalog.Branch; @@ -31,6 +35,7 @@ import java.nio.file.Path; import java.time.Duration; import java.util.ArrayList; +import java.util.HexFormat; import java.util.Optional; import java.util.concurrent.atomic.AtomicLong; import java.util.Comparator; @@ -701,6 +706,80 @@ void signInsAreThrottledPerUserAndAddress() { assertThrows(HStoreException.ResourceLimit.class, () -> accounts.attempt("local", Optional.empty(), () -> Optional.of("never"))); } + @Test + void passwordsAreStoredAsScramVerifiersAndLegacyHashesUpgrade() { + Security.User alice = database.read(reader -> Security.user(reader.view(), "alice")).orElseThrow(); + assertTrue(alice.hash().startsWith("SCRAM-SHA-256$600000:"), alice.hash()); + byte[] salt = new byte[16]; + byte[] salted = Scram.saltedPassword("legacy-pass", salt, 4096); + database.write(writer -> { + writer.transaction().put(Security.USERS, Hashing.of("old"), new Security.User("old", HexFormat.of().formatHex(salt), + HexFormat.of().formatHex(salted), 4096, Principal.DEFAULT_TENANT, Role.READER)); + return null; + }); + assertTrue(Security.legacy(database.read(reader -> Security.user(reader.view(), "old")).orElseThrow())); + assertTrue(new Session(database).authenticate("old", "legacy-pass")); + admin.close(); + database.close(); + database = HypergraphDatabase.open(directory, options()); + admin = new Session(database); + Security.User upgraded = database.read(reader -> Security.user(reader.view(), "old")).orElseThrow(); + assertEquals(Scram.Verifier.of("legacy-pass", salt, 4096).encode(), upgraded.hash()); + assertTrue(new Session(database).authenticate("old", "legacy-pass")); + assertFalse(new Session(database).authenticate("old", "wrong-pass")); + long generation = database.engine().transactions().current().id(); + admin.close(); + database.close(); + database = HypergraphDatabase.open(directory, options()); + admin = new Session(database); + assertEquals(generation, database.engine().transactions().current().id()); + + String verifier = Scram.Verifier.of("chosen-elsewhere", new byte[]{1, 2, 3, 4, 5, 6, 7, 8}, 4096).encode(); + admin.execute("CREATE USER remote PASSWORD '" + verifier + "' ROLE reader"); + assertTrue(new Session(database).authenticate("remote", "chosen-elsewhere")); + String weak = Scram.Verifier.of("chosen-elsewhere", new byte[]{1, 2, 3, 4, 5, 6, 7, 8}, 1000).encode(); + assertThrows(HStoreException.InvalidSchema.class, () -> admin.execute("ALTER USER remote PASSWORD '" + weak + "'")); + String shortSalt = Scram.Verifier.of("chosen-elsewhere", new byte[]{1, 2}, 4096).encode(); + assertThrows(HStoreException.InvalidSchema.class, () -> admin.execute("ALTER USER remote PASSWORD '" + shortSalt + "'")); + } + + @Test + void scramExchangesSignSessionsInAndCountFailures() { + try (Session session = new Session(database)) { + session.origin("192.0.2.10"); + Scram.Client client = new Scram.Client("alice", "s3cret-pass", Optional.empty()); + String serverFirst = session.scramFirst(Scram.MECHANISM, client.first(), Optional.empty()); + String serverFinal = session.scramFinal(client.last(serverFirst)).orElseThrow(); + client.verify(serverFinal); + assertTrue(session.authenticated()); + assertEquals("alice", session.principal().user()); + assertTrue(session.principal().tenant() != Principal.DEFAULT_TENANT); + } + for (int i = 0; i < SignIns.ACCOUNT_LIMIT; i++) { + try (Session session = new Session(database)) { + session.origin("192.0.2.11"); + Scram.Client client = new Scram.Client("bob", "wrong-" + i, Optional.empty()); + assertTrue(session.scramFinal(client.last(session.scramFirst(Scram.MECHANISM, client.first(), Optional.empty()))).isEmpty()); + assertFalse(session.authenticated()); + } + } + try (Session session = new Session(database)) { + session.origin("192.0.2.11"); + Scram.Client client = new Scram.Client("bob", "hunter2-pass", Optional.empty()); + assertThrows(HStoreException.ResourceLimit.class, () -> session.scramFirst(Scram.MECHANISM, client.first(), Optional.empty())); + } + try (Session session = new Session(database)) { + session.origin("192.0.2.12"); + Scram.Client client = new Scram.Client("nobody", "anything-at-all", Optional.empty()); + assertTrue(session.scramFinal(client.last(session.scramFirst(Scram.MECHANISM, client.first(), Optional.empty()))).isEmpty()); + assertThrows(HStoreException.InvalidSchema.class, () -> session.scramFinal("c=biws,r=x,p=AAAA")); + } + try (Session abandoned = new Session(database)) { + abandoned.origin("192.0.2.13"); + abandoned.scramFirst(Scram.MECHANISM, new Scram.Client("bob", "x", Optional.empty()).first(), Optional.empty()); + } + } + @Test void passwordsNeedEightToAThousandCharacters() { for (String password : new String[]{"", "short", "seven77", "x".repeat(1025)}) { diff --git a/database/src/test/java/io/hstore/db/security/ScramTest.java b/database/src/test/java/io/hstore/db/security/ScramTest.java new file mode 100644 index 00000000..5f7251c5 --- /dev/null +++ b/database/src/test/java/io/hstore/db/security/ScramTest.java @@ -0,0 +1,170 @@ +// SPDX-FileCopyrightText: MetaCognition India Pvt Limited +// SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0 + +package io.hstore.db.security; + +import io.hstore.engine.HStoreException; + +import org.junit.jupiter.api.Test; + +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.HexFormat; +import java.util.List; +import java.util.Optional; +import java.util.function.Function; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class ScramTest { + + private static final byte[] RFC_SALT = Base64.getDecoder().decode("W22ZaJ0SNY7soEsUEjb6gQ=="); + private static final String RFC_CLIENT_NONCE = "rOprNGfwEbeRWgbNEkqO"; + private static final String RFC_SERVER_NONCE = "%hvYDpWUa2RaTCAfuxFIlj)hNlF$k0"; + private static final byte[] SECRET = new byte[32]; + + private static Function> only(String user, Scram.Verifier verifier) { + return name -> name.equals(user) ? Optional.of(verifier) : Optional.empty(); + } + + @Test + void matchesTheRfc7677Example() { + Scram.Verifier verifier = Scram.Verifier.of("pencil", RFC_SALT, 4096); + Scram.Client client = new Scram.Client("user", "pencil", Optional.empty(), RFC_CLIENT_NONCE); + Scram.Server server = new Scram.Server(only("user", verifier), SECRET, 600_000, Optional.empty(), RFC_SERVER_NONCE); + String first = client.first(); + assertEquals("n,,n=user,r=rOprNGfwEbeRWgbNEkqO", first); + String serverFirst = server.first(Scram.MECHANISM, first); + assertEquals("r=rOprNGfwEbeRWgbNEkqO%hvYDpWUa2RaTCAfuxFIlj)hNlF$k0,s=W22ZaJ0SNY7soEsUEjb6gQ==,i=4096", serverFirst); + String last = client.last(serverFirst); + assertEquals("c=biws,r=rOprNGfwEbeRWgbNEkqO%hvYDpWUa2RaTCAfuxFIlj)hNlF$k0,p=dHzbZapWIk4jUhN+Ute9ytag9zjfMHgsqmmiz7AndVQ=", last); + Scram.Outcome.Accepted accepted = assertInstanceOf(Scram.Outcome.Accepted.class, server.last(last)); + assertEquals("v=6rriTRBi23WpRR/wtup+mMhUZUn/dB5nLTJRsjl95G4=", accepted.serverFinal()); + client.verify(accepted.serverFinal()); + assertThrows(HStoreException.class, () -> client.verify("v=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=")); + assertThrows(HStoreException.class, () -> client.verify("e=invalid-proof")); + } + + @Test + void wrongPasswordsAndUnknownUsersAreRejectedAlike() { + Scram.Verifier verifier = Scram.Verifier.of("correct horse", RFC_SALT, 4096); + for (String[] attempt : new String[][]{{"alice", "wrong horse"}, {"mallory", "correct horse"}}) { + Scram.Client client = new Scram.Client(attempt[0], attempt[1], Optional.empty()); + Scram.Server server = new Scram.Server(only("alice", verifier), SECRET, 4096, Optional.empty()); + Scram.Outcome outcome = server.last(client.last(server.first(Scram.MECHANISM, client.first()))); + assertEquals(new Scram.Outcome.Rejected(attempt[0]), outcome); + } + Scram.Server unknown = new Scram.Server(name -> Optional.empty(), SECRET, 600_000, Optional.empty()); + String first = unknown.first(Scram.MECHANISM, "n,,n=nobody,r=abc"); + Scram.Server again = new Scram.Server(name -> Optional.empty(), SECRET, 600_000, Optional.empty()); + String repeated = again.first(Scram.MECHANISM, "n,,n=nobody,r=abc"); + assertEquals(first.substring(first.indexOf(",s=")), repeated.substring(repeated.indexOf(",s="))); + assertTrue(first.endsWith(",i=600000")); + Scram.Server other = new Scram.Server(name -> Optional.empty(), new byte[]{1}, 600_000, Optional.empty()); + String otherSalt = other.first(Scram.MECHANISM, "n,,n=nobody,r=abc"); + assertNotEquals(first.substring(first.indexOf(",s=")), otherSalt.substring(otherSalt.indexOf(",s="))); + } + + @Test + void channelBindingTiesTheProofToTheCertificate() { + Scram.Verifier verifier = Scram.Verifier.of("pencil-pencil", RFC_SALT, 4096); + byte[] certificate = new byte[32]; + certificate[0] = 7; + Scram.Client bound = new Scram.Client("user", "pencil-pencil", Optional.of(certificate)); + Scram.Server server = new Scram.Server(only("user", verifier), SECRET, 4096, Optional.of(certificate)); + assertEquals(Scram.MECHANISM_PLUS, bound.mechanism()); + Scram.Outcome.Accepted accepted = assertInstanceOf(Scram.Outcome.Accepted.class, + server.last(bound.last(server.first(bound.mechanism(), bound.first())))); + bound.verify(accepted.serverFinal()); + + byte[] relayed = new byte[32]; + Scram.Client fooled = new Scram.Client("user", "pencil-pencil", Optional.of(relayed)); + Scram.Server real = new Scram.Server(only("user", verifier), SECRET, 4096, Optional.of(certificate)); + assertInstanceOf(Scram.Outcome.Rejected.class, real.last(fooled.last(real.first(fooled.mechanism(), fooled.first())))); + + Scram.Client unbound = new Scram.Client("user", "pencil-pencil", Optional.empty()); + Scram.Server tls = new Scram.Server(only("user", verifier), SECRET, 4096, Optional.of(certificate)); + assertInstanceOf(Scram.Outcome.Accepted.class, tls.last(unbound.last(tls.first(Scram.MECHANISM, unbound.first())))); + + Scram.Server downgraded = new Scram.Server(only("user", verifier), SECRET, 4096, Optional.of(certificate)); + assertThrows(HStoreException.class, () -> downgraded.first(Scram.MECHANISM, "y,,n=user,r=abc")); + Scram.Server plain = new Scram.Server(only("user", verifier), SECRET, 4096, Optional.empty()); + assertThrows(HStoreException.class, () -> plain.first(Scram.MECHANISM_PLUS, "p=tls-server-end-point,,n=user,r=abc")); + Scram.Server mismatched = new Scram.Server(only("user", verifier), SECRET, 4096, Optional.of(certificate)); + assertThrows(HStoreException.class, () -> mismatched.first(Scram.MECHANISM, "p=tls-server-end-point,,n=user,r=abc")); + Scram.Server yes = new Scram.Server(only("user", verifier), SECRET, 4096, Optional.empty()); + yes.first(Scram.MECHANISM, "y,,n=user,r=abc"); + } + + @Test + void malformedMessagesFailCleanly() { + Scram.Verifier verifier = Scram.Verifier.of("pencil-pencil", RFC_SALT, 4096); + for (String bad : List.of("", "n,,", "n,,r=abc", "n,,n=user", "x,,n=user,r=abc", "n,a=admin,n=user,r=abc", "n,,n=us=er,r=abc", + "n,,n=user,r=a,b", "n,,n=user,r=abc,m=ext", "n,," + "x".repeat(5000))) { + Scram.Server server = new Scram.Server(only("user", verifier), SECRET, 4096, Optional.empty()); + assertThrows(HStoreException.class, () -> server.first(Scram.MECHANISM, bad), bad); + } + assertThrows(HStoreException.class, () -> new Scram.Server(only("user", verifier), SECRET, 4096, Optional.empty()) + .first("SCRAM-SHA-1", "n,,n=user,r=abc")); + for (String bad : List.of("c=biws,r=abc", "c=biws,r=abc,p=!!!", "")) { + Scram.Server server = new Scram.Server(only("user", verifier), SECRET, 4096, Optional.empty()); + server.first(Scram.MECHANISM, "n,,n=user,r=abc"); + assertThrows(HStoreException.class, () -> server.last(bad), bad); + } + Scram.Server replayed = new Scram.Server(only("user", verifier), SECRET, 4096, Optional.empty()); + Scram.Client client = new Scram.Client("user", "pencil-pencil", Optional.empty()); + String last = client.last(replayed.first(Scram.MECHANISM, client.first())); + assertInstanceOf(Scram.Outcome.Accepted.class, replayed.last(last)); + assertThrows(HStoreException.class, () -> replayed.last(last)); + Scram.Server wrongNonce = new Scram.Server(only("user", verifier), SECRET, 4096, Optional.empty()); + wrongNonce.first(Scram.MECHANISM, "n,,n=user,r=abc"); + assertInstanceOf(Scram.Outcome.Rejected.class, wrongNonce.last(last)); + Scram.Client cautious = new Scram.Client("user", "pencil-pencil", Optional.empty(), "abc"); + cautious.first(); + assertThrows(HStoreException.class, () -> cautious.last("r=zzz,s=W22ZaJ0SNY7soEsUEjb6gQ==,i=4096")); + assertThrows(HStoreException.class, () -> cautious.last("r=abcdef,s=W22ZaJ0SNY7soEsUEjb6gQ==,i=1")); + assertThrows(HStoreException.class, () -> cautious.last("r=abcdef,s=@@,i=4096")); + } + + @Test + void verifiersRoundTripAndLegacyHashesConvert() throws Exception { + Scram.Verifier verifier = Scram.Verifier.of("pencil-pencil", RFC_SALT, 4096); + assertEquals(Optional.of(verifier), Scram.Verifier.parse(verifier.encode())); + assertTrue(verifier.encode().startsWith("SCRAM-SHA-256$4096:W22ZaJ0SNY7soEsUEjb6gQ==$")); + for (String bad : List.of("", "SCRAM-SHA-256$", "SCRAM-SHA-256$x:AA==$AA==:AA==", "SCRAM-SHA-256$4096:AA==$AA==:AA==", + "SCRAM-SHA-256$0:" + verifier.encode().substring(verifier.encode().indexOf(':') + 1), "md5abc")) { + assertEquals(Optional.empty(), Scram.Verifier.parse(bad), bad); + } + assertTrue(verifier.matches("pencil-pencil")); + assertTrue(!verifier.matches("pencil-pencil ")); + String password = "pässwörd-🔒"; + byte[] salted = Scram.saltedPassword(password, RFC_SALT, 4096); + assertArrayEquals(pbkdf2(password.getBytes(StandardCharsets.UTF_8), RFC_SALT, 4096), salted); + Security.User legacy = new Security.User("legacy", HexFormat.of().formatHex(RFC_SALT), HexFormat.of().formatHex(salted), 4096, 0, Role.READER); + assertTrue(Security.legacy(legacy)); + assertEquals(Scram.Verifier.of(password, RFC_SALT, 4096), Security.verifier(legacy)); + assertTrue(Security.verifier(legacy).matches(password)); + } + + private static byte[] pbkdf2(byte[] password, byte[] salt, int iterations) throws Exception { + Mac mac = Mac.getInstance("HmacSHA256"); + mac.init(new SecretKeySpec(password, "HmacSHA256")); + mac.update(salt); + byte[] block = mac.doFinal(new byte[]{0, 0, 0, 1}); + byte[] result = block.clone(); + for (int i = 1; i < iterations; i++) { + block = mac.doFinal(block); + for (int j = 0; j < result.length; j++) { + result[j] ^= block[j]; + } + } + return result; + } +} diff --git a/docs/database/hql.md b/docs/database/hql.md index 04686dc4..e7c9e2fc 100644 --- a/docs/database/hql.md +++ b/docs/database/hql.md @@ -684,6 +684,12 @@ show = "SHOW" ( "TENANTS" | "USERS" ) ; `SHOW TENANTS` columns: `id, name, atoms, edges, payload bytes, max atoms, max edges`; `SHOW USERS`: `user, tenant, role` (tenant id). The tenancy model is specified in [security](security.md). +A password must be 8 to 1,024 characters long. In its place `CREATE USER` and `ALTER USER` accept a SCRAM-SHA-256 +verifier (`hstore verifier` prints one), so the password never reaches the server. `AUTHENTICATE` works in +embedded sessions (`hstore shell`, `hstore exec`, the Java API) and in Studio's console. The wire protocol refuses +it, because it would carry the password in clear text; clients sign in there with SCRAM, and `hstore connect` +turns an `AUTHENTICATE` typed at its prompt into that exchange ([wire protocol](../operations/wire-protocol.md#authentication)). + ```sql CREATE TENANT clinic_b QUOTA (atoms 3, edges 1); ALTER TENANT clinic_b QUOTA (payload_bytes 1048576, query_pages 100000); diff --git a/docs/database/security.md b/docs/database/security.md index 81a086cc..7ddda537 100644 --- a/docs/database/security.md +++ b/docs/database/security.md @@ -126,12 +126,27 @@ record User(String name, String salt, String hash, int iterations, int tenant, R - Passwords set by `CREATE USER`, `ALTER USER … PASSWORD` and `hstore init --superuser` must be 8 to 1,024 characters long (`Security.requirePassword`). Passwords stored before this rule keep working until they are changed. -- Password hashing: **PBKDF2WithHmacSHA256**, 600,000 iterations, 16-byte salt from `SecureRandom`, 256-bit - derived key, stored hex-encoded together with the iteration count. Users created before the count was raised - from 120,000 keep their old hash until their password is next changed. -- `Security.authenticate` re-derives with the stored salt and iterations and compares with - `MessageDigest.isEqual` (constant time with respect to content). For a name that does not exist it still runs - PBKDF2 against a fixed salt, so the response time does not tell which names exist. +- Passwords are stored as SCRAM-SHA-256 verifiers (`Scram.Verifier`, [RFC 5802](https://www.rfc-editor.org/rfc/rfc5802)), + written in the PostgreSQL format `SCRAM-SHA-256$:$:`. `SaltedPassword` + is PBKDF2-HMAC-SHA256 of the UTF-8 password with a 16-byte `SecureRandom` salt and 600,000 iterations; + `StoredKey` is `SHA-256(HMAC(SaltedPassword, "Client Key"))` and `ServerKey` is + `HMAC(SaltedPassword, "Server Key")`. Neither the password nor `SaltedPassword` is stored, so a copy of the + users slot does not let anyone sign in without guessing the password first. +- Older versions stored the hex `SaltedPassword` itself. The first open by this version turns every such record + into a verifier in one transaction (`Security.upgradeLegacyUsers`); the password is not needed, nothing changes + for the user, and a version from before the change can no longer read the upgraded records. Users created + before the iteration count was raised from 120,000 keep their count until their password is next changed. +- `CREATE USER` and `ALTER USER … PASSWORD` accept a verifier in place of a password (made by `hstore verifier` + or any SCRAM library), with at least 4,096 iterations and an 8-byte salt; the length rule cannot apply to it. +- The wire protocol signs in with SCRAM ([wire protocol](../operations/wire-protocol.md#authentication)): the + client proves knowledge of the password against `StoredKey` and the server proves knowledge of `ServerKey`. + For a name that does not exist the server answers with a salt derived from the name and a 256-bit secret + created when the database is first opened (stored under `EngineSlots.SETTINGS`), so the exchange looks the + same for every name and only fails at its last step. +- `Security.authenticate`, used by Studio's login, `AUTHENTICATE` statements in embedded and Studio sessions and + the Java API, derives `StoredKey` from the given password and compares with `MessageDigest.isEqual`. For a name + that does not exist it still runs PBKDF2 against a fixed salt, so the response time does not tell which names + exist. - Every sign-in, from the wire protocol, Studio or an `AUTHENTICATE` statement, goes through one throttle per database (`SignIns`): at most 10 failed attempts for one user name from one client address, and 20 from one address over all user names, in 15 minutes. Sign-ins without an address (the embedded API, `hstore exec` and @@ -156,7 +171,8 @@ record User(String name, String salt, String hash, int iterations, int tenant, R `Principal(user, user.tenant, user.role)`. The derivation cost is deliberate (≈0.2–0.5 s per attempt on current CPUs) to slow offline guessing if the data -directory leaks; failed attempts are logged by the server with the peer address. +directory leaks; over the wire protocol the client pays it, not the server. Failed attempts are logged by the +server with the peer address. ## Quotas @@ -235,9 +251,9 @@ allocator but are not charged as atoms. | cross-tenant read through ids or keys | tenant in index keys plus `visible` checks on every path, incident edges of other tenants skipped (also for memberships an older version let through), untyped matches scan only the tenant's own types, per-tenant planner estimates and view staleness | type names, user names, generation numbers and the global atom id sequence are visible to every tenant; cache hits and latency depend on what other tenants do, and semantic search walks one shared HNSW graph | | cross-tenant write or reference | `require` before mutations; members must be visible | none known | | resource exhaustion by a tenant | atom/edge/payload quotas (payload bytes are charged in their own transaction, so a rollback still counts them, as the payload store never shrinks), the tenant's page budget on every read statement, per-transaction page budget, HORA budgets that `LIMIT` cannot raise past 1,000,000 rows, `DEPTH` 1-64, `SIMILAR TO … TOP` 1-1000, expression and JSON nesting up to 256 levels, number literals up to 400 characters with exponents up to 1000, 32 views per tenant, 128-character names, new embedding models only by a global admin | CPU in planning and projection is not budgeted; the HNSW index is shared; a WRITER can still add new property names, role names (on edge types that declare no roles) and state schemas, which stay in the shared dictionary | -| password database theft | PBKDF2-SHA256, 600k iterations, per-user salt | offline guessing of weak passwords remains possible | +| password database theft | SCRAM verifiers: PBKDF2-SHA256, 600k iterations, per-user salt, no stored password or salted password | offline guessing of weak passwords remains possible | | online password guessing | one throttle for every sign-in path: 10 failures per user and address, 20 per address in 15 minutes; a delay before every failed answer; a warning when one user collects 50 failures; passwords of at least 8 characters | guessing spread over many addresses is slowed and logged, not stopped | -| credential sniffing | `tls = on` for the wire protocol and Studio | plaintext when TLS is off | +| credential sniffing | SCRAM-SHA-256 on the wire protocol, with channel binding over TLS; clear-text `AUTHENTICATE` refused there; `tls = on` for Studio | Studio's login form and `CREATE USER`/`ALTER USER` passwords are clear text without TLS (pass a verifier instead) | | privilege escalation | role checks in both HQL and the Java API; tenant admins cannot leave their tenant; sessions that never authenticated must authenticate once the first user exists; sessions end when their user is dropped or its password changes; tenant admins cannot drop the last user | an embedded application holding `Principal.SYSTEM` is fully trusted | | tampering with data files | CRC32C on pages, WAL records and payloads detects corruption | no cryptographic integrity or encryption at rest; protect the data directory with file-system permissions (the container runs as a non-root user with mode 0700) | diff --git a/docs/operations/cli.md b/docs/operations/cli.md index 5e546fba..19575ce8 100644 --- a/docs/operations/cli.md +++ b/docs/operations/cli.md @@ -12,6 +12,7 @@ usage: hstore [options] exec (-c STATEMENTS | FILE) run HQL against an embedded database connect HOST:PORT [--user NAME --password PW] interactive shell against a server ping HOST:PORT exit 0 when a server accepts connections + verifier [--password PW] print a SCRAM-SHA-256 verifier to use as a CREATE USER password bench [--scenario NAME] [--scale N] adversarial storage benchmarks check verify checksums and tree invariants config [] print effective settings @@ -136,9 +137,12 @@ error [INVALID_SCHEMA]: syntax error at offset 5: expected VIEWS but found 'NOPE ### `connect` An interactive shell over the [wire protocol](wire-protocol.md). When the server banner says -`authentication required` and credentials are available, `connect` sends `AUTHENTICATE` before showing the -prompt, and the banner it prints ends in `authenticated as ` instead. A rejected login exits with code `2` -and prints the server's `error [AUTHENTICATION_FAILED]: authentication failed`. +`authentication required` and credentials are available, `connect` signs in with SCRAM-SHA-256 (with channel +binding over TLS) before showing the prompt, so the password never leaves the client, and the banner it prints +ends in `authenticated as ` instead. It also checks that the server knows the password, and refuses a +server that cannot prove it. A rejected login exits with code `2` and prints the server's +`error [AUTHENTICATION_FAILED]: authentication failed`. A single `AUTHENTICATE user PASSWORD '…'` statement typed +at the prompt is turned into the same exchange. | Source of credentials | Order | |---|---| @@ -177,6 +181,19 @@ $ hstore ping 127.0.0.1:7499; echo $? 2 ``` +### `verifier` + +Prints a SCRAM-SHA-256 verifier for a password, read from `--password`, then `$HSTORE_PASSWORD`, then a no-echo +prompt. The password must be 8 to 1,024 characters long. Use the output in place of the password in +`CREATE USER … PASSWORD '…'` or `ALTER USER … PASSWORD '…'`, so that the password itself never reaches the server +or its logs, even without TLS: + +``` +$ hstore verifier +password for the new user: +SCRAM-SHA-256$600000:5Ezq2HNhB0F2o/5xgXlS7Q==$Q0pM…:dLh0… +``` + ### `check` Opens the database (performing recovery if needed) and verifies every tree of every active branch. It diff --git a/docs/operations/logging.md b/docs/operations/logging.md index 22a91ef0..0ca871c9 100644 --- a/docs/operations/logging.md +++ b/docs/operations/logging.md @@ -14,7 +14,7 @@ last step of shutdown, which also removes the `hstore-0.log.lck` lock file. ## Line format ``` -2026-10-04 06:33:30.334 IST [28997] LOG: [server] connection 2 authorized: user=admin tenant=0 role=ADMIN +2026-10-04 06:33:30.334 IST [28997] LOG: [server] connection 2 authorized: user=admin tenant=0 role=ADMIN method=SCRAM-SHA-256 └──────────── timestamp ───────────┘ └pid┘ └sev┘ └component┘ └──────────────── message ────────────────┘ ``` @@ -129,7 +129,9 @@ and in Docker it is the last line of `docker logs`. The storage-integrity failur | Severity | Message | |---|---| | LOG | `connection {n} received from /{ip}:{port}` | -| LOG | `connection {n} authorized: user={user} tenant={tenantId} role={ROLE}` | +| LOG | `connection {n} authorized: user={user} tenant={tenantId} role={ROLE} method={SCRAM-SHA-256 or SCRAM-SHA-256-PLUS}` | +| WARNING | `connection {n} from /{ip}:{port} sent a clear-text password, which is refused; the client needs an upgrade`: a client older than 0.2.0 tried `AUTHENTICATE`. | +| WARNING | `connection {n} from /{ip}:{port}: SASL sign-in failed: {reason}`: a malformed or throttled SCRAM exchange. | | WARNING | `password authentication failed for user "{user}" from /{ip}:{port}` (logged whatever `log_connections` says). Control characters in the user name, as in every log message, are written as `\n`, `\r` or `\uXXXX`. | | WARNING | `connection {n} from /{ip}:{port} closed after 5 failed authentication attempts`: any five rejected requests (wrong password or not `AUTHENTICATE`) before login close the connection. | | LOG | `disconnection: connection {n} from /{ip}:{port} session time {ms} ms` | diff --git a/docs/operations/wire-protocol.md b/docs/operations/wire-protocol.md index 794f0c88..3248eb8a 100644 --- a/docs/operations/wire-protocol.md +++ b/docs/operations/wire-protocol.md @@ -56,8 +56,10 @@ sequenceDiagram else accepted S-->>C: banner "hstore connection 6 at generation 159; authentication required" loop until authenticated - C->>S: "AUTHENTICATE admin PASSWORD 'pw-secret'" - S-->>C: "authenticated as admin" or "error [AUTHENTICATION_FAILED]: authentication failed" + C->>S: "SASL SCRAM-SHA-256 " + S-->>C: "sasl " + C->>S: "SASL " + S-->>C: "authenticated as admin" and "sasl ", or "error [AUTHENTICATION_FAILED]: authentication failed" end Note over C,S: the 5th rejected request closes the connection loop requests @@ -80,23 +82,52 @@ or when it is `auto` and at least one user exists. ### Authentication -While a connection is unauthenticated, each request must contain exactly one `AUTHENTICATE` statement: +Clients sign in with SCRAM-SHA-256 ([RFC 5802](https://www.rfc-editor.org/rfc/rfc5802), +[RFC 7677](https://www.rfc-editor.org/rfc/rfc7677)), the same mechanism PostgreSQL uses. The password never +crosses the connection: the client proves that it knows it, and the server proves in turn that it holds the +user's verifier, so a client also detects a server that only pretends to be the database. The exchange takes two +requests. Each SCRAM message travels base64-encoded on one line: -``` -AUTHENTICATE PASSWORD '' -``` +| Step | Client sends | Server answers | +|---|---|---| +| 1 | `SASL ` | `sasl ` | +| 2 | `SASL ` | `authenticated as ` and, on a second line, `sasl ` | + +`` is `SCRAM-SHA-256`, or `SCRAM-SHA-256-PLUS` on a TLS connection. With `-PLUS` the client uses +the `tls-server-end-point` channel binding ([RFC 5929](https://www.rfc-editor.org/rfc/rfc5929)): it hashes the +server's certificate, with SHA-256 unless the certificate is signed with SHA-384 or SHA-512, and mixes the hash +into its proof. A proof relayed through a man in the middle, which presents a different certificate, then fails. +The server rejects `-PLUS` without TLS, and on a TLS connection it rejects a client that announces channel binding +support (gs2 flag `y`) without using it, since that means someone stripped it. -`` is an identifier or a quoted string. Inside quoted strings, `'` is escaped as `''`. Possible -replies: +The client must check the server's final message, `v=`, before it trusts the connection. +Passwords are hashed as UTF-8 without SASLprep normalisation, authorization identities and SCRAM extensions are +not supported, and the iteration count is 600,000 for passwords set by this version (at least 4,096 for verifiers +created elsewhere), so a client spends about half a second deriving the key. + +A user that does not exist gets a well-formed `server-first-message` with a salt derived from the user name and a +secret kept in the database, the same every time, and fails only at the second step, exactly like a wrong +password. + +Possible replies: | Reply | Meaning | |---|---| -| `authenticated as ` | The session now runs as that user's principal (tenant and role). | -| `error [AUTHENTICATION_FAILED]: authentication failed` | Unknown user or wrong password. | -| `error [AUTHENTICATION_REQUIRED]: authenticate first: AUTHENTICATE PASSWORD ''` | Any other statement. | -| `error [INVALID_SCHEMA]: syntax error …` | The request did not parse. | - -Every rejected request counts as a failed attempt. After `MAX_AUTHENTICATION_ATTEMPTS` = 5 failures, the +| `sasl …` | Continue with the next step. | +| `authenticated as ` / `sasl …` | The session now runs as that user's principal (tenant and role). | +| `error [AUTHENTICATION_FAILED]: authentication failed` | Unknown user or wrong password, answered after `authentication_failure_delay_ms`. | +| `error [AUTHENTICATION_FAILED]: malformed SCRAM message: …`, `unsupported SASL mechanism …` | The client sent something the server cannot use. | +| `error [AUTHENTICATION_FAILED]: this server no longer accepts passwords in clear text; upgrade your HStore client to version 0.2.0 or later, which signs in with SCRAM-SHA-256` | The client sent `AUTHENTICATE PASSWORD '…'`, which older clients used. The password is not checked. | +| `error [AUTHENTICATION_REQUIRED]: authenticate first with SASL SCRAM-SHA-256; see the wire protocol documentation` | Any other request. | +| `error [ABORTED_RESOURCE_LIMIT]: too many failed sign-in attempts; …` | The throttle described below. | + +An `AUTHENTICATE` statement is refused with the same upgrade message on an authenticated connection too; send a +new SASL exchange to switch users. `hstore connect` turns an `AUTHENTICATE` typed in its shell into that +exchange. `CREATE USER` and `ALTER USER … PASSWORD` still carry the password in the statement; use TLS for them, +or pass a SCRAM verifier made on the client (`hstore verifier`, the format is +`SCRAM-SHA-256$:$:` as in PostgreSQL) in place of the password. + +Every rejected request before sign-in counts as a failed attempt. After `MAX_AUTHENTICATION_ATTEMPTS` = 5 failures, the server logs a WARNING and closes the connection without replying further. Clients must reconnect to try again. `AUTHENTICATION_FAILED` and `AUTHENTICATION_REQUIRED` occur only in this handshake. They are protocol codes, not `HStoreException` codes. @@ -107,7 +138,6 @@ request gets `AUTHENTICATION_REQUIRED`. An authenticated session may send `AUTHE outside a transaction only. That clears its variables and returns it to main. A session that fails 5 such attempts refuses further `AUTHENTICATE` statements. -Passwords are verified with PBKDF2-HMAC-SHA256 (600,000 iterations for passwords set since the count was raised). A failed attempt is answered after `authentication_failure_delay_ms` (default 500). Every sign-in path shares one throttle: after 10 failures for a user name from one address, or 20 from one address over all names, within 15 minutes, further attempts from that address get `error [ABORTED_RESOURCE_LIMIT]: too many failed sign-in @@ -206,11 +236,16 @@ S: hstore connection 6 at generation 159; authentication required S: . C: SHOW TYPES; C: . -S: error [AUTHENTICATION_REQUIRED]: authenticate first: AUTHENTICATE PASSWORD '' +S: error [AUTHENTICATION_REQUIRED]: authenticate first with SASL SCRAM-SHA-256; see the wire protocol documentation S: . -C: AUTHENTICATE admin PASSWORD 'pw-secret' +C: SASL SCRAM-SHA-256 biwsbj1hZG1pbixyPWVwSmszWWlmWFVzSHVOaG51a2llMlExSw== +C: . +S: sasl cj1lcEprM1lpZlhVc0h1TmhudWtpZTJRMUtEZ2xDdGNNOFpLUTNpc3dtaGh1M2Fkdysscz1wZklhOHhpVVBFQUFlUnUreW9tMElBPT0saT02MDAwMDA= +S: . +C: SASL Yz1iaXdzLHI9ZXBKazNZaWZYVXNIdU5obnVraWUyUTFLRGdsQ3RjTThaS1EzaXN3bWhodTNhZHcrLHA9STNPN1V0alNHNlFFSXRJYURYalFzY1pXd25RUTNLMFphMm9VMUdJbk1KVT0= C: . S: authenticated as admin +S: sasl dj1TbG1KOXMrVmVpZWVjMGprUjg0ZUU3ZHAxclVlbmNIVFVjUTJTb1l5eUpJPQ== S: . C: FORMAT JSON; MATCH NODE d:Drug WHERE d.class = 'statin' RETURN d.name; C: . @@ -234,24 +269,72 @@ S: . ## Writing a client -A complete client needs only a socket and the two framing functions. In Python: +A complete client needs only a socket, the two framing functions and SCRAM, all from the Python standard +library. Pass `tls_ca` (a CA or the server's own certificate) when the server runs with `tls = on`; the client +then uses channel binding. The certificate hash below is SHA-256, which is right for certificates signed with +SHA-256 (the usual case). ```python +import base64 +import hashlib +import hmac +import os import socket +import ssl + + +def b64(text): + return base64.b64encode(text.encode()).decode() + + +def unb64(text): + return base64.b64decode(text).decode() + class HStore: - def __init__(self, host="127.0.0.1", port=7432, user=None, password=None): - self.stream = socket.create_connection((host, port)).makefile("rwb") + def __init__(self, host="127.0.0.1", port=7432, user=None, password=None, tls_ca=None): + self.sock = socket.create_connection((host, port)) + if tls_ca: + context = ssl.create_default_context(cafile=tls_ca) + self.sock = context.wrap_socket(self.sock, server_hostname=host) + self.stream = self.sock.makefile("rwb") self.banner = self._receive() if self.banner.startswith("error"): raise ConnectionError(self.banner) if self.banner.endswith("authentication required"): - quote = lambda s: "'" + s.replace("'", "''") + "'" - reply = self.execute(f"AUTHENTICATE {quote(user)} PASSWORD {quote(password)}") - if reply.startswith("error"): - raise PermissionError(reply) + self._sign_in(user, password) self.execute("FORMAT JSON;") + def _sign_in(self, user, password): + binding = None + if isinstance(self.sock, ssl.SSLSocket): + binding = hashlib.sha256(self.sock.getpeercert(binary_form=True)).digest() + header = "p=tls-server-end-point,," if binding else "n,," + nonce = base64.b64encode(os.urandom(18)).decode() + first_bare = "n=" + user.replace("=", "=3D").replace(",", "=2C") + ",r=" + nonce + mechanism = "SCRAM-SHA-256-PLUS" if binding else "SCRAM-SHA-256" + reply = self.execute(f"SASL {mechanism} {b64(header + first_bare)}") + if not reply.startswith("sasl "): + raise PermissionError(reply) + server_first = unb64(reply[5:]) + fields = dict(part.split("=", 1) for part in server_first.split(",")) + if not fields["r"].startswith(nonce) or int(fields["i"]) < 4096: + raise PermissionError("unexpected server-first-message") + salted = hashlib.pbkdf2_hmac("sha256", password.encode(), base64.b64decode(fields["s"]), int(fields["i"])) + client_key = hmac.digest(salted, b"Client Key", "sha256") + without_proof = "c=" + base64.b64encode(header.encode() + (binding or b"")).decode() + ",r=" + fields["r"] + auth_message = f"{first_bare},{server_first},{without_proof}".encode() + signature = hmac.digest(hashlib.sha256(client_key).digest(), auth_message, "sha256") + proof = base64.b64encode(bytes(a ^ b for a, b in zip(client_key, signature))).decode() + reply = self.execute(f"SASL {b64(without_proof + ',p=' + proof)}") + if reply.startswith("error"): + raise PermissionError(reply) + status, final = reply.split("\n", 1) + server_key = hmac.digest(salted, b"Server Key", "sha256") + expected = "v=" + base64.b64encode(hmac.digest(server_key, auth_message, "sha256")).decode() + if not hmac.compare_digest(unb64(final[5:]), expected): + raise PermissionError("the server could not prove that it knows the password") + def execute(self, script): for line in script.split("\n"): self.stream.write((("." + line) if line.startswith(".") else line).encode() + b"\n") diff --git a/engine/pom.xml b/engine/pom.xml index 7f4140bf..46e17107 100644 --- a/engine/pom.xml +++ b/engine/pom.xml @@ -7,7 +7,7 @@ io.hstore hstore-parent - 0.1.0 + 0.2.0 hstore-engine diff --git a/pom.xml b/pom.xml index 0c4b4b12..c490a77c 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ io.hstore hstore-parent - 0.1.0 + 0.2.0 pom HStore https://github.com/metacoglabs/hypergraph diff --git a/server/pom.xml b/server/pom.xml index 4b0a58e7..81f69aea 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -7,7 +7,7 @@ io.hstore hstore-parent - 0.1.0 + 0.2.0 hstore-server diff --git a/server/src/main/java/io/hstore/server/Main.java b/server/src/main/java/io/hstore/server/Main.java index fb9e1958..cb066e0f 100644 --- a/server/src/main/java/io/hstore/server/Main.java +++ b/server/src/main/java/io/hstore/server/Main.java @@ -30,7 +30,7 @@ public final class Main { - static final String VERSION = "0.1.0"; + static final String VERSION = "0.2.0"; static final String NOTICE = "Licensed under PolyForm Noncommercial 1.0.0. Commercial use needs a license from MetaCognition India Pvt Limited: krish@getmetacognition.com"; private static final String USAGE = """ @@ -41,6 +41,7 @@ public final class Main { exec (-c STATEMENTS | FILE) run HQL against an embedded database connect HOST:PORT [--user NAME --password PW] interactive shell against a server ping HOST:PORT exit 0 when a server accepts connections + verifier [--password PW] print a SCRAM-SHA-256 verifier to use as a CREATE USER password bench [--scenario NAME] [--scale N] adversarial storage benchmarks check verify checksums and tree invariants config [] print effective settings @@ -67,6 +68,9 @@ record Connect(InetSocketAddress address, Optional c record Ping(InetSocketAddress address) implements Command { } + record Verifier(Optional password) implements Command { + } + record Bench(Path directory, String scenario, int scale) implements Command { } @@ -152,6 +156,7 @@ private static Command command(Arguments arguments) { arguments.option("c").orElseGet(() -> read(Path.of(arguments.require(2, "script file"))))); case "connect" -> new Command.Connect(address(arguments.require(1, "host:port")), credentials(arguments, environment)); case "ping" -> new Command.Ping(address(arguments.require(1, "host:port"))); + case "verifier" -> new Command.Verifier(arguments.option("password").or(() -> Optional.ofNullable(environment.get("HSTORE_PASSWORD")))); case "bench" -> new Command.Bench(directory(arguments), arguments.option("scenario").orElse("all"), Integer.parseInt(arguments.option("scale").orElse("1"))); case "check" -> new Command.Check(directory(arguments)); @@ -235,6 +240,10 @@ private static int run(Command command, Arguments arguments) { yield 2; } } + case Command.Verifier(Optional password) -> { + IO.println(Security.newVerifier(password.orElseGet(() -> prompt("the new user"))).encode()); + yield 0; + } case Command.Bench(Path directory, String scenario, int scale) -> { Logging.configure(Level.WARNING, "", 1, 1); IO.println(Bench.header()); diff --git a/server/src/main/java/io/hstore/server/RemoteEndpoint.java b/server/src/main/java/io/hstore/server/RemoteEndpoint.java index 7940acb0..fc55633a 100644 --- a/server/src/main/java/io/hstore/server/RemoteEndpoint.java +++ b/server/src/main/java/io/hstore/server/RemoteEndpoint.java @@ -3,6 +3,11 @@ package io.hstore.server; +import io.hstore.db.query.Ast.Statement; +import io.hstore.db.query.Parser; +import io.hstore.db.security.Scram; +import io.hstore.engine.HStoreException; + import java.io.BufferedInputStream; import java.io.IOException; import java.io.InputStream; @@ -12,13 +17,18 @@ import java.net.InetSocketAddress; import java.net.Socket; import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.List; import java.util.Optional; +import javax.net.ssl.SSLPeerUnverifiedException; +import javax.net.ssl.SSLSocket; final class RemoteEndpoint implements Endpoint { record Credentials(String user, String password) { - String statement() { - return "AUTHENTICATE '" + user.replace("'", "''") + "' PASSWORD '" + password.replace("'", "''") + "'"; + @Override + public String toString() { + return "Credentials[user=" + user + "]"; } } @@ -51,7 +61,13 @@ String statement() { throw new IllegalStateException(banner); } if (banner.endsWith("authentication required") && credentials.isPresent()) { - String reply = execute(credentials.get().statement()); + String reply; + try { + reply = signIn(credentials.get()); + } catch (RuntimeException failure) { + close(); + throw failure; + } if (reply.startsWith("error")) { close(); throw new IllegalArgumentException(reply); @@ -60,6 +76,49 @@ String statement() { } } + String signIn(Credentials credentials) { + Optional binding = socket instanceof SSLSocket secure ? peerEndPoint(secure) : Optional.empty(); + Scram.Client client = new Scram.Client(credentials.user(), credentials.password(), binding); + String first = send(Server.SASL + client.mechanism() + " " + encode(client.first())); + if (!first.startsWith("sasl ")) { + return first; + } + String last = send(Server.SASL + encode(client.last(decode(first.substring("sasl ".length()))))); + if (last.startsWith("error")) { + return last; + } + String[] lines = last.split("\n", 2); + if (lines.length != 2 || !lines[0].startsWith("authenticated as ") || !lines[1].startsWith("sasl ")) { + return "error: unexpected reply to the sign-in: " + last; + } + try { + client.verify(decode(lines[1].substring("sasl ".length()))); + } catch (HStoreException failure) { + return "error: " + failure.getMessage(); + } + return lines[0]; + } + + private static Optional peerEndPoint(SSLSocket socket) { + try { + return Scram.endPointOf(socket.getSession().getPeerCertificates()); + } catch (SSLPeerUnverifiedException e) { + return Optional.empty(); + } + } + + private static String encode(String message) { + return Base64.getEncoder().encodeToString(message.getBytes(StandardCharsets.UTF_8)); + } + + private static String decode(String message) { + try { + return new String(Base64.getDecoder().decode(message.strip()), StandardCharsets.UTF_8); + } catch (IllegalArgumentException e) { + throw new IllegalStateException("the server sent a malformed SASL message"); + } + } + private static Socket plain(InetSocketAddress address) throws IOException { Socket socket = new Socket(); socket.connect(new InetSocketAddress(address.getHostString(), address.getPort()), TIMEOUT_MILLIS); @@ -68,6 +127,29 @@ private static Socket plain(InetSocketAddress address) throws IOException { @Override public String execute(String script) { + Optional signIn = authentication(script); + if (signIn.isPresent()) { + String reply = signIn(signIn.get()); + if (!reply.startsWith("error")) { + banner = banner.replaceFirst("authentication required$", reply); + } + return reply; + } + return send(script); + } + + private static Optional authentication(String script) { + try { + List statements = Parser.script(script); + return statements.size() == 1 && statements.getFirst() instanceof Statement.Authenticate(String user, String password) + ? Optional.of(new Credentials(user, password)) + : Optional.empty(); + } catch (HStoreException unparsable) { + return Optional.empty(); + } + } + + private String send(String script) { try { WireProtocol.send(out, script); return WireProtocol.receive(in, Long.MAX_VALUE).orElse("error: server closed the connection"); diff --git a/server/src/main/java/io/hstore/server/Server.java b/server/src/main/java/io/hstore/server/Server.java index 54280309..9d200e5c 100644 --- a/server/src/main/java/io/hstore/server/Server.java +++ b/server/src/main/java/io/hstore/server/Server.java @@ -8,6 +8,7 @@ import io.hstore.db.query.Parser; import io.hstore.db.query.QueryResult; import io.hstore.db.query.Session; +import io.hstore.db.security.Scram; import io.hstore.engine.HStoreException; import java.io.BufferedInputStream; @@ -25,6 +26,7 @@ import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.ArrayList; +import java.util.Base64; import java.util.List; import java.util.Map; import java.util.Optional; @@ -220,6 +222,17 @@ private void handle(Socket socket) { refuseOtherHost(out, connection, peer); return; } + if (request.get().startsWith(SASL)) { + boolean before = session.authenticated(); + if (!sasl(session, request.get(), out, socket, connection, peer) && !before && ++failures >= MAX_AUTHENTICATION_ATTEMPTS) { + LOG.log(System.Logger.Level.WARNING, "connection {0} from {1} closed after {2} failed authentication attempts", + connection, peer, failures); + return; + } + timeout(socket, session); + unsettled = unsettled(socket, session, unsettled); + continue; + } if (!session.authenticated() && policy.authenticationRequired(database)) { if (!authenticate(session, request.get(), out, connection, peer) && ++failures >= MAX_AUTHENTICATION_ATTEMPTS) { LOG.log(System.Logger.Level.WARNING, "connection {0} from {1} closed after {2} failed authentication attempts", @@ -307,29 +320,71 @@ private boolean authenticate(Session session, String request, Writer out, long c WireProtocol.respond(out, "error [" + failure.code() + "]: " + failure.getMessage()); return false; } - if (statements.size() == 1 && statements.getFirst() instanceof Statement.Authenticate(String user, String password)) { - boolean accepted; - try { - accepted = session.authenticate(user, password); - } catch (HStoreException refused) { - WireProtocol.respond(out, "error [" + refused.code() + "]: " + refused.getMessage()); - return false; + if (statements.stream().anyMatch(statement -> statement instanceof Statement.Authenticate)) { + refuseCleartext(out, connection, peer); + return false; + } + WireProtocol.respond(out, "error [AUTHENTICATION_REQUIRED]: authenticate first with SASL " + Scram.MECHANISM + + "; see the wire protocol documentation"); + return false; + } + + static final String SASL = "SASL "; + static final String UPGRADE = "this server no longer accepts passwords in clear text; upgrade your HStore client to version " + + Main.VERSION + " or later, which signs in with " + Scram.MECHANISM; + + private void refuseCleartext(Writer out, long connection, String peer) throws IOException { + LOG.log(System.Logger.Level.WARNING, "connection {0} from {1} sent a clear-text password, which is refused; the client needs an upgrade", + connection, peer); + WireProtocol.respond(out, "error [AUTHENTICATION_FAILED]: " + UPGRADE); + } + + private boolean sasl(Session session, String request, Writer out, Socket socket, long connection, String peer) throws IOException { + String[] parts = request.substring(SASL.length()).strip().split(" "); + try { + if (parts.length == 2) { + String serverFirst = session.scramFirst(parts[0], decode(parts[1]), channelBinding(socket)); + WireProtocol.respond(out, "sasl " + Base64.getEncoder().encodeToString(serverFirst.getBytes(StandardCharsets.UTF_8))); + return true; } - if (accepted) { - if (policy.logConnections()) { - LOG.log(System.Logger.Level.INFO, "connection {0} authorized: user={1} tenant={2} role={3}", - connection, user, session.principal().tenant(), session.principal().role()); + if (parts.length == 1 && !parts[0].isEmpty()) { + Optional serverFinal = session.scramFinal(decode(parts[0])); + if (serverFinal.isPresent()) { + if (policy.logConnections()) { + LOG.log(System.Logger.Level.INFO, "connection {0} authorized: user={1} tenant={2} role={3} method={4}", + connection, session.principal().user(), session.principal().tenant(), session.principal().role(), + socket instanceof SSLSocket ? Scram.MECHANISM_PLUS : Scram.MECHANISM); + } + WireProtocol.respond(out, "authenticated as " + session.principal().user() + "\nsasl " + + Base64.getEncoder().encodeToString(serverFinal.get().getBytes(StandardCharsets.UTF_8))); + return true; } - WireProtocol.respond(out, "authenticated as " + user); - return true; + LOG.log(System.Logger.Level.WARNING, "password authentication failed for user \"{0}\" from {1}", + session.scramUser().orElse(""), peer); + policy.delayFailure(); + WireProtocol.respond(out, "error [AUTHENTICATION_FAILED]: authentication failed"); + return false; } - LOG.log(System.Logger.Level.WARNING, "password authentication failed for user \"{0}\" from {1}", user, peer); - policy.delayFailure(); - WireProtocol.respond(out, "error [AUTHENTICATION_FAILED]: authentication failed"); + WireProtocol.respond(out, "error [AUTHENTICATION_FAILED]: expected SASL or SASL "); + return false; + } catch (HStoreException failure) { + LOG.log(System.Logger.Level.WARNING, "connection {0} from {1}: SASL sign-in failed: {2}", connection, peer, failure.getMessage()); + WireProtocol.respond(out, "error [" + (failure instanceof HStoreException.ResourceLimit ? failure.code() : "AUTHENTICATION_FAILED") + + "]: " + failure.getMessage()); return false; } - WireProtocol.respond(out, "error [AUTHENTICATION_REQUIRED]: authenticate first: AUTHENTICATE PASSWORD ''"); - return false; + } + + private static String decode(String base64) { + try { + return new String(Base64.getDecoder().decode(base64), StandardCharsets.UTF_8); + } catch (IllegalArgumentException e) { + throw HStoreException.invalid("malformed SCRAM message: not base64"); + } + } + + private static Optional channelBinding(Socket socket) { + return socket instanceof SSLSocket secure ? Scram.endPointOf(secure.getSession().getLocalCertificates()) : Optional.empty(); } private String execute(Session session, String script, long connection) { @@ -341,6 +396,11 @@ private String execute(Session session, String script, long connection) { LOG.log(System.Logger.Level.INFO, "connection {0}: {1}", connection, failure.getMessage()); return "error [" + failure.code() + "]: " + failure.getMessage(); } + if (statements.stream().anyMatch(sourced -> sourced.statement() instanceof Statement.Authenticate)) { + LOG.log(System.Logger.Level.WARNING, "connection {0} sent a clear-text password, which is refused; the client needs an upgrade", + connection); + return "error [AUTHENTICATION_FAILED]: " + UPGRADE; + } for (Parser.Sourced sourced : statements) { Statement statement = sourced.statement(); long started = System.nanoTime(); diff --git a/server/src/test/java/io/hstore/server/ServerTest.java b/server/src/test/java/io/hstore/server/ServerTest.java index 9fe668e8..bbc9c9b3 100644 --- a/server/src/test/java/io/hstore/server/ServerTest.java +++ b/server/src/test/java/io/hstore/server/ServerTest.java @@ -165,7 +165,14 @@ void wireProtocolRunsOverTls() throws Exception { try (Endpoint endpoint = new RemoteEndpoint(address, Optional.empty(), Optional.of(trusting))) { assertTrue(endpoint.describe().startsWith("hstore connection")); assertTrue(endpoint.execute("CREATE NODE TYPE Person (name STRING); SHOW TYPES;").contains("Person")); + endpoint.execute("CREATE USER admin PASSWORD 'pw-secret' ROLE admin;"); } + try (Endpoint bound = new RemoteEndpoint(address, Optional.of(new RemoteEndpoint.Credentials("admin", "pw-secret")), Optional.of(trusting))) { + assertTrue(bound.describe().endsWith("authenticated as admin"), bound.describe()); + assertTrue(bound.execute("WHOAMI;").contains("ADMIN")); + } + assertThrows(IllegalArgumentException.class, () -> new RemoteEndpoint(address, + Optional.of(new RemoteEndpoint.Credentials("admin", "wrong-password")), Optional.of(trusting))); assertThrows(UncheckedIOException.class, () -> new RemoteEndpoint(address, Optional.empty(), Optional.of(new Tls.Client(Tls.system(), true)))); assertTimeoutPreemptively(Duration.ofSeconds(20), @@ -323,7 +330,8 @@ void hostileConnectionsAreClosedEarly() throws Exception { assertTrue(admin.execute("MATCH NODE d:Doc RETURN d.text;").contains("CREATE USER injected")); admin.execute("ALTER USER alice PASSWORD 'rotated-pass';"); assertTrue(alice.execute("WHOAMI;").contains("authenticate again")); - assertTrue(alice.execute("AUTHENTICATE alice PASSWORD 'rotated-pass'; WHOAMI;").contains("acme")); + assertTrue(alice.execute("AUTHENTICATE alice PASSWORD 'rotated-pass';").contains("authenticated as alice")); + assertTrue(alice.execute("WHOAMI;").contains("acme")); String crashed = admin.execute("MATCH NODE p WHERE " + "(".repeat(50_000) + "p = @1" + ")".repeat(50_000) + " RETURN p;"); assertTrue(crashed.startsWith("error [INVALID_SCHEMA]") && !crashed.contains("Exception"), crashed); assertFalse(admin.execute("WHOAMI;").startsWith("error")); @@ -408,6 +416,44 @@ void failedSignInsAreAnsweredAfterTheConfiguredDelay() throws Exception { } } + @Test + void clearTextPasswordsAreRefusedWithAnUpgradeMessage() throws Exception { + DatabaseOptions options = DatabaseOptions.defaults().withEngine(EngineOptions.defaults().withPageSize(4096)); + try (HypergraphDatabase database = HypergraphDatabase.open(directory, options)) { + new Session(database).execute("CREATE USER admin PASSWORD 'pw-secret' ROLE admin"); + Thread acceptor; + try (Server server = new Server(database, new InetSocketAddress("127.0.0.1", 0))) { + acceptor = Thread.ofVirtual().start(server::serve); + InetSocketAddress address = new InetSocketAddress("127.0.0.1", server.port()); + try (Socket socket = new Socket(address.getHostString(), address.getPort())) { + socket.setSoTimeout(10_000); + InputStream in = new java.io.BufferedInputStream(socket.getInputStream()); + java.io.Writer out = new java.io.OutputStreamWriter(socket.getOutputStream(), StandardCharsets.UTF_8); + assertTrue(WireProtocol.receive(in, 1 << 20).orElseThrow().endsWith("authentication required")); + WireProtocol.send(out, "AUTHENTICATE admin PASSWORD 'pw-secret'"); + String old = WireProtocol.receive(in, 1 << 20).orElseThrow(); + assertTrue(old.startsWith("error [AUTHENTICATION_FAILED]: this server no longer accepts passwords in clear text; upgrade your " + + "HStore client to version " + Main.VERSION), old); + WireProtocol.send(out, "SHOW USERS;"); + assertTrue(WireProtocol.receive(in, 1 << 20).orElseThrow().contains("SASL SCRAM-SHA-256")); + WireProtocol.send(out, "SASL SCRAM-SHA-1 bixuPXgscj1hYmM="); + assertTrue(WireProtocol.receive(in, 1 << 20).orElseThrow().contains("unsupported SASL mechanism")); + WireProtocol.send(out, "SASL @@@"); + assertTrue(WireProtocol.receive(in, 1 << 20).orElseThrow().startsWith("error [AUTHENTICATION_FAILED]")); + } + try (Endpoint admin = new RemoteEndpoint(address, Optional.of(new RemoteEndpoint.Credentials("admin", "pw-secret")))) { + assertTrue(admin.execute("AUTHENTICATE admin PASSWORD 'pw-secret'; WHOAMI;").contains("upgrade your HStore client")); + assertTrue(admin.execute("AUTHENTICATE admin PASSWORD 'pw-secret';").startsWith("authenticated as admin")); + assertTrue(admin.execute("WHOAMI;").contains("ADMIN")); + } + try (Endpoint nobody = new RemoteEndpoint(address, Optional.empty())) { + assertTrue(nobody.execute("AUTHENTICATE nobody PASSWORD 'whatever-it-is';").startsWith("error [AUTHENTICATION_FAILED]: authentication failed")); + } + } + acceptor.join(); + } + } + private static List exchange(InetSocketAddress address, String payload) throws Exception { try (Socket socket = new Socket(address.getHostString(), address.getPort())) { socket.setSoTimeout(10_000); From 16a1d7bc9bd7a17977b9c8987ffecf5c5d9f8d8f Mon Sep 17 00:00:00 2001 From: venkat1701 Date: Sun, 25 Oct 2026 22:01:15 +0530 Subject: [PATCH 2/5] Count the commit that a new database makes when it is first opened --- database/src/test/java/io/hstore/db/ChangeLogReplayTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/database/src/test/java/io/hstore/db/ChangeLogReplayTest.java b/database/src/test/java/io/hstore/db/ChangeLogReplayTest.java index 9f95f2a8..be7d230a 100644 --- a/database/src/test/java/io/hstore/db/ChangeLogReplayTest.java +++ b/database/src/test/java/io/hstore/db/ChangeLogReplayTest.java @@ -39,6 +39,7 @@ void replayingTheChangeLogRebuildsPropertiesIndexesAndDocuments(long seed) { try (HypergraphDatabase database = HypergraphDatabase.open(directory, DatabaseOptions.defaults() .withEngine(EngineOptions.defaults().withPageSize(4096).withHistoryLimit(100_000).withCheckpointWalBytes(Long.MAX_VALUE)))) { long first = database.engine().stats().generation(); + long logged = database.engine().verifyChangeLog().commits(); database.write(writer -> { writer.defineNode("Person", List.of(new PropertyDef("name", TypeTag.STRING, true, false), new PropertyDef("age", TypeTag.INT, false, false))); @@ -160,7 +161,7 @@ void replayingTheChangeLogRebuildsPropertiesIndexesAndDocuments(long seed) { } } long commits = database.engine().stats().generation() - first - branchChanges; - assertEquals(commits, database.engine().verifyChangeLog().commits()); + assertEquals(logged + commits, database.engine().verifyChangeLog().commits()); assertTrue(commits > 150); } } From e7e0816afd91591246c48f148a6c1f2fcddca195 Mon Sep 17 00:00:00 2001 From: venkat1701 Date: Tue, 27 Oct 2026 15:19:17 +0530 Subject: [PATCH 3/5] Make clients insist on a complete SCRAM exchange and stop reusing old hashes The client now refuses a server that does not ask for the password, answers the first SASL message with anything but a challenge, names another user, or skips or fails its proof, and disconnects. A stored 0.1 hash equals SCRAM's SaltedPassword, so it is no longer turned into a verifier: such users are refused over SCRAM and get a fresh salt on their next clear-text sign-in through Studio or the embedded API. AUTHENTICATE wrapped in AT GENERATION or AS OF is refused, verifiers must use 4096 to 10000000 iterations, and passwords go through SASLprep on both sides. --- .../java/io/hstore/db/HypergraphDatabase.java | 31 ++++- .../main/java/io/hstore/db/query/Parser.java | 11 +- .../main/java/io/hstore/db/query/Session.java | 8 ++ .../java/io/hstore/db/security/SaslPrep.java | 124 ++++++++++++++++++ .../java/io/hstore/db/security/Scram.java | 16 ++- .../java/io/hstore/db/security/Security.java | 44 ++++--- .../test/java/io/hstore/db/SecurityTest.java | 79 +++++++++-- .../java/io/hstore/db/security/ScramTest.java | 58 +++++++- docs/database/security.md | 29 +++- docs/operations/logging.md | 3 + docs/operations/wire-protocol.md | 68 ++++++++-- .../java/io/hstore/server/RemoteEndpoint.java | 36 +++-- .../main/java/io/hstore/server/Server.java | 20 ++- .../java/io/hstore/server/FakeServerTest.java | 109 +++++++++++++++ .../java/io/hstore/server/ServerTest.java | 16 +++ 15 files changed, 568 insertions(+), 84 deletions(-) create mode 100644 database/src/main/java/io/hstore/db/security/SaslPrep.java create mode 100644 server/src/test/java/io/hstore/server/FakeServerTest.java diff --git a/database/src/main/java/io/hstore/db/HypergraphDatabase.java b/database/src/main/java/io/hstore/db/HypergraphDatabase.java index 3b855f96..ca1df673 100644 --- a/database/src/main/java/io/hstore/db/HypergraphDatabase.java +++ b/database/src/main/java/io/hstore/db/HypergraphDatabase.java @@ -40,6 +40,10 @@ public final class HypergraphDatabase implements AutoCloseable { + private static final System.Logger LOG = System.getLogger("hstore.security"); + public static final String LEGACY_PASSWORD = "user \"{0}\" has a password stored by HStore 0.1, which cannot be used with SCRAM-SHA-256; " + + "it can sign in once through Studio to convert it, or an administrator can set a new one with ALTER USER"; + private static final int MAX_ATTEMPTS = 8; private final DatabaseOptions options; @@ -68,14 +72,11 @@ private HypergraphDatabase(Path directory, DatabaseOptions options) { } private byte[] prepareSignIns() { - boolean ready = read(reader -> Security.scramSecret(reader.view()).isPresent() - && Security.users(reader.view()).stream().noneMatch(Security::legacy)); - if (!ready) { + if (read(reader -> Security.scramSecret(reader.view())).isEmpty()) { write(writer -> { if (Security.scramSecret(writer.transaction()).isEmpty()) { Security.createScramSecret(writer.transaction()); } - Security.upgradeLegacyUsers(writer.transaction()); return null; }); } @@ -83,8 +84,12 @@ private byte[] prepareSignIns() { } public Scram.Server scram(Optional channelBinding) { - return new Scram.Server(name -> read(reader -> Security.user(reader.view(), name)).map(Security::verifier), scramSecret, - Security.ITERATIONS, channelBinding); + return new Scram.Server(name -> read(reader -> Security.user(reader.view(), name)).flatMap(user -> { + if (Security.legacy(user)) { + LOG.log(System.Logger.Level.WARNING, LEGACY_PASSWORD, user.name()); + } + return Security.verifier(user); + }), scramSecret, Security.ITERATIONS, channelBinding); } public SignIns signIns() { @@ -197,7 +202,19 @@ public Optional authenticate(String user, String password) { } public Optional authenticate(String user, String password, Optional origin) { - return signIns.attempt(user, origin, () -> read(reader -> Security.authenticate(reader.view(), user, password))); + Optional found = signIns.attempt(user, origin, () -> read(reader -> Security.authenticate(reader.view(), user, password))); + found.ifPresent(_ -> read(reader -> Security.user(reader.view(), user)).filter(Security::legacy).ifPresent(legacy -> { + try { + write(writer -> { + Security.replaceLegacy(writer.transaction(), legacy, password); + return null; + }); + LOG.log(System.Logger.Level.INFO, "the password of user \"{0}\" is now stored as a SCRAM-SHA-256 verifier", user); + } catch (HStoreException changed) { + LOG.log(System.Logger.Level.INFO, "the password of user \"{0}\" changed while it was being converted: {1}", user, changed.getMessage()); + } + })); + return found; } void chargePayload(int tenant, long bytes) { diff --git a/database/src/main/java/io/hstore/db/query/Parser.java b/database/src/main/java/io/hstore/db/query/Parser.java index a710b511..d7bc0e32 100644 --- a/database/src/main/java/io/hstore/db/query/Parser.java +++ b/database/src/main/java/io/hstore/db/query/Parser.java @@ -89,6 +89,13 @@ public static Statement parse(String source) { return statements.getFirst(); } + private static Statement historical(Statement inner) { + if (inner instanceof Statement.Authenticate) { + throw HStoreException.invalid("AUTHENTICATE cannot run at a past generation"); + } + return inner; + } + private Statement statement() { if (accept("BEGIN")) { return new Statement.Begin(accept("SERIALIZABLE")); @@ -111,11 +118,11 @@ private Statement statement() { } if (accept("AT")) { expect("GENERATION"); - return new Statement.At(longValue(), statement()); + return new Statement.At(longValue(), historical(statement())); } if (peek().is("AS") && peek(1).is("OF")) { index += 2; - return new Statement.AsOf(instant(), statement()); + return new Statement.AsOf(instant(), historical(statement())); } if (accept("CREATE")) { return create(); diff --git a/database/src/main/java/io/hstore/db/query/Session.java b/database/src/main/java/io/hstore/db/query/Session.java index 85ef2bda..70b0e9e5 100644 --- a/database/src/main/java/io/hstore/db/query/Session.java +++ b/database/src/main/java/io/hstore/db/query/Session.java @@ -42,6 +42,7 @@ public final class Session implements AutoCloseable { private int failedAuthentications; private Optional scram = Optional.empty(); private Optional scramUser = Optional.empty(); + private Optional clearTextRefusal = Optional.empty(); private record ScramAttempt(Scram.Server server, SignIns.Reservation reservation) { } @@ -127,6 +128,10 @@ public Optional scramUser() { return scramUser; } + public void refuseClearTextPasswords(String reason) { + clearTextRefusal = Optional.of(reason); + } + private void requireCanAuthenticate() { if (transaction.isPresent()) { throw HStoreException.invalid("cannot authenticate inside a transaction"); @@ -219,6 +224,9 @@ private QueryResult dispatch(Statement statement) { yield QueryResult.message("using tenant " + tenant.name()); } case Statement.Authenticate(String user, String password) -> { + if (clearTextRefusal.isPresent()) { + throw HStoreException.invalid(clearTextRefusal.get()); + } if (!authenticate(user, password)) { throw HStoreException.invalid("authentication failed for user " + user); } diff --git a/database/src/main/java/io/hstore/db/security/SaslPrep.java b/database/src/main/java/io/hstore/db/security/SaslPrep.java new file mode 100644 index 00000000..e1e48647 --- /dev/null +++ b/database/src/main/java/io/hstore/db/security/SaslPrep.java @@ -0,0 +1,124 @@ +// SPDX-FileCopyrightText: MetaCognition India Pvt Limited +// SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0 + +package io.hstore.db.security; + +import java.text.Normalizer; +import java.util.Arrays; + +final class SaslPrep { + + // Tables B.1, C.1.2, C.2 to C.9, A.1, D.1 and D.2 of RFC 3454, which are fixed at Unicode 3.2. + private static final Ranges MAPPED_TO_NOTHING = Ranges.parse("AD,34F,1806,180B-180D,200B-200D,2060,FE00-FE0F,FEFF"); + private static final Ranges NON_ASCII_SPACE = Ranges.parse("A0,1680,2000-200B,202F,205F,3000"); + private static final Ranges PROHIBITED = Ranges.parse("0-1F,7F-A0,340-341,6DD,70F,1680,180E,2000-200F,2028-202F,205F-2063,206A-206F,2FF0-2FFB,3000,E000-F8FF," + + "FDD0-FDEF,FEFF,FFF9-FFFF,1D173-1D17A,1FFFE-1FFFF,2FFFE-2FFFF,3FFFE-3FFFF,4FFFE-4FFFF,5FFFE-5FFFF,6FFFE-6FFFF," + + "7FFFE-7FFFF,8FFFE-8FFFF,9FFFE-9FFFF,AFFFE-AFFFF,BFFFE-BFFFF,CFFFE-CFFFF,DFFFE-DFFFF,E0001,E0020-E007F," + + "EFFFE-10FFFF"); + private static final Ranges UNASSIGNED = Ranges.parse("221,234-24F,2AE-2AF,2EF-2FF,350-35F,370-373,376-379,37B-37D,37F-383,38B,38D,3A2,3CF,3F7-3FF,487,4CF,4F6-4F7," + + "4FA-4FF,510-530,557-558,560,588,58B-590,5A2,5BA,5C5-5CF,5EB-5EF,5F5-60B,60D-61A,61C-61E,620,63B-63F,656-65F," + + "6EE-6EF,6FF,70E,72D-72F,74B-77F,7B2-900,904,93A-93B,94E-94F,955-957,971-980,984,98D-98E,991-992,9A9,9B1," + + "9B3-9B5,9BA-9BB,9BD,9C5-9C6,9C9-9CA,9CE-9D6,9D8-9DB,9DE,9E4-9E5,9FB-A01,A03-A04,A0B-A0E,A11-A12,A29,A31,A34," + + "A37,A3A-A3B,A3D,A43-A46,A49-A4A,A4E-A58,A5D,A5F-A65,A75-A80,A84,A8C,A8E,A92,AA9,AB1,AB4,ABA-ABB,AC6,ACA," + + "ACE-ACF,AD1-ADF,AE1-AE5,AF0-B00,B04,B0D-B0E,B11-B12,B29,B31,B34-B35,B3A-B3B,B44-B46,B49-B4A,B4E-B55,B58-B5B," + + "B5E,B62-B65,B71-B81,B84,B8B-B8D,B91,B96-B98,B9B,B9D,BA0-BA2,BA5-BA7,BAB-BAD,BB6,BBA-BBD,BC3-BC5,BC9,BCE-BD6," + + "BD8-BE6,BF3-C00,C04,C0D,C11,C29,C34,C3A-C3D,C45,C49,C4E-C54,C57-C5F,C62-C65,C70-C81,C84,C8D,C91,CA9,CB4," + + "CBA-CBD,CC5,CC9,CCE-CD4,CD7-CDD,CDF,CE2-CE5,CF0-D01,D04,D0D,D11,D29,D3A-D3D,D44-D45,D49,D4E-D56,D58-D5F," + + "D62-D65,D70-D81,D84,D97-D99,DB2,DBC,DBE-DBF,DC7-DC9,DCB-DCE,DD5,DD7,DE0-DF1,DF5-E00,E3B-E3E,E5C-E80,E83," + + "E85-E86,E89,E8B-E8C,E8E-E93,E98,EA0,EA4,EA6,EA8-EA9,EAC,EBA,EBE-EBF,EC5,EC7,ECE-ECF,EDA-EDB,EDE-EFF,F48," + + "F6B-F70,F8C-F8F,F98,FBD,FCD-FCE,FD0-FFF,1022,1028,102B,1033-1035,103A-103F,105A-109F,10C6-10CF,10F9-10FA," + + "10FC-10FF,115A-115E,11A3-11A7,11FA-11FF,1207,1247,1249,124E-124F,1257,1259,125E-125F,1287,1289,128E-128F,12AF," + + "12B1,12B6-12B7,12BF,12C1,12C6-12C7,12CF,12D7,12EF,130F,1311,1316-1317,131F,1347,135B-1360,137D-139F,13F5-1400," + + "1677-167F,169D-169F,16F1-16FF,170D,1715-171F,1737-173F,1754-175F,176D,1771,1774-177F,17DD-17DF,17EA-17FF,180F," + + "181A-181F,1878-187F,18AA-1DFF,1E9C-1E9F,1EFA-1EFF,1F16-1F17,1F1E-1F1F,1F46-1F47,1F4E-1F4F,1F58,1F5A,1F5C,1F5E," + + "1F7E-1F7F,1FB5,1FC5,1FD4-1FD5,1FDC,1FF0-1FF1,1FF5,1FFF,2053-2056,2058-205E,2064-2069,2072-2073,208F-209F," + + "20B2-20CF,20EB-20FF,213B-213C,214C-2152,2184-218F,23CF-23FF,2427-243F,244B-245F,24FF,2614-2615,2618,267E-267F," + + "268A-2700,2705,270A-270B,2728,274C,274E,2753-2755,2757,275F-2760,2795-2797,27B0,27BF-27CF,27EC-27EF,2B00-2E7F," + + "2E9A,2EF4-2EFF,2FD6-2FEF,2FFC-2FFF,3040,3097-3098,3100-3104,312D-3130,318F,31B8-31EF,321D-321F,3244-3250," + + "327C-327E,32CC-32CF,32FF,3377-337A,33DE-33DF,33FF,4DB6-4DFF,9FA6-9FFF,A48D-A48F,A4C7-ABFF,D7A4-D7FF,FA2E-FA2F," + + "FA6B-FAFF,FB07-FB12,FB18-FB1C,FB37,FB3D,FB3F,FB42,FB45,FBB2-FBD2,FD40-FD4F,FD90-FD91,FDC8-FDCF,FDFD-FDFF," + + "FE10-FE1F,FE24-FE2F,FE47-FE48,FE53,FE67,FE6C-FE6F,FE75,FEFD-FEFE,FF00,FFBF-FFC1,FFC8-FFC9,FFD0-FFD1,FFD8-FFD9," + + "FFDD-FFDF,FFE7,FFEF-FFF8,10000-102FF,1031F,10324-1032F,1034B-103FF,10426-10427,1044E-1CFFF,1D0F6-1D0FF," + + "1D127-1D129,1D1DE-1D3FF,1D455,1D49D,1D4A0-1D4A1,1D4A3-1D4A4,1D4A7-1D4A8,1D4AD,1D4BA,1D4BC,1D4C1,1D4C4,1D506," + + "1D50B-1D50C,1D515,1D51D,1D53A,1D53F,1D545,1D547-1D549,1D551,1D6A4-1D6A7,1D7CA-1D7CD,1D800-1FFFD,2A6D7-2F7FF," + + "2FA1E-2FFFD,30000-3FFFD,40000-4FFFD,50000-5FFFD,60000-6FFFD,70000-7FFFD,80000-8FFFD,90000-9FFFD,A0000-AFFFD," + + "B0000-BFFFD,C0000-CFFFD,D0000-DFFFD,E0000,E0002-E001F,E0080-EFFFD"); + private static final Ranges RAND_AL_CAT = Ranges.parse("5BE,5C0,5C3,5D0-5EA,5F0-5F4,61B,61F,621-63A,640-64A,66D-66F,671-6D5,6DD,6E5-6E6,6FA-6FE,700-70D,710,712-72C," + + "780-7A5,7B1,200F,FB1D,FB1F-FB28,FB2A-FB36,FB38-FB3C,FB3E,FB40-FB41,FB43-FB44,FB46-FBB1,FBD3-FD3D,FD50-FD8F," + + "FD92-FDC7,FDF0-FDFC,FE70-FE74,FE76-FEFC"); + private static final Ranges L_CAT = Ranges.parse("41-5A,61-7A,AA,B5,BA,C0-D6,D8-F6,F8-220,222-233,250-2AD,2B0-2B8,2BB-2C1,2D0-2D1,2E0-2E4,2EE,37A,386,388-38A," + + "38C,38E-3A1,3A3-3CE,3D0-3F5,400-482,48A-4CE,4D0-4F5,4F8-4F9,500-50F,531-556,559-55F,561-587,589,903,905-939," + + "93D-940,949-94C,950,958-961,964-970,982-983,985-98C,98F-990,993-9A8,9AA-9B0,9B2,9B6-9B9,9BE-9C0,9C7-9C8," + + "9CB-9CC,9D7,9DC-9DD,9DF-9E1,9E6-9F1,9F4-9FA,A05-A0A,A0F-A10,A13-A28,A2A-A30,A32-A33,A35-A36,A38-A39,A3E-A40," + + "A59-A5C,A5E,A66-A6F,A72-A74,A83,A85-A8B,A8D,A8F-A91,A93-AA8,AAA-AB0,AB2-AB3,AB5-AB9,ABD-AC0,AC9,ACB-ACC,AD0," + + "AE0,AE6-AEF,B02-B03,B05-B0C,B0F-B10,B13-B28,B2A-B30,B32-B33,B36-B39,B3D-B3E,B40,B47-B48,B4B-B4C,B57,B5C-B5D," + + "B5F-B61,B66-B70,B83,B85-B8A,B8E-B90,B92-B95,B99-B9A,B9C,B9E-B9F,BA3-BA4,BA8-BAA,BAE-BB5,BB7-BB9,BBE-BBF," + + "BC1-BC2,BC6-BC8,BCA-BCC,BD7,BE7-BF2,C01-C03,C05-C0C,C0E-C10,C12-C28,C2A-C33,C35-C39,C41-C44,C60-C61,C66-C6F," + + "C82-C83,C85-C8C,C8E-C90,C92-CA8,CAA-CB3,CB5-CB9,CBE,CC0-CC4,CC7-CC8,CCA-CCB,CD5-CD6,CDE,CE0-CE1,CE6-CEF," + + "D02-D03,D05-D0C,D0E-D10,D12-D28,D2A-D39,D3E-D40,D46-D48,D4A-D4C,D57,D60-D61,D66-D6F,D82-D83,D85-D96,D9A-DB1," + + "DB3-DBB,DBD,DC0-DC6,DCF-DD1,DD8-DDF,DF2-DF4,E01-E30,E32-E33,E40-E46,E4F-E5B,E81-E82,E84,E87-E88,E8A,E8D," + + "E94-E97,E99-E9F,EA1-EA3,EA5,EA7,EAA-EAB,EAD-EB0,EB2-EB3,EBD,EC0-EC4,EC6,ED0-ED9,EDC-EDD,F00-F17,F1A-F34,F36," + + "F38,F3E-F47,F49-F6A,F7F,F85,F88-F8B,FBE-FC5,FC7-FCC,FCF,1000-1021,1023-1027,1029-102A,102C,1031,1038," + + "1040-1057,10A0-10C5,10D0-10F8,10FB,1100-1159,115F-11A2,11A8-11F9,1200-1206,1208-1246,1248,124A-124D,1250-1256," + + "1258,125A-125D,1260-1286,1288,128A-128D,1290-12AE,12B0,12B2-12B5,12B8-12BE,12C0,12C2-12C5,12C8-12CE,12D0-12D6," + + "12D8-12EE,12F0-130E,1310,1312-1315,1318-131E,1320-1346,1348-135A,1361-137C,13A0-13F4,1401-1676,1681-169A," + + "16A0-16F0,1700-170C,170E-1711,1720-1731,1735-1736,1740-1751,1760-176C,176E-1770,1780-17B6,17BE-17C5,17C7-17C8," + + "17D4-17DA,17DC,17E0-17E9,1810-1819,1820-1877,1880-18A8,1E00-1E9B,1EA0-1EF9,1F00-1F15,1F18-1F1D,1F20-1F45," + + "1F48-1F4D,1F50-1F57,1F59,1F5B,1F5D,1F5F-1F7D,1F80-1FB4,1FB6-1FBC,1FBE,1FC2-1FC4,1FC6-1FCC,1FD0-1FD3,1FD6-1FDB," + + "1FE0-1FEC,1FF2-1FF4,1FF6-1FFC,200E,2071,207F,2102,2107,210A-2113,2115,2119-211D,2124,2126,2128,212A-212D," + + "212F-2131,2133-2139,213D-213F,2145-2149,2160-2183,2336-237A,2395,249C-24E9,3005-3007,3021-3029,3031-3035," + + "3038-303C,3041-3096,309D-309F,30A1-30FA,30FC-30FF,3105-312C,3131-318E,3190-31B7,31F0-321C,3220-3243,3260-327B," + + "327F-32B0,32C0-32CB,32D0-32FE,3300-3376,337B-33DD,33E0-33FE,3400-4DB5,4E00-9FA5,A000-A48C,AC00-D7A3,E000-FA2D," + + "FA30-FA6A,FB00-FB06,FB13-FB17,FF21-FF3A,FF41-FF5A,FF66-FFBE,FFC2-FFC7,FFCA-FFCF,FFD2-FFD7,FFDA-FFDC," + + "10300-1031E,10320-10323,10330-1034A,10400-10425,10428-1044D,1D000-1D0F5,1D100-1D126,1D12A-1D166,1D16A-1D172," + + "1D183-1D184,1D18C-1D1A9,1D1AE-1D1DD,1D400-1D454,1D456-1D49C,1D49E-1D49F,1D4A2,1D4A5-1D4A6,1D4A9-1D4AC," + + "1D4AE-1D4B9,1D4BB,1D4BD-1D4C0,1D4C2-1D4C3,1D4C5-1D505,1D507-1D50A,1D50D-1D514,1D516-1D51C,1D51E-1D539," + + "1D53B-1D53E,1D540-1D544,1D546,1D54A-1D550,1D552-1D6A3,1D6A8-1D7C9,20000-2A6D6,2F800-2FA1D,F0000-FFFFD," + + "100000-10FFFD"); + + private SaslPrep() { + } + + static String prepare(String password) { + if (password.chars().allMatch(c -> c < 0x80)) { + return password; + } + int[] mapped = password.codePoints() + .filter(code -> !MAPPED_TO_NOTHING.contains(code) || NON_ASCII_SPACE.contains(code)) + .map(code -> NON_ASCII_SPACE.contains(code) ? ' ' : code) + .toArray(); + if (mapped.length == 0 || Arrays.stream(mapped).anyMatch(UNASSIGNED::contains)) { + return password; + } + String normalized = Normalizer.normalize(new String(mapped, 0, mapped.length), Normalizer.Form.NFKC); + int[] codes = normalized.codePoints().toArray(); + if (Arrays.stream(codes).anyMatch(code -> PROHIBITED.contains(code) || code >= 0xD800 && code <= 0xDFFF || UNASSIGNED.contains(code))) { + return password; + } + if (Arrays.stream(codes).anyMatch(RAND_AL_CAT::contains) + && (Arrays.stream(codes).anyMatch(L_CAT::contains) || !RAND_AL_CAT.contains(codes[0]) || !RAND_AL_CAT.contains(codes[codes.length - 1]))) { + return password; + } + return normalized; + } + + private record Ranges(int[] starts, int[] ends) { + static Ranges parse(String text) { + String[] parts = text.split(","); + int[] starts = new int[parts.length]; + int[] ends = new int[parts.length]; + for (int i = 0; i < parts.length; i++) { + int dash = parts[i].indexOf('-'); + starts[i] = Integer.parseInt(dash < 0 ? parts[i] : parts[i].substring(0, dash), 16); + ends[i] = dash < 0 ? starts[i] : Integer.parseInt(parts[i].substring(dash + 1), 16); + } + return new Ranges(starts, ends); + } + + boolean contains(int code) { + int at = Arrays.binarySearch(starts, code); + int range = at >= 0 ? at : -at - 2; + return range >= 0 && code <= ends[range]; + } + } +} diff --git a/database/src/main/java/io/hstore/db/security/Scram.java b/database/src/main/java/io/hstore/db/security/Scram.java index 09ba43a9..3f8750cc 100644 --- a/database/src/main/java/io/hstore/db/security/Scram.java +++ b/database/src/main/java/io/hstore/db/security/Scram.java @@ -29,7 +29,9 @@ public final class Scram { public static final String MECHANISM = "SCRAM-SHA-256"; public static final String MECHANISM_PLUS = "SCRAM-SHA-256-PLUS"; public static final String CHANNEL_BINDING = "tls-server-end-point"; - private static final String PREFIX = "SCRAM-SHA-256$"; + public static final String PREFIX = "SCRAM-SHA-256$"; + public static final int MIN_ITERATIONS = 4096; + public static final int MAX_ITERATIONS = 10_000_000; private static final int NONCE_BYTES = 18; private static final int MAX_MESSAGE = 4096; private static final SecureRandom RANDOM = new SecureRandom(); @@ -42,7 +44,7 @@ private Scram() { public record Verifier(int iterations, byte[] salt, byte[] storedKey, byte[] serverKey) { public static Verifier of(String password, byte[] salt, int iterations) { - return fromSaltedPassword(saltedPassword(password, salt, iterations), salt, iterations); + return fromSaltedPassword(saltedPassword(SaslPrep.prepare(password), salt, iterations), salt, iterations); } public static Verifier fromSaltedPassword(byte[] saltedPassword, byte[] salt, int iterations) { @@ -67,7 +69,7 @@ public static Optional parse(String text) { byte[] salt = DECODE.decode(head[1]); byte[] stored = DECODE.decode(keys[0]); byte[] server = DECODE.decode(keys[1]); - if (iterations < 1 || salt.length == 0 || stored.length != 32 || server.length != 32) { + if (iterations < MIN_ITERATIONS || iterations > MAX_ITERATIONS || salt.length == 0 || stored.length != 32 || server.length != 32) { return Optional.empty(); } return Optional.of(new Verifier(iterations, salt, stored, server)); @@ -82,7 +84,7 @@ public String encode() { } public boolean matches(String password) { - byte[] salted = saltedPassword(password, salt, iterations); + byte[] salted = saltedPassword(SaslPrep.prepare(password), salt, iterations); return MessageDigest.isEqual(sha256(hmac(salted, "Client Key")), storedKey); } @@ -248,13 +250,13 @@ public String last(String serverFirst) { } catch (NumberFormatException e) { throw malformed("iteration count is not a number"); } - if (iterations < 4096 || iterations > 10_000_000) { - throw malformed("iteration count " + iterations + " is outside 4096 to 10000000"); + if (iterations < MIN_ITERATIONS || iterations > MAX_ITERATIONS) { + throw malformed("iteration count " + iterations + " is outside " + MIN_ITERATIONS + " to " + MAX_ITERATIONS); } byte[] bindData = concat(header.getBytes(StandardCharsets.UTF_8), channelBinding.orElse(new byte[0])); String withoutProof = "c=" + ENCODE.encodeToString(bindData) + ",r=" + combined; String authMessage = firstBare + "," + serverFirst + "," + withoutProof; - byte[] salted = saltedPassword(password, salt, iterations); + byte[] salted = saltedPassword(SaslPrep.prepare(password), salt, iterations); byte[] clientKey = hmac(salted, "Client Key"); byte[] proof = xor(clientKey, hmac(sha256(clientKey), authMessage)); serverSignature = hmac(hmac(salted, "Server Key"), authMessage); diff --git a/database/src/main/java/io/hstore/db/security/Security.java b/database/src/main/java/io/hstore/db/security/Security.java index f5ac054b..bc31bc43 100644 --- a/database/src/main/java/io/hstore/db/security/Security.java +++ b/database/src/main/java/io/hstore/db/security/Security.java @@ -16,6 +16,7 @@ import io.hstore.engine.txn.Transaction; import io.hstore.engine.txn.View; +import java.security.MessageDigest; import java.security.SecureRandom; import java.util.HexFormat; import java.util.List; @@ -31,7 +32,6 @@ public record User(String name, String salt, String hash, int iterations, int te } public static final int ITERATIONS = 600_000; - private static final int MIN_VERIFIER_ITERATIONS = 4096; private static final long SCRAM_SECRET = 0x5343_5241_4D00_0000L; private static final int SCRAM_SECRET_WORDS = 4; public static final int MIN_PASSWORD_LENGTH = 8; @@ -156,13 +156,10 @@ public static void setPassword(Transaction txn, String name, String password) { } static Scram.Verifier secret(String password) { - Optional given = Scram.Verifier.parse(password); - if (given.isPresent()) { - if (given.get().iterations() < MIN_VERIFIER_ITERATIONS || given.get().salt().length < 8) { - throw HStoreException.invalid("a SCRAM-SHA-256 verifier needs at least " + MIN_VERIFIER_ITERATIONS - + " iterations and an 8-byte salt"); - } - return given.get(); + if (password.startsWith(Scram.PREFIX)) { + return Scram.Verifier.parse(password).filter(given -> given.salt().length >= 8) + .orElseThrow(() -> HStoreException.invalid("not a valid SCRAM-SHA-256 verifier: it needs " + Scram.MIN_ITERATIONS + + " to " + Scram.MAX_ITERATIONS + " iterations, a salt of at least 8 bytes and 32-byte keys")); } return newVerifier(password); } @@ -174,22 +171,31 @@ public static Scram.Verifier newVerifier(String password) { return Scram.Verifier.of(password, salt, ITERATIONS); } - public static Scram.Verifier verifier(User user) { - return Scram.Verifier.parse(user.hash()).orElseGet(() -> Scram.Verifier.fromSaltedPassword(HEX.parseHex(user.hash()), - HEX.parseHex(user.salt()), user.iterations())); + public static Optional verifier(User user) { + return Scram.Verifier.parse(user.hash()); } public static boolean legacy(User user) { - return Scram.Verifier.parse(user.hash()).isEmpty(); + return !user.hash().startsWith(Scram.PREFIX); + } + + public static List legacyUsers(View view) { + return users(view).stream().filter(Security::legacy).map(User::name).sorted().toList(); + } + + public static void replaceLegacy(Transaction txn, User legacy, String password) { + byte[] salt = new byte[16]; + RANDOM.nextBytes(salt); + Scram.Verifier verifier = Scram.Verifier.of(password, salt, ITERATIONS); + txn.merge(USERS, userKey(legacy.name()), current -> current.equals(Optional.of(legacy)), + _ -> Optional.of(new User(legacy.name(), HEX.formatHex(salt), verifier.encode(), ITERATIONS, legacy.tenant(), legacy.role()))); } - public static int upgradeLegacyUsers(Transaction txn) { - List legacy = users(txn).stream().filter(Security::legacy).toList(); - for (User user : legacy) { - txn.put(USERS, userKey(user.name()), new User(user.name(), user.salt(), verifier(user).encode(), user.iterations(), - user.tenant(), user.role())); + private static boolean matches(User user, String password) { + if (legacy(user)) { + return MessageDigest.isEqual(derive(password, HEX.parseHex(user.salt()), user.iterations()), HEX.parseHex(user.hash())); } - return legacy.size(); + return verifier(user).filter(verifier -> verifier.matches(password)).isPresent(); } public static Optional scramSecret(View view) { @@ -242,7 +248,7 @@ public static Optional authenticate(View view, String name, String pa if (stored.isEmpty()) { derive(password, UNKNOWN_USER_SALT, ITERATIONS); } - return stored.filter(user -> verifier(user).matches(password)).map(Security::principal); + return stored.filter(user -> matches(user, password)).map(Security::principal); } public static boolean hasUsers(View view) { diff --git a/database/src/test/java/io/hstore/db/SecurityTest.java b/database/src/test/java/io/hstore/db/SecurityTest.java index 9fd0c22c..9a04cf9c 100644 --- a/database/src/test/java/io/hstore/db/SecurityTest.java +++ b/database/src/test/java/io/hstore/db/SecurityTest.java @@ -30,11 +30,17 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.security.MessageDigest; import java.time.Duration; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Base64; import java.util.HexFormat; import java.util.Optional; import java.util.concurrent.atomic.AtomicLong; @@ -761,33 +767,78 @@ void signInsAreThrottledPerUserAndAddress() { assertThrows(HStoreException.ResourceLimit.class, () -> accounts.attempt("local", Optional.empty(), () -> Optional.of("never"))); } + private static byte[] hmac(byte[] key, byte[] message) throws Exception { + Mac mac = Mac.getInstance("HmacSHA256"); + mac.init(new SecretKeySpec(key, "HmacSHA256")); + return mac.doFinal(message); + } + + private static Optional signInWithSaltedPassword(Session session, String user, byte[] salted) throws Exception { + String firstBare = "n=" + user + ",r=stolen-hash-nonce"; + String serverFirst = session.scramFirst(Scram.MECHANISM, "n,," + firstBare, Optional.empty()); + String withoutProof = "c=biws,r=" + serverFirst.substring(2, serverFirst.indexOf(',')); + byte[] authMessage = (firstBare + "," + serverFirst + "," + withoutProof).getBytes(StandardCharsets.UTF_8); + byte[] clientKey = hmac(salted, "Client Key".getBytes(StandardCharsets.UTF_8)); + byte[] signature = hmac(MessageDigest.getInstance("SHA-256").digest(clientKey), authMessage); + byte[] proof = new byte[clientKey.length]; + for (int i = 0; i < proof.length; i++) { + proof[i] = (byte) (clientKey[i] ^ signature[i]); + } + return session.scramFinal(withoutProof + ",p=" + Base64.getEncoder().encodeToString(proof)); + } + + private Optional scram(String user, String password) { + try (Session session = new Session(database)) { + Scram.Client client = new Scram.Client(user, password, Optional.empty()); + return session.scramFinal(client.last(session.scramFirst(Scram.MECHANISM, client.first(), Optional.empty()))); + } + } + @Test - void passwordsAreStoredAsScramVerifiersAndLegacyHashesUpgrade() { + void anOldPasswordHashIsNeverEnoughToSignIn() throws Exception { Security.User alice = database.read(reader -> Security.user(reader.view(), "alice")).orElseThrow(); assertTrue(alice.hash().startsWith("SCRAM-SHA-256$600000:"), alice.hash()); + Scram.Verifier aliceVerifier = Security.verifier(alice).orElseThrow(); + try (Session session = new Session(database)) { + assertTrue(signInWithSaltedPassword(session, "alice", Scram.saltedPassword("s3cret-pass", aliceVerifier.salt(), + aliceVerifier.iterations())).isPresent(), "the forged exchange works when the key is right"); + } + byte[] salt = new byte[16]; - byte[] salted = Scram.saltedPassword("legacy-pass", salt, 4096); + byte[] stolen = Scram.saltedPassword("legacy-pass", salt, 4096); + Security.User legacy = new Security.User("old", HexFormat.of().formatHex(salt), HexFormat.of().formatHex(stolen), 4096, + Principal.DEFAULT_TENANT, Role.READER); database.write(writer -> { - writer.transaction().put(Security.USERS, Hashing.of("old"), new Security.User("old", HexFormat.of().formatHex(salt), - HexFormat.of().formatHex(salted), 4096, Principal.DEFAULT_TENANT, Role.READER)); + writer.transaction().put(Security.USERS, Hashing.of("old"), legacy); return null; }); - assertTrue(Security.legacy(database.read(reader -> Security.user(reader.view(), "old")).orElseThrow())); - assertTrue(new Session(database).authenticate("old", "legacy-pass")); admin.close(); database.close(); database = HypergraphDatabase.open(directory, options()); admin = new Session(database); - Security.User upgraded = database.read(reader -> Security.user(reader.view(), "old")).orElseThrow(); - assertEquals(Scram.Verifier.of("legacy-pass", salt, 4096).encode(), upgraded.hash()); + assertEquals(legacy, database.read(reader -> Security.user(reader.view(), "old")).orElseThrow(), "opening converts nothing"); + assertEquals(List.of("old"), database.read(reader -> Security.legacyUsers(reader.view()))); + + try (Session session = new Session(database)) { + assertEquals(Optional.empty(), signInWithSaltedPassword(session, "old", stolen)); + } + assertEquals(Optional.empty(), scram("old", "legacy-pass"), "an old hash cannot be used with SCRAM, even with the password"); + assertFalse(new Session(database).authenticate("old", "wrong-pass")); + assertEquals(legacy, database.read(reader -> Security.user(reader.view(), "old")).orElseThrow()); + + assertTrue(new Session(database).authenticate("old", "legacy-pass")); + Security.User converted = database.read(reader -> Security.user(reader.view(), "old")).orElseThrow(); + assertFalse(Security.legacy(converted)); + Scram.Verifier fresh = Security.verifier(converted).orElseThrow(); + assertFalse(Arrays.equals(salt, fresh.salt()), "the conversion picks a new salt"); + assertEquals(Security.ITERATIONS, fresh.iterations()); + assertEquals(List.of(), database.read(reader -> Security.legacyUsers(reader.view()))); + try (Session session = new Session(database)) { + assertEquals(Optional.empty(), signInWithSaltedPassword(session, "old", stolen)); + } + assertTrue(scram("old", "legacy-pass").isPresent()); assertTrue(new Session(database).authenticate("old", "legacy-pass")); assertFalse(new Session(database).authenticate("old", "wrong-pass")); - long generation = database.engine().transactions().current().id(); - admin.close(); - database.close(); - database = HypergraphDatabase.open(directory, options()); - admin = new Session(database); - assertEquals(generation, database.engine().transactions().current().id()); String verifier = Scram.Verifier.of("chosen-elsewhere", new byte[]{1, 2, 3, 4, 5, 6, 7, 8}, 4096).encode(); admin.execute("CREATE USER remote PASSWORD '" + verifier + "' ROLE reader"); diff --git a/database/src/test/java/io/hstore/db/security/ScramTest.java b/database/src/test/java/io/hstore/db/security/ScramTest.java index 5f7251c5..7528c042 100644 --- a/database/src/test/java/io/hstore/db/security/ScramTest.java +++ b/database/src/test/java/io/hstore/db/security/ScramTest.java @@ -149,8 +149,7 @@ void verifiersRoundTripAndLegacyHashesConvert() throws Exception { assertArrayEquals(pbkdf2(password.getBytes(StandardCharsets.UTF_8), RFC_SALT, 4096), salted); Security.User legacy = new Security.User("legacy", HexFormat.of().formatHex(RFC_SALT), HexFormat.of().formatHex(salted), 4096, 0, Role.READER); assertTrue(Security.legacy(legacy)); - assertEquals(Scram.Verifier.of(password, RFC_SALT, 4096), Security.verifier(legacy)); - assertTrue(Security.verifier(legacy).matches(password)); + assertEquals(Optional.empty(), Security.verifier(legacy)); } private static byte[] pbkdf2(byte[] password, byte[] salt, int iterations) throws Exception { @@ -167,4 +166,59 @@ private static byte[] pbkdf2(byte[] password, byte[] salt, int iterations) throw } return result; } + + @Test + void saslPrepMatchesTheStringprepProfile() { + List> vectors = List.of( + List.of("I\u00adX", "IX"), + List.of("user", "user"), + List.of("USER", "USER"), + List.of("\u00aa", "a"), + List.of("\u2168", "IX"), + List.of("\u0007x", "\u0007x"), + List.of("\u06271", "\u06271"), + List.of("\u0627\u0628", "\u0627\u0628"), + List.of("p\u00e4ssw\u00f6rd", "p\u00e4ssw\u00f6rd"), + List.of("pa\u0308ssword", "p\u00e4ssword"), + List.of("\u00a0lead", " lead"), + List.of("ab\u200bcd", "ab cd"), + List.of("\u2000\u3000", " "), + List.of("\u00ad", "\u00ad"), + List.of("\ufb01ne", "fine"), + List.of("\uff21\uff22", "AB"), + List.of("\u0221", "\u0221"), + List.of("\ud83d\ude00pass", "\ud83d\ude00pass"), + List.of("\ue000secret", "\ue000secret"), + List.of("\u05d0x\u05d1", "\u05d0x\u05d1"), + List.of("\u05d0\u05d1", "\u05d0\u05d1"), + List.of("\u212b", "\u00c5"), + List.of("\u1e9b\u0323", "\u1e69"), + List.of("caf\u00e9-\u2163", "caf\u00e9-IV"), + List.of("\u0340a", "\u0300a"), + List.of("\u200eltr\u00e9", "\u200eltr\u00e9"), + List.of("\ud83c\udd00x", "\ud83c\udd00x")); + for (List vector : vectors) { + assertEquals(vector.get(1), SaslPrep.prepare(vector.get(0)), vector.get(0)); + } + } + + @Test + void passwordsAreComparedAfterSaslPrep() { + Scram.Verifier composed = Scram.Verifier.of("p\u00e4ssword-\u2163", "pepper-salt".getBytes(StandardCharsets.UTF_8), 4096); + assertTrue(composed.matches("pa\u0308ssword-IV")); + Scram.Client client = new Scram.Client("ann", "pa\u0308ssword-IV", Optional.empty()); + Scram.Server server = new Scram.Server(_ -> Optional.of(composed), new byte[32], 4096, Optional.empty()); + String serverFirst = server.first(Scram.MECHANISM, client.first()); + Scram.Outcome.Accepted accepted = assertInstanceOf(Scram.Outcome.Accepted.class, server.last(client.last(serverFirst))); + client.verify(accepted.serverFinal()); + } + + @Test + void verifiersMustUseABoundedIterationCount() { + Scram.Verifier sound = Scram.Verifier.of("pw-secret", "pepper-salt".getBytes(StandardCharsets.UTF_8), 4096); + assertTrue(Scram.Verifier.parse(sound.encode()).isPresent()); + for (String iterations : List.of("1", "4095", "10000001", "2000000000")) { + assertTrue(Scram.Verifier.parse(sound.encode().replace("$4096:", "$" + iterations + ":")).isEmpty(), iterations); + } + } } diff --git a/docs/database/security.md b/docs/database/security.md index 7ddda537..b2c4e92f 100644 --- a/docs/database/security.md +++ b/docs/database/security.md @@ -132,17 +132,34 @@ record User(String name, String salt, String hash, int iterations, int tenant, R `StoredKey` is `SHA-256(HMAC(SaltedPassword, "Client Key"))` and `ServerKey` is `HMAC(SaltedPassword, "Server Key")`. Neither the password nor `SaltedPassword` is stored, so a copy of the users slot does not let anyone sign in without guessing the password first. -- Older versions stored the hex `SaltedPassword` itself. The first open by this version turns every such record - into a verifier in one transaction (`Security.upgradeLegacyUsers`); the password is not needed, nothing changes - for the user, and a version from before the change can no longer read the upgraded records. Users created - before the iteration count was raised from 120,000 keep their count until their password is next changed. +- Passwords go through SASLprep ([RFC 4013](https://www.rfc-editor.org/rfc/rfc4013)) before PBKDF2, as in + PostgreSQL: a password made only of ASCII characters is used as it is; otherwise non-ASCII spaces become + spaces, characters such as the soft hyphen are dropped and the result is NFKC-normalized. When the result would + contain a character that SASLprep prohibits, or one unassigned in Unicode 3.2, the original password is used + unchanged. The client applies the same rule, so `p\u00e4ssword` typed precomposed or decomposed signs in either + way, and passwords set through other SCRAM clients and libraries match. +- Older versions stored the hex `SaltedPassword` itself. That value is exactly what a SCRAM client derives from + the password, so a copy of it from an old backup would be as good as the password. This version therefore never + uses an old record for SCRAM: such a user cannot sign in over the wire protocol, the server answers it as it + answers an unknown name, and logs a WARNING naming the user. The user's next sign-in through Studio, or an + `AUTHENTICATE` statement in an embedded or Studio session, checks the password against the old record and then + replaces the record with a verifier under a new random salt and 600,000 iterations + (`Security.replaceLegacy`), so old backups give nothing beyond an offline guess. An administrator can instead + set a new password with `ALTER USER`. `hstore serve` logs a WARNING at startup listing the users that still + have an old record. A version from before this change cannot read the converted records. - `CREATE USER` and `ALTER USER … PASSWORD` accept a verifier in place of a password (made by `hstore verifier` - or any SCRAM library), with at least 4,096 iterations and an 8-byte salt; the length rule cannot apply to it. + or any SCRAM library), with 4,096 to 10,000,000 iterations and a salt of at least 8 bytes; the length rule cannot + apply to it. A password that starts with `SCRAM-SHA-256$` but is not such a verifier is refused rather than + stored as a password. The upper bound keeps one Studio or embedded sign-in, where the server runs PBKDF2, to a + few seconds of CPU; clients refuse a server that asks for more. - The wire protocol signs in with SCRAM ([wire protocol](../operations/wire-protocol.md#authentication)): the client proves knowledge of the password against `StoredKey` and the server proves knowledge of `ServerKey`. For a name that does not exist the server answers with a salt derived from the name and a 256-bit secret created when the database is first opened (stored under `EngineSlots.SETTINGS`), so the exchange looks the - same for every name and only fails at its last step. + same for every name and only fails at its last step. The iteration count offered for an unknown name is + 600,000, so a user whose verifier was made with another count (for example one copied from PostgreSQL, which + defaults to 4,096) can be told apart from an unknown name; PostgreSQL behaves the same way. Give such users a + new password if their names must not be guessable. - `Security.authenticate`, used by Studio's login, `AUTHENTICATE` statements in embedded and Studio sessions and the Java API, derives `StoredKey` from the given password and compares with `MessageDigest.isEqual`. For a name that does not exist it still runs PBKDF2 against a fixed salt, so the response time does not tell which names diff --git a/docs/operations/logging.md b/docs/operations/logging.md index 0ca871c9..8be61bb1 100644 --- a/docs/operations/logging.md +++ b/docs/operations/logging.md @@ -142,7 +142,10 @@ and in Docker it is the last line of `docker logs`. The storage-integrity failur | WARNING | `connection from /{ip}:{port} refused: {reason} (max_connections={n})`: the server is full, or 16 connections from this address are still waiting to authenticate. | | WARNING | `connection {n} from /{ip}:{port} refused: no users exist and the client is not on this host` (Studio: `studio request from … refused: …`). | | WARNING | `user "{user}" failed to sign in 50 times within 15 minutes; someone may be guessing its password from several addresses` (logger `hstore.security`). | +| WARNING | `user "{user}" has a password stored by HStore 0.1, which cannot be used with SCRAM-SHA-256; it can sign in once through Studio to convert it, or an administrator can set a new one with ALTER USER` (logger `hstore.security`): the client only sees `authentication failed`. | +| INFO | `the password of user "{user}" is now stored as a SCRAM-SHA-256 verifier` (logger `hstore.security`): a user with a password from HStore 0.1 signed in through Studio or an `AUTHENTICATE` statement. | | WARNING | at startup, `authentication = off and the server listens on {address}: …` or `the database has no users, so connections from other hosts are refused until CREATE USER creates one`, when the listen address is not loopback. | +| WARNING | at startup, `{n} users have passwords stored by HStore 0.1 and cannot sign in over this port until each signs in once through Studio or gets a new password with ALTER USER: {names}`. | | LOG | `studio session opened for user={user} role={ROLE} from /{ip}:{port}` | | WARNING | `studio password authentication failed for user "{user}" from /{ip}:{port}` | diff --git a/docs/operations/wire-protocol.md b/docs/operations/wire-protocol.md index 3248eb8a..bead6db7 100644 --- a/docs/operations/wire-protocol.md +++ b/docs/operations/wire-protocol.md @@ -85,8 +85,8 @@ or when it is `auto` and at least one user exists. Clients sign in with SCRAM-SHA-256 ([RFC 5802](https://www.rfc-editor.org/rfc/rfc5802), [RFC 7677](https://www.rfc-editor.org/rfc/rfc7677)), the same mechanism PostgreSQL uses. The password never crosses the connection: the client proves that it knows it, and the server proves in turn that it holds the -user's verifier, so a client also detects a server that only pretends to be the database. The exchange takes two -requests. Each SCRAM message travels base64-encoded on one line: +user's verifier. A client that follows the rules below therefore detects a server that only pretends to be the +database, unless that server has a copy of the user's verifier. The exchange takes two requests. Each SCRAM message travels base64-encoded on one line: | Step | Client sends | Server answers | |---|---|---| @@ -100,14 +100,27 @@ into its proof. A proof relayed through a man in the middle, which presents a di The server rejects `-PLUS` without TLS, and on a TLS connection it rejects a client that announces channel binding support (gs2 flag `y`) without using it, since that means someone stripped it. -The client must check the server's final message, `v=`, before it trusts the connection. -Passwords are hashed as UTF-8 without SASLprep normalisation, authorization identities and SCRAM extensions are -not supported, and the iteration count is 600,000 for passwords set by this version (at least 4,096 for verifiers -created elsewhere), so a client spends about half a second deriving the key. +A client that has a password to send must not trust the connection until the exchange has finished, and +`hstore connect` disconnects when any of these fails: + +- the banner ends with `; authentication required`. A server that does not ask for a password cannot prove that + it knows it, so a client with credentials refuses to go on rather than send statements to it; +- the answer to step 1 is `sasl …` (or an `error`), and its nonce extends the client's nonce; +- the iteration count is between 4,096 and 10,000,000; +- the answer to step 2 is exactly `authenticated as ` followed by `sasl v=`, and the signature is the one the client computes from the password. + +When the last check fails the server already considers the session signed in, so the client must close the +connection. Passwords go through SASLprep ([RFC 4013](https://www.rfc-editor.org/rfc/rfc4013)) and are then hashed +as UTF-8, as in PostgreSQL: an all-ASCII password is used as it is, and a password that SASLprep rejects is used +unchanged. Authorization identities and SCRAM extensions are not supported, and the iteration count is 600,000 +for passwords set by this version, so a client spends about half a second deriving the key. A user that does not exist gets a well-formed `server-first-message` with a salt derived from the user name and a secret kept in the database, the same every time, and fails only at the second step, exactly like a wrong -password. +password. A user whose password was stored by HStore 0.1 is answered the same way until it signs in once through +Studio or gets a new password ([security](../database/security.md#users)); the server log +says which case it was. Possible replies: @@ -281,6 +294,8 @@ import hmac import os import socket import ssl +import stringprep +import unicodedata def b64(text): @@ -291,6 +306,25 @@ def unb64(text): return base64.b64decode(text).decode() +def saslprep(password): + if password.isascii(): + return password + mapped = "".join(" " if stringprep.in_table_c12(c) else "" if stringprep.in_table_b1(c) else c for c in password) + if not mapped: + return password + prepared = unicodedata.ucd_3_2_0.normalize("NFKC", mapped) + prohibited = (stringprep.in_table_a1, stringprep.in_table_c12, stringprep.in_table_c21_c22, stringprep.in_table_c3, + stringprep.in_table_c4, stringprep.in_table_c5, stringprep.in_table_c6, stringprep.in_table_c7, + stringprep.in_table_c8, stringprep.in_table_c9) + if any(test(c) for c in prepared for test in prohibited): + return password + if any(map(stringprep.in_table_d1, prepared)) and (any(map(stringprep.in_table_d2, prepared)) + or not stringprep.in_table_d1(prepared[0]) + or not stringprep.in_table_d1(prepared[-1])): + return password + return prepared + + class HStore: def __init__(self, host="127.0.0.1", port=7432, user=None, password=None, tls_ca=None): self.sock = socket.create_connection((host, port)) @@ -301,8 +335,15 @@ class HStore: self.banner = self._receive() if self.banner.startswith("error"): raise ConnectionError(self.banner) - if self.banner.endswith("authentication required"): - self._sign_in(user, password) + if user is not None: + if not self.banner.endswith("authentication required"): + self.sock.close() + raise PermissionError("the server did not ask for a password, so it cannot prove who it is") + try: + self._sign_in(user, password) + except Exception: + self.sock.close() + raise self.execute("FORMAT JSON;") def _sign_in(self, user, password): @@ -318,9 +359,9 @@ class HStore: raise PermissionError(reply) server_first = unb64(reply[5:]) fields = dict(part.split("=", 1) for part in server_first.split(",")) - if not fields["r"].startswith(nonce) or int(fields["i"]) < 4096: + if not fields["r"].startswith(nonce) or not 4096 <= int(fields["i"]) <= 10_000_000: raise PermissionError("unexpected server-first-message") - salted = hashlib.pbkdf2_hmac("sha256", password.encode(), base64.b64decode(fields["s"]), int(fields["i"])) + salted = hashlib.pbkdf2_hmac("sha256", saslprep(password).encode(), base64.b64decode(fields["s"]), int(fields["i"])) client_key = hmac.digest(salted, b"Client Key", "sha256") without_proof = "c=" + base64.b64encode(header.encode() + (binding or b"")).decode() + ",r=" + fields["r"] auth_message = f"{first_bare},{server_first},{without_proof}".encode() @@ -329,10 +370,11 @@ class HStore: reply = self.execute(f"SASL {b64(without_proof + ',p=' + proof)}") if reply.startswith("error"): raise PermissionError(reply) - status, final = reply.split("\n", 1) + status, _, final = reply.partition("\n") server_key = hmac.digest(salted, b"Server Key", "sha256") expected = "v=" + base64.b64encode(hmac.digest(server_key, auth_message, "sha256")).decode() - if not hmac.compare_digest(unb64(final[5:]), expected): + if (status != "authenticated as " + user or not final.startswith("sasl ") + or not hmac.compare_digest(unb64(final[5:]).encode(), expected.encode())): raise PermissionError("the server could not prove that it knows the password") def execute(self, script): diff --git a/server/src/main/java/io/hstore/server/RemoteEndpoint.java b/server/src/main/java/io/hstore/server/RemoteEndpoint.java index fc55633a..6657d56c 100644 --- a/server/src/main/java/io/hstore/server/RemoteEndpoint.java +++ b/server/src/main/java/io/hstore/server/RemoteEndpoint.java @@ -60,7 +60,13 @@ public String toString() { close(); throw new IllegalStateException(banner); } - if (banner.endsWith("authentication required") && credentials.isPresent()) { + if (credentials.isPresent() && !banner.endsWith("authentication required")) { + close(); + throw new IllegalArgumentException("the server at " + address.getHostString() + ":" + address.getPort() + + " did not ask for a password, so it cannot prove that it is the server you meant to reach; not signing in as " + + credentials.get().user()); + } + if (credentials.isPresent()) { String reply; try { reply = signIn(credentials.get()); @@ -80,23 +86,29 @@ String signIn(Credentials credentials) { Optional binding = socket instanceof SSLSocket secure ? peerEndPoint(secure) : Optional.empty(); Scram.Client client = new Scram.Client(credentials.user(), credentials.password(), binding); String first = send(Server.SASL + client.mechanism() + " " + encode(client.first())); - if (!first.startsWith("sasl ")) { + if (first.startsWith("error")) { return first; } - String last = send(Server.SASL + encode(client.last(decode(first.substring("sasl ".length()))))); - if (last.startsWith("error")) { - return last; - } - String[] lines = last.split("\n", 2); - if (lines.length != 2 || !lines[0].startsWith("authenticated as ") || !lines[1].startsWith("sasl ")) { - return "error: unexpected reply to the sign-in: " + last; + if (!first.startsWith("sasl ") || first.contains("\n")) { + close(); + return "error: the server did not answer with a SCRAM challenge, so it cannot prove that it knows the password; disconnected"; } try { + String last = send(Server.SASL + encode(client.last(decode(first.substring("sasl ".length()))))); + if (last.startsWith("error")) { + return last; + } + String[] lines = last.split("\n", 2); + if (lines.length != 2 || !lines[0].equals("authenticated as " + credentials.user()) || !lines[1].startsWith("sasl ")) { + close(); + return "error: unexpected reply to the sign-in, so the server did not prove that it knows the password; disconnected"; + } client.verify(decode(lines[1].substring("sasl ".length()))); - } catch (HStoreException failure) { - return "error: " + failure.getMessage(); + return lines[0]; + } catch (HStoreException | IllegalStateException failure) { + close(); + return "error: " + failure.getMessage() + "; disconnected"; } - return lines[0]; } private static Optional peerEndPoint(SSLSocket socket) { diff --git a/server/src/main/java/io/hstore/server/Server.java b/server/src/main/java/io/hstore/server/Server.java index 9d200e5c..0fe1cfad 100644 --- a/server/src/main/java/io/hstore/server/Server.java +++ b/server/src/main/java/io/hstore/server/Server.java @@ -9,6 +9,7 @@ import io.hstore.db.query.QueryResult; import io.hstore.db.query.Session; import io.hstore.db.security.Scram; +import io.hstore.db.security.Security; import io.hstore.engine.HStoreException; import java.io.BufferedInputStream; @@ -149,6 +150,11 @@ void serve() { + "CREATE USER creates one"); } } + List legacy = database.read(reader -> Security.legacyUsers(reader.view())); + if (!legacy.isEmpty()) { + LOG.log(System.Logger.Level.WARNING, "{0} users have passwords stored by HStore 0.1 and cannot sign in over this port until each " + + "signs in once through Studio or gets a new password with ALTER USER: {1}", legacy.size(), String.join(", ", legacy)); + } while (running) { try { Socket socket = listener.accept(); @@ -191,6 +197,7 @@ private void handle(Socket socket) { boolean active = false; boolean unsettled = true; Session session = new Session(database); + session.refuseClearTextPasswords(UPGRADE); try (socket; session) { if (socket instanceof SSLSocket secure) { secure.setSoTimeout(HANDSHAKE_TIMEOUT_MILLIS); @@ -320,7 +327,7 @@ private boolean authenticate(Session session, String request, Writer out, long c WireProtocol.respond(out, "error [" + failure.code() + "]: " + failure.getMessage()); return false; } - if (statements.stream().anyMatch(statement -> statement instanceof Statement.Authenticate)) { + if (statements.stream().anyMatch(Server::signsIn)) { refuseCleartext(out, connection, peer); return false; } @@ -333,6 +340,15 @@ private boolean authenticate(Session session, String request, Writer out, long c static final String UPGRADE = "this server no longer accepts passwords in clear text; upgrade your HStore client to version " + Main.VERSION + " or later, which signs in with " + Scram.MECHANISM; + private static boolean signsIn(Statement statement) { + return switch (statement) { + case Statement.Authenticate _ -> true; + case Statement.At(long _, Statement inner) -> signsIn(inner); + case Statement.AsOf(long _, Statement inner) -> signsIn(inner); + default -> false; + }; + } + private void refuseCleartext(Writer out, long connection, String peer) throws IOException { LOG.log(System.Logger.Level.WARNING, "connection {0} from {1} sent a clear-text password, which is refused; the client needs an upgrade", connection, peer); @@ -396,7 +412,7 @@ private String execute(Session session, String script, long connection) { LOG.log(System.Logger.Level.INFO, "connection {0}: {1}", connection, failure.getMessage()); return "error [" + failure.code() + "]: " + failure.getMessage(); } - if (statements.stream().anyMatch(sourced -> sourced.statement() instanceof Statement.Authenticate)) { + if (statements.stream().map(Parser.Sourced::statement).anyMatch(Server::signsIn)) { LOG.log(System.Logger.Level.WARNING, "connection {0} sent a clear-text password, which is refused; the client needs an upgrade", connection); return "error [AUTHENTICATION_FAILED]: " + UPGRADE; diff --git a/server/src/test/java/io/hstore/server/FakeServerTest.java b/server/src/test/java/io/hstore/server/FakeServerTest.java new file mode 100644 index 00000000..fa4844c5 --- /dev/null +++ b/server/src/test/java/io/hstore/server/FakeServerTest.java @@ -0,0 +1,109 @@ +// SPDX-FileCopyrightText: MetaCognition India Pvt Limited +// SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0 + +package io.hstore.server; + +import org.junit.jupiter.api.Test; + +import java.io.BufferedInputStream; +import java.io.InputStream; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.net.Socket; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.function.UnaryOperator; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class FakeServerTest { + + private static final String REQUIRED = "hstore connection 1 at generation 2; authentication required"; + private static final RemoteEndpoint.Credentials ADMIN = new RemoteEndpoint.Credentials("admin", "pw-secret"); + + private static String base64(String text) { + return Base64.getEncoder().encodeToString(text.getBytes(StandardCharsets.UTF_8)); + } + + private static String challenge(String clientFirst) { + String message = new String(Base64.getDecoder().decode(clientFirst.substring(clientFirst.lastIndexOf(' ') + 1)), StandardCharsets.UTF_8); + String nonce = message.substring(message.indexOf("r=") + 2); + return "sasl " + base64("r=" + nonce + "fake,s=" + base64("salt-salt") + ",i=4096"); + } + + private static String forgedProof() { + return "authenticated as admin\nsasl " + base64("v=" + Base64.getEncoder().encodeToString(new byte[32])); + } + + private record Fake(InetSocketAddress address, CompletableFuture clientHungUp) { + } + + private static Fake fake(String banner, UnaryOperator replies) throws Exception { + ServerSocket listener = new ServerSocket(0, 1, java.net.InetAddress.getLoopbackAddress()); + CompletableFuture hungUp = new CompletableFuture<>(); + Thread.ofVirtual().start(() -> { + try (listener; Socket socket = listener.accept()) { + socket.setSoTimeout(10_000); + InputStream in = new BufferedInputStream(socket.getInputStream()); + Writer out = new OutputStreamWriter(socket.getOutputStream(), StandardCharsets.UTF_8); + WireProtocol.respond(out, banner); + for (Optional request = WireProtocol.receive(in, 1 << 20); request.isPresent(); request = WireProtocol.receive(in, 1 << 20)) { + WireProtocol.respond(out, replies.apply(request.get())); + } + hungUp.complete(true); + } catch (Exception e) { + hungUp.complete(e instanceof java.net.SocketException); + } + }); + return new Fake(new InetSocketAddress("127.0.0.1", listener.getLocalPort()), hungUp); + } + + private static void refused(String banner, UnaryOperator replies, String reason) throws Exception { + Fake fake = fake(banner, replies); + IllegalArgumentException refused = assertThrows(IllegalArgumentException.class, + () -> new RemoteEndpoint(fake.address(), Optional.of(ADMIN))); + assertTrue(refused.getMessage().contains(reason), refused.getMessage()); + assertEquals(true, fake.clientHungUp().get(10, TimeUnit.SECONDS)); + } + + @Test + void aServerThatClaimsSuccessWithoutScramIsRefused() throws Exception { + refused(REQUIRED, _ -> "authenticated as admin", "did not answer with a SCRAM challenge"); + } + + @Test + void aServerThatDoesNotAskForThePasswordIsRefused() throws Exception { + refused("hstore connection 1 at generation 2", _ -> "authenticated as admin", "did not ask for a password"); + } + + @Test + void aServerThatCannotProveItKnowsThePasswordIsRefused() throws Exception { + refused(REQUIRED, request -> request.startsWith("SASL SCRAM") ? challenge(request) : forgedProof(), + "could not prove that it knows the password"); + } + + @Test + void aServerThatSkipsItsProofIsRefused() throws Exception { + refused(REQUIRED, request -> request.startsWith("SASL SCRAM") ? challenge(request) : "authenticated as admin", + "did not prove that it knows the password"); + } + + @Test + void aServerThatNamesAnotherUserIsRefused() throws Exception { + refused(REQUIRED, request -> request.startsWith("SASL SCRAM") ? challenge(request) + : "authenticated as root\nsasl " + base64("v=AAAA"), "did not prove that it knows the password"); + } + + @Test + void aWeakIterationCountIsRefused() throws Exception { + refused(REQUIRED, request -> "sasl " + base64(new String(Base64.getDecoder().decode(challenge(request).substring(5)), + StandardCharsets.UTF_8).replace("i=4096", "i=1")), "iteration count 1 "); + } +} diff --git a/server/src/test/java/io/hstore/server/ServerTest.java b/server/src/test/java/io/hstore/server/ServerTest.java index bbc9c9b3..0ca73f75 100644 --- a/server/src/test/java/io/hstore/server/ServerTest.java +++ b/server/src/test/java/io/hstore/server/ServerTest.java @@ -441,8 +441,24 @@ void clearTextPasswordsAreRefusedWithAnUpgradeMessage() throws Exception { WireProtocol.send(out, "SASL @@@"); assertTrue(WireProtocol.receive(in, 1 << 20).orElseThrow().startsWith("error [AUTHENTICATION_FAILED]")); } + try (Socket socket = new Socket(address.getHostString(), address.getPort())) { + socket.setSoTimeout(10_000); + InputStream in = new java.io.BufferedInputStream(socket.getInputStream()); + java.io.Writer out = new java.io.OutputStreamWriter(socket.getOutputStream(), StandardCharsets.UTF_8); + assertTrue(WireProtocol.receive(in, 1 << 20).orElseThrow().endsWith("authentication required")); + for (String wrapped : List.of("AT GENERATION 1 AUTHENTICATE admin PASSWORD 'pw-secret'", + "AS OF '2020-01-01T00:00:00Z' AUTHENTICATE admin PASSWORD 'pw-secret'")) { + WireProtocol.send(out, wrapped); + assertTrue(WireProtocol.receive(in, 1 << 20).orElseThrow().startsWith("error")); + WireProtocol.send(out, "WHOAMI;"); + assertTrue(WireProtocol.receive(in, 1 << 20).orElseThrow().contains("authenticate first")); + } + } try (Endpoint admin = new RemoteEndpoint(address, Optional.of(new RemoteEndpoint.Credentials("admin", "pw-secret")))) { assertTrue(admin.execute("AUTHENTICATE admin PASSWORD 'pw-secret'; WHOAMI;").contains("upgrade your HStore client")); + admin.execute("CREATE USER other PASSWORD 'other-secret' ROLE reader;"); + assertTrue(admin.execute("AT GENERATION 1 AUTHENTICATE other PASSWORD 'other-secret'; WHOAMI;").startsWith("error")); + assertTrue(admin.execute("WHOAMI;").contains("ADMIN")); assertTrue(admin.execute("AUTHENTICATE admin PASSWORD 'pw-secret';").startsWith("authenticated as admin")); assertTrue(admin.execute("WHOAMI;").contains("ADMIN")); } From 4258a693fcb271fdaeac6bd63a992374da14ee0a Mon Sep 17 00:00:00 2001 From: venkat1701 Date: Tue, 27 Oct 2026 15:20:19 +0530 Subject: [PATCH 4/5] Ask the database for users with old password records --- database/src/main/java/io/hstore/db/HypergraphDatabase.java | 4 ++++ server/src/main/java/io/hstore/server/Server.java | 3 +-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/database/src/main/java/io/hstore/db/HypergraphDatabase.java b/database/src/main/java/io/hstore/db/HypergraphDatabase.java index ca1df673..7ad761cd 100644 --- a/database/src/main/java/io/hstore/db/HypergraphDatabase.java +++ b/database/src/main/java/io/hstore/db/HypergraphDatabase.java @@ -92,6 +92,10 @@ public Scram.Server scram(Optional channelBinding) { }), scramSecret, Security.ITERATIONS, channelBinding); } + public List usersWithOldPasswords() { + return read(reader -> Security.legacyUsers(reader.view())); + } + public SignIns signIns() { return signIns; } diff --git a/server/src/main/java/io/hstore/server/Server.java b/server/src/main/java/io/hstore/server/Server.java index 0fe1cfad..a8756f5a 100644 --- a/server/src/main/java/io/hstore/server/Server.java +++ b/server/src/main/java/io/hstore/server/Server.java @@ -9,7 +9,6 @@ import io.hstore.db.query.QueryResult; import io.hstore.db.query.Session; import io.hstore.db.security.Scram; -import io.hstore.db.security.Security; import io.hstore.engine.HStoreException; import java.io.BufferedInputStream; @@ -150,7 +149,7 @@ void serve() { + "CREATE USER creates one"); } } - List legacy = database.read(reader -> Security.legacyUsers(reader.view())); + List legacy = database.usersWithOldPasswords(); if (!legacy.isEmpty()) { LOG.log(System.Logger.Level.WARNING, "{0} users have passwords stored by HStore 0.1 and cannot sign in over this port until each " + "signs in once through Studio or gets a new password with ALTER USER: {1}", legacy.size(), String.join(", ", legacy)); From f3a33b0864c5c49b828b458f3fe740f53034288d Mon Sep 17 00:00:00 2001 From: venkat1701 Date: Tue, 27 Oct 2026 16:51:38 +0530 Subject: [PATCH 5/5] Keep the shell from sending AUTHENTICATE in clear text and bind sign-ins to the record - The shell refuses any script that mentions AUTHENTICATE unless it is a single statement it can turn into a SCRAM exchange, and answers 'disconnected' after it dropped a connection whose proof failed. - A SCRAM exchange only succeeds while the user's record is the one its verifier came from, and clear-text sign-ins keep the exact record they checked, so a password change or a drop and re-create during a sign-in is caught. - The Python client in the docs normalizes like the server; both agree on every code point. - Parse errors of scripts with AUTHENTICATE are not logged verbatim, and the old-password warning is logged once per user per 10 minutes. --- .../java/io/hstore/db/HypergraphDatabase.java | 56 ++++++++++++++----- .../main/java/io/hstore/db/query/Parser.java | 8 +++ .../main/java/io/hstore/db/query/Session.java | 20 ++++--- .../java/io/hstore/db/security/Security.java | 13 +++-- .../test/java/io/hstore/db/SecurityTest.java | 12 ++++ docs/database/security.md | 11 ++-- docs/operations/logging.md | 2 +- docs/operations/wire-protocol.md | 15 +++-- .../java/io/hstore/server/RemoteEndpoint.java | 7 +++ .../main/java/io/hstore/server/Server.java | 3 +- .../java/io/hstore/server/FakeServerTest.java | 31 ++++++++++ .../java/io/hstore/server/ServerTest.java | 2 +- 12 files changed, 140 insertions(+), 40 deletions(-) diff --git a/database/src/main/java/io/hstore/db/HypergraphDatabase.java b/database/src/main/java/io/hstore/db/HypergraphDatabase.java index 7ad761cd..d36f7367 100644 --- a/database/src/main/java/io/hstore/db/HypergraphDatabase.java +++ b/database/src/main/java/io/hstore/db/HypergraphDatabase.java @@ -32,15 +32,19 @@ import java.nio.file.Path; import java.time.Duration; import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; import java.util.function.Function; import java.util.stream.Stream; public final class HypergraphDatabase implements AutoCloseable { private static final System.Logger LOG = System.getLogger("hstore.security"); + private static final long LEGACY_WARNING_INTERVAL_NANOS = Duration.ofMinutes(10).toNanos(); public static final String LEGACY_PASSWORD = "user \"{0}\" has a password stored by HStore 0.1, which cannot be used with SCRAM-SHA-256; " + "it can sign in once through Studio to convert it, or an administrator can set a new one with ALTER USER"; @@ -55,6 +59,7 @@ public final class HypergraphDatabase implements AutoCloseable { private final MaterializedViews views; private final SignIns signIns = new SignIns(); private final byte[] scramSecret; + private final Map legacyWarnings = new ConcurrentHashMap<>(); private HypergraphDatabase(Path directory, DatabaseOptions options) { this.options = options; @@ -84,14 +89,30 @@ private byte[] prepareSignIns() { } public Scram.Server scram(Optional channelBinding) { + return scram(channelBinding, _ -> { }); + } + + public Scram.Server scram(Optional channelBinding, Consumer used) { return new Scram.Server(name -> read(reader -> Security.user(reader.view(), name)).flatMap(user -> { if (Security.legacy(user)) { - LOG.log(System.Logger.Level.WARNING, LEGACY_PASSWORD, user.name()); + warnLegacy(user.name()); + return Optional.empty(); } - return Security.verifier(user); + Optional verifier = Security.verifier(user); + verifier.ifPresent(_ -> used.accept(user)); + return verifier; }), scramSecret, Security.ITERATIONS, channelBinding); } + private void warnLegacy(String user) { + long now = System.nanoTime(); + Long last = legacyWarnings.get(user); + if (last == null || now - last > LEGACY_WARNING_INTERVAL_NANOS) { + legacyWarnings.put(user, now); + LOG.log(System.Logger.Level.WARNING, LEGACY_PASSWORD, user); + } + } + public List usersWithOldPasswords() { return read(reader -> Security.legacyUsers(reader.view())); } @@ -206,19 +227,24 @@ public Optional authenticate(String user, String password) { } public Optional authenticate(String user, String password, Optional origin) { - Optional found = signIns.attempt(user, origin, () -> read(reader -> Security.authenticate(reader.view(), user, password))); - found.ifPresent(_ -> read(reader -> Security.user(reader.view(), user)).filter(Security::legacy).ifPresent(legacy -> { - try { - write(writer -> { - Security.replaceLegacy(writer.transaction(), legacy, password); - return null; - }); - LOG.log(System.Logger.Level.INFO, "the password of user \"{0}\" is now stored as a SCRAM-SHA-256 verifier", user); - } catch (HStoreException changed) { - LOG.log(System.Logger.Level.INFO, "the password of user \"{0}\" changed while it was being converted: {1}", user, changed.getMessage()); - } - })); - return found; + return signIn(user, password, origin).map(Security::principal); + } + + public Optional signIn(String user, String password, Optional origin) { + return signIns.attempt(user, origin, () -> read(reader -> Security.signIn(reader.view(), user, password))) + .map(found -> Security.legacy(found) ? convert(found, password) : found); + } + + private Security.User convert(Security.User legacy, String password) { + try { + Security.User converted = write(writer -> Security.replaceLegacy(writer.transaction(), legacy, password)); + LOG.log(System.Logger.Level.INFO, "the password of user \"{0}\" is now stored as a SCRAM-SHA-256 verifier", legacy.name()); + return converted; + } catch (HStoreException changed) { + LOG.log(System.Logger.Level.INFO, "the password of user \"{0}\" changed while it was being converted: {1}", legacy.name(), + changed.getMessage()); + return legacy; + } } void chargePayload(int tenant, long bytes) { diff --git a/database/src/main/java/io/hstore/db/query/Parser.java b/database/src/main/java/io/hstore/db/query/Parser.java index d7bc0e32..c5365630 100644 --- a/database/src/main/java/io/hstore/db/query/Parser.java +++ b/database/src/main/java/io/hstore/db/query/Parser.java @@ -89,6 +89,14 @@ public static Statement parse(String source) { return statements.getFirst(); } + public static boolean mentionsAuthenticate(String source) { + try { + return Lexer.tokenize(source).stream().anyMatch(token -> token.is("AUTHENTICATE")); + } catch (HStoreException unlexable) { + return source.toUpperCase(Locale.ROOT).matches("(?s).*\\bAUTHENTICATE\\b.*"); + } + } + private static Statement historical(Statement inner) { if (inner instanceof Statement.Authenticate) { throw HStoreException.invalid("AUTHENTICATE cannot run at a past generation"); diff --git a/database/src/main/java/io/hstore/db/query/Session.java b/database/src/main/java/io/hstore/db/query/Session.java index 70b0e9e5..3f877234 100644 --- a/database/src/main/java/io/hstore/db/query/Session.java +++ b/database/src/main/java/io/hstore/db/query/Session.java @@ -24,6 +24,7 @@ import java.util.Map; import java.util.Optional; import java.util.OptionalLong; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; public final class Session implements AutoCloseable { @@ -44,7 +45,7 @@ public final class Session implements AutoCloseable { private Optional scramUser = Optional.empty(); private Optional clearTextRefusal = Optional.empty(); - private record ScramAttempt(Scram.Server server, SignIns.Reservation reservation) { + private record ScramAttempt(Scram.Server server, SignIns.Reservation reservation, Optional user) { } private Format format = Format.TABLE; @@ -79,7 +80,7 @@ public String render(QueryResult result) { public boolean authenticate(String user, String password) { requireCanAuthenticate(); - Optional found = database.authenticate(user, password, origin); + Optional found = database.signIn(user, password, origin); if (found.isEmpty()) { failedAuthentications++; return false; @@ -92,11 +93,12 @@ public String scramFirst(String mechanism, String clientFirst, Optional requireCanAuthenticate(); scram.ifPresent(pending -> pending.reservation().failed()); scram = Optional.empty(); - Scram.Server server = database.scram(channelBinding); + AtomicReference used = new AtomicReference<>(); + Scram.Server server = database.scram(channelBinding, used::set); String serverFirst = server.first(mechanism, clientFirst); scramUser = Optional.of(server.user()); SignIns.Reservation reservation = database.signIns().reserve(server.user(), origin); - scram = Optional.of(new ScramAttempt(server, reservation)); + scram = Optional.of(new ScramAttempt(server, reservation, Optional.ofNullable(used.get()))); return serverFirst; } @@ -111,8 +113,8 @@ public Optional scramFinal(String clientFinal) { failedAuthentications++; throw malformed; } - Optional found = outcome instanceof Scram.Outcome.Accepted(String user, String _) - ? database.read(reader -> Security.user(reader.view(), user)).map(Security::principal) + Optional found = outcome instanceof Scram.Outcome.Accepted(String user, String _) + ? database.read(reader -> Security.user(reader.view(), user)).filter(current -> attempt.user().equals(Optional.of(current))) : Optional.empty(); if (found.isEmpty()) { attempt.reservation().failed(); @@ -141,10 +143,10 @@ private void requireCanAuthenticate() { } } - private void signIn(Principal found) { - principal = found; + private void signIn(Security.User found) { + principal = Security.principal(found); authenticated = true; - credentials = database.read(reader -> Security.user(reader.view(), found.user())); + credentials = Optional.of(found); renewing = false; variables.clear(); branch = Branch.MAIN; diff --git a/database/src/main/java/io/hstore/db/security/Security.java b/database/src/main/java/io/hstore/db/security/Security.java index bc31bc43..983b5a5b 100644 --- a/database/src/main/java/io/hstore/db/security/Security.java +++ b/database/src/main/java/io/hstore/db/security/Security.java @@ -183,12 +183,13 @@ public static List legacyUsers(View view) { return users(view).stream().filter(Security::legacy).map(User::name).sorted().toList(); } - public static void replaceLegacy(Transaction txn, User legacy, String password) { + public static User replaceLegacy(Transaction txn, User legacy, String password) { byte[] salt = new byte[16]; RANDOM.nextBytes(salt); Scram.Verifier verifier = Scram.Verifier.of(password, salt, ITERATIONS); - txn.merge(USERS, userKey(legacy.name()), current -> current.equals(Optional.of(legacy)), - _ -> Optional.of(new User(legacy.name(), HEX.formatHex(salt), verifier.encode(), ITERATIONS, legacy.tenant(), legacy.role()))); + User converted = new User(legacy.name(), HEX.formatHex(salt), verifier.encode(), ITERATIONS, legacy.tenant(), legacy.role()); + txn.merge(USERS, userKey(legacy.name()), current -> current.equals(Optional.of(legacy)), _ -> Optional.of(converted)); + return converted; } private static boolean matches(User user, String password) { @@ -244,11 +245,15 @@ public static List users(View view) { } public static Optional authenticate(View view, String name, String password) { + return signIn(view, name, password).map(Security::principal); + } + + public static Optional signIn(View view, String name, String password) { Optional stored = user(view, name); if (stored.isEmpty()) { derive(password, UNKNOWN_USER_SALT, ITERATIONS); } - return stored.filter(user -> matches(user, password)).map(Security::principal); + return stored.filter(user -> matches(user, password)); } public static boolean hasUsers(View view) { diff --git a/database/src/test/java/io/hstore/db/SecurityTest.java b/database/src/test/java/io/hstore/db/SecurityTest.java index 9a04cf9c..acf138ff 100644 --- a/database/src/test/java/io/hstore/db/SecurityTest.java +++ b/database/src/test/java/io/hstore/db/SecurityTest.java @@ -849,6 +849,18 @@ void anOldPasswordHashIsNeverEnoughToSignIn() throws Exception { assertThrows(HStoreException.InvalidSchema.class, () -> admin.execute("ALTER USER remote PASSWORD '" + shortSalt + "'")); } + @Test + void aPasswordChangeEndsAnExchangeAlreadyInFlight() { + try (Session session = new Session(database)) { + Scram.Client client = new Scram.Client("alice", "s3cret-pass", Optional.empty()); + String serverFirst = session.scramFirst(Scram.MECHANISM, client.first(), Optional.empty()); + admin.execute("ALTER USER alice PASSWORD 'rotated-pass'"); + assertEquals(Optional.empty(), session.scramFinal(client.last(serverFirst))); + assertFalse(session.authenticated()); + } + assertTrue(scram("alice", "rotated-pass").isPresent()); + } + @Test void scramExchangesSignSessionsInAndCountFailures() { try (Session session = new Session(database)) { diff --git a/docs/database/security.md b/docs/database/security.md index b2c4e92f..3d0de34b 100644 --- a/docs/database/security.md +++ b/docs/database/security.md @@ -134,10 +134,13 @@ record User(String name, String salt, String hash, int iterations, int tenant, R users slot does not let anyone sign in without guessing the password first. - Passwords go through SASLprep ([RFC 4013](https://www.rfc-editor.org/rfc/rfc4013)) before PBKDF2, as in PostgreSQL: a password made only of ASCII characters is used as it is; otherwise non-ASCII spaces become - spaces, characters such as the soft hyphen are dropped and the result is NFKC-normalized. When the result would - contain a character that SASLprep prohibits, or one unassigned in Unicode 3.2, the original password is used - unchanged. The client applies the same rule, so `p\u00e4ssword` typed precomposed or decomposed signs in either - way, and passwords set through other SCRAM clients and libraries match. + spaces, characters such as the soft hyphen are dropped and the result is NFKC-normalized with the current + Unicode tables, as PostgreSQL does. When the password holds a character unassigned in Unicode 3.2 (checked before + and after normalizing), or the result holds one that SASLprep prohibits, the original password is used + unchanged. Five CJK compatibility ideographs whose decomposition Unicode corrected after 3.2 (U+2F868, U+2F874, + U+2F91F, U+2F95F, U+2F9BF) therefore normalize as in PostgreSQL rather than as in Unicode 3.2. The client and the + Python client in the [wire protocol](../operations/wire-protocol.md#writing-a-client) apply the same rule (they + agree on every code point), so `p\u00e4ssword` typed precomposed or decomposed signs in either way. - Older versions stored the hex `SaltedPassword` itself. That value is exactly what a SCRAM client derives from the password, so a copy of it from an old backup would be as good as the password. This version therefore never uses an old record for SCRAM: such a user cannot sign in over the wire protocol, the server answers it as it diff --git a/docs/operations/logging.md b/docs/operations/logging.md index 8be61bb1..53480d00 100644 --- a/docs/operations/logging.md +++ b/docs/operations/logging.md @@ -142,7 +142,7 @@ and in Docker it is the last line of `docker logs`. The storage-integrity failur | WARNING | `connection from /{ip}:{port} refused: {reason} (max_connections={n})`: the server is full, or 16 connections from this address are still waiting to authenticate. | | WARNING | `connection {n} from /{ip}:{port} refused: no users exist and the client is not on this host` (Studio: `studio request from … refused: …`). | | WARNING | `user "{user}" failed to sign in 50 times within 15 minutes; someone may be guessing its password from several addresses` (logger `hstore.security`). | -| WARNING | `user "{user}" has a password stored by HStore 0.1, which cannot be used with SCRAM-SHA-256; it can sign in once through Studio to convert it, or an administrator can set a new one with ALTER USER` (logger `hstore.security`): the client only sees `authentication failed`. | +| WARNING | `user "{user}" has a password stored by HStore 0.1, which cannot be used with SCRAM-SHA-256; it can sign in once through Studio to convert it, or an administrator can set a new one with ALTER USER` (logger `hstore.security`, at most once every 10 minutes per user): the client only sees `authentication failed`. | | INFO | `the password of user "{user}" is now stored as a SCRAM-SHA-256 verifier` (logger `hstore.security`): a user with a password from HStore 0.1 signed in through Studio or an `AUTHENTICATE` statement. | | WARNING | at startup, `authentication = off and the server listens on {address}: …` or `the database has no users, so connections from other hosts are refused until CREATE USER creates one`, when the listen address is not loopback. | | WARNING | at startup, `{n} users have passwords stored by HStore 0.1 and cannot sign in over this port until each signs in once through Studio or gets a new password with ALTER USER: {names}`. | diff --git a/docs/operations/wire-protocol.md b/docs/operations/wire-protocol.md index bead6db7..b8fc975f 100644 --- a/docs/operations/wire-protocol.md +++ b/docs/operations/wire-protocol.md @@ -136,7 +136,10 @@ Possible replies: An `AUTHENTICATE` statement is refused with the same upgrade message on an authenticated connection too; send a new SASL exchange to switch users. `hstore connect` turns an `AUTHENTICATE` typed in its shell into that -exchange. `CREATE USER` and `ALTER USER … PASSWORD` still carry the password in the statement; use TLS for them, +exchange when it is the only statement it sends; any other script that contains `AUTHENTICATE` is refused in the +shell without sending anything, so a mistyped sign-in cannot put the password on the wire. A server's `error` +reply to a SASL message leaves the connection as it was; when the server's proof is missing or wrong the shell +disconnects. `CREATE USER` and `ALTER USER … PASSWORD` still carry the password in the statement; use TLS for them, or pass a SCRAM verifier made on the client (`hstore verifier`, the format is `SCRAM-SHA-256$:$:` as in PostgreSQL) in place of the password. @@ -177,9 +180,11 @@ without a reply, so a web page cannot use a browser to post HQL to the port. If a user is dropped or its password changes, the next statement on each of its open connections fails with `user NAME was dropped or its password changed; authenticate again`, and so does every statement after it until the -connection sends a successful `AUTHENTICATE`. +connection signs in again with a SASL exchange. An exchange that started before the change fails at its last step, +since the server only accepts it while the user's record is the one it took the verifier from. -Without `tls = on` the password travels in clear text. With TLS the server completes the handshake before it +Signing in never sends the password, with or without TLS. Without `tls = on`, everything else travels in clear +text, including the passwords in `CREATE USER` and `ALTER USER … PASSWORD`. With TLS the server completes the handshake before it sends the banner, and gives the client 10 seconds to do so. A client that fails or stalls the handshake (for example a plaintext client on the TLS port) is logged at LOG level and disconnected. `hstore connect` and `hstore ping` likewise give up after 10 seconds without a banner. @@ -310,9 +315,9 @@ def saslprep(password): if password.isascii(): return password mapped = "".join(" " if stringprep.in_table_c12(c) else "" if stringprep.in_table_b1(c) else c for c in password) - if not mapped: + if not mapped or any(map(stringprep.in_table_a1, mapped)): return password - prepared = unicodedata.ucd_3_2_0.normalize("NFKC", mapped) + prepared = unicodedata.normalize("NFKC", mapped) prohibited = (stringprep.in_table_a1, stringprep.in_table_c12, stringprep.in_table_c21_c22, stringprep.in_table_c3, stringprep.in_table_c4, stringprep.in_table_c5, stringprep.in_table_c6, stringprep.in_table_c7, stringprep.in_table_c8, stringprep.in_table_c9) diff --git a/server/src/main/java/io/hstore/server/RemoteEndpoint.java b/server/src/main/java/io/hstore/server/RemoteEndpoint.java index 6657d56c..1808b407 100644 --- a/server/src/main/java/io/hstore/server/RemoteEndpoint.java +++ b/server/src/main/java/io/hstore/server/RemoteEndpoint.java @@ -147,6 +147,10 @@ public String execute(String script) { } return reply; } + if (Parser.mentionsAuthenticate(script)) { + return "error: AUTHENTICATE must be sent on its own, so that it signs in with SCRAM and the password never crosses the " + + "connection; nothing was sent"; + } return send(script); } @@ -162,6 +166,9 @@ private static Optional authentication(String script) { } private String send(String script) { + if (socket.isClosed()) { + return "error: disconnected; connect again to continue"; + } try { WireProtocol.send(out, script); return WireProtocol.receive(in, Long.MAX_VALUE).orElse("error: server closed the connection"); diff --git a/server/src/main/java/io/hstore/server/Server.java b/server/src/main/java/io/hstore/server/Server.java index a8756f5a..5e24db44 100644 --- a/server/src/main/java/io/hstore/server/Server.java +++ b/server/src/main/java/io/hstore/server/Server.java @@ -408,7 +408,8 @@ private String execute(Session session, String script, long connection) { try { statements = Parser.sourced(script); } catch (HStoreException failure) { - LOG.log(System.Logger.Level.INFO, "connection {0}: {1}", connection, failure.getMessage()); + LOG.log(System.Logger.Level.INFO, "connection {0}: {1}", connection, + Parser.mentionsAuthenticate(script) ? "a script with AUTHENTICATE could not be parsed" : failure.getMessage()); return "error [" + failure.code() + "]: " + failure.getMessage(); } if (statements.stream().map(Parser.Sourced::statement).anyMatch(Server::signsIn)) { diff --git a/server/src/test/java/io/hstore/server/FakeServerTest.java b/server/src/test/java/io/hstore/server/FakeServerTest.java index fa4844c5..63d5756a 100644 --- a/server/src/test/java/io/hstore/server/FakeServerTest.java +++ b/server/src/test/java/io/hstore/server/FakeServerTest.java @@ -14,8 +14,10 @@ import java.net.Socket; import java.nio.charset.StandardCharsets; import java.util.Base64; +import java.util.List; import java.util.Optional; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.TimeUnit; import java.util.function.UnaryOperator; @@ -106,4 +108,33 @@ void aWeakIterationCountIsRefused() throws Exception { refused(REQUIRED, request -> "sasl " + base64(new String(Base64.getDecoder().decode(challenge(request).substring(5)), StandardCharsets.UTF_8).replace("i=4096", "i=1")), "iteration count 1 "); } + + @Test + void theShellNeverSendsAPasswordOutsideScram() throws Exception { + List received = new CopyOnWriteArrayList<>(); + Fake fake = fake("hstore connection 1 at generation 2", request -> { + received.add(request); + return "authenticated as admin"; + }); + try (RemoteEndpoint shell = new RemoteEndpoint(fake.address(), Optional.empty())) { + for (String script : List.of("AUTHENTICATE admin PASSWORD 'pw-secret'; SHOW TYPES;", + "AT GENERATION 1 AUTHENTICATE admin PASSWORD 'pw-secret';", + "authenticate 'admin' 'pw-secret' ;;x", + "SHOW TYPES; Authenticate admin PASSWORD 'pw-secret")) { + assertTrue(shell.execute(script).startsWith("error: AUTHENTICATE must be sent on its own"), script); + } + assertTrue(shell.execute("INSERT NODE Note {text: 'authenticate later'};").startsWith("authenticated")); + } + assertTrue(received.stream().noneMatch(request -> request.contains("pw-secret")), received.toString()); + } + + @Test + void theShellStopsAfterAFailedProof() throws Exception { + Fake fake = fake(REQUIRED, request -> request.startsWith("SASL SCRAM") ? challenge(request) + : request.startsWith("SASL ") ? forgedProof() : "[]"); + try (RemoteEndpoint shell = new RemoteEndpoint(fake.address(), Optional.empty())) { + assertTrue(shell.execute("AUTHENTICATE admin PASSWORD 'pw-secret';").contains("could not prove")); + assertEquals("error: disconnected; connect again to continue", shell.execute("SHOW TYPES;")); + } + } } diff --git a/server/src/test/java/io/hstore/server/ServerTest.java b/server/src/test/java/io/hstore/server/ServerTest.java index 0ca73f75..f09d4c7f 100644 --- a/server/src/test/java/io/hstore/server/ServerTest.java +++ b/server/src/test/java/io/hstore/server/ServerTest.java @@ -455,7 +455,7 @@ void clearTextPasswordsAreRefusedWithAnUpgradeMessage() throws Exception { } } try (Endpoint admin = new RemoteEndpoint(address, Optional.of(new RemoteEndpoint.Credentials("admin", "pw-secret")))) { - assertTrue(admin.execute("AUTHENTICATE admin PASSWORD 'pw-secret'; WHOAMI;").contains("upgrade your HStore client")); + assertTrue(admin.execute("AUTHENTICATE admin PASSWORD 'pw-secret'; WHOAMI;").contains("must be sent on its own")); admin.execute("CREATE USER other PASSWORD 'other-secret' ROLE reader;"); assertTrue(admin.execute("AT GENERATION 1 AUTHENTICATE other PASSWORD 'other-secret'; WHOAMI;").startsWith("error")); assertTrue(admin.execute("WHOAMI;").contains("ADMIN"));