From 1cebabb8b1be404da69ce52e373001c7903842d5 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 6 Sep 2026 21:29:08 +0100 Subject: [PATCH 1/9] Fetch public keys asynchronously and remove the waiting forms PublicKeyFetch.publicKeyPem and PublicKeyFetch.verify returned only once the creator had answered, holding whichever thread asked, which on a request thread or an event loop is a stall of up to fifteen seconds for each key not yet held. Both now return a CompletableFuture at once and the waiting forms are gone, with no deprecated twin left behind. The request is made by a new PublicKeyTransport interface, and the new HttpUrlConnectionTransport is the one used where a caller names none. It runs the JDK's blocking HttpURLConnection on an Executor, either one the caller gives or a shared pool of daemon threads bounded at twice the processors available, because Java 8, which the library still targets, has no non-blocking HTTP client of its own. On Java 11 and later a transport over java.net.http.HttpClient.sendAsync can be supplied instead. The redirect refusal and the date parameter are kept exactly as they were. The cache now holds the future of each fetch rather than the key, so two requests for the same key made while the first is in flight share one request, and a fetch that fails is dropped so the next request asks again. The tests join the futures, and five are added, covering the shared in-flight request, a failure not being held, the request running on the executor given, an executor that refuses, and a missing transport. The README example and the class list are updated to the new shape. --- README.md | 55 ++- .../owid/HttpUrlConnectionTransport.java | 285 ++++++++++++++ .../swancommunity/owid/PublicKeyFetch.java | 357 ++++++++++-------- .../owid/PublicKeyFetchException.java | 3 +- .../owid/PublicKeyTransport.java | 66 ++++ .../owidconsumer/ReadmeExampleTest.java | 10 +- .../swancommunity/owid/DatedKeyFetchTest.java | 264 +++++++++++-- 7 files changed, 846 insertions(+), 194 deletions(-) create mode 100644 src/main/java/com/swancommunity/owid/HttpUrlConnectionTransport.java create mode 100644 src/main/java/com/swancommunity/owid/PublicKeyTransport.java diff --git a/README.md b/README.md index ef2fb0a..aa69ae7 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,10 @@ creates, signs, serializes, and verifies OWIDs. to any web framework. - Fetching the public key of another creator uses `HttpURLConnection` from the JDK, so verifying over the network adds no dependency and still runs on - Java 8. + Java 8. Every method that reaches the network answers with a + `CompletableFuture`, and the blocking connection runs on a background + thread. A transport over `java.net.http.HttpClient.sendAsync` can be + supplied on Java 11 and later. ## Payload size and application limits @@ -159,15 +162,37 @@ identifier it signed under an earlier key reads as not matching, which is why a creator that rotates its key has to honour the date. Keys already fetched are held against the URL they came from, which names the domain, the version and the minute, up to 1024 of them before the store is emptied, and -`clearCache` empties it on demand. +`clearCache` empties it on demand. Two requests for the same key made while +the first is still on its way share one request, and a fetch that fails is +not held, so the next request asks again. + +Every method that reaches the network answers with a `CompletableFuture` and +returns at once. There is no form that waits, so a request thread or an event +loop is never held while a creator answers, and a caller that wants to wait +joins the future itself. The request is made by a `PublicKeyTransport`, and +where none is named `HttpUrlConnectionTransport` is used, which runs the +JDK's blocking `HttpURLConnection` on a background thread. The pool it uses +has daemon threads, never more of them than twice the processors available, +and requests beyond that wait in a queue. An `Executor` of your own can be +given to its constructor instead. On Java 11 and later supply a transport of +your own over `java.net.http.HttpClient.sendAsync`, which blocks no thread at +all. Any transport must never follow a redirect and must request the URL +exactly as given, for the reasons the interface comment sets out. ```java import com.swancommunity.owid.OwidSignatureStatus; import com.swancommunity.owid.OwidVerificationResult; import com.swancommunity.owid.PublicKeyFetch; -OwidVerificationResult result = PublicKeyFetch.verify( - owid, "https", Collections.emptyList()); +import java.util.concurrent.CompletableFuture; + +CompletableFuture pending = + PublicKeyFetch.verify( + owid, "https", Collections.emptyList()); +// The call returns at once and the request runs on a background +// thread. Continue from the future, or join it where waiting is +// acceptable, as it is here. +OwidVerificationResult result = pending.join(); if (result.getStatus() == OwidSignatureStatus.KEY_UNAVAILABLE) { // The key could not be obtained, so the signature was never examined. // Only SIGNATURE_INVALID means the identifier should be distrusted. @@ -387,12 +412,24 @@ domain, a null payload, or a field that cannot be serialized. point on the domain the OWID carries. - `publicKeyUrl` builds the request, naming the version of the OWID and the minute the OWID was signed. - - `publicKeyPem` returns the key, raising `PublicKeyFetchException`, which - carries the status to report, the domain and the response code. - - `verify` answers with the status, so a key that could not be fetched is - `KEY_UNAVAILABLE`, one that could not be read is `INVALID_KEY`, and - neither is mistaken for a signature that does not match. + - `publicKeyPem` returns a `CompletableFuture` of the key. The future fails + with `PublicKeyFetchException`, which carries the status to report, the + domain and the response code, where the key could not be obtained. + - `verify` returns a `CompletableFuture` of the status, so a key that + could not be fetched is `KEY_UNAVAILABLE`, one that could not be read is + `INVALID_KEY`, and neither is mistaken for a signature that does not + match. The future never fails. + - Both take an optional `PublicKeyTransport`, and use + `HttpUrlConnectionTransport` on its shared pool where none is given. - `clearCache` empties the keys already fetched. +- `PublicKeyTransport` makes the request and answers with a + `CompletableFuture` of the body, so a transport over any HTTP client can + be supplied. It must never follow a redirect and must request the URL + exactly as given. +- `HttpUrlConnectionTransport` is the transport used where none is named, + running `HttpURLConnection` on an `Executor`, either one given to its + constructor or a shared pool of daemon threads bounded at twice the + processors available. - `PublicKeySchedule` holds the keys a creator has published and chooses between them. - `PublicKeySchedule.of` takes the keys in any order. diff --git a/src/main/java/com/swancommunity/owid/HttpUrlConnectionTransport.java b/src/main/java/com/swancommunity/owid/HttpUrlConnectionTransport.java new file mode 100644 index 0000000..28cac7e --- /dev/null +++ b/src/main/java/com/swancommunity/owid/HttpUrlConnectionTransport.java @@ -0,0 +1,285 @@ +/* **************************************************************************** + * Copyright 2026 51 Degrees Mobile Experts Limited (51degrees.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * ***************************************************************************/ + +package com.swancommunity.owid; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.URL; +import java.net.URLConnection; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executor; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * The transport used where a caller names none, making the request with + * {@link HttpURLConnection} from the JDK. + * + *

HttpURLConnection blocks the thread that calls it, so the request is + * handed to an {@link Executor} and the future returned completes when the + * thread the executor gave it has the response. The thread that asked is + * never held. That is blocking I/O on a background thread, which is the + * best Java 8 offers without a dependency. On Java 11 and later supply a + * {@link PublicKeyTransport} of your own over + * {@code java.net.http.HttpClient.sendAsync} instead, which blocks no + * thread at all, and keep the two rules that interface describes.

+ * + *

Where no executor is given a pool shared by every instance is used. Its + * threads are daemon threads, so a process that is otherwise finished is not + * kept alive by a fetch still in flight, and there are never more of them + * than twice the processors available, with at least two. Requests beyond + * that wait in a queue rather than being refused, and a thread that has had + * nothing to do for a minute ends.

+ * + *

Only the JDK is used, so the library keeps its promise of no runtime + * dependencies.

+ */ +public final class HttpUrlConnectionTransport implements PublicKeyTransport { + + /** How long to wait for the connection to be made, in milliseconds. */ + private static final int CONNECT_TIMEOUT_MILLISECONDS = 5000; + + /** How long to wait for the response, in milliseconds. */ + private static final int READ_TIMEOUT_MILLISECONDS = 10000; + + /** How long an idle thread of the shared pool lives, in seconds. */ + private static final long IDLE_THREAD_SECONDS = 60; + + /** The executor the blocking request is handed to. */ + private final Executor executor; + + /** + * Creates a transport that runs each request on the shared pool of + * daemon threads described in the class comment. + */ + public HttpUrlConnectionTransport() { + this(SharedPool.INSTANCE); + } + + /** + * Creates a transport that runs each request on the executor given. + * + *

The executor is the caller's own, so its threads, their number and + * whether they are daemon threads are the caller's choice, and ending + * it when the process ends is the caller's job too.

+ * + * @param executor the executor to run each request on + * @throws IllegalArgumentException if the executor is missing + */ + public HttpUrlConnectionTransport(Executor executor) { + if (executor == null) { + throw new IllegalArgumentException("the executor is missing"); + } + this.executor = executor; + } + + @Override + public CompletableFuture fetch(final String url, + final String domain) { + final CompletableFuture future = + new CompletableFuture(); + try { + executor.execute(() -> { + try { + future.complete(read(url, domain)); + } catch (PublicKeyFetchException e) { + future.completeExceptionally(e); + } catch (Throwable e) { + // Nothing in read is expected to throw anything else, + // but a future that is never completed would hold a + // caller for ever, so whatever escaped is carried out + // as the key being unavailable. An Error is then + // rethrown, because the thread it happened on has to + // know as well. + future.completeExceptionally(unexpected(domain, e)); + if (e instanceof Error) { + throw (Error) e; + } + } + }); + } catch (RejectedExecutionException e) { + // An executor that has been shut down, or one with a bounded + // queue that is full, refuses the task at once. The refusal + // arrives through the future like every other failure, so a + // caller has one place to look. + future.completeExceptionally(new PublicKeyFetchException( + "the request for the public key of domain " + + quoted(domain) + + " was refused by the executor", + OwidSignatureStatus.KEY_UNAVAILABLE, + domain, + 0, + e)); + } + return future; + } + + /** Performs the request and returns the body as text. */ + private static String read(String url, String domain) + throws PublicKeyFetchException { + HttpURLConnection connection = null; + try { + URLConnection opened = new URL(url).openConnection(); + if ((opened instanceof HttpURLConnection) == false) { + // A scheme the caller chose that does not make an HTTP + // request, such as file. Reported as a key that could not be + // obtained rather than allowed to escape as a cast failure, + // because every route into this class promises a status. + throw new PublicKeyFetchException( + "the scheme used for domain " + quoted(domain) + + " does not make an HTTP request", + OwidSignatureStatus.KEY_UNAVAILABLE, + domain, + 0, + null); + } + connection = (HttpURLConnection) opened; + // Never follow a redirect. HttpURLConnection follows one to + // any other host by default, so a creator whose domain + // answered 302 to some other place would have that other + // place's key trusted as its own, and a network attacker able + // to bend the creator's DNS, or a creator that was simply + // misconfigured, could put a key there and have forgeries + // verify. Left alone, the 3xx is the response code, and the + // check below reads it as the key being unavailable, which it + // is. + connection.setInstanceFollowRedirects(false); + connection.setRequestMethod("GET"); + connection.setConnectTimeout(CONNECT_TIMEOUT_MILLISECONDS); + connection.setReadTimeout(READ_TIMEOUT_MILLISECONDS); + connection.setRequestProperty("Accept", "text/plain"); + int code = connection.getResponseCode(); + if (code != HttpURLConnection.HTTP_OK) { + drain(connection.getErrorStream()); + throw new PublicKeyFetchException( + "domain " + quoted(domain) + " returned code '" + code + + "' for the public key", + OwidSignatureStatus.KEY_UNAVAILABLE, + domain, + code, + null); + } + InputStream body = connection.getInputStream(); + try { + return new String(readAll(body), StandardCharsets.UTF_8); + } finally { + body.close(); + } + } catch (IOException e) { + // A refused connection, a name that does not resolve and a + // timeout all arrive here, and all of them mean the signature + // was never examined. + throw new PublicKeyFetchException( + "the public key could not be fetched from domain " + + quoted(domain), + OwidSignatureStatus.KEY_UNAVAILABLE, + domain, + 0, + e); + } finally { + if (connection != null) { + connection.disconnect(); + } + } + } + + /** The failure to carry for something read was not expected to throw. */ + private static PublicKeyFetchException unexpected(String domain, + Throwable cause) { + return new PublicKeyFetchException( + "the request for the public key of domain " + quoted(domain) + + " failed unexpectedly", + OwidSignatureStatus.KEY_UNAVAILABLE, + domain, + 0, + cause); + } + + /** The value in single quotes, for a message. */ + private static String quoted(String value) { + return "'" + value + "'"; + } + + /** Reads a stream to its end. */ + private static byte[] readAll(InputStream stream) throws IOException { + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + byte[] block = new byte[4096]; + int read = stream.read(block); + while (read > 0) { + buffer.write(block, 0, read); + read = stream.read(block); + } + return buffer.toByteArray(); + } + + /** Closes the error body of a refused request, where there is one. */ + private static void drain(InputStream stream) { + if (stream == null) { + return; + } + try { + stream.close(); + } catch (IOException e) { + // Nothing useful can be done about a body that will not close, + // and the refusal itself is what the caller is told about. + } + } + + /** + * The pool shared by every transport created without an executor. The + * pool starts no thread until a request is handed to it, so a process + * that only ever verifies with keys it already holds starts none. + */ + private static final class SharedPool { + + static final Executor INSTANCE = create(); + + private SharedPool() { + } + + private static Executor create() { + int threads = Math.max(2, + Runtime.getRuntime().availableProcessors() * 2); + ThreadPoolExecutor pool = new ThreadPoolExecutor(threads, threads, + IDLE_THREAD_SECONDS, TimeUnit.SECONDS, + new LinkedBlockingQueue(), + new ThreadFactory() { + private final AtomicInteger made = new AtomicInteger(); + + @Override + public Thread newThread(Runnable task) { + Thread thread = new Thread(task, + "owid-public-key-fetch-" + + made.incrementAndGet()); + thread.setDaemon(true); + return thread; + } + }); + // Core threads are the whole pool, so without this they would + // live for ever once started. + pool.allowCoreThreadTimeOut(true); + return pool; + } + } +} diff --git a/src/main/java/com/swancommunity/owid/PublicKeyFetch.java b/src/main/java/com/swancommunity/owid/PublicKeyFetch.java index 47aa86b..e94a666 100644 --- a/src/main/java/com/swancommunity/owid/PublicKeyFetch.java +++ b/src/main/java/com/swancommunity/owid/PublicKeyFetch.java @@ -16,15 +16,10 @@ package com.swancommunity.owid; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.net.HttpURLConnection; -import java.net.URL; -import java.net.URLConnection; -import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; import java.util.concurrent.ConcurrentHashMap; /** @@ -43,6 +38,18 @@ * earlier key reads as not matching, which is why a creator that rotates its * key has to honour the date.

* + *

Every method here that reaches the network answers with a + * {@link CompletableFuture} and returns at once. There is no form that + * waits, so a request thread or an event loop is never held while a creator + * answers, and a caller that wants to wait joins the future itself. The + * request is made by a {@link PublicKeyTransport}, and where the caller + * names none {@link HttpUrlConnectionTransport} is used, which runs the + * JDK's blocking connection on a background thread. On Java 11 and later a + * caller can supply a transport over + * {@code java.net.http.HttpClient.sendAsync} instead, which blocks no thread + * at all. Building the URL is pure text and reaches nothing, so + * {@link #publicKeyUrl(Owid, String)} answers in the ordinary way.

+ * *

Only the JDK is used, so the library keeps its promise of no runtime * dependencies and still runs on Java 8, which has no HTTP client of its * own.

@@ -52,12 +59,6 @@ */ public final class PublicKeyFetch { - /** How long to wait for the connection to be made, in milliseconds. */ - private static final int CONNECT_TIMEOUT_MILLISECONDS = 5000; - - /** How long to wait for the response, in milliseconds. */ - private static final int READ_TIMEOUT_MILLISECONDS = 10000; - /** * The most keys held in the cache before the cache is emptied and filled * again. A bound is needed because a verifier sees identifiers from many @@ -67,16 +68,27 @@ public final class PublicKeyFetch { private static final int MAXIMUM_CACHED_KEYS = 1024; /** - * Keys already fetched, held against the URL the keys were fetched from. + * Keys fetched, or on their way, held against the URL they were asked + * for. * *

The specification asks implementations to cache so that verifying * many identifiers does not mean repeating requests to another * processor. Holding the key against the whole URL is safe because the * URL names the domain, the version and the minute, and the key a * creator published for a minute in the past does not change.

+ * + *

The value is the future of the fetch rather than the key itself, so + * a second request for a key that is still on its way joins the request + * already made instead of making another. A fetch that fails is removed + * the moment it fails, so an outage is never remembered and the next + * request tries again.

*/ - private static final Map CACHE = - new ConcurrentHashMap(); + private static final Map> CACHE = + new ConcurrentHashMap>(); + + /** The transport used where the caller names none. */ + private static final PublicKeyTransport DEFAULT_TRANSPORT = + new HttpUrlConnectionTransport(); private PublicKeyFetch() { } @@ -122,198 +134,239 @@ public static String publicKeyUrl(Owid owid, String scheme) } /** - * Returns the public key PEM of the creator of the OWID, for the date - * the OWID carries. + * Fetches the public key PEM of the creator of the OWID, for the date + * the OWID carries, using {@link HttpUrlConnectionTransport} on its + * shared pool. + * + *

Returns at once. The future completes with the key in PEM form, + * fails with a {@link PublicKeyFetchException} where the key could not + * be obtained, carrying the status to report for the identifier, and + * fails with an {@link OwidException} where the OWID, the scheme or the + * domain is not usable. Nothing is thrown from the call itself.

* * @param owid the OWID whose creator key is wanted * @param scheme the scheme to use, normally {@code https} - * @return the public key in PEM form - * @throws PublicKeyFetchException if the key could not be obtained, with - * the status to report for the - * identifier - * @throws OwidException if the OWID, the scheme or the domain - * is not usable + * @return the public key in PEM form, through a future */ - public static String publicKeyPem(Owid owid, String scheme) - throws OwidException { - return publicKeyPemAtUrl(publicKeyUrl(owid, scheme), - owid.getDomain()); + public static CompletableFuture publicKeyPem(Owid owid, + String scheme) { + return publicKeyPem(owid, scheme, DEFAULT_TRANSPORT); + } + + /** + * Fetches the public key PEM of the creator of the OWID, for the date + * the OWID carries, using the transport given. + * + *

Returns at once. The future completes with the key in PEM form, + * fails with a {@link PublicKeyFetchException} where the key could not + * be obtained, carrying the status to report for the identifier, and + * fails with an {@link OwidException} where the OWID, the scheme, the + * domain or the transport is not usable. Nothing is thrown from the + * call itself.

+ * + * @param owid the OWID whose creator key is wanted + * @param scheme the scheme to use, normally {@code https} + * @param transport the transport to make the request with + * @return the public key in PEM form, through a future + */ + public static CompletableFuture publicKeyPem(Owid owid, + String scheme, PublicKeyTransport transport) { + String url; + try { + url = publicKeyUrl(owid, scheme); + } catch (OwidException e) { + return failed(e); + } + return publicKeyPemAtUrl(url, owid.getDomain(), transport); } /** * Asks whether the signature on the OWID is genuine, fetching the key - * that was in force when the OWID was signed from the creator domain. + * that was in force when the OWID was signed from the creator domain + * using {@link HttpUrlConnectionTransport} on its shared pool. * - *

A key that cannot be fetched is + *

Returns at once, and the future never fails, because every route + * out of the fetch promises a status. A key that cannot be fetched is * {@link OwidSignatureStatus#KEY_UNAVAILABLE} and one that arrives in a * form this library cannot read is * {@link OwidSignatureStatus#INVALID_KEY}. Neither is * {@link OwidSignatureStatus#SIGNATURE_INVALID}, because an outage or a * badly served key leaves the signature unjudged, and reporting either - * as invalid would read as an attack.

+ * as invalid would read as an attack. The signature is examined on the + * thread that completes the fetch, which for the default transport is + * one of its pool, or on the caller's own thread where the key is + * already held.

* * @param owid the OWID to check * @param scheme the scheme to use, normally {@code https} * @param others the other OWIDs that were signed together with this one, * in the same order as when signed - * @return the outcome of the check + * @return the outcome of the check, through a future + */ + public static CompletableFuture verify(Owid owid, + String scheme, List others) { + return verify(owid, scheme, others, DEFAULT_TRANSPORT); + } + + /** + * Asks whether the signature on the OWID is genuine, fetching the key + * that was in force when the OWID was signed from the creator domain + * using the transport given. + * + *

Returns at once, and the future never fails, because every route + * out of the fetch promises a status. A key that cannot be fetched, a + * URL that cannot be built and a transport that is missing are all + * {@link OwidSignatureStatus#KEY_UNAVAILABLE}, and a key that arrives in + * a form this library cannot read is + * {@link OwidSignatureStatus#INVALID_KEY}. Neither is + * {@link OwidSignatureStatus#SIGNATURE_INVALID}, because an outage or a + * badly served key leaves the signature unjudged, and reporting either + * as invalid would read as an attack. The signature is examined on the + * thread that completes the fetch, or on the caller's own thread where + * the key is already held.

+ * + * @param owid the OWID to check + * @param scheme the scheme to use, normally {@code https} + * @param others the other OWIDs that were signed together with this + * one, in the same order as when signed + * @param transport the transport to make the request with + * @return the outcome of the check, through a future */ - public static OwidVerificationResult verify(Owid owid, String scheme, - List others) { + public static CompletableFuture verify(Owid owid, + String scheme, List others, PublicKeyTransport transport) { String url; try { url = publicKeyUrl(owid, scheme); } catch (OwidException e) { - return OwidVerificationResult.of( - OwidSignatureStatus.KEY_UNAVAILABLE); + return CompletableFuture.completedFuture( + OwidVerificationResult.of( + OwidSignatureStatus.KEY_UNAVAILABLE)); } - return verifyAtUrl(owid, url, others); + return verifyAtUrl(owid, url, others, transport); } /** * Empties the cache of keys already fetched. Provided so that a long * running process can release the memory, and so that a test can start - * from a known state. + * from a known state. A fetch still on its way is forgotten here but + * still completes for whoever holds its future. */ public static void clearCache() { CACHE.clear(); } /** - * The work {@link #verify(Owid, String, List)} does once the URL is - * known, kept apart so that the tests drive the real fetch against a key - * end point the tests can stand up locally rather than against a near - * copy of the fetch. + * The work {@link #verify(Owid, String, List, PublicKeyTransport)} does + * once the URL is known, kept apart so that the tests drive the real + * fetch against a key end point the tests can stand up locally rather + * than against a near copy of the fetch. */ - static OwidVerificationResult verifyAtUrl(Owid owid, String url, - List others) { - String pem; - try { - pem = publicKeyPemAtUrl(url, owid.getDomain()); - } catch (PublicKeyFetchException e) { - return OwidVerificationResult.of(e.getStatus()); - } catch (OwidException e) { - return OwidVerificationResult.of( - OwidSignatureStatus.KEY_UNAVAILABLE); - } - return owid.verify(pem, others); + static CompletableFuture verifyAtUrl( + final Owid owid, String url, final List others, + PublicKeyTransport transport) { + return publicKeyPemAtUrl(url, owid.getDomain(), transport) + .handle((pem, failure) -> { + if (failure != null) { + return OwidVerificationResult.of(statusOf(failure)); + } + return owid.verify(pem, others); + }); } /** * Fetches the PEM at the URL, answering from the cache where the same - * URL has already been fetched. + * URL has already been fetched or is being fetched now. + * + *

The future held in the cache is this class's own rather than the + * transport's, so that the transport's completion can be watched and a + * failure dropped from the cache without touching the map from inside + * one of its own operations, which a concurrent map does not allow.

*/ - static String publicKeyPemAtUrl(String url, String domain) - throws OwidException { - String cached = CACHE.get(url); - if (cached != null) { - return cached; + static CompletableFuture publicKeyPemAtUrl(final String url, + String domain, PublicKeyTransport transport) { + if (transport == null) { + return failed(new OwidException("the transport is missing")); + } + CompletableFuture held = CACHE.get(url); + if (held != null) { + return held; } - String pem = read(url, domain); if (CACHE.size() >= MAXIMUM_CACHED_KEYS) { CACHE.clear(); } - CACHE.put(url, pem); - return pem; - } - - /** Performs the request and returns the body as text. */ - private static String read(String url, String domain) - throws OwidException { - HttpURLConnection connection = null; + final CompletableFuture fetch = new CompletableFuture(); + held = CACHE.putIfAbsent(url, fetch); + if (held != null) { + // Another thread asked for the same key between the lookup and + // the insert, and its fetch is the one both callers share. + return held; + } + CompletableFuture started; try { - URLConnection opened = new URL(url).openConnection(); - if ((opened instanceof HttpURLConnection) == false) { - // A scheme the caller chose that does not make an HTTP - // request, such as file. Reported as a key that could not be - // obtained rather than allowed to escape as a cast failure, - // because every route into this class promises a status. - throw new PublicKeyFetchException( - "the scheme used for domain " + quoted(domain) - + " does not make an HTTP request", - OwidSignatureStatus.KEY_UNAVAILABLE, - domain, - 0, - null); - } - connection = (HttpURLConnection) opened; - // Never follow a redirect. HttpURLConnection follows one to - // any other host by default, so a creator whose domain - // answered 302 to some other place would have that other - // place's key trusted as its own, and a network attacker able - // to bend the creator's DNS, or a creator that was simply - // misconfigured, could put a key there and have forgeries - // verify. Left alone, the 3xx is the response code, and the - // check below reads it as the key being unavailable, which it - // is. - connection.setInstanceFollowRedirects(false); - connection.setRequestMethod("GET"); - connection.setConnectTimeout(CONNECT_TIMEOUT_MILLISECONDS); - connection.setReadTimeout(READ_TIMEOUT_MILLISECONDS); - connection.setRequestProperty("Accept", "text/plain"); - int code = connection.getResponseCode(); - if (code != HttpURLConnection.HTTP_OK) { - drain(connection.getErrorStream()); - throw new PublicKeyFetchException( - "domain " + quoted(domain) + " returned code '" + code - + "' for the public key", - OwidSignatureStatus.KEY_UNAVAILABLE, - domain, - code, - null); - } - InputStream body = connection.getInputStream(); - try { - return new String(readAll(body), StandardCharsets.UTF_8); - } finally { - body.close(); - } - } catch (IOException e) { - // A refused connection, a name that does not resolve and a - // timeout all arrive here, and all of them mean the signature - // was never examined. - throw new PublicKeyFetchException( - "the public key could not be fetched from domain " - + quoted(domain), - OwidSignatureStatus.KEY_UNAVAILABLE, - domain, - 0, - e); - } finally { - if (connection != null) { - connection.disconnect(); - } + started = transport.fetch(url, domain); + } catch (RuntimeException e) { + // A transport keeps its promise by failing the future rather + // than throwing, but one that breaks the promise must not leave + // a future in the cache that never completes. + started = failed(e); + } + if (started == null) { + started = failed(new PublicKeyFetchException( + "the transport returned no future for domain '" + domain + + "'", + OwidSignatureStatus.KEY_UNAVAILABLE, domain, 0, null)); } + started.whenComplete((pem, failure) -> { + if (failure == null && pem != null) { + fetch.complete(pem); + return; + } + // Only this fetch is removed, never whatever replaced it after + // the cache was emptied and filled again in the meantime. + CACHE.remove(url, fetch); + fetch.completeExceptionally(failure != null + ? unwrap(failure) + : new PublicKeyFetchException( + "the transport returned no key for domain '" + + domain + "'", + OwidSignatureStatus.KEY_UNAVAILABLE, domain, 0, + null)); + }); + return fetch; } - /** The value in single quotes, for a message. */ - private static String quoted(String value) { - return "'" + value + "'"; + /** A future that has already failed with the exception given. */ + private static CompletableFuture failed(Throwable failure) { + CompletableFuture future = new CompletableFuture(); + future.completeExceptionally(failure); + return future; } - /** Reads a stream to its end. */ - private static byte[] readAll(InputStream stream) throws IOException { - ByteArrayOutputStream buffer = new ByteArrayOutputStream(); - byte[] block = new byte[4096]; - int read = stream.read(block); - while (read > 0) { - buffer.write(block, 0, read); - read = stream.read(block); + /** + * The exception a failed future carries, with the wrapper a dependent + * future adds taken off so the one the transport raised is what a + * caller sees. + */ + private static Throwable unwrap(Throwable failure) { + Throwable cause = failure; + while (cause instanceof CompletionException + && cause.getCause() != null) { + cause = cause.getCause(); } - return buffer.toByteArray(); + return cause; } - /** Closes the error body of a refused request, where there is one. */ - private static void drain(InputStream stream) { - if (stream == null) { - return; - } - try { - stream.close(); - } catch (IOException e) { - // Nothing useful can be done about a body that will not close, - // and the refusal itself is what the caller is told about. + /** + * The status to report for a fetch that failed. A fetch failure carries + * its own status, and anything else, such as a URL that could not be + * built, means the key was never obtained. + */ + private static OwidSignatureStatus statusOf(Throwable failure) { + Throwable cause = unwrap(failure); + if (cause instanceof PublicKeyFetchException) { + return ((PublicKeyFetchException) cause).getStatus(); } + return OwidSignatureStatus.KEY_UNAVAILABLE; } /** diff --git a/src/main/java/com/swancommunity/owid/PublicKeyFetchException.java b/src/main/java/com/swancommunity/owid/PublicKeyFetchException.java index 22cc71a..20e04da 100644 --- a/src/main/java/com/swancommunity/owid/PublicKeyFetchException.java +++ b/src/main/java/com/swancommunity/owid/PublicKeyFetchException.java @@ -17,7 +17,8 @@ package com.swancommunity.owid; /** - * Raised when the public key of a creator could not be obtained. + * Carried by the failed future when the public key of a creator could not + * be obtained. * *

The status to report is decided where the failure happens and carried * here, so a caller never has to read message text to tell an outage from a diff --git a/src/main/java/com/swancommunity/owid/PublicKeyTransport.java b/src/main/java/com/swancommunity/owid/PublicKeyTransport.java new file mode 100644 index 0000000..60abf99 --- /dev/null +++ b/src/main/java/com/swancommunity/owid/PublicKeyTransport.java @@ -0,0 +1,66 @@ +/* **************************************************************************** + * Copyright 2026 51 Degrees Mobile Experts Limited (51degrees.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * ***************************************************************************/ + +package com.swancommunity.owid; + +import java.util.concurrent.CompletableFuture; + +/** + * Makes the request for a creator's public key and answers with a future, + * so that {@link PublicKeyFetch} never holds the thread that asked. + * + *

{@link HttpUrlConnectionTransport} is the one used where a caller names + * none. It runs the JDK's blocking connection on a background thread, which + * is the best Java 8 offers without a dependency. On Java 11 and later a + * caller can supply a transport of its own over + * {@code java.net.http.HttpClient.sendAsync}, which blocks no thread at all, + * and it must keep the two rules below.

+ * + *

The first rule is that a redirect is never followed. A creator whose + * domain answers 3xx must read as the key being unavailable, with no request + * made to wherever the redirect points, because otherwise a network attacker + * able to bend the creator's DNS, or a creator that was simply + * misconfigured, could have some other place's key trusted as the creator's + * own and forgeries would verify.

+ * + *

The second rule is that the URL is requested exactly as given. The + * query names the minute the identifier was signed as the {@code date} + * parameter, and a creator that rotates its key chooses the key by that + * parameter, so dropping or rewriting the query fetches the wrong key and + * every identifier signed under an earlier key reads as not matching.

+ */ +public interface PublicKeyTransport { + + /** + * Requests the URL and answers with the body as text. + * + *

The future completes with the body where the response code is 200, + * and fails with a {@link PublicKeyFetchException} carrying + * {@link OwidSignatureStatus#KEY_UNAVAILABLE} for anything else, which + * covers any other response code, a redirect, a connection that is + * refused, a name that does not resolve and a timeout. The method itself + * returns at once and never throws, and it never returns null.

+ * + * @param url the URL to request, exactly as given + * @param domain the creator domain the key is asked of, carried by the + * exception where the fetch fails so a caller can say whose + * key was wanted. The URL normally names the same host, + * but need not, because a test stands up an end point on + * the loopback address in place of the creator + * @return the body of the response, or the failure, through a future + */ + CompletableFuture fetch(String url, String domain); +} diff --git a/src/test/java/com/example/owidconsumer/ReadmeExampleTest.java b/src/test/java/com/example/owidconsumer/ReadmeExampleTest.java index 3ca594c..cab7c3e 100644 --- a/src/test/java/com/example/owidconsumer/ReadmeExampleTest.java +++ b/src/test/java/com/example/owidconsumer/ReadmeExampleTest.java @@ -36,6 +36,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.concurrent.CompletableFuture; import org.junit.jupiter.api.Test; /** @@ -179,8 +180,13 @@ void fetchingTheKeyFromTheCreatorDomain() throws OwidException { Owid owid = Creator.create("owid.invalid", Crypto.generate()) .createString("signed by a creator that cannot be reached"); - OwidVerificationResult result = PublicKeyFetch.verify( - owid, "https", Collections.emptyList()); + CompletableFuture pending = + PublicKeyFetch.verify( + owid, "https", Collections.emptyList()); + // The call returns at once and the request runs on a background + // thread. Continue from the future, or join it where waiting is + // acceptable, as it is here. + OwidVerificationResult result = pending.join(); if (result.getStatus() == OwidSignatureStatus.KEY_UNAVAILABLE) { // The key could not be obtained, so the signature was never // examined. Only SIGNATURE_INVALID means the identifier should diff --git a/src/test/java/com/swancommunity/owid/DatedKeyFetchTest.java b/src/test/java/com/swancommunity/owid/DatedKeyFetchTest.java index 65dd0cc..0cffb49 100644 --- a/src/test/java/com/swancommunity/owid/DatedKeyFetchTest.java +++ b/src/test/java/com/swancommunity/owid/DatedKeyFetchTest.java @@ -17,8 +17,11 @@ package com.swancommunity.owid; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -30,6 +33,12 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.Executor; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -43,12 +52,19 @@ * 51d.es schedule. The URL under test is the one the library builds, with * only the host replaced, so a fault in the path or the query is caught * here.

+ * + *

Every fetch answers with a future. The tests join the future, which is + * fine here and is not what a caller on a request thread would do.

*/ class DatedKeyFetchTest { /** No other OWIDs were covered by the signature on the fixture. */ private static final List ALONE = Collections.emptyList(); + /** The transport a caller gets without naming one. */ + private static final PublicKeyTransport HTTP = + new HttpUrlConnectionTransport(); + /** The end points started by a test, stopped when the test ends. */ private final List started = new ArrayList(); @@ -76,6 +92,29 @@ private KeyEndPoint endPoint(KeyEndPoint.Answer answer) return endPoint; } + /** The status a fetch through the default transport ends with. */ + private static OwidSignatureStatus statusAt(Owid owid, String url) { + return PublicKeyFetch.verifyAtUrl(owid, url, ALONE, HTTP).join() + .getStatus(); + } + + /** The PEM a fetch through the default transport ends with. */ + private static String pemAt(String url, String domain) { + return PublicKeyFetch.publicKeyPemAtUrl(url, domain, HTTP).join(); + } + + /** + * The exception a failed future carries. Joining wraps it in a + * completion exception, and the one inside is the one the library + * raised. + */ + private static T failureOf( + CompletableFuture future, Class type, String message) { + CompletionException wrapped = assertThrows(CompletionException.class, + future::join, message); + return assertInstanceOf(type, wrapped.getCause(), message); + } + /** * The URL names the minute the identifier was created, which is the value * the end point selects a key by, and it names the well known path from @@ -135,8 +174,7 @@ void datedFetchVerifiesAnIdentifierFromAnEarlierKeyWeek() Owid owid = KeyFixtures.identifier(); KeyEndPoint endPoint = endPoint(KeyEndPoint.Answer.SCHEDULE); assertEquals(OwidSignatureStatus.SIGNATURE_VALID, - PublicKeyFetch.verifyAtUrl(owid, endPoint.urlFor(owid), ALONE) - .getStatus(), + statusAt(owid, endPoint.urlFor(owid)), "should verify against the key in force when it was signed"); assertEquals( Collections.singletonList( @@ -161,7 +199,7 @@ void undatedFetchLeavesAnEarlierWeeksIdentifierUnverified() String undated = endPoint.base() + "/owid/api/v3/public-key?format=pkcs"; assertEquals(OwidSignatureStatus.SIGNATURE_INVALID, - PublicKeyFetch.verifyAtUrl(owid, undated, ALONE).getStatus(), + statusAt(owid, undated), "an undated request gets the key in force at the request, " + "which did not sign it"); assertEquals(Collections.singletonList((String) null), @@ -185,7 +223,7 @@ void aKeyTheEndPointCannotServeIsKeyUnavailable() String url = endPoint.base() + "/owid/api/v3/public-key?date=" + Io.minutesSinceBase(before) + "&format=pkcs"; assertEquals(OwidSignatureStatus.KEY_UNAVAILABLE, - PublicKeyFetch.verifyAtUrl(owid, url, ALONE).getStatus(), + statusAt(owid, url), "no key means the signature was never examined"); } @@ -194,11 +232,11 @@ void aKeyTheEndPointCannotServeIsKeyUnavailable() void aRefusedRequestCarriesTheStatusAndTheCode() throws IOException, OwidException { KeyEndPoint endPoint = endPoint(KeyEndPoint.Answer.SCHEDULE); - final String url = endPoint.base() + "/owid/api/v3/public-key?date=0" + String url = endPoint.base() + "/owid/api/v3/public-key?date=0" + "&format=pkcs"; - PublicKeyFetchException failure = assertThrows( + PublicKeyFetchException failure = failureOf( + PublicKeyFetch.publicKeyPemAtUrl(url, "51d.es", HTTP), PublicKeyFetchException.class, - () -> PublicKeyFetch.publicKeyPemAtUrl(url, "51d.es"), "a date the schedule does not reach is refused"); assertEquals(OwidSignatureStatus.KEY_UNAVAILABLE, failure.getStatus(), @@ -222,18 +260,11 @@ void anEndPointThatCannotBeReachedIsKeyUnavailable() String url = endPoint.urlFor(owid); endPoint.stop(); assertEquals(OwidSignatureStatus.KEY_UNAVAILABLE, - PublicKeyFetch.verifyAtUrl(owid, url, ALONE).getStatus(), + statusAt(owid, url), "a connection that is refused leaves the signature " + "unjudged"); } - /** - * Key material that arrives but cannot be read is the fault of the key - * and not of the identifier, so it is reported apart from a signature - * that does not match. This is the 30 August 2026 fault, where the key - * end points served PEM a strict parser refused and every offline check - * against them failed while the keys and the identifiers were both fine. - */ /** * A creator whose domain answers with a redirect does not get the key * at the other end trusted as its own. The other end here serves the @@ -251,8 +282,7 @@ void aRedirectIsNotFollowed() throws IOException, OwidException { KeyEndPoint.Answer.REDIRECT, elsewhere.urlFor(owid)); started.add(creator); assertEquals(OwidSignatureStatus.KEY_UNAVAILABLE, - PublicKeyFetch.verifyAtUrl(owid, creator.urlFor(owid), ALONE) - .getStatus(), + statusAt(owid, creator.urlFor(owid)), "a redirect is the key being unavailable, never a key from " + "wherever it points"); assertEquals(1, creator.dates().size(), @@ -262,14 +292,20 @@ void aRedirectIsNotFollowed() throws IOException, OwidException { + "never made"); } + /** + * Key material that arrives but cannot be read is the fault of the key + * and not of the identifier, so it is reported apart from a signature + * that does not match. This is the 30 August 2026 fault, where the key + * end points served PEM a strict parser refused and every offline check + * against them failed while the keys and the identifiers were both fine. + */ @Test void aKeyThatCannotBeReadIsInvalidKey() throws IOException, OwidException { Owid owid = KeyFixtures.identifier(); KeyEndPoint endPoint = endPoint(KeyEndPoint.Answer.BROKEN_KEY); assertEquals(OwidSignatureStatus.INVALID_KEY, - PublicKeyFetch.verifyAtUrl(owid, endPoint.urlFor(owid), ALONE) - .getStatus(), + statusAt(owid, endPoint.urlFor(owid)), "a key that cannot be read is not a signature that does not " + "match"); } @@ -286,21 +322,165 @@ void theKeyIsFetchedOnceAndHeldAfterThat() KeyEndPoint endPoint = endPoint(KeyEndPoint.Answer.SCHEDULE); String url = endPoint.urlFor(owid); assertEquals(OwidSignatureStatus.SIGNATURE_VALID, - PublicKeyFetch.verifyAtUrl(owid, url, ALONE).getStatus(), + statusAt(owid, url), "the first check fetches the key"); assertEquals(OwidSignatureStatus.SIGNATURE_VALID, - PublicKeyFetch.verifyAtUrl(owid, url, ALONE).getStatus(), + statusAt(owid, url), "the second check answers from the cache"); assertEquals(1, endPoint.dates().size(), "the end point was asked once"); PublicKeyFetch.clearCache(); assertEquals(OwidSignatureStatus.SIGNATURE_VALID, - PublicKeyFetch.verifyAtUrl(owid, url, ALONE).getStatus(), + statusAt(owid, url), "the check still works once the cache is emptied"); assertEquals(2, endPoint.dates().size(), "emptying the cache means the key is fetched again"); } + /** + * Two requests for the same key made while the first is still on its + * way share one request. The transport here answers only when the test + * lets it, so both requests are in flight together for certain, and + * the count of requests the transport saw is the whole point. + */ + @Test + void twoRequestsInFlightForOneKeyMakeOneRequest() + throws IOException, OwidException { + Owid owid = KeyFixtures.identifier(); + KeyEndPoint endPoint = endPoint(KeyEndPoint.Answer.SCHEDULE); + String url = endPoint.urlFor(owid); + HeldTransport held = new HeldTransport(); + CompletableFuture first = PublicKeyFetch.publicKeyPemAtUrl( + url, owid.getDomain(), held); + CompletableFuture second = PublicKeyFetch.publicKeyPemAtUrl( + url, owid.getDomain(), held); + assertEquals(1, held.requests.get(), + "the second request joins the first rather than asking " + + "again"); + assertSame(first, second, "both callers hold the same fetch"); + assertFalse(first.isDone(), "nothing has answered yet"); + // The genuine key, fetched through the transport itself rather than + // through the cache, because the cache holds the fetch still on its + // way and would hand back that same waiting future. + held.answer.complete(HTTP.fetch(url, owid.getDomain()).join()); + assertEquals(first.join(), second.join(), + "both callers get the one key that was fetched"); + assertEquals(OwidSignatureStatus.SIGNATURE_VALID, + PublicKeyFetch.verifyAtUrl(owid, url, ALONE, held).join() + .getStatus(), + "the key that arrived verifies the identifier"); + assertEquals(1, held.requests.get(), + "a key already held is not asked for again"); + } + + /** + * A fetch that fails is not held, so the next request for the same key + * asks again rather than repeating the failure for as long as the + * process runs. An outage is not a fact about the key. + */ + @Test + void aFetchThatFailsIsNotHeld() throws IOException, OwidException { + Owid owid = KeyFixtures.identifier(); + KeyEndPoint endPoint = endPoint(KeyEndPoint.Answer.SCHEDULE); + String url = endPoint.urlFor(owid); + HeldTransport held = new HeldTransport(); + CompletableFuture first = PublicKeyFetch.publicKeyPemAtUrl( + url, owid.getDomain(), held); + held.answer.completeExceptionally(new PublicKeyFetchException( + "the creator is away", OwidSignatureStatus.KEY_UNAVAILABLE, + owid.getDomain(), 503, null)); + PublicKeyFetchException failure = failureOf(first, + PublicKeyFetchException.class, + "the failure reaches the caller as the library raised it"); + assertEquals(503, failure.getStatusCode(), + "the failure is the one the transport gave"); + assertEquals(OwidSignatureStatus.SIGNATURE_VALID, + statusAt(owid, url), + "the next request asks again and the key arrives"); + assertEquals(1, held.requests.get(), + "the failed transport was asked once"); + assertEquals(1, endPoint.dates().size(), + "the end point was asked once, by the request that came " + + "after the failure"); + } + + /** + * The request runs on the executor the caller gave the transport, and + * not on the thread that asked, which is what makes the fetch safe to + * call from a request thread or an event loop. + */ + @Test + void theRequestRunsOnTheExecutorGiven() + throws IOException, OwidException { + Owid owid = KeyFixtures.identifier(); + KeyEndPoint endPoint = endPoint(KeyEndPoint.Answer.SCHEDULE); + final AtomicReference ran = new AtomicReference(); + Executor executor = task -> { + Thread thread = new Thread(task, "the executor given"); + ran.set(thread); + thread.start(); + }; + PublicKeyTransport transport = + new HttpUrlConnectionTransport(executor); + String url = endPoint.urlFor(owid); + CompletableFuture fetch = transport.fetch(url, + owid.getDomain()); + assertNotNull(ran.get(), "the executor was given the request"); + assertNotEquals(Thread.currentThread(), ran.get(), + "the thread that asked is not the one that fetches"); + assertEquals(OwidSignatureStatus.SIGNATURE_VALID, + owid.verify(fetch.join(), ALONE).getStatus(), + "the key fetched on the executor verifies the identifier"); + } + + /** + * An executor that refuses the request, because it has been shut down + * or is full, fails the future rather than throwing at the caller, so a + * caller has one place to look for every failure. + */ + @Test + void aRequestTheExecutorRefusesIsKeyUnavailable() + throws IOException, OwidException { + Owid owid = KeyFixtures.identifier(); + KeyEndPoint endPoint = endPoint(KeyEndPoint.Answer.SCHEDULE); + Executor refusing = task -> { + throw new RejectedExecutionException("shut down"); + }; + PublicKeyTransport transport = + new HttpUrlConnectionTransport(refusing); + PublicKeyFetchException failure = failureOf( + transport.fetch(endPoint.urlFor(owid), owid.getDomain()), + PublicKeyFetchException.class, + "the refusal arrives through the future"); + assertEquals(OwidSignatureStatus.KEY_UNAVAILABLE, + failure.getStatus(), "no request means no key"); + assertEquals(owid.getDomain(), failure.getDomain(), + "the domain asked of is carried"); + assertEquals(OwidSignatureStatus.KEY_UNAVAILABLE, + PublicKeyFetch.verifyAtUrl(owid, endPoint.urlFor(owid), + ALONE, transport).join().getStatus(), + "a check through the refusing executor is unjudged"); + assertTrue(endPoint.dates().isEmpty(), + "the end point was never reached"); + } + + /** A transport has to be given where the caller names one. */ + @Test + void aMissingTransportIsRefused() + throws IOException, OwidException { + Owid owid = KeyFixtures.identifier(); + failureOf(PublicKeyFetch.publicKeyPem(owid, "https", null), + OwidException.class, + "the key cannot be fetched with no transport"); + assertEquals(OwidSignatureStatus.KEY_UNAVAILABLE, + PublicKeyFetch.verify(owid, "https", ALONE, null).join() + .getStatus(), + "a check with no transport is unjudged"); + assertThrows(IllegalArgumentException.class, + () -> new HttpUrlConnectionTransport(null), + "the default transport needs an executor"); + } + /** * Keys are held against the URL they came from, which names the minute, * so two identifiers from different weeks fetch two different keys, and @@ -315,14 +495,14 @@ void keysAreHeldPerRequestAndNotPerDomain() Instant.parse("2026-08-20T00:00:00Z")); Owid later = crafted(Version.VERSION3, KeyFixtures.IDENTIFIER_DOMAIN, Instant.parse("2026-09-04T00:00:00Z")); - String first = PublicKeyFetch.publicKeyPemAtUrl( - endPoint.urlFor(earlier), KeyFixtures.IDENTIFIER_DOMAIN); - String second = PublicKeyFetch.publicKeyPemAtUrl( - endPoint.urlFor(later), KeyFixtures.IDENTIFIER_DOMAIN); + String first = pemAt(endPoint.urlFor(earlier), + KeyFixtures.IDENTIFIER_DOMAIN); + String second = pemAt(endPoint.urlFor(later), + KeyFixtures.IDENTIFIER_DOMAIN); assertNotEquals(first, second, "two weeks, two keys"); assertEquals(2, endPoint.dates().size(), "one request per week"); - assertEquals(first, PublicKeyFetch.publicKeyPemAtUrl( - endPoint.urlFor(earlier), KeyFixtures.IDENTIFIER_DOMAIN), + assertEquals(first, pemAt(endPoint.urlFor(earlier), + KeyFixtures.IDENTIFIER_DOMAIN), "the held key is the one fetched for that week"); assertEquals(2, endPoint.dates().size(), "a week already held is not asked for again"); @@ -341,8 +521,12 @@ void aDomainThatIsNotADomainNameIsRefused() throws OwidException { assertThrows(OwidException.class, () -> PublicKeyFetch.publicKeyUrl(owid, "https"), "a domain carrying a path and a query is refused"); + failureOf(PublicKeyFetch.publicKeyPem(owid, "https"), + OwidException.class, + "a URL that cannot be built fails the fetch"); assertEquals(OwidSignatureStatus.KEY_UNAVAILABLE, - PublicKeyFetch.verify(owid, "https", ALONE).getStatus(), + PublicKeyFetch.verify(owid, "https", ALONE).join() + .getStatus(), "a URL that cannot be built leaves the signature unjudged"); } @@ -361,7 +545,8 @@ void aDomainThatIsNotADomainNameIsRefused() throws OwidException { void aSchemeThatIsNotHttpIsKeyUnavailable() throws OwidException { assertEquals(OwidSignatureStatus.KEY_UNAVAILABLE, PublicKeyFetch.verify( - KeyFixtures.identifier(), "mailto", ALONE).getStatus(), + KeyFixtures.identifier(), "mailto", ALONE).join() + .getStatus(), "a scheme that fetches no key leaves the signature unjudged"); } @@ -377,6 +562,25 @@ void missingValuesAreRefused() { "there is no URL without a scheme"); } + /** + * A transport that answers only when the test lets it, counting the + * requests made of it, so a test can hold two requests in flight + * together and say how many reached the wire. + */ + private static final class HeldTransport implements PublicKeyTransport { + + final AtomicInteger requests = new AtomicInteger(); + + final CompletableFuture answer = + new CompletableFuture(); + + @Override + public CompletableFuture fetch(String url, String domain) { + requests.incrementAndGet(); + return answer; + } + } + /** * Builds an OWID with the version, domain and date given and a signature * of zeroes, for the cases that are about the URL rather than about the From 010b13304d37549ae4eb9f1a97907353d558c0a8 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sun, 6 Sep 2026 23:13:26 +0100 Subject: [PATCH 2/9] Build the key url without the deprecated URL constructor The URL(String) constructor is deprecated from Java 20, and pipeline-java compiles this source into its 51Did module with -Xlint:all and -Werror, excluding only Endpoints and PublicKeyFetch, so the new transport would have failed that build on the Java 21 leg of its matrix. URI then toURL replaces it. URI.toURL has been present since 1.0, so the Java 8 floor is unaffected. A url that will not parse now raises URISyntaxException rather than MalformedURLException, so the catch takes both and a bad url is still reported as the key being unavailable with the same status as before. --- .../owid/HttpUrlConnectionTransport.java | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/swancommunity/owid/HttpUrlConnectionTransport.java b/src/main/java/com/swancommunity/owid/HttpUrlConnectionTransport.java index 28cac7e..2874db0 100644 --- a/src/main/java/com/swancommunity/owid/HttpUrlConnectionTransport.java +++ b/src/main/java/com/swancommunity/owid/HttpUrlConnectionTransport.java @@ -20,6 +20,8 @@ import java.io.IOException; import java.io.InputStream; import java.net.HttpURLConnection; +import java.net.URI; +import java.net.URISyntaxException; import java.net.URL; import java.net.URLConnection; import java.nio.charset.StandardCharsets; @@ -140,7 +142,10 @@ private static String read(String url, String domain) throws PublicKeyFetchException { HttpURLConnection connection = null; try { - URLConnection opened = new URL(url).openConnection(); + // URI then toURL rather than the URL(String) constructor, + // which is deprecated from Java 20 and would fail a consumer + // compiling this source with warnings as errors. + URLConnection opened = new URI(url).toURL().openConnection(); if ((opened instanceof HttpURLConnection) == false) { // A scheme the caller chose that does not make an HTTP // request, such as file. Reported as a key that could not be @@ -186,10 +191,10 @@ private static String read(String url, String domain) } finally { body.close(); } - } catch (IOException e) { - // A refused connection, a name that does not resolve and a - // timeout all arrive here, and all of them mean the signature - // was never examined. + } catch (IOException | URISyntaxException e) { + // A refused connection, a name that does not resolve, a + // timeout and a url that will not parse all arrive here, and + // all of them mean the signature was never examined. throw new PublicKeyFetchException( "the public key could not be fetched from domain " + quoted(domain), From 68f8d8b640776d7a2f6294403bfce8d987b12ae9 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 7 Sep 2026 09:01:58 +0100 Subject: [PATCH 3/9] Hold each key against the span of minutes the creator confirmed it for The cache was keyed by the whole key url, and the url carries the date of the identifier being verified in minutes. A creator's key changes on the order of a week, so two identifiers signed a minute apart never shared an entry and a hundred identifiers over a hundred minutes made a hundred requests for one key. The cache only ever served a repeat verification of one identifier. Keys are now held by end point, which is the key url without its date, and each key carries the span of minutes the creator has confirmed it for. A key is in force from the start of its period until the next key starts, so a key the creator answers with at two minutes was in force at every minute between them. An identifier dated inside a confirmed span is verified without a request. One dated outside every span is asked about, and the answer widens the span when the same key comes back or adds a key when it does not. The span is never widened across a minute the creator has answered with another key for. A date later than now is held against now, because that is how a creator reads it. Held against the future minute, the key would still be served for that minute after the creator had rotated. A request without a date is read as now for the same reason. The bound of 1024 keys and the empty and refill are unchanged, as is the sharing of one request between callers arriving together and the refusal to hold a failure. clearCache now also forgets the requests under way, as the Python and Rust ports do. The same change is made to every port that caches, so the ports behave the same way. Tests cover a minute between two confirmed minutes being served without a request, a hundred identifiers inside one confirmed period making none, a key never being served outside its span across a rotation, a future date being read as now, and the bound against a creator that answers every minute with a different key. The bound test and the shared request test were checked by breaking the code and watching them fail. The main sources compile clean with -Xlint:all,-try,-options -Werror. --- README.md | 16 +- .../swancommunity/owid/PublicKeyFetch.java | 298 +++++++++++++++--- .../swancommunity/owid/DatedKeyFetchTest.java | 194 ++++++++++++ 3 files changed, 459 insertions(+), 49 deletions(-) diff --git a/README.md b/README.md index aa69ae7..9e35c88 100644 --- a/README.md +++ b/README.md @@ -159,10 +159,18 @@ the path is the version byte of the identifier being checked and the minutes are counted from 2020-01-01 in the same way the identifier stores its date. A creator that ignores the parameter returns its current key, so every identifier it signed under an earlier key reads as not matching, which is why -a creator that rotates its key has to honour the date. Keys already fetched -are held against the URL they came from, which names the domain, the version -and the minute, up to 1024 of them before the store is emptied, and -`clearCache` empties it on demand. Two requests for the same key made while +a creator that rotates its key has to honour the date. + +Keys already fetched are held by creator, each against the span of minutes +the creator has confirmed it for. A key is in force from the start of its +period until the next key starts, so a key the creator answers with at two +minutes was in force at every minute between them, and an identifier dated +inside a confirmed span is verified without a request whichever minute it +carries. One dated outside every span is asked about, which widens the span +when the same key comes back. At most 1024 keys are held across every +creator before the store is emptied and filled again, and `clearCache` +empties it on demand, which is how a long running process drops a key it has +learned it should no longer trust. Two requests for the same key made while the first is still on its way share one request, and a fetch that fails is not held, so the next request asks again. diff --git a/src/main/java/com/swancommunity/owid/PublicKeyFetch.java b/src/main/java/com/swancommunity/owid/PublicKeyFetch.java index e94a666..6a52e3f 100644 --- a/src/main/java/com/swancommunity/owid/PublicKeyFetch.java +++ b/src/main/java/com/swancommunity/owid/PublicKeyFetch.java @@ -16,11 +16,13 @@ package com.swancommunity.owid; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; -import java.util.concurrent.ConcurrentHashMap; /** * Fetches the signing public key of a creator from the well known end point @@ -61,30 +63,79 @@ public final class PublicKeyFetch { /** * The most keys held in the cache before the cache is emptied and filled - * again. A bound is needed because a verifier sees identifiers from many - * domains and many weeks, and an unbounded map would grow for as long as - * the process runs. + * again, across every creator. A bound is needed because a verifier sees + * identifiers from many domains and many weeks, and an unbounded store + * would grow for as long as the process runs. */ private static final int MAXIMUM_CACHED_KEYS = 1024; /** - * Keys fetched, or on their way, held against the URL they were asked - * for. + * One key a creator has answered with, and the span of minutes the + * creator has confirmed it was in force for. + * + *

A creator's key is in force from the start of its period until the + * next key starts, so a key the creator confirms at two minutes was in + * force at every minute between them. The span grows as the creator + * confirms the same key for more minutes, and an identifier dated inside + * it is verified without a request.

+ */ + private static final class HeldKey { + /** The key in PEM form, as the creator served it. */ + final String pem; + /** The earliest minute the creator has confirmed the key for. */ + long first; + /** The latest minute the creator has confirmed the key for. */ + long last; + + HeldKey(String pem, long minute) { + this.pem = pem; + this.first = minute; + this.last = minute; + } + + /** Whether the minute lies within the confirmed span. */ + boolean covers(long minute) { + return first <= minute && minute <= last; + } + } + + /** + * Guards {@link #CACHE}, {@link #heldKeys} and {@link #IN_FLIGHT}. Held + * across a few map and list operations only, never across a request. + */ + private static final Object LOCK = new Object(); + + /** + * Keys already fetched, by the creator's key end point, which is the key + * URL without its date. Each end point holds the keys the creator has + * answered with, each with the span of minutes the creator has confirmed + * it for. * *

The specification asks implementations to cache so that verifying * many identifiers does not mean repeating requests to another - * processor. Holding the key against the whole URL is safe because the - * URL names the domain, the version and the minute, and the key a - * creator published for a minute in the past does not change.

- * - *

The value is the future of the fetch rather than the key itself, so - * a second request for a key that is still on its way joins the request - * already made instead of making another. A fetch that fails is removed - * the moment it fails, so an outage is never remembered and the next - * request tries again.

+ * processor. The key URL carries the date of the identifier being + * verified, in minutes, and a creator's key changes on the order of a + * week. Keyed by the whole URL, as this cache once was, two identifiers + * signed a minute apart never shared an entry, so a hundred identifiers + * over a hundred minutes made a hundred requests for one key. Keyed by + * end point and span, an identifier dated between two minutes the + * creator has already answered for is verified without a request.

*/ - private static final Map> CACHE = - new ConcurrentHashMap>(); + private static final Map> CACHE = + new HashMap>(); + + /** How many keys are held across every end point. */ + private static int heldKeys; + + /** + * Requests under way, by the dated URL asked for, so that a second + * request for a key that is still on its way joins the request already + * made instead of making another. An entry is removed the moment its + * request ends, whatever the outcome, so a failure is never handed to a + * later caller and an outage is never remembered. + */ + private static final Map> IN_FLIGHT = + new HashMap>(); /** The transport used where the caller names none. */ private static final PublicKeyTransport DEFAULT_TRANSPORT = @@ -246,13 +297,27 @@ public static CompletableFuture verify(Owid owid, } /** - * Empties the cache of keys already fetched. Provided so that a long - * running process can release the memory, and so that a test can start - * from a known state. A fetch still on its way is forgotten here but - * still completes for whoever holds its future. + * Empties the cache of keys already fetched, and forgets the requests + * under way so that the next caller for any key starts a request of its + * own. A request already under way is not stopped and still completes + * for whoever holds its future. This is how a long running process drops + * a key it has learned it should no longer trust, after a creator + * rotates its key following a compromise, and how a test starts from a + * known state. */ public static void clearCache() { - CACHE.clear(); + synchronized (LOCK) { + CACHE.clear(); + heldKeys = 0; + IN_FLIGHT.clear(); + } + } + + /** How many keys the cache holds, for the tests. */ + static int cachedKeyCount() { + synchronized (LOCK) { + return heldKeys; + } } /** @@ -274,32 +339,37 @@ static CompletableFuture verifyAtUrl( } /** - * Fetches the PEM at the URL, answering from the cache where the same - * URL has already been fetched or is being fetched now. + * Fetches the PEM at the URL. Answered from the cache where the creator + * has already confirmed a key for the minute the URL names, from a + * request already under way for the same URL where there is one, and + * otherwise through the transport. * - *

The future held in the cache is this class's own rather than the - * transport's, so that the transport's completion can be watched and a - * failure dropped from the cache without touching the map from inside - * one of its own operations, which a concurrent map does not allow.

+ *

The future held for a request under way is this class's own rather + * than the transport's, so that the transport's completion can be + * watched, the key held against the minute it was asked for, and a + * failure forgotten, all before the callers waiting are answered.

*/ static CompletableFuture publicKeyPemAtUrl(final String url, String domain, PublicKeyTransport transport) { if (transport == null) { return failed(new OwidException("the transport is missing")); } - CompletableFuture held = CACHE.get(url); - if (held != null) { - return held; - } - if (CACHE.size() >= MAXIMUM_CACHED_KEYS) { - CACHE.clear(); - } - final CompletableFuture fetch = new CompletableFuture(); - held = CACHE.putIfAbsent(url, fetch); - if (held != null) { - // Another thread asked for the same key between the lookup and - // the insert, and its fetch is the one both callers share. - return held; + final String endPoint = endPointOf(url); + final long minute = minuteOf(url); + final CompletableFuture fetch; + synchronized (LOCK) { + String pem = heldPem(endPoint, minute); + if (pem != null) { + return CompletableFuture.completedFuture(pem); + } + CompletableFuture held = IN_FLIGHT.get(url); + if (held != null) { + // Another caller asked for the same key and its fetch is + // the one both callers share. + return held; + } + fetch = new CompletableFuture(); + IN_FLIGHT.put(url, fetch); } CompletableFuture started; try { @@ -307,7 +377,7 @@ static CompletableFuture publicKeyPemAtUrl(final String url, } catch (RuntimeException e) { // A transport keeps its promise by failing the future rather // than throwing, but one that breaks the promise must not leave - // a future in the cache that never completes. + // a future among the requests under way that never completes. started = failed(e); } if (started == null) { @@ -318,12 +388,19 @@ static CompletableFuture publicKeyPemAtUrl(final String url, } started.whenComplete((pem, failure) -> { if (failure == null && pem != null) { + // Held before the callers are answered, so a caller arriving + // between the two finds the key rather than starting a + // request of its own. + synchronized (LOCK) { + hold(endPoint, minute, pem); + forget(url, fetch); + } fetch.complete(pem); return; } - // Only this fetch is removed, never whatever replaced it after - // the cache was emptied and filled again in the meantime. - CACHE.remove(url, fetch); + synchronized (LOCK) { + forget(url, fetch); + } fetch.completeExceptionally(failure != null ? unwrap(failure) : new PublicKeyFetchException( @@ -335,6 +412,137 @@ static CompletableFuture publicKeyPemAtUrl(final String url, return fetch; } + /** + * Removes the request from those under way. Only this request is + * removed, never whatever replaced it after the cache was emptied and a + * fresh request started for the same URL in the meantime. Called under + * the lock. + */ + private static void forget(String url, CompletableFuture fetch) { + if (IN_FLIGHT.get(url) == fetch) { + IN_FLIGHT.remove(url); + } + } + + /** + * The key URL without its query, which names the scheme, the creator and + * the version, and so the key end point being asked. + */ + private static String endPointOf(String url) { + int query = url.indexOf('?'); + return query < 0 ? url : url.substring(0, query); + } + + /** + * The minute the cache reads the URL as asking about. + * + *

The date parameter where the URL carries one, and otherwise now, + * because a creator answers a request without a date with the key in + * force now. A date later than now is read as now as well, because that + * is how a creator reads it. A schedule is published ahead of time and a + * key that has not started has signed nothing, so the creator answers a + * future date with the key in force now, and that answer must be held + * against now rather than against a minute the creator has not spoken + * for. Held against the future minute, the key would still be served + * for that minute after the creator had rotated, and a genuine + * identifier signed then would read as not matching.

+ */ + private static long minuteOf(String url) { + long now = Io.minutesSinceBase(Instant.now()); + int query = url.indexOf('?'); + if (query < 0) { + return now; + } + for (String pair : url.substring(query + 1).split("&")) { + if (pair.startsWith("date=")) { + try { + return Math.min(Long.parseLong(pair.substring(5)), now); + } catch (NumberFormatException notANumber) { + return now; + } + } + } + return now; + } + + /** + * The key held for the end point whose confirmed span covers the minute, + * or null where no held key does. Called under the lock. + */ + private static String heldPem(String endPoint, long minute) { + List keys = CACHE.get(endPoint); + if (keys != null) { + for (HeldKey key : keys) { + if (key.covers(minute)) { + return key.pem; + } + } + } + return null; + } + + /** + * Records that the creator answered the minute with the key. Called + * under the lock. + * + *

A key already held for the end point has its span widened to take + * in the minute. A key not held before is added, emptying the cache + * first when it is full, because the domains and dates asked about come + * from the identifiers presented to this process and the cache must not + * grow on their input.

+ */ + private static void hold(String endPoint, long minute, String pem) { + List keys = CACHE.get(endPoint); + if (keys != null) { + for (HeldKey key : keys) { + if (key.pem.equals(pem) && widen(keys, key, minute)) { + return; + } + } + } + if (heldKeys >= MAXIMUM_CACHED_KEYS) { + CACHE.clear(); + heldKeys = 0; + keys = null; + } + if (keys == null) { + keys = new ArrayList(); + CACHE.put(endPoint, keys); + } + keys.add(new HeldKey(pem, minute)); + heldKeys++; + } + + /** + * Widens the span of a held key to take in the minute, and says whether + * the minute is now within it. + * + *

The span is not widened across a minute the creator has answered + * with another key for, because that would mean the creator had gone + * back to a key it had left, and the minutes between the two spans are + * then not this key's to claim. The key is held again as a separate span + * instead.

+ */ + private static boolean widen(List keys, HeldKey key, + long minute) { + if (key.covers(minute)) { + return true; + } + long from = Math.min(minute, key.first); + long to = Math.max(minute, key.last); + for (HeldKey other : keys) { + if (other != key && other.last > from && other.first < to) { + return false; + } + } + if (minute < key.first) { + key.first = minute; + } else { + key.last = minute; + } + return true; + } + /** A future that has already failed with the exception given. */ private static CompletableFuture failed(Throwable failure) { CompletableFuture future = new CompletableFuture(); diff --git a/src/test/java/com/swancommunity/owid/DatedKeyFetchTest.java b/src/test/java/com/swancommunity/owid/DatedKeyFetchTest.java index 0cffb49..4831e8c 100644 --- a/src/test/java/com/swancommunity/owid/DatedKeyFetchTest.java +++ b/src/test/java/com/swancommunity/owid/DatedKeyFetchTest.java @@ -24,9 +24,11 @@ import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeTrue; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.lang.reflect.Field; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; @@ -605,4 +607,196 @@ private static Owid crafted(Version version, String domain, Instant date) "the crafted OWID carries no real signature"); return owid; } + + /** The minute count for a moment, counted the way the key URL counts it. */ + private static long minutes(String moment) { + return Io.minutesSinceBase(Instant.parse(moment)); + } + + /** A key URL on the end point for the minute given. */ + private static String urlFor(KeyEndPoint endPoint, long minute) { + return endPoint.base() + "/owid/api/v3/public-key?date=" + minute + + "&format=pkcs"; + } + + /** The PEM the published schedule says was in force at the minute. */ + private static String inForce(long minute) throws OwidException { + return KeyFixtures.schedule() + .keyInForce(Io.baseDate().plus(Duration.ofMinutes(minute))) + .getPublicKeyPem(); + } + + /** + * A key the creator has confirmed for two minutes is served for every + * minute between them without a request, because a key is in force from + * the start of its period until the next key starts. A minute outside + * every confirmed span is asked about. + */ + @Test + void aMinuteBetweenTwoConfirmedMinutesIsServedFromTheCache() + throws IOException, OwidException { + KeyEndPoint endPoint = endPoint(KeyEndPoint.Answer.SCHEDULE); + // The week of 31 August 2026, which the fixture identifier was + // signed in, and which is wholly in the past so the cache reads + // each minute as itself rather than as now. + long first = minutes("2026-08-31T00:01:00Z"); + long last = minutes("2026-09-06T23:00:00Z"); + String pem = pemAt(urlFor(endPoint, first), + KeyFixtures.IDENTIFIER_DOMAIN); + assertEquals(pem, pemAt(urlFor(endPoint, last), + KeyFixtures.IDENTIFIER_DOMAIN), "one key covers the week"); + assertEquals(2, endPoint.dates().size(), + "the two ends of the span were asked about"); + for (long between : new long[] { + first + 1, first + 3 * 24 * 60, last - 1 }) { + assertEquals(pem, pemAt(urlFor(endPoint, between), + KeyFixtures.IDENTIFIER_DOMAIN), + "the key served for minute " + between); + } + assertEquals(2, endPoint.dates().size(), + "a minute between two confirmed minutes is not asked about"); + assertEquals(1, PublicKeyFetch.cachedKeyCount(), + "one key is held however many minutes it covers"); + assertNotEquals(pem, pemAt(urlFor(endPoint, first - 2), + KeyFixtures.IDENTIFIER_DOMAIN), + "a minute in the week before is the earlier week's key"); + assertEquals(3, endPoint.dates().size(), + "a minute before the span is asked about"); + assertEquals(2, PublicKeyFetch.cachedKeyCount(), + "the earlier week's key is held as a second key"); + } + + /** + * The case that made the cache almost useless when it was keyed by the + * whole URL. A hundred identifiers with a hundred different minutes + * inside one key's period cost a hundred requests then. With the ends + * of the period confirmed they cost none. + */ + @Test + void aHundredIdentifiersInOneConfirmedPeriodMakeNoRequest() + throws IOException, OwidException { + KeyEndPoint endPoint = endPoint(KeyEndPoint.Answer.SCHEDULE); + long start = minutes("2026-09-01T00:00:00Z"); + pemAt(urlFor(endPoint, start), KeyFixtures.IDENTIFIER_DOMAIN); + pemAt(urlFor(endPoint, start + 100), KeyFixtures.IDENTIFIER_DOMAIN); + for (int i = 1; i <= 100; i++) { + pemAt(urlFor(endPoint, start + i), KeyFixtures.IDENTIFIER_DOMAIN); + } + assertEquals(2, endPoint.dates().size(), + "a hundred identifiers over a hundred minutes made no " + + "request once both ends of the span were known"); + } + + /** + * A key is only ever served for a minute inside the span the creator + * has confirmed it for. Where the creator rotated between two confirmed + * minutes, the minutes between them belong to neither key until the + * creator is asked, and every answer agrees with the published + * schedule. + */ + @Test + void aKeyIsNeverServedForAMinuteOutsideItsConfirmedSpan() + throws IOException, OwidException { + KeyEndPoint endPoint = endPoint(KeyEndPoint.Answer.SCHEDULE); + long rotation = minutes("2026-08-31T00:00:00Z"); + long week = 7 * 24 * 60; + // The start of the week before the rotation and the end of the week + // after it, so the two keys are held with the rotation between. + pemAt(urlFor(endPoint, rotation - week), KeyFixtures.IDENTIFIER_DOMAIN); + pemAt(urlFor(endPoint, rotation + week - 1), + KeyFixtures.IDENTIFIER_DOMAIN); + assertEquals(2, endPoint.dates().size()); + assertEquals(2, PublicKeyFetch.cachedKeyCount()); + + // Every minute across the rotation, in an order that walks in from + // both sides, is answered with the key the schedule gives, whether + // from the cache or by asking. + long[] minutes = { + rotation - 1, rotation, rotation - 2, rotation + 1, + rotation - week / 2, rotation + week / 2, + rotation - 3, rotation + 2, rotation - 1, rotation }; + for (long minute : minutes) { + assertEquals(inForce(minute), pemAt(urlFor(endPoint, minute), + KeyFixtures.IDENTIFIER_DOMAIN), + "the key served for minute " + minute); + } + assertEquals(2, PublicKeyFetch.cachedKeyCount(), + "two keys are held, each with its own span"); + int asked = endPoint.dates().size(); + assertTrue(asked > 2 && asked < 2 + minutes.length, + "some minutes were asked about and some were served: " + + asked); + + // The minute either side of the rotation is now confirmed, so + // nothing across the whole fortnight needs asking. + for (long minute = rotation - week; minute < rotation + week; + minute += 60) { + assertEquals(inForce(minute), pemAt(urlFor(endPoint, minute), + KeyFixtures.IDENTIFIER_DOMAIN), + "the key served for minute " + minute); + } + assertEquals(asked, endPoint.dates().size(), + "both spans are fully confirmed, so nothing was asked"); + } + + /** + * A date later than now is held against now, because a creator answers + * a future date with the key in force now and a key held against a + * minute the creator has not spoken for would be served for that minute + * after the creator had rotated. Two future dates therefore share one + * request, and so does a request with no date. + */ + @Test + void aFutureDateIsHeldAgainstNow() throws IOException, OwidException { + KeyEndPoint endPoint = endPoint(KeyEndPoint.Answer.SCHEDULE); + long started = Io.minutesSinceBase(Instant.now()); + long week = 7 * 24 * 60; + pemAt(urlFor(endPoint, started + week), KeyFixtures.IDENTIFIER_DOMAIN); + pemAt(urlFor(endPoint, started + 2 * week), + KeyFixtures.IDENTIFIER_DOMAIN); + pemAt(endPoint.base() + "/owid/api/v3/public-key?format=pkcs", + KeyFixtures.IDENTIFIER_DOMAIN); + assumeTrue(Io.minutesSinceBase(Instant.now()) == started, + "the minute changed during the test, so the calls were not " + + "all about the same now"); + assertEquals(1, endPoint.dates().size(), + "two future dates and no date are all now, and now was " + + "asked about once"); + } + + /** + * The cache does not grow without limit. The number of distinct keys a + * verifier is shown is chosen by whoever presents the identifiers rather + * than by this process, so the stand in creator here answers every + * minute with a different key, which is the worst a creator can do to + * the cache. The bound is read from the library so the test cannot + * drift from it. + */ + @Test + void theCacheIsBounded() throws Exception { + Field bound = PublicKeyFetch.class.getDeclaredField( + "MAXIMUM_CACHED_KEYS"); + bound.setAccessible(true); + int maximum = bound.getInt(null); + final AtomicInteger requests = new AtomicInteger(); + PublicKeyTransport distinct = (url, domain) -> { + requests.incrementAndGet(); + String minute = url.substring(url.indexOf("date=") + 5, + url.indexOf("&format")); + return CompletableFuture.completedFuture( + "-----BEGIN PUBLIC KEY-----\n" + minute + + "\n-----END PUBLIC KEY-----\n"); + }; + for (int i = 0; i <= maximum; i++) { + PublicKeyFetch.publicKeyPemAtUrl( + "https://example.invalid/owid/api/v3/public-key?date=" + i + + "&format=pkcs", + "example.invalid", distinct).join(); + } + assertEquals(maximum + 1, requests.get(), + "every minute was a different key, so every one was asked"); + assertTrue(PublicKeyFetch.cachedKeyCount() <= maximum, + "held " + PublicKeyFetch.cachedKeyCount() + " of at most " + + maximum); + } } From 57010d372d0afe85ee64203dda26d3ec5c061427 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 7 Sep 2026 10:12:39 +0100 Subject: [PATCH 4/9] Ask about minutes within the clock drift allowance rather than holding them The span cache read a date later than now as now and held the answer against now. A creator whose clock runs ahead of this one's signs identifiers dated in this process's future, and just after a rotation the cache would have served such an identifier the old key from a span confirmed up to now. It would have read as not matching until this clock caught up, for as long as the two clocks differ. The cache keyed by url did not have this fault, because it asked the creator for the exact minute. A minute within fifteen minutes of now, or later, is now neither served from the cache nor held in it. The creator may have read such a minute as its present rather than as the minute named, so its answer says nothing certain about the minute. A request with no date is treated the same way. Live identifiers therefore cost one request per minute per creator, which is what they always cost, and every identifier older than the allowance is served from the spans. The same allowance is applied on every port that caches. The test that read a future date as now is replaced by one that shows a recent minute asked about twice, a future minute and an undated request asked about, and a minute beyond the allowance held after its first request. --- README.md | 26 ++++---- .../swancommunity/owid/PublicKeyFetch.java | 59 +++++++++++++------ .../swancommunity/owid/DatedKeyFetchTest.java | 36 +++++++---- 3 files changed, 81 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index 9e35c88..bb53749 100644 --- a/README.md +++ b/README.md @@ -161,18 +161,22 @@ creator that ignores the parameter returns its current key, so every identifier it signed under an earlier key reads as not matching, which is why a creator that rotates its key has to honour the date. -Keys already fetched are held by creator, each against the span of minutes -the creator has confirmed it for. A key is in force from the start of its -period until the next key starts, so a key the creator answers with at two -minutes was in force at every minute between them, and an identifier dated -inside a confirmed span is verified without a request whichever minute it -carries. One dated outside every span is asked about, which widens the span -when the same key comes back. At most 1024 keys are held across every -creator before the store is emptied and filled again, and `clearCache` +Keys already fetched are held by creator, each against the span of minutes the +creator has confirmed it for. A key is in force from the start of its period +until the next key starts, so a key the creator answers with at two minutes was +in force at every minute between them, and an identifier dated inside a +confirmed span is verified without a request whichever minute it carries. One +dated outside every span is asked about, which widens the span when the same +key comes back. One dated within fifteen minutes of now, or later, is asked +about every time and never held, because a creator whose clock differs from +this one's may have read that minute as its present rather than as the minute +named. Live identifiers therefore cost one request per minute per creator, as +they always did, and older ones cost none. At most 1024 keys are held across +every creator before the store is emptied and filled again, and `clearCache` empties it on demand, which is how a long running process drops a key it has -learned it should no longer trust. Two requests for the same key made while -the first is still on its way share one request, and a fetch that fails is -not held, so the next request asks again. +learned it should no longer trust. Two requests for the same key made while the +first is still on its way share one request, and a fetch that fails is not +held, so the next request asks again. Every method that reaches the network answers with a `CompletableFuture` and returns at once. There is no form that waits, so a request thread or an event diff --git a/src/main/java/com/swancommunity/owid/PublicKeyFetch.java b/src/main/java/com/swancommunity/owid/PublicKeyFetch.java index 6a52e3f..914cb99 100644 --- a/src/main/java/com/swancommunity/owid/PublicKeyFetch.java +++ b/src/main/java/com/swancommunity/owid/PublicKeyFetch.java @@ -69,6 +69,27 @@ public final class PublicKeyFetch { */ private static final int MAXIMUM_CACHED_KEYS = 1024; + /** + * How far a creator's clock may run ahead of or behind this one's, in + * minutes. A minute closer to now than this, or later, is asked about + * rather than served from the cache, and is not held. + * + *

A creator reads a date later than its own now as now, and answers + * with the key in force now. Within this window this process cannot tell + * whether the creator read the minute as its past or as its present, so + * the answer says nothing certain about the minute. An identifier signed + * just after a rotation by a creator whose clock runs ahead would + * otherwise be served the old key from a span confirmed up to now, and + * would read as not matching until this clock caught up. Identifiers + * dated within the window are asked about once per minute per creator, + * as they always were, and every older identifier is served from the + * spans.

+ */ + private static final long CLOCK_DRIFT_ALLOWANCE_MINUTES = 15; + + /** The minute {@link #minuteOf} answers where the cache must not be used. */ + private static final long NOT_HELD = -1; + /** * One key a creator has answered with, and the span of minutes the * creator has confirmed it was in force for. @@ -358,7 +379,7 @@ static CompletableFuture publicKeyPemAtUrl(final String url, final long minute = minuteOf(url); final CompletableFuture fetch; synchronized (LOCK) { - String pem = heldPem(endPoint, minute); + String pem = minute == NOT_HELD ? null : heldPem(endPoint, minute); if (pem != null) { return CompletableFuture.completedFuture(pem); } @@ -392,7 +413,9 @@ static CompletableFuture publicKeyPemAtUrl(final String url, // between the two finds the key rather than starting a // request of its own. synchronized (LOCK) { - hold(endPoint, minute, pem); + if (minute != NOT_HELD) { + hold(endPoint, minute, pem); + } forget(url, fetch); } fetch.complete(pem); @@ -434,35 +457,37 @@ private static String endPointOf(String url) { } /** - * The minute the cache reads the URL as asking about. + * The minute the cache reads the URL as asking about, or + * {@link #NOT_HELD} where the cache must not be used for the request. * - *

The date parameter where the URL carries one, and otherwise now, - * because a creator answers a request without a date with the key in - * force now. A date later than now is read as now as well, because that - * is how a creator reads it. A schedule is published ahead of time and a - * key that has not started has signed nothing, so the creator answers a - * future date with the key in force now, and that answer must be held - * against now rather than against a minute the creator has not spoken - * for. Held against the future minute, the key would still be served - * for that minute after the creator had rotated, and a genuine - * identifier signed then would read as not matching.

+ *

The date parameter where the URL carries one and it is at least + * {@link #CLOCK_DRIFT_ALLOWANCE_MINUTES} behind now. A request without a + * date asks for the key in force now, and one dated within the + * allowance, or later, may be read by the creator as its present rather + * than as the minute named, so neither is served from the cache nor held + * in it.

*/ private static long minuteOf(String url) { long now = Io.minutesSinceBase(Instant.now()); int query = url.indexOf('?'); if (query < 0) { - return now; + return NOT_HELD; } for (String pair : url.substring(query + 1).split("&")) { if (pair.startsWith("date=")) { try { - return Math.min(Long.parseLong(pair.substring(5)), now); + long minute = Long.parseLong(pair.substring(5)); + if (minute >= 0 + && minute <= now - CLOCK_DRIFT_ALLOWANCE_MINUTES) { + return minute; + } } catch (NumberFormatException notANumber) { - return now; + // Not a count of minutes, so nothing to hold against. } + return NOT_HELD; } } - return now; + return NOT_HELD; } /** diff --git a/src/test/java/com/swancommunity/owid/DatedKeyFetchTest.java b/src/test/java/com/swancommunity/owid/DatedKeyFetchTest.java index 4831e8c..12edcc0 100644 --- a/src/test/java/com/swancommunity/owid/DatedKeyFetchTest.java +++ b/src/test/java/com/swancommunity/owid/DatedKeyFetchTest.java @@ -740,28 +740,40 @@ void aKeyIsNeverServedForAMinuteOutsideItsConfirmedSpan() } /** - * A date later than now is held against now, because a creator answers - * a future date with the key in force now and a key held against a - * minute the creator has not spoken for would be served for that minute - * after the creator had rotated. Two future dates therefore share one - * request, and so does a request with no date. + * A minute within the clock drift allowance of now, or later, is asked + * about every time and never held, because a creator whose clock differs + * from this one's may have read it as its present rather than as the + * minute named. A minute beyond the allowance is held as usual. Live + * identifiers therefore cost one request per minute per creator, as they + * always did, and older ones cost none. */ @Test - void aFutureDateIsHeldAgainstNow() throws IOException, OwidException { + void aMinuteWithinTheDriftAllowanceIsNotHeld() throws Exception { KeyEndPoint endPoint = endPoint(KeyEndPoint.Answer.SCHEDULE); + Field field = PublicKeyFetch.class.getDeclaredField( + "CLOCK_DRIFT_ALLOWANCE_MINUTES"); + field.setAccessible(true); + long allowance = field.getLong(null); long started = Io.minutesSinceBase(Instant.now()); - long week = 7 * 24 * 60; - pemAt(urlFor(endPoint, started + week), KeyFixtures.IDENTIFIER_DOMAIN); - pemAt(urlFor(endPoint, started + 2 * week), + long recent = started - 1; + pemAt(urlFor(endPoint, recent), KeyFixtures.IDENTIFIER_DOMAIN); + pemAt(urlFor(endPoint, recent), KeyFixtures.IDENTIFIER_DOMAIN); + pemAt(urlFor(endPoint, started + 7 * 24 * 60), KeyFixtures.IDENTIFIER_DOMAIN); pemAt(endPoint.base() + "/owid/api/v3/public-key?format=pkcs", KeyFixtures.IDENTIFIER_DOMAIN); + long old = started - allowance - 1; + pemAt(urlFor(endPoint, old), KeyFixtures.IDENTIFIER_DOMAIN); + pemAt(urlFor(endPoint, old), KeyFixtures.IDENTIFIER_DOMAIN); assumeTrue(Io.minutesSinceBase(Instant.now()) == started, "the minute changed during the test, so the calls were not " + "all about the same now"); - assertEquals(1, endPoint.dates().size(), - "two future dates and no date are all now, and now was " - + "asked about once"); + assertEquals(5, endPoint.dates().size(), + "the recent minute was asked about twice, the future minute " + + "and the request with no date once each, and the " + + "old minute once with the second call held"); + assertEquals(1, PublicKeyFetch.cachedKeyCount(), + "only the old minute's key is held"); } /** From 8d25a3aea489fef49b1d4cf839da8a75b1de95f7 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 7 Sep 2026 13:39:57 +0100 Subject: [PATCH 5/9] Answer the public key request with the span the key covers, and hold it The public key end point now answers with a JSON object carrying the key as publicKeySPKI together with validFrom and validTo, the UTC moments the key came into force and the next key starts. validFrom is null for a creator with one key and no schedule, and validTo is null for the last key in a schedule. The PEM alone as text is not a valid answer, and a client that receives it reports the key as one it cannot read. The answer is checked before it is sent, by the same code a client checks it with. The key must be a public key this library can read, a key valid to a moment must be valid from an earlier one, and the key must have been in force at the moment asked about. A creator whose store or schedule fails that check answers with a server error rather than a bad answer, so a fault on the server side shows up in the server's own tests and never reaches a client. The client holds the key for the whole span the creator stated, so an identifier dated anywhere in it is verified without a request whatever the clock drift, and live identifiers cost one request per key rather than one per minute. A creator that states no span has its key held against the minutes it confirms, with the fifteen minute clock drift allowance kept out of that cache as before. A signature that does not verify under the key selected for the identifier's minute, where that minute is within the drift allowance of an edge of the key's span, is checked against the neighbouring key before it is reported as not matching, because a creator's signing machines may not agree with its schedule to the minute. Where the key tried was never in force at the identifier's minute the neighbours are not tried. Callers verifying the same identifier at the same moment make one request between them, and a test with many threads proves it. The stand in creator in the tests answers with what this library's own server side code builds, so every client test runs against the response a creator built on this library sends, and the loop between the two halves is closed. --- README.md | 35 +- .../com/swancommunity/owid/Endpoints.java | 131 +++++- .../swancommunity/owid/PublicKeyFetch.java | 413 +++++++++++++----- .../swancommunity/owid/PublicKeyResponse.java | 336 ++++++++++++++ .../swancommunity/owid/PublicKeySchedule.java | 29 +- .../swancommunity/owid/DatedKeyFetchTest.java | 257 ++++++++++- .../com/swancommunity/owid/EndpointsTest.java | 6 +- .../com/swancommunity/owid/KeyEndPoint.java | 76 +++- 8 files changed, 1096 insertions(+), 187 deletions(-) create mode 100644 src/main/java/com/swancommunity/owid/PublicKeyResponse.java diff --git a/README.md b/README.md index bb53749..47e09b7 100644 --- a/README.md +++ b/README.md @@ -167,16 +167,24 @@ until the next key starts, so a key the creator answers with at two minutes was in force at every minute between them, and an identifier dated inside a confirmed span is verified without a request whichever minute it carries. One dated outside every span is asked about, which widens the span when the same -key comes back. One dated within fifteen minutes of now, or later, is asked -about every time and never held, because a creator whose clock differs from -this one's may have read that minute as its present rather than as the minute -named. Live identifiers therefore cost one request per minute per creator, as -they always did, and older ones cost none. At most 1024 keys are held across -every creator before the store is emptied and filled again, and `clearCache` -empties it on demand, which is how a long running process drops a key it has -learned it should no longer trust. Two requests for the same key made while the -first is still on its way share one request, and a fetch that fails is not -held, so the next request asks again. +key comes back. The creator answers with the key and the moments it is valid +from and to, so the whole span is held from one answer and an identifier dated +anywhere in it is verified without a request whatever the clock drift. An +answer in any other form, the PEM alone among them, is reported as a key that +cannot be read. A signature that does not verify under the key selected, where +the identifier is dated within fifteen minutes of the edge of that key's span, +is checked against the neighbouring key before it is reported as not matching, +because a creator's signing machines may not agree with its schedule to the +minute. One dated within fifteen minutes of now, or later, is asked about every +time and never held, because a creator whose clock differs from this one's may +have read that minute as its present rather than as the minute named. Live +identifiers therefore cost one request per minute per creator, as they always +did, and older ones cost none. At most 1024 keys are held across every creator +before the store is emptied and filled again, and `clearCache` empties it on +demand, which is how a long running process drops a key it has learned it +should no longer trust. Two requests for the same key made while the first is +still on its way share one request, and a fetch that fails is not held, so the +next request asks again. Every method that reaches the network answers with a `CompletableFuture` and returns at once. There is no form that waits, so a request thread or an event @@ -434,6 +442,13 @@ domain, a null payload, or a field that cannot be serialized. - Both take an optional `PublicKeyTransport`, and use `HttpUrlConnectionTransport` on its shared pool where none is given. - `clearCache` empties the keys already fetched. + - `Endpoints.publicKeyResponse` and `Endpoints.publicKeyResponseAt` return the + JSON body of the public key end point, the key as `publicKeySPKI` with + `validFrom` and `validTo`, the UTC moments the key came into force and the + next key starts, and `Endpoints.publicKeyAnswer` builds and checks any such + answer so a key that cannot be read or a schedule that contradicts itself is + refused before it is sent. `PublicKeyResponse` reads and writes the body. The + PEM alone as text is no longer a valid answer. - `PublicKeyTransport` makes the request and answers with a `CompletableFuture` of the body, so a transport over any HTTP client can be supplied. It must never follow a redirect and must request the URL diff --git a/src/main/java/com/swancommunity/owid/Endpoints.java b/src/main/java/com/swancommunity/owid/Endpoints.java index 8ccc1ed..7220c15 100644 --- a/src/main/java/com/swancommunity/owid/Endpoints.java +++ b/src/main/java/com/swancommunity/owid/Endpoints.java @@ -16,6 +16,8 @@ package com.swancommunity.owid; +import java.time.Duration; +import java.time.Instant; /** * Helpers for hosting the well known end points required by the OWID * specification. These are framework agnostic. They return the path and body @@ -86,27 +88,130 @@ public static String creatorResponse(Creator creator, String name, } /** - * Returns the text body for the public key end point. The specification - * allows the key to be requested in SPKI or PKCS form. This - * implementation returns the SPKI PEM for both values because the - * importers accept it. + * Returns the JSON body for the public key end point of a creator with + * one key and no schedule. The key is stated as {@code publicKeySPKI} and + * both {@code validFrom} and {@code validTo} are null, because the + * creator knows nothing about when the key started or will stop. + * + *

The specification allows the key to be requested in SPKI or PKCS + * form. This implementation returns the SPKI PEM for both values because + * the importers accept it.

* * @param creator the creator * @param format the format parameter, {@code spki} or {@code pkcs} - * @return the public key PEM + * @return the JSON body * @throws OwidException if the format is not valid, or the public key - * cannot be exported + * cannot be exported or read back */ public static String publicKeyResponse(Creator creator, String format) throws OwidException { - if ("spki".equals(format) || "pkcs".equals(format)) { - return creator.crypto().subjectPublicKeyInfo(); + if ("spki".equals(format) == false && "pkcs".equals(format) == false) { + // The value is not repeated back, because it arrives on a query + // string from whoever called the end point and a refusal is often + // logged. + throw new OwidException( + "format parameter 'spki' or 'pkcs' must be provided"); + } + return publicKeyAnswer(creator.crypto().subjectPublicKeyInfo(), null, + null, null); + } + + /** + * Returns the JSON body of the public key end point for the key and the + * span it covers, checked with {@link PublicKeyResponse#validate(Instant)} + * first so that a creator never sends an answer it would itself refuse. + * + * @param publicKeyPem the key in PEM form + * @param validFrom the UTC moment the key came into force, or null + * @param validTo the UTC moment the next key starts, or null + * @param asked the moment the request asks about, or null + * @return the JSON body + * @throws OwidException if the answer would not be valid + */ + public static String publicKeyAnswer(String publicKeyPem, Instant validFrom, + Instant validTo, Instant asked) throws OwidException { + PublicKeyResponse answer = PublicKeyResponse.of(publicKeyPem, validFrom, + validTo); + answer.validate(asked); + return answer.toJson(); + } + + /** + * The status code and body a public key end point answers a request + * with. + */ + public static final class Response { + private final int status; + private final String body; + + Response(int status, String body) { + this.status = status; + this.body = body; + } + + /** The HTTP status code. */ + public int getStatus() { + return status; + } + + /** The body, empty where the status is not 200. */ + public String getBody() { + return body; + } + } + + /** + * Returns the status code and JSON body for the public key end point of + * a creator that rotates its key, chosen from the schedule the way the + * specification requires. + * + *

The date parameter is the OWID's own date, counted in whole minutes + * since 2020-01-01, and the key served is the one in force then, being + * the latest key whose start is at or before it. A request without a + * date, or with a date later than the moment of the request, is served + * the key in force at that moment, so a caller cannot ask for a key whose + * period has not begun. The answer is 200 with the body from + * {@link #publicKeyAnswer}, stating the key and the moments it is valid + * from and to, 404 with an empty body where no key is in force at the + * date, and 400 with an empty body where the date is not a count of + * minutes.

+ * + * @param schedule the published schedule + * @param format the format parameter, {@code spki} or {@code pkcs} + * @param date the date parameter, or null where the request has none + * @param now the moment of the request + * @return the status and body + * @throws OwidException if the format is not valid, or the answer would + * fail its check, which is a fault in the schedule + */ + public static Response publicKeyResponseAt(PublicKeySchedule schedule, + String format, String date, Instant now) throws OwidException { + if ("spki".equals(format) == false && "pkcs".equals(format) == false) { + throw new OwidException( + "format parameter 'spki' or 'pkcs' must be provided"); + } + Instant asked = now; + if (date != null && date.isEmpty() == false) { + long minutes; + try { + minutes = Long.parseLong(date); + } catch (NumberFormatException e) { + return new Response(400, ""); + } + if (minutes < 0 || minutes > 0xFFFFFFFFL) { + return new Response(400, ""); + } + asked = Io.baseDate().plus(Duration.ofMinutes(minutes)); + if (asked.isAfter(now)) { + asked = now; + } + } + DatedPublicKey key = schedule.keyInForce(asked); + if (key == null) { + return new Response(404, ""); } - // The value is not repeated back, because it arrives on a query - // string from whoever called the end point and a refusal is often - // logged. - throw new OwidException( - "format parameter 'spki' or 'pkcs' must be provided"); + return new Response(200, publicKeyAnswer(key.getPublicKeyPem(), + key.getStartsAt(), schedule.nextStartAfter(key), asked)); } private static void appendField(StringBuilder json, String name, diff --git a/src/main/java/com/swancommunity/owid/PublicKeyFetch.java b/src/main/java/com/swancommunity/owid/PublicKeyFetch.java index 914cb99..eab5405 100644 --- a/src/main/java/com/swancommunity/owid/PublicKeyFetch.java +++ b/src/main/java/com/swancommunity/owid/PublicKeyFetch.java @@ -71,55 +71,84 @@ public final class PublicKeyFetch { /** * How far a creator's clock may run ahead of or behind this one's, in - * minutes. A minute closer to now than this, or later, is asked about - * rather than served from the cache, and is not held. + * minutes. * - *

A creator reads a date later than its own now as now, and answers - * with the key in force now. Within this window this process cannot tell - * whether the creator read the minute as its past or as its present, so - * the answer says nothing certain about the minute. An identifier signed - * just after a rotation by a creator whose clock runs ahead would - * otherwise be served the old key from a span confirmed up to now, and - * would read as not matching until this clock caught up. Identifiers - * dated within the window are asked about once per minute per creator, - * as they always were, and every older identifier is served from the - * spans.

+ *

It is used in two places. A creator that does not state the span of + * the key it answers with reads a date later than its own now as now, so + * within this window of now this process cannot tell whether the creator + * read the minute as its past or as its present, and nothing learned from + * such an answer is held or served. And a creator's signing machines may + * not agree with the creator's own schedule to the minute, so an + * identifier dated within this window of a key's edge that does not + * verify under that key is checked against the neighbouring key before + * it is reported as not matching.

*/ private static final long CLOCK_DRIFT_ALLOWANCE_MINUTES = 15; - /** The minute {@link #minuteOf} answers where the cache must not be used. */ - private static final long NOT_HELD = -1; + /** The minute {@link #minuteOf} answers where the URL names none. */ + private static final long NO_MINUTE = -1; /** - * One key a creator has answered with, and the span of minutes the - * creator has confirmed it was in force for. + * One key a creator has answered with, and the span of minutes the key + * is known to cover. * *

A creator's key is in force from the start of its period until the * next key starts, so a key the creator confirms at two minutes was in - * force at every minute between them. The span grows as the creator - * confirms the same key for more minutes, and an identifier dated inside - * it is verified without a request.

+ * force at every minute between them. Where the creator stated the span + * in its answer the span is explicit and complete, and an identifier + * dated anywhere inside it is verified without a request. Otherwise the + * span grows as the creator confirms the same key for more minutes.

*/ private static final class HeldKey { /** The key in PEM form, as the creator served it. */ final String pem; - /** The earliest minute the creator has confirmed the key for. */ + /** The earliest minute the key is known to cover. */ long first; - /** The latest minute the creator has confirmed the key for. */ + /** The latest minute the key is known to cover. */ long last; + /** Whether the creator stated the whole span itself. */ + boolean explicit; - HeldKey(String pem, long minute) { + HeldKey(String pem, long first, long last, boolean explicit) { this.pem = pem; - this.first = minute; - this.last = minute; + this.first = first; + this.last = last; + this.explicit = explicit; } - /** Whether the minute lies within the confirmed span. */ + /** Whether the minute lies within the known span. */ boolean covers(long minute) { return first <= minute && minute <= last; } } + /** + * What the cache or a fetch answers with. The key, and where it is known, + * the span of minutes the key covers, so that a caller can tell whether + * the identifier it is checking sits near the edge of the span. + */ + private static final class KeyAnswer { + final String pem; + final long first; + final long last; + final boolean known; + + KeyAnswer(String pem, long first, long last, boolean known) { + this.pem = pem; + this.first = first; + this.last = last; + this.known = known; + } + + static KeyAnswer unknown(String pem) { + return new KeyAnswer(pem, 0, 0, false); + } + + boolean covers(long minute) { + return known && first <= minute && minute <= last; + } + } + /** * Guards {@link #CACHE}, {@link #heldKeys} and {@link #IN_FLIGHT}. Held * across a few map and list operations only, never across a request. @@ -136,10 +165,7 @@ boolean covers(long minute) { * many identifiers does not mean repeating requests to another * processor. The key URL carries the date of the identifier being * verified, in minutes, and a creator's key changes on the order of a - * week. Keyed by the whole URL, as this cache once was, two identifiers - * signed a minute apart never shared an entry, so a hundred identifiers - * over a hundred minutes made a hundred requests for one key. Keyed by - * end point and span, an identifier dated between two minutes the + * week. Keyed by end point and span rather than by the whole URL, an identifier dated between two minutes the * creator has already answered for is verified without a request.

*/ private static final Map> CACHE = @@ -155,8 +181,8 @@ boolean covers(long minute) { * request ends, whatever the outcome, so a failure is never handed to a * later caller and an outage is never remembered. */ - private static final Map> IN_FLIGHT = - new HashMap>(); + private static final Map> IN_FLIGHT = + new HashMap>(); /** The transport used where the caller names none. */ private static final PublicKeyTransport DEFAULT_TRANSPORT = @@ -273,6 +299,7 @@ public static CompletableFuture publicKeyPem(Owid owid, * @param scheme the scheme to use, normally {@code https} * @param others the other OWIDs that were signed together with this one, * in the same order as when signed + * * @return the outcome of the check, through a future */ public static CompletableFuture verify(Owid owid, @@ -301,6 +328,7 @@ public static CompletableFuture verify(Owid owid, * @param scheme the scheme to use, normally {@code https} * @param others the other OWIDs that were signed together with this * one, in the same order as when signed + * * @param transport the transport to make the request with * @return the outcome of the check, through a future */ @@ -348,48 +376,119 @@ static int cachedKeyCount() { * than against a near copy of the fetch. */ static CompletableFuture verifyAtUrl( - final Owid owid, String url, final List others, - PublicKeyTransport transport) { - return publicKeyPemAtUrl(url, owid.getDomain(), transport) - .handle((pem, failure) -> { + final Owid owid, final String url, final List others, + final PublicKeyTransport transport) { + return keyAtUrl(url, owid.getDomain(), transport) + .handle((answer, failure) -> { if (failure != null) { - return OwidVerificationResult.of(statusOf(failure)); + return CompletableFuture.completedFuture( + OwidVerificationResult.of(statusOf(failure))); + } + OwidVerificationResult result = owid.verify(answer.pem, + others); + if (result.getStatus() + != OwidSignatureStatus.SIGNATURE_INVALID) { + return CompletableFuture.completedFuture(result); } - return owid.verify(pem, others); - }); + return neighbourVerifies(owid, url, answer, others, + transport).thenApply(verified -> verified + ? OwidVerificationResult.of( + OwidSignatureStatus.SIGNATURE_VALID) + : result); + }) + .thenCompose(future -> future); + } + + /** + * Whether a key neighbouring the one the OWID's own minute selected + * verifies the signature instead. + * + *

A creator's signing machines may not agree with its own schedule to + * the minute, so an identifier dated just after a key started may have + * been signed with the key before it, and one dated just before may have + * been signed with the key after. Where the signature does not verify + * under the key selected and the OWID's minute is within the clock drift + * allowance of the edge of the span that key is known to cover, the key + * for the minute just beyond that edge is asked for and tried. A key + * already known to cover the neighbouring minute is not asked for again, + * and a neighbour that turns out to be the same key is not tried again. + * This costs at most two more requests, and only for a signature that + * has already failed.

+ */ + private static CompletableFuture neighbourVerifies( + final Owid owid, String url, final KeyAnswer tried, + final List others, PublicKeyTransport transport) { + long minute = Io.minutesSinceBase(owid.getDate()); + if (minute < 0 || (tried.known && tried.covers(minute) == false)) { + // Either the OWID's minute cannot be counted, or the key tried + // was never in force at that minute, so the OWID is not near an + // edge of that key's span. + return CompletableFuture.completedFuture(false); + } + final String endPoint = endPointOf(url); + CompletableFuture verified = + CompletableFuture.completedFuture(false); + for (final long at : new long[] { + minute - CLOCK_DRIFT_ALLOWANCE_MINUTES, + minute + CLOCK_DRIFT_ALLOWANCE_MINUTES}) { + if (at < 0 || at > 0xFFFFFFFFL || tried.covers(at)) { + continue; + } + verified = verified.thenCompose(already -> { + if (already) { + return CompletableFuture.completedFuture(true); + } + return keyAtUrl(endPoint + "?date=" + at + "&format=pkcs", + owid.getDomain(), transport) + .handle((neighbour, failure) -> failure == null + && neighbour.pem.equals(tried.pem) == false + && owid.verify(neighbour.pem, others).getStatus() + == OwidSignatureStatus.SIGNATURE_VALID); + }); + } + return verified; + } + + /** + * Fetches the PEM at the URL. See {@link #keyAtUrl}. + */ + static CompletableFuture publicKeyPemAtUrl(String url, + String domain, PublicKeyTransport transport) { + return keyAtUrl(url, domain, transport).thenApply(answer -> answer.pem); } /** - * Fetches the PEM at the URL. Answered from the cache where the creator - * has already confirmed a key for the minute the URL names, from a - * request already under way for the same URL where there is one, and - * otherwise through the transport. + * Fetches the key the URL asks for, with the span it is known to cover. + * Answered from the cache where a held key is known to cover the minute + * the URL names, from a request already under way for the same URL where + * there is one, and otherwise through the transport. The creator's + * answer states the moments the key is valid from and to, so the whole + * span is held from that one answer. * *

The future held for a request under way is this class's own rather * than the transport's, so that the transport's completion can be - * watched, the key held against the minute it was asked for, and a - * failure forgotten, all before the callers waiting are answered.

+ * watched, the answer read and held, and a failure forgotten, all before + * the callers waiting are answered.

*/ - static CompletableFuture publicKeyPemAtUrl(final String url, - String domain, PublicKeyTransport transport) { + static CompletableFuture keyAtUrl(final String url, + final String domain, PublicKeyTransport transport) { if (transport == null) { return failed(new OwidException("the transport is missing")); } final String endPoint = endPointOf(url); - final long minute = minuteOf(url); - final CompletableFuture fetch; + final CompletableFuture fetch; synchronized (LOCK) { - String pem = minute == NOT_HELD ? null : heldPem(endPoint, minute); - if (pem != null) { - return CompletableFuture.completedFuture(pem); - } - CompletableFuture held = IN_FLIGHT.get(url); + KeyAnswer held = heldFor(endPoint, url); if (held != null) { + return CompletableFuture.completedFuture(held); + } + CompletableFuture shared = IN_FLIGHT.get(url); + if (shared != null) { // Another caller asked for the same key and its fetch is // the one both callers share. - return held; + return shared; } - fetch = new CompletableFuture(); + fetch = new CompletableFuture(); IN_FLIGHT.put(url, fetch); } CompletableFuture started; @@ -407,18 +506,22 @@ static CompletableFuture publicKeyPemAtUrl(final String url, + "'", OwidSignatureStatus.KEY_UNAVAILABLE, domain, 0, null)); } - started.whenComplete((pem, failure) -> { - if (failure == null && pem != null) { - // Held before the callers are answered, so a caller arriving - // between the two finds the key rather than starting a - // request of its own. - synchronized (LOCK) { - if (minute != NOT_HELD) { - hold(endPoint, minute, pem); + started.whenComplete((body, failure) -> { + if (failure == null && body != null) { + KeyAnswer answer; + try { + answer = readAnswer(body, domain, endPoint, url); + } catch (PublicKeyFetchException unreadable) { + synchronized (LOCK) { + forget(url, fetch); } + fetch.completeExceptionally(unreadable); + return; + } + synchronized (LOCK) { forget(url, fetch); } - fetch.complete(pem); + fetch.complete(answer); return; } synchronized (LOCK) { @@ -435,13 +538,53 @@ static CompletableFuture publicKeyPemAtUrl(final String url, return fetch; } + /** + * Reads a public key answer and holds the key it carries against the + * span it states, or against the minute asked about where it states + * none. An answer that is not the JSON form the specification requires, + * the PEM alone among the other forms, or that fails the checks a + * creator applies before sending it, is reported as a key that cannot be + * read. + */ + private static KeyAnswer readAnswer(String body, String domain, + String endPoint, String url) throws PublicKeyFetchException { + PublicKeyResponse answer; + try { + answer = PublicKeyResponse.parse(body); + answer.validate(null); + } catch (OwidException e) { + throw new PublicKeyFetchException( + "domain " + quoted(domain) + " answered with a public key " + + "answer that is not valid: " + e.getMessage(), + OwidSignatureStatus.INVALID_KEY, domain, 0, e); + } + synchronized (LOCK) { + return hold(endPoint, url, answer.getPublicKeySpki(), + minutesOrNull(answer.getValidFrom()), + minutesOrNull(answer.getValidTo())); + } + } + + /** The moment as minutes since the base date, or null. */ + private static Long minutesOrNull(Instant moment) { + if (moment == null) { + return null; + } + long minutes = Io.minutesSinceBase(moment); + return minutes < 0 ? null : Long.valueOf(minutes); + } + + private static String quoted(String value) { + return "'" + value + "'"; + } + /** * Removes the request from those under way. Only this request is * removed, never whatever replaced it after the cache was emptied and a * fresh request started for the same URL in the meantime. Called under * the lock. */ - private static void forget(String url, CompletableFuture fetch) { + private static void forget(String url, CompletableFuture fetch) { if (IN_FLIGHT.get(url) == fetch) { IN_FLIGHT.remove(url); } @@ -457,49 +600,56 @@ private static String endPointOf(String url) { } /** - * The minute the cache reads the URL as asking about, or - * {@link #NOT_HELD} where the cache must not be used for the request. - * - *

The date parameter where the URL carries one and it is at least - * {@link #CLOCK_DRIFT_ALLOWANCE_MINUTES} behind now. A request without a - * date asks for the key in force now, and one dated within the - * allowance, or later, may be read by the creator as its present rather - * than as the minute named, so neither is served from the cache nor held - * in it.

+ * The minute the URL asks about, or {@link #NO_MINUTE} where it names + * none. */ private static long minuteOf(String url) { - long now = Io.minutesSinceBase(Instant.now()); int query = url.indexOf('?'); if (query < 0) { - return NOT_HELD; + return NO_MINUTE; } for (String pair : url.substring(query + 1).split("&")) { if (pair.startsWith("date=")) { try { long minute = Long.parseLong(pair.substring(5)); - if (minute >= 0 - && minute <= now - CLOCK_DRIFT_ALLOWANCE_MINUTES) { - return minute; - } + return minute < 0 ? NO_MINUTE : minute; } catch (NumberFormatException notANumber) { - // Not a count of minutes, so nothing to hold against. + return NO_MINUTE; } - return NOT_HELD; } } - return NOT_HELD; + return NO_MINUTE; } /** - * The key held for the end point whose confirmed span covers the minute, - * or null where no held key does. Called under the lock. + * Whether the minute lies within the clock drift allowance of now or + * later, which is a minute a creator that does not state its spans may + * have read as its present rather than as the minute named. */ - private static String heldPem(String endPoint, long minute) { + private static boolean recent(long minute) { + return minute > Io.minutesSinceBase(Instant.now()) + - CLOCK_DRIFT_ALLOWANCE_MINUTES; + } + + /** + * The key held for the end point that is known to cover the minute the + * URL asks about, or null where none is. Called under the lock. + * + *

A minute within the drift allowance of now is only served where the + * creator itself stated the span, because a span confirmed minute by + * minute says nothing certain about such a minute.

+ */ + private static KeyAnswer heldFor(String endPoint, String url) { + long minute = minuteOf(url); + if (minute == NO_MINUTE) { + return null; + } List keys = CACHE.get(endPoint); if (keys != null) { + boolean recent = recent(minute); for (HeldKey key : keys) { - if (key.covers(minute)) { - return key.pem; + if (key.covers(minute) && (key.explicit || recent == false)) { + return new KeyAnswer(key.pem, key.first, key.last, true); } } } @@ -507,21 +657,58 @@ private static String heldPem(String endPoint, long minute) { } /** - * Records that the creator answered the minute with the key. Called - * under the lock. + * Records the creator's answer to the URL, being the key and, where the + * creator stated it, the span the key covers as the minute it came into + * force and the minute the next key starts. Returns the key with the span + * it is now known to cover. Called under the lock. * - *

A key already held for the end point has its span widened to take - * in the minute. A key not held before is added, emptying the cache - * first when it is full, because the domains and dates asked about come - * from the identifiers presented to this process and the cache must not - * grow on their input.

+ *

With both the start and the end the whole span is held as the + * creator's own statement. With the start alone the key is held from the + * start up to the drift allowance behind now, because no later key can + * have started before then. With neither the minute asked about is held + * on its own, as long as it is not within the drift allowance of now. A + * key already held for the end point has its span widened to take in the + * new one. A key not held before is added, emptying the cache first when + * it is full, because the cache must not grow on the input of whoever + * presents the identifiers.

*/ - private static void hold(String endPoint, long minute, String pem) { + private static KeyAnswer hold(String endPoint, String url, String pem, + Long start, Long end) { + long minute = minuteOf(url); + long first; + long last; + boolean explicit = false; + if (start != null && end != null && end > start) { + first = start; + last = end - 1; + explicit = true; + } else if (start != null) { + first = start; + last = Math.max(start, Io.minutesSinceBase(Instant.now()) + - CLOCK_DRIFT_ALLOWANCE_MINUTES); + } else if (minute != NO_MINUTE && recent(minute) == false) { + first = minute; + last = minute; + } else { + return KeyAnswer.unknown(pem); + } List keys = CACHE.get(endPoint); if (keys != null) { for (HeldKey key : keys) { - if (key.pem.equals(pem) && widen(keys, key, minute)) { - return; + if (key.pem.equals(pem)) { + if (widen(keys, key, first, last)) { + key.explicit = key.explicit || explicit; + return new KeyAnswer(pem, key.first, key.last, true); + } + // The creator has answered with another key inside this + // span before, which it does not do unless it went back + // to a key it had left. Nothing more is held about it. + return KeyAnswer.unknown(pem); + } + } + for (HeldKey other : keys) { + if (other.last >= first && other.first <= last) { + return KeyAnswer.unknown(pem); } } } @@ -534,37 +721,31 @@ private static void hold(String endPoint, long minute, String pem) { keys = new ArrayList(); CACHE.put(endPoint, keys); } - keys.add(new HeldKey(pem, minute)); + keys.add(new HeldKey(pem, first, last, explicit)); heldKeys++; + return new KeyAnswer(pem, first, last, true); } /** - * Widens the span of a held key to take in the minute, and says whether - * the minute is now within it. + * Widens the span of a held key to take in the span given, and says + * whether it did. * *

The span is not widened across a minute the creator has answered * with another key for, because that would mean the creator had gone * back to a key it had left, and the minutes between the two spans are - * then not this key's to claim. The key is held again as a separate span - * instead.

+ * then not this key's to claim.

*/ - private static boolean widen(List keys, HeldKey key, - long minute) { - if (key.covers(minute)) { - return true; - } - long from = Math.min(minute, key.first); - long to = Math.max(minute, key.last); + private static boolean widen(List keys, HeldKey key, long first, + long last) { + first = Math.min(first, key.first); + last = Math.max(last, key.last); for (HeldKey other : keys) { - if (other != key && other.last > from && other.first < to) { + if (other != key && other.last >= first && other.first <= last) { return false; } } - if (minute < key.first) { - key.first = minute; - } else { - key.last = minute; - } + key.first = first; + key.last = last; return true; } diff --git a/src/main/java/com/swancommunity/owid/PublicKeyResponse.java b/src/main/java/com/swancommunity/owid/PublicKeyResponse.java new file mode 100644 index 0000000..1d8ed7f --- /dev/null +++ b/src/main/java/com/swancommunity/owid/PublicKeyResponse.java @@ -0,0 +1,336 @@ +/* **************************************************************************** + * Copyright 2026 51 Degrees Mobile Experts Limited (51degrees.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * ***************************************************************************/ + +package com.swancommunity.owid; + +import java.time.Instant; +import java.time.format.DateTimeParseException; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * The JSON body of the public key end point. It carries the key together with + * the moments it is valid from and to, in UTC, so a client holds the key for + * the whole span from one answer rather than asking again for every minute. + * + *

{@code validFrom} is null where the creator has a single key and no + * schedule, and {@code validTo} is null where no later key has been + * scheduled. Both the creator that sends the answer and the client that reads + * it check it with {@link #validate(Instant)}, so a fault in a creator's + * schedule or store is a server error at the creator rather than a bad answer + * a client then has to refuse.

+ * + *

Only the JDK is used, so the library keeps its promise of no runtime + * dependencies. The answer is a flat object of three fields, each a string or + * null, which is all the reading and writing here supports.

+ */ +public final class PublicKeyResponse { + + private final String publicKeySpki; + private final Instant validFrom; + private final Instant validTo; + + private PublicKeyResponse(String publicKeySpki, Instant validFrom, + Instant validTo) { + this.publicKeySpki = publicKeySpki; + this.validFrom = validFrom; + this.validTo = validTo; + } + + /** + * An answer for the key and the moments it is valid from and to, either + * of which may be null. + * + * @param publicKeySpki the key in PEM form + * @param validFrom the UTC moment the key came into force, or null + * @param validTo the UTC moment the next key starts, or null + * @return the answer, not yet checked + */ + public static PublicKeyResponse of(String publicKeySpki, Instant validFrom, + Instant validTo) { + return new PublicKeyResponse(publicKeySpki, validFrom, validTo); + } + + /** The key in PEM form. */ + public String getPublicKeySpki() { + return publicKeySpki; + } + + /** The UTC moment the key came into force, or null where not known. */ + public Instant getValidFrom() { + return validFrom; + } + + /** The UTC moment the next key starts, or null where none is scheduled. */ + public Instant getValidTo() { + return validTo; + } + + /** + * Checks the answer the way both the creator that sends it and the client + * that reads it must. The key must be a public key this library can read, + * a key valid to a moment must be valid from an earlier one, and where the + * moment asked about is known the key must have come into force by then + * and, if it has an end, not have ended. + * + * @param asked the moment asked about, or null where it is not known + * @throws OwidException if the answer is not valid + */ + public void validate(Instant asked) throws OwidException { + if (publicKeySpki == null || publicKeySpki.trim().isEmpty()) { + throw new OwidException("the public key answer holds no key"); + } + try { + Crypto.newVerifyOnly(publicKeySpki); + } catch (OwidException e) { + throw new OwidException( + "the public key answer holds a key that cannot be read"); + } + if (validTo != null) { + if (validFrom == null) { + throw new OwidException("the public key answer states when " + + "the key ends but not when it started"); + } + if (validTo.isAfter(validFrom) == false) { + throw new OwidException("the public key answer states a key " + + "that ends before it starts"); + } + } + if (asked != null) { + if (validFrom != null && validFrom.isAfter(asked)) { + throw new OwidException("the public key answer states a key " + + "that had not started at the moment asked about"); + } + if (validTo != null && validTo.isAfter(asked) == false) { + throw new OwidException("the public key answer states a key " + + "that had ended at the moment asked about"); + } + } + } + + /** + * The answer as JSON, with the moments as RFC 3339 strings in UTC and + * null where there is no moment. + * + * @return the JSON body + */ + public String toJson() { + StringBuilder json = new StringBuilder("{\"publicKeySPKI\":"); + appendString(json, publicKeySpki); + json.append(",\"validFrom\":"); + appendMoment(json, validFrom); + json.append(",\"validTo\":"); + appendMoment(json, validTo); + return json.append('}').toString(); + } + + /** + * Reads an answer from its JSON body. + * + * @param json the body + * @return the answer, not yet checked with {@link #validate(Instant)} + * @throws OwidException if the body is not a JSON object of the three + * fields, each a string or null + */ + public static PublicKeyResponse parse(String json) throws OwidException { + Map fields = readFlatObject(json); + return new PublicKeyResponse( + fields.get("publicKeySPKI"), + moment(fields.get("validFrom"), "validFrom"), + moment(fields.get("validTo"), "validTo")); + } + + private static Instant moment(String text, String field) + throws OwidException { + if (text == null) { + return null; + } + try { + return Instant.parse(text); + } catch (DateTimeParseException e) { + throw new OwidException("the public key answer's " + field + + " is not a moment in UTC"); + } + } + + private static void appendMoment(StringBuilder json, Instant moment) { + if (moment == null) { + json.append("null"); + } else { + appendString(json, moment.toString()); + } + } + + private static void appendString(StringBuilder json, String value) { + if (value == null) { + json.append("null"); + return; + } + json.append('"'); + for (int i = 0; i < value.length(); i++) { + char c = value.charAt(i); + switch (c) { + case '"': + json.append("\\\""); + break; + case '\\': + json.append("\\\\"); + break; + case '\n': + json.append("\\n"); + break; + case '\r': + json.append("\\r"); + break; + case '\t': + json.append("\\t"); + break; + default: + if (c < 0x20) { + json.append(String.format("\\u%04x", (int) c)); + } else { + json.append(c); + } + } + } + json.append('"'); + } + + /** + * Reads a JSON object whose values are strings or null. Anything else is + * refused, because the answer has no other shape. + */ + private static Map readFlatObject(String json) + throws OwidException { + Map fields = new LinkedHashMap(); + if (json == null) { + throw notJson(); + } + int[] at = {skipSpace(json, 0)}; + if (at[0] >= json.length() || json.charAt(at[0]) != '{') { + throw notJson(); + } + at[0]++; + at[0] = skipSpace(json, at[0]); + if (at[0] < json.length() && json.charAt(at[0]) == '}') { + return fields; + } + while (true) { + at[0] = skipSpace(json, at[0]); + String name = readString(json, at); + at[0] = skipSpace(json, at[0]); + if (at[0] >= json.length() || json.charAt(at[0]) != ':') { + throw notJson(); + } + at[0]++; + at[0] = skipSpace(json, at[0]); + if (json.startsWith("null", at[0])) { + fields.put(name, null); + at[0] += 4; + } else { + fields.put(name, readString(json, at)); + } + at[0] = skipSpace(json, at[0]); + if (at[0] >= json.length()) { + throw notJson(); + } + char next = json.charAt(at[0]); + at[0]++; + if (next == '}') { + break; + } + if (next != ',') { + throw notJson(); + } + } + if (skipSpace(json, at[0]) != json.length()) { + throw notJson(); + } + return fields; + } + + private static int skipSpace(String json, int at) { + while (at < json.length() && Character.isWhitespace(json.charAt(at))) { + at++; + } + return at; + } + + private static String readString(String json, int[] at) + throws OwidException { + if (at[0] >= json.length() || json.charAt(at[0]) != '"') { + throw notJson(); + } + at[0]++; + StringBuilder value = new StringBuilder(); + while (at[0] < json.length()) { + char c = json.charAt(at[0]++); + if (c == '"') { + return value.toString(); + } + if (c != '\\') { + value.append(c); + continue; + } + if (at[0] >= json.length()) { + throw notJson(); + } + char escaped = json.charAt(at[0]++); + switch (escaped) { + case '"': + case '\\': + case '/': + value.append(escaped); + break; + case 'b': + value.append('\b'); + break; + case 'f': + value.append('\f'); + break; + case 'n': + value.append('\n'); + break; + case 'r': + value.append('\r'); + break; + case 't': + value.append('\t'); + break; + case 'u': + if (at[0] + 4 > json.length()) { + throw notJson(); + } + try { + value.append((char) Integer.parseInt( + json.substring(at[0], at[0] + 4), 16)); + } catch (NumberFormatException e) { + throw notJson(); + } + at[0] += 4; + break; + default: + throw notJson(); + } + } + throw notJson(); + } + + private static OwidException notJson() { + return new OwidException("the public key answer is not the JSON " + + "object of three fields the specification requires"); + } +} diff --git a/src/main/java/com/swancommunity/owid/PublicKeySchedule.java b/src/main/java/com/swancommunity/owid/PublicKeySchedule.java index d6745b3..1bf0a38 100644 --- a/src/main/java/com/swancommunity/owid/PublicKeySchedule.java +++ b/src/main/java/com/swancommunity/owid/PublicKeySchedule.java @@ -37,9 +37,7 @@ * start is at or before the date asked about. Keys are generated in batches, * often many weeks ahead of the weeks the keys cover, so the moment key * material was generated says nothing about which key signed anything and is - * not held here at all. Selecting on a generation moment picks a key that has - * not started yet and reports a genuine identifier as not matching, which is - * what the .NET port did before that port was fixed.

+ * not held here at all. Selecting on a generation moment picks a key that has not started yet and reports a genuine identifier as not matching.

* *

A date the schedule does not reach, being one earlier than the first * start, has no key. That answer is reported as @@ -142,13 +140,29 @@ public DatedPublicKey keyInForce(Instant date) { *

This is not the key in force now. A creator publishes its schedule * ahead of time, so the last key by start is usually one whose period * has not begun and which has signed nothing yet. The key in force now - * is {@link #current()}. Serving the last key where the current one was - * meant is the same fault as selecting by the generation moment, being - * a key from a period that has not started, and it is the fault the - * .NET port carried in its answer to a request that named no date.

+ * is {@link #current()}. Serving the last key where the current one was meant is the same fault as selecting by the generation moment, being a key from a period that has not started.

* * @return the key with the latest start, or null when there are none */ + /** + * Returns the earliest start in the schedule after the key's own, being + * the moment the key stops being in force, or null where the key is the + * last in the schedule and is in force until further notice. + * + * @param key a key of this schedule + * @return the moment the next key starts, or null + */ + public Instant nextStartAfter(DatedPublicKey key) { + Instant next = null; + for (DatedPublicKey other : keys) { + if (other.getStartsAt().isAfter(key.getStartsAt()) + && (next == null || other.getStartsAt().isBefore(next))) { + next = other.getStartsAt(); + } + } + return next; + } + public DatedPublicKey last() { if (keys.isEmpty()) { return null; @@ -189,6 +203,7 @@ public DatedPublicKey keyFor(Owid owid) { * @param owid the OWID to check * @param others the other OWIDs that were signed together with this one, * in the same order as when signed + * * @return the outcome of the check, which is * {@link OwidSignatureStatus#KEY_UNAVAILABLE} where the schedule * holds no key for the date diff --git a/src/test/java/com/swancommunity/owid/DatedKeyFetchTest.java b/src/test/java/com/swancommunity/owid/DatedKeyFetchTest.java index 12edcc0..1087e5b 100644 --- a/src/test/java/com/swancommunity/owid/DatedKeyFetchTest.java +++ b/src/test/java/com/swancommunity/owid/DatedKeyFetchTest.java @@ -29,6 +29,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.lang.reflect.Field; +import java.nio.charset.StandardCharsets; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; @@ -36,6 +37,7 @@ import java.util.Collections; import java.util.List; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CyclicBarrier; import java.util.concurrent.CompletionException; import java.util.concurrent.Executor; import java.util.concurrent.RejectedExecutionException; @@ -297,9 +299,7 @@ void aRedirectIsNotFollowed() throws IOException, OwidException { /** * Key material that arrives but cannot be read is the fault of the key * and not of the identifier, so it is reported apart from a signature - * that does not match. This is the 30 August 2026 fault, where the key - * end points served PEM a strict parser refused and every offline check - * against them failed while the keys and the identifiers were both fine. + * that does not match. */ @Test void aKeyThatCannotBeReadIsInvalidKey() @@ -359,7 +359,6 @@ void twoRequestsInFlightForOneKeyMakeOneRequest() assertEquals(1, held.requests.get(), "the second request joins the first rather than asking " + "again"); - assertSame(first, second, "both callers hold the same fetch"); assertFalse(first.isDone(), "nothing has answered yet"); // The genuine key, fetched through the transport itself rather than // through the cache, because the cache holds the fetch still on its @@ -431,7 +430,8 @@ void theRequestRunsOnTheExecutorGiven() assertNotEquals(Thread.currentThread(), ran.get(), "the thread that asked is not the one that fetches"); assertEquals(OwidSignatureStatus.SIGNATURE_VALID, - owid.verify(fetch.join(), ALONE).getStatus(), + owid.verify(PublicKeyResponse.parse(fetch.join()) + .getPublicKeySpki(), ALONE).getStatus(), "the key fetched on the executor verifies the identifier"); } @@ -635,7 +635,7 @@ private static String inForce(long minute) throws OwidException { @Test void aMinuteBetweenTwoConfirmedMinutesIsServedFromTheCache() throws IOException, OwidException { - KeyEndPoint endPoint = endPoint(KeyEndPoint.Answer.SCHEDULE); + KeyEndPoint endPoint = endPoint(KeyEndPoint.Answer.SPANLESS); // The week of 31 August 2026, which the fixture identifier was // signed in, and which is wholly in the past so the cache reads // each minute as itself rather than as now. @@ -675,7 +675,7 @@ void aMinuteBetweenTwoConfirmedMinutesIsServedFromTheCache() @Test void aHundredIdentifiersInOneConfirmedPeriodMakeNoRequest() throws IOException, OwidException { - KeyEndPoint endPoint = endPoint(KeyEndPoint.Answer.SCHEDULE); + KeyEndPoint endPoint = endPoint(KeyEndPoint.Answer.SPANLESS); long start = minutes("2026-09-01T00:00:00Z"); pemAt(urlFor(endPoint, start), KeyFixtures.IDENTIFIER_DOMAIN); pemAt(urlFor(endPoint, start + 100), KeyFixtures.IDENTIFIER_DOMAIN); @@ -697,7 +697,7 @@ void aHundredIdentifiersInOneConfirmedPeriodMakeNoRequest() @Test void aKeyIsNeverServedForAMinuteOutsideItsConfirmedSpan() throws IOException, OwidException { - KeyEndPoint endPoint = endPoint(KeyEndPoint.Answer.SCHEDULE); + KeyEndPoint endPoint = endPoint(KeyEndPoint.Answer.SPANLESS); long rotation = minutes("2026-08-31T00:00:00Z"); long week = 7 * 24 * 60; // The start of the week before the rotation and the end of the week @@ -743,13 +743,11 @@ void aKeyIsNeverServedForAMinuteOutsideItsConfirmedSpan() * A minute within the clock drift allowance of now, or later, is asked * about every time and never held, because a creator whose clock differs * from this one's may have read it as its present rather than as the - * minute named. A minute beyond the allowance is held as usual. Live - * identifiers therefore cost one request per minute per creator, as they - * always did, and older ones cost none. + * minute named. A minute beyond the allowance is held as usual. Live identifiers therefore cost one request per minute per creator and older ones cost none. */ @Test void aMinuteWithinTheDriftAllowanceIsNotHeld() throws Exception { - KeyEndPoint endPoint = endPoint(KeyEndPoint.Answer.SCHEDULE); + KeyEndPoint endPoint = endPoint(KeyEndPoint.Answer.SPANLESS); Field field = PublicKeyFetch.class.getDeclaredField( "CLOCK_DRIFT_ALLOWANCE_MINUTES"); field.setAccessible(true); @@ -793,11 +791,14 @@ void theCacheIsBounded() throws Exception { final AtomicInteger requests = new AtomicInteger(); PublicKeyTransport distinct = (url, domain) -> { requests.incrementAndGet(); - String minute = url.substring(url.indexOf("date=") + 5, - url.indexOf("&format")); - return CompletableFuture.completedFuture( - "-----BEGIN PUBLIC KEY-----\n" + minute - + "\n-----END PUBLIC KEY-----\n"); + try { + return CompletableFuture.completedFuture( + Endpoints.publicKeyAnswer( + Crypto.generate().subjectPublicKeyInfo(), + null, null, null)); + } catch (OwidException e) { + throw new IllegalStateException(e); + } }; for (int i = 0; i <= maximum; i++) { PublicKeyFetch.publicKeyPemAtUrl( @@ -811,4 +812,226 @@ void theCacheIsBounded() throws Exception { "held " + PublicKeyFetch.cachedKeyCount() + " of at most " + maximum); } + + /** A JSON answer for the key alone, as a creator with no schedule sends. */ + private static String spanless(String pem) throws OwidException { + return Endpoints.publicKeyAnswer(pem, null, null, null); + } + + /** + * An identifier for the domain dated at the moment and signed with the + * crypto given, standing for one whose signing machine's clock did not + * agree with the creator's schedule to the minute. + */ + private static Owid signedAt(String domain, Instant moment, Crypto crypto) + throws OwidException { + byte[] payload = "payload".getBytes(StandardCharsets.UTF_8); + byte[] data = Owid.dataForCrypto(Version.VERSION3, domain, moment, + payload, ALONE); + return new Owid(Version.VERSION3, domain, moment, payload, + crypto.signByteArray(data)); + } + + /** + * A creator that states the moments the key is valid from and to, which + * is what the library's own server side helper answers, has the whole + * span held from that one answer, so every other minute of the span is + * served without a request. + */ + @Test + void aKeyAnsweredWithItsSpanIsHeldForTheWholeSpan() + throws IOException, OwidException { + KeyEndPoint endPoint = endPoint(KeyEndPoint.Answer.SCHEDULE); + String pem = pemAt(urlFor(endPoint, minutes("2026-08-31T00:01:00Z")), + KeyFixtures.IDENTIFIER_DOMAIN); + for (String moment : new String[] {"2026-09-06T23:59:00Z", + "2026-09-03T12:00:00Z", "2026-08-31T00:00:00Z"}) { + assertEquals(pem, pemAt(urlFor(endPoint, minutes(moment)), + KeyFixtures.IDENTIFIER_DOMAIN), moment); + } + assertEquals(1, endPoint.dates().size(), + "the whole week was held from one answer"); + assertEquals(1, PublicKeyFetch.cachedKeyCount()); + assertNotEquals(pem, pemAt(urlFor(endPoint, + minutes("2026-08-30T23:59:00Z")), KeyFixtures.IDENTIFIER_DOMAIN), + "the minute before the week is the earlier week's key"); + pemAt(urlFor(endPoint, minutes("2026-08-24T00:00:00Z")), + KeyFixtures.IDENTIFIER_DOMAIN); + assertEquals(2, endPoint.dates().size(), + "the earlier week was held from its one answer"); + } + + /** + * The drift allowance, which keeps minutes near now out of a cache built + * from confirmed minutes, does not apply to a span the creator stated + * itself, so live identifiers cost one request per key rather than one + * per minute. + */ + @Test + void aRecentMinuteIsServedWhereTheCreatorStatedTheSpan() + throws IOException, OwidException { + KeyEndPoint endPoint = endPoint(KeyEndPoint.Answer.SCHEDULE); + Instant now = Instant.now(); + DatedPublicKey current = KeyFixtures.schedule().keyInForce(now); + assumeTrue(current != null + && KeyFixtures.schedule().nextStartAfter(current) != null, + "the fixture schedule has no key after the one in force now"); + long started = Io.minutesSinceBase(now); + pemAt(urlFor(endPoint, started - 1), KeyFixtures.IDENTIFIER_DOMAIN); + pemAt(urlFor(endPoint, started), KeyFixtures.IDENTIFIER_DOMAIN); + pemAt(urlFor(endPoint, started - 10), KeyFixtures.IDENTIFIER_DOMAIN); + assertEquals(1, endPoint.dates().size(), + "the current key was served for every recent minute from one " + + "answer"); + } + + /** + * An identifier dated just after a key started, but signed with the key + * before it, verifies, and one dated just before a key started but + * signed with it verifies too, because the neighbouring key is tried + * when the selected key fails within the drift allowance of the span's + * edge. Further from the edge the failure stands. The stand in creator + * answers with the library's own server side helper, so the loop between + * the two halves of the library is closed. + */ + @Test + void aSignatureFailingNearTheEdgeOfASpanIsCheckedAgainstTheNeighbour() + throws OwidException { + Crypto first = Crypto.generate(); + Crypto second = Crypto.generate(); + Crypto third = Crypto.generate(); + Instant rotation = Instant.parse("2026-08-31T00:00:00Z"); + Duration week = Duration.ofDays(7); + final PublicKeySchedule schedule = PublicKeySchedule.of(Arrays.asList( + DatedPublicKey.of(rotation.minus(week), + first.subjectPublicKeyInfo()), + DatedPublicKey.of(rotation, second.subjectPublicKeyInfo()), + DatedPublicKey.of(rotation.plus(week), + third.subjectPublicKeyInfo()))); + final List requests = new ArrayList(); + PublicKeyTransport creator = (url, domain) -> { + requests.add(url); + String date = null; + int at = url.indexOf("date="); + if (at >= 0) { + date = url.substring(at + 5, url.indexOf('&', at)); + } + try { + Endpoints.Response response = Endpoints.publicKeyResponseAt( + schedule, "pkcs", date, Instant.now()); + return CompletableFuture.completedFuture(response.getBody()); + } catch (OwidException e) { + throw new IllegalStateException(e); + } + }; + Owid late = signedAt("creator.test", rotation.plus(Duration.ofMinutes(5)), + first); + assertEquals(OwidSignatureStatus.SIGNATURE_VALID, + statusOf(late, creator), + "signed with the earlier key just after the rotation"); + assertEquals(2, requests.size(), + "the selected key and then the earlier key were asked for"); + Owid early = signedAt("creator.test", + rotation.minus(Duration.ofMinutes(5)), second); + assertEquals(OwidSignatureStatus.SIGNATURE_VALID, + statusOf(early, creator), + "signed with the later key just before the rotation"); + assertEquals(2, requests.size(), "both keys are held with their spans"); + Owid far = signedAt("creator.test", rotation.plus(Duration.ofMinutes(20)), + first); + assertEquals(OwidSignatureStatus.SIGNATURE_INVALID, + statusOf(far, creator), "well inside the later key's span"); + assertEquals(2, requests.size(), + "the neighbouring minutes lie inside the spans held"); + Owid genuine = signedAt("creator.test", rotation.plus(Duration.ofDays(3)), + second); + assertEquals(OwidSignatureStatus.SIGNATURE_VALID, + statusOf(genuine, creator)); + Owid forged = signedAt("creator.test", rotation.plus(Duration.ofDays(3)), + third); + assertEquals(OwidSignatureStatus.SIGNATURE_INVALID, + statusOf(forged, creator), + "signed with a key not in force at its date"); + } + + /** The status of an OWID checked through the transport given. */ + private static OwidSignatureStatus statusOf(Owid owid, + PublicKeyTransport transport) throws OwidException { + return PublicKeyFetch.verifyAtUrl(owid, + PublicKeyFetch.publicKeyUrl(owid, "https"), ALONE, transport) + .join().getStatus(); + } + + /** + * The PEM alone as text is reported as a key this library cannot read + * rather than used, and so is a span that ends before it starts. + */ + @Test + void anAnswerThatIsNotTheJsonFormIsAKeyThatCannotBeRead() + throws IOException, OwidException { + Owid owid = KeyFixtures.identifier(); + KeyEndPoint endPoint = endPoint(KeyEndPoint.Answer.PEM_ONLY); + assertEquals(OwidSignatureStatus.INVALID_KEY, + statusAt(owid, endPoint.urlFor(owid))); + final String pem = KeyFixtures.schedule().getKeys().get(0).getPublicKeyPem(); + PublicKeyTransport contradictory = (url, domain) -> + CompletableFuture.completedFuture(PublicKeyResponse.of(pem, + Instant.parse("2026-08-31T00:00:00Z"), + Instant.parse("2026-08-24T00:00:00Z")).toJson()); + assertEquals(OwidSignatureStatus.INVALID_KEY, + statusOf(owid, contradictory)); + } + + /** + * Threads verifying the same OWID at the same moment make one request for + * its key between them, and every one of them gets the answer. The stand + * in transport holds its answer until every thread has asked, so all of + * them are in flight together against one request. + */ + @Test + void manyThreadsVerifyingOneOwidTogetherMakeOneRequest() + throws Exception { + final int callers = 32; + final Owid owid = KeyFixtures.identifier(); + final String answer = spanless( + KeyFixtures.schedule().keyFor(owid).getPublicKeyPem()); + final AtomicInteger requests = new AtomicInteger(); + final CompletableFuture held = new CompletableFuture(); + PublicKeyTransport transport = (url, domain) -> { + requests.incrementAndGet(); + return held; + }; + final String url = PublicKeyFetch.publicKeyUrl(owid, "https"); + final CyclicBarrier start = new CyclicBarrier(callers + 1); + final List statuses = Collections.synchronizedList( + new ArrayList()); + List threads = new ArrayList(); + for (int i = 0; i < callers; i++) { + Thread thread = new Thread(() -> { + try { + start.await(); + statuses.add(PublicKeyFetch.verifyAtUrl(owid, url, ALONE, + transport).join().getStatus()); + } catch (Exception e) { + throw new IllegalStateException(e); + } + }); + thread.start(); + threads.add(thread); + } + // Every thread goes at the same moment, and the transport only + // answers once they are all waiting on it. + start.await(); + Thread.sleep(200); + held.complete(answer); + for (Thread thread : threads) { + thread.join(30_000); + } + assertEquals(callers, statuses.size(), "every thread finished"); + for (OwidSignatureStatus status : statuses) { + assertEquals(OwidSignatureStatus.SIGNATURE_VALID, status, + "every thread verified the OWID"); + } + assertEquals(1, requests.get(), "one request for " + callers + " threads"); + } } diff --git a/src/test/java/com/swancommunity/owid/EndpointsTest.java b/src/test/java/com/swancommunity/owid/EndpointsTest.java index 1ec9106..da77510 100644 --- a/src/test/java/com/swancommunity/owid/EndpointsTest.java +++ b/src/test/java/com/swancommunity/owid/EndpointsTest.java @@ -18,6 +18,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import org.junit.jupiter.api.Test; @@ -61,8 +62,11 @@ void publicKeyResponseFormats() throws OwidException { Creator creator = newCreator(); for (String format : new String[] {"spki", "pkcs"}) { String body = Endpoints.publicKeyResponse(creator, format); - assertTrue(body.contains("BEGIN PUBLIC KEY"), + PublicKeyResponse answer = PublicKeyResponse.parse(body); + assertTrue(answer.getPublicKeySpki().contains("BEGIN PUBLIC KEY"), "should return the PEM for format " + format); + assertNull(answer.getValidFrom(), "a single key has no schedule"); + assertNull(answer.getValidTo()); } assertThrows(OwidException.class, () -> Endpoints.publicKeyResponse(creator, "other"), diff --git a/src/test/java/com/swancommunity/owid/KeyEndPoint.java b/src/test/java/com/swancommunity/owid/KeyEndPoint.java index bab46e5..cdad305 100644 --- a/src/test/java/com/swancommunity/owid/KeyEndPoint.java +++ b/src/test/java/com/swancommunity/owid/KeyEndPoint.java @@ -65,6 +65,15 @@ enum Answer { /** The published schedule, chosen by the date requested. */ SCHEDULE, + /** + * The key alone as JSON with no moments, as a creator with one key + * and no schedule answers. + */ + SPANLESS, + + /** The key alone as text, which the specification does not allow. */ + PEM_ONLY, + /** Text shaped like a PEM that no key can be read out of. */ BROKEN_KEY, @@ -119,24 +128,24 @@ public void handle(HttpExchange exchange) throws IOException { exchange.close(); return; } - String body; + Endpoints.Response response; try { - body = body(schedule, answer, date); - } catch (NumberFormatException malformed) { - // A date that is not a number is refused, as the cloud - // refuses it, rather than failing inside the handler. - exchange.sendResponseHeaders(400, -1); + response = body(schedule, answer, date); + } catch (OwidException fault) { + exchange.sendResponseHeaders(500, -1); exchange.close(); return; } - if (body == null) { - exchange.sendResponseHeaders(404, -1); + if (response.getStatus() != 200) { + exchange.sendResponseHeaders(response.getStatus(), -1); exchange.close(); return; } - byte[] bytes = body.getBytes(StandardCharsets.UTF_8); - exchange.getResponseHeaders() - .set("Content-Type", "text/plain"); + byte[] bytes = response.getBody().getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().set("Content-Type", + answer == Answer.PEM_ONLY + ? "text/plain" + : "application/json"); exchange.sendResponseHeaders(200, bytes.length); OutputStream stream = exchange.getResponseBody(); try { @@ -179,29 +188,50 @@ List dates() { } /** The body to serve, or null where the end point has no key. */ - private static String body(PublicKeySchedule schedule, Answer answer, - String date) { + /** + * The answer for the request, built by the library's own server side + * helper so the client is tested against what a creator built on it + * sends. A creator stating no moments, and the key alone as text, are + * built here for the tests that need them. + */ + private static Endpoints.Response body(PublicKeySchedule schedule, + Answer answer, String date) throws OwidException { if (answer == Answer.BROKEN_KEY) { - // Shaped like a PEM, with a body no key can be read out of. This - // is the 30 August 2026 fault, where the end points served PEM a - // strict parser refused and good identifiers went unverified. - return "-----BEGIN PUBLIC KEY-----\n" - + "bm90IGEga2V5\n" - + "-----END PUBLIC KEY-----\n"; + // Shaped like a PEM, with a body no key can be read out of, sent + // as the JSON form without the check a creator applies, because + // that check is what catches it. + return new Endpoints.Response(200, + PublicKeyResponse.of("-----BEGIN PUBLIC KEY-----\n" + + "bm90IGEga2V5\n" + + "-----END PUBLIC KEY-----\n", null, null) + .toJson()); + } + if (answer == Answer.SCHEDULE) { + return Endpoints.publicKeyResponseAt(schedule, "pkcs", date, + REQUEST_MOMENT); } Instant asked = REQUEST_MOMENT; if (date != null) { - asked = Io.baseDate() - .plus(Duration.ofMinutes(Long.parseLong(date))); + try { + asked = Io.baseDate() + .plus(Duration.ofMinutes(Long.parseLong(date))); + } catch (NumberFormatException malformed) { + return new Endpoints.Response(400, ""); + } if (asked.isAfter(REQUEST_MOMENT)) { asked = REQUEST_MOMENT; } } DatedPublicKey key = schedule.keyInForce(asked); if (key == null) { - return null; + return new Endpoints.Response(404, ""); + } + if (answer == Answer.PEM_ONLY) { + return new Endpoints.Response(200, key.getPublicKeyPem()); } - return key.getPublicKeyPem(); + return new Endpoints.Response(200, + Endpoints.publicKeyAnswer(key.getPublicKeyPem(), null, null, + null)); } /** The value of a parameter in a query, or null where there is none. */ From ea5f7e061bf112e0de1be1367643659bf7a722c2 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 7 Sep 2026 16:14:28 +0100 Subject: [PATCH 6/9] Align the key cache with the specification on the neighbouring key and the stated span The neighbouring key is asked for by the minute just beyond the edge of the span the creator stated, rather than by a minute a fixed distance from the identifier, so a key in force for less than the drift allowance is still the one tried. The span a caller sees is the one the creator stated. A key stated with a start and no end has no later edge, whatever the cache holds it for, so a live identifier dated just after a rotation is checked against the key before it. A creator that stated no span has one key and no neighbour to try. Where the creator's own statement puts the identifier's date outside the span of the key it answered with and nothing verifies, the key is reported as unavailable rather than the signature as not matching, because a key that was not in force proves nothing about the identifier. The request asks for the JSON form by name, and the README and the end point javadoc describe the answer as the JSON object the specification requires rather than the PEM alone. --- README.md | 46 +++-- .../com/swancommunity/owid/Endpoints.java | 7 +- .../owid/HttpUrlConnectionTransport.java | 2 +- .../swancommunity/owid/PublicKeyFetch.java | 176 +++++++++++++----- .../swancommunity/owid/DatedKeyFetchTest.java | 151 ++++++++++++++- 5 files changed, 304 insertions(+), 78 deletions(-) diff --git a/README.md b/README.md index 47e09b7..0dab28a 100644 --- a/README.md +++ b/README.md @@ -161,25 +161,30 @@ creator that ignores the parameter returns its current key, so every identifier it signed under an earlier key reads as not matching, which is why a creator that rotates its key has to honour the date. -Keys already fetched are held by creator, each against the span of minutes the -creator has confirmed it for. A key is in force from the start of its period -until the next key starts, so a key the creator answers with at two minutes was -in force at every minute between them, and an identifier dated inside a -confirmed span is verified without a request whichever minute it carries. One -dated outside every span is asked about, which widens the span when the same -key comes back. The creator answers with the key and the moments it is valid -from and to, so the whole span is held from one answer and an identifier dated -anywhere in it is verified without a request whatever the clock drift. An -answer in any other form, the PEM alone among them, is reported as a key that -cannot be read. A signature that does not verify under the key selected, where -the identifier is dated within fifteen minutes of the edge of that key's span, -is checked against the neighbouring key before it is reported as not matching, -because a creator's signing machines may not agree with its schedule to the -minute. One dated within fifteen minutes of now, or later, is asked about every -time and never held, because a creator whose clock differs from this one's may -have read that minute as its present rather than as the minute named. Live -identifiers therefore cost one request per minute per creator, as they always -did, and older ones cost none. At most 1024 keys are held across every creator +Keys fetched from a creator are held in memory. The request names the minute +the identifier was created, so a creator that rotates its key answers with the +key in force then, and the answer is the JSON form, which carries the moments +the key is valid from and to as well as the key. A creator built on this +library states both, so the whole span is held from one answer and an +identifier dated anywhere in it is verified without a request whatever the +clock drift. An answer that states the start alone is held from the start up +to fifteen minutes behind now, because no later key can have started before +then. An answer that states no span comes from a creator with one key and no +schedule, and is held against the minute asked about and every minute between +two such answers for the same key, but never for a minute within fifteen +minutes of now, because a creator whose clock differs from this one's may have +read that minute as its present rather than as the minute named. The PEM alone +as text is not a valid answer and is refused. A signature that does not verify +under the key selected, where the identifier is dated within fifteen minutes +of an edge of the span the creator stated for that key, is checked against the +key for the minute just beyond that edge before it is reported as not +matching, because a creator's signing machines may not agree with its schedule +to the minute. Where the creator's own statement puts the identifier's date +outside the span of the key it answered with and nothing verifies, the key is +reported as unavailable rather than the signature as not matching, because a +key that was not in force proves nothing about the identifier. Live +identifiers from a creator that states its spans cost one request per key, +and older ones cost none. At most 1024 keys are held across every creator before the store is emptied and filled again, and `clearCache` empties it on demand, which is how a long running process drops a key it has learned it should no longer trust. Two requests for the same key made while the first is @@ -473,7 +478,8 @@ domain, a null payload, or a field that cannot be serialized. points. - `creatorResponse` returns JSON with the fields `domain`, `name`, `publicKeySPKI`, and `contractURL`. The path is `/owid/api/v{n}/creator`. - - `publicKeyResponse` returns the PEM. The path is + - `publicKeyResponse` returns the JSON body of the public key end point + for a creator with one key and no schedule. The path is `/owid/api/v{n}/public-key` with a `format` parameter of `spki` or `pkcs`. ## Data structure notes diff --git a/src/main/java/com/swancommunity/owid/Endpoints.java b/src/main/java/com/swancommunity/owid/Endpoints.java index 7220c15..99a92ef 100644 --- a/src/main/java/com/swancommunity/owid/Endpoints.java +++ b/src/main/java/com/swancommunity/owid/Endpoints.java @@ -28,9 +28,10 @@ *
    *
  • {@code /owid/api/v{version}/creator} returning JSON with the domain, * common name, and public key of the creator.
    • - *
  • {@code /owid/api/v{version}/public-key} returning the public key as - * PEM text. The {@code format} query parameter must be {@code spki} or - * {@code pkcs}.
    • + *
  • {@code /owid/api/v{version}/public-key} returning a JSON object + * carrying the public key as {@code publicKeySPKI} together with the + * moments it is valid from and to. The {@code format} query parameter + * must be {@code spki} or {@code pkcs}.
    • *
*/ public final class Endpoints { diff --git a/src/main/java/com/swancommunity/owid/HttpUrlConnectionTransport.java b/src/main/java/com/swancommunity/owid/HttpUrlConnectionTransport.java index 2874db0..296ed7e 100644 --- a/src/main/java/com/swancommunity/owid/HttpUrlConnectionTransport.java +++ b/src/main/java/com/swancommunity/owid/HttpUrlConnectionTransport.java @@ -173,7 +173,7 @@ private static String read(String url, String domain) connection.setRequestMethod("GET"); connection.setConnectTimeout(CONNECT_TIMEOUT_MILLISECONDS); connection.setReadTimeout(READ_TIMEOUT_MILLISECONDS); - connection.setRequestProperty("Accept", "text/plain"); + connection.setRequestProperty("Accept", "application/json"); int code = connection.getResponseCode(); if (code != HttpURLConnection.HTTP_OK) { drain(connection.getErrorStream()); diff --git a/src/main/java/com/swancommunity/owid/PublicKeyFetch.java b/src/main/java/com/swancommunity/owid/PublicKeyFetch.java index eab5405..bb66658 100644 --- a/src/main/java/com/swancommunity/owid/PublicKeyFetch.java +++ b/src/main/java/com/swancommunity/owid/PublicKeyFetch.java @@ -73,14 +73,15 @@ public final class PublicKeyFetch { * How far a creator's clock may run ahead of or behind this one's, in * minutes. * - *

It is used in two places. A creator that does not state the span of - * the key it answers with reads a date later than its own now as now, so - * within this window of now this process cannot tell whether the creator - * read the minute as its past or as its present, and nothing learned from - * such an answer is held or served. And a creator's signing machines may - * not agree with the creator's own schedule to the minute, so an - * identifier dated within this window of a key's edge that does not - * verify under that key is checked against the neighbouring key before + *

It is used in two places. A creator that does not state the end of + * the span of the key it answers with reads a date later than its own + * now as now, so within this window of now this process cannot tell + * whether the creator read the minute as its past or as its present, and + * nothing learned from such an answer is held or served. And a creator's + * signing machines may not agree with the creator's own schedule to the + * minute, so an identifier dated within this window of an edge of the + * span the creator stated for a key that does not verify under that key + * is checked against the key for the minute just beyond that edge before * it is reported as not matching.

*/ private static final long CLOCK_DRIFT_ALLOWANCE_MINUTES = 15; @@ -88,6 +89,9 @@ public final class PublicKeyFetch { /** The minute {@link #minuteOf} answers where the URL names none. */ private static final long NO_MINUTE = -1; + /** The last minute an OWID can carry, being an unsigned 32 bit count. */ + private static final long MAXIMUM_MINUTE = 0xFFFFFFFFL; + /** * One key a creator has answered with, and the span of minutes the key * is known to cover. @@ -108,12 +112,20 @@ private static final class HeldKey { long last; /** Whether the creator stated the whole span itself. */ boolean explicit; - - HeldKey(String pem, long first, long last, boolean explicit) { + /** + * Whether the creator stated the start of the span and no end, so + * that as far as the creator has said the key is in force until + * further notice, whatever this cache holds it for. + */ + boolean openEnded; + + HeldKey(String pem, long first, long last, boolean explicit, + boolean openEnded) { this.pem = pem; this.first = first; this.last = last; this.explicit = explicit; + this.openEnded = openEnded; } /** Whether the minute lies within the known span. */ @@ -123,14 +135,20 @@ boolean covers(long minute) { } /** - * What the cache or a fetch answers with. The key, and where it is known, - * the span of minutes the key covers, so that a caller can tell whether - * the identifier it is checking sits near the edge of the span. + * What the cache or a fetch answers with. The key and, where the creator + * stated one, the span of minutes the creator says the key covers, so + * that a caller can tell whether the identifier it is checking sits near + * an edge of the span, or outside it altogether. A span stated with a + * start and no end runs to the last minute there is. */ private static final class KeyAnswer { + /** The key in PEM form. */ final String pem; + /** The first minute the creator says the key covers. */ final long first; + /** The last minute the creator says the key covers. */ final long last; + /** Whether the creator stated a span at all. */ final boolean known; KeyAnswer(String pem, long first, long last, boolean known) { @@ -144,6 +162,7 @@ static KeyAnswer unknown(String pem) { return new KeyAnswer(pem, 0, 0, false); } + /** Whether the minute lies within the stated span. */ boolean covers(long minute) { return known && first <= minute && minute <= last; } @@ -374,10 +393,19 @@ static int cachedKeyCount() { * once the URL is known, kept apart so that the tests drive the real * fetch against a key end point the tests can stand up locally rather * than against a near copy of the fetch. + * + *

The signature is checked under the key the end point serves for the + * OWID's own minute, and under the neighbouring key where that minute is + * within the clock drift allowance of an edge of the span the creator + * stated. A key the creator says was not in force at the OWID's minute + * proves nothing about the identifier, so where nothing verifies under + * such a key the answer is that the key is unavailable and not that the + * signature does not match.

*/ static CompletableFuture verifyAtUrl( final Owid owid, final String url, final List others, final PublicKeyTransport transport) { + final long minute = Io.minutesSinceBase(owid.getDate()); return keyAtUrl(url, owid.getDomain(), transport) .handle((answer, failure) -> { if (failure != null) { @@ -387,14 +415,23 @@ static CompletableFuture verifyAtUrl( OwidVerificationResult result = owid.verify(answer.pem, others); if (result.getStatus() - != OwidSignatureStatus.SIGNATURE_INVALID) { + != OwidSignatureStatus.SIGNATURE_INVALID + || minute < 0) { return CompletableFuture.completedFuture(result); } - return neighbourVerifies(owid, url, answer, others, - transport).thenApply(verified -> verified - ? OwidVerificationResult.of( - OwidSignatureStatus.SIGNATURE_VALID) - : result); + return neighbourVerifies(owid, minute, url, answer, + others, transport).thenApply(verified -> { + if (verified) { + return OwidVerificationResult.of( + OwidSignatureStatus.SIGNATURE_VALID); + } + if (answer.known + && answer.covers(minute) == false) { + return OwidVerificationResult.of( + OwidSignatureStatus.KEY_UNAVAILABLE); + } + return result; + }); }) .thenCompose(future -> future); } @@ -408,32 +445,31 @@ static CompletableFuture verifyAtUrl( * been signed with the key before it, and one dated just before may have * been signed with the key after. Where the signature does not verify * under the key selected and the OWID's minute is within the clock drift - * allowance of the edge of the span that key is known to cover, the key - * for the minute just beyond that edge is asked for and tried. A key - * already known to cover the neighbouring minute is not asked for again, - * and a neighbour that turns out to be the same key is not tried again. - * This costs at most two more requests, and only for a signature that - * has already failed.

+ * allowance of an edge of the span the creator stated for that key, the + * key for the minute just beyond that edge is asked for and tried. A key + * already held for that minute is not asked for again, and a neighbour + * that turns out to be the same key is not tried again. A creator that + * stated no span has one key and no schedule, so there is no neighbour + * to try. This costs at most two more requests, and only for a signature + * that has already failed.

*/ private static CompletableFuture neighbourVerifies( - final Owid owid, String url, final KeyAnswer tried, + final Owid owid, long minute, String url, final KeyAnswer tried, final List others, PublicKeyTransport transport) { - long minute = Io.minutesSinceBase(owid.getDate()); - if (minute < 0 || (tried.known && tried.covers(minute) == false)) { - // Either the OWID's minute cannot be counted, or the key tried - // was never in force at that minute, so the OWID is not near an - // edge of that key's span. + if (tried.known == false) { return CompletableFuture.completedFuture(false); } + List beyond = new ArrayList(2); + if (tried.first > 0 && nearEdge(minute, tried.first)) { + beyond.add(tried.first - 1); + } + if (tried.last < MAXIMUM_MINUTE && nearEdge(minute, tried.last)) { + beyond.add(tried.last + 1); + } final String endPoint = endPointOf(url); CompletableFuture verified = CompletableFuture.completedFuture(false); - for (final long at : new long[] { - minute - CLOCK_DRIFT_ALLOWANCE_MINUTES, - minute + CLOCK_DRIFT_ALLOWANCE_MINUTES}) { - if (at < 0 || at > 0xFFFFFFFFL || tried.covers(at)) { - continue; - } + for (final long at : beyond) { verified = verified.thenCompose(already -> { if (already) { return CompletableFuture.completedFuture(true); @@ -449,6 +485,15 @@ private static CompletableFuture neighbourVerifies( return verified; } + /** + * Whether the minute is no further from the edge minute than the clocks + * of a creator's signing machines are allowed to differ from its + * schedule. + */ + private static boolean nearEdge(long minute, long edge) { + return Math.abs(minute - edge) <= CLOCK_DRIFT_ALLOWANCE_MINUTES; + } + /** * Fetches the PEM at the URL. See {@link #keyAtUrl}. */ @@ -649,18 +694,48 @@ private static KeyAnswer heldFor(String endPoint, String url) { boolean recent = recent(minute); for (HeldKey key : keys) { if (key.covers(minute) && (key.explicit || recent == false)) { - return new KeyAnswer(key.pem, key.first, key.last, true); + return statedFor(key); } } } return null; } + /** + * The span the creator stated for a held key, which is the whole held + * span where the creator stated it, runs to the last minute there is + * where the creator stated a start and no end, and is nothing where the + * creator stated no span. + */ + private static KeyAnswer statedFor(HeldKey key) { + if (key.explicit) { + return new KeyAnswer(key.pem, key.first, key.last, true); + } + if (key.openEnded) { + return new KeyAnswer(key.pem, key.first, MAXIMUM_MINUTE, true); + } + return KeyAnswer.unknown(key.pem); + } + + /** + * The span the creator stated in its answer. See + * {@link #statedFor(HeldKey)}. + */ + private static KeyAnswer stated(String pem, Long start, Long end) { + if (start == null) { + return KeyAnswer.unknown(pem); + } + if (end != null && end > start) { + return new KeyAnswer(pem, start, end - 1, true); + } + return new KeyAnswer(pem, start, MAXIMUM_MINUTE, true); + } + /** * Records the creator's answer to the URL, being the key and, where the * creator stated it, the span the key covers as the minute it came into * force and the minute the next key starts. Returns the key with the span - * it is now known to cover. Called under the lock. + * the creator stated for it. Called under the lock. * *

With both the start and the end the whole span is held as the * creator's own statement. With the start alone the key is held from the @@ -674,10 +749,12 @@ private static KeyAnswer heldFor(String endPoint, String url) { */ private static KeyAnswer hold(String endPoint, String url, String pem, Long start, Long end) { + KeyAnswer stated = stated(pem, start, end); long minute = minuteOf(url); long first; long last; boolean explicit = false; + boolean openEnded = false; if (start != null && end != null && end > start) { first = start; last = end - 1; @@ -686,11 +763,12 @@ private static KeyAnswer hold(String endPoint, String url, String pem, first = start; last = Math.max(start, Io.minutesSinceBase(Instant.now()) - CLOCK_DRIFT_ALLOWANCE_MINUTES); + openEnded = true; } else if (minute != NO_MINUTE && recent(minute) == false) { first = minute; last = minute; } else { - return KeyAnswer.unknown(pem); + return stated; } List keys = CACHE.get(endPoint); if (keys != null) { @@ -698,17 +776,19 @@ private static KeyAnswer hold(String endPoint, String url, String pem, if (key.pem.equals(pem)) { if (widen(keys, key, first, last)) { key.explicit = key.explicit || explicit; - return new KeyAnswer(pem, key.first, key.last, true); + key.openEnded = key.explicit == false + && (key.openEnded || openEnded); } - // The creator has answered with another key inside this - // span before, which it does not do unless it went back - // to a key it had left. Nothing more is held about it. - return KeyAnswer.unknown(pem); + // Where the span was not widened the creator has + // answered with another key inside it before, which it + // does not do unless it went back to a key it had left, + // and nothing more is held about this key. + return stated; } } for (HeldKey other : keys) { if (other.last >= first && other.first <= last) { - return KeyAnswer.unknown(pem); + return stated; } } } @@ -721,9 +801,9 @@ private static KeyAnswer hold(String endPoint, String url, String pem, keys = new ArrayList(); CACHE.put(endPoint, keys); } - keys.add(new HeldKey(pem, first, last, explicit)); + keys.add(new HeldKey(pem, first, last, explicit, openEnded)); heldKeys++; - return new KeyAnswer(pem, first, last, true); + return stated; } /** diff --git a/src/test/java/com/swancommunity/owid/DatedKeyFetchTest.java b/src/test/java/com/swancommunity/owid/DatedKeyFetchTest.java index 1087e5b..9b68e2f 100644 --- a/src/test/java/com/swancommunity/owid/DatedKeyFetchTest.java +++ b/src/test/java/com/swancommunity/owid/DatedKeyFetchTest.java @@ -192,8 +192,10 @@ void datedFetchVerifiesAnIdentifierFromAnEarlierKeyWeek() * The same identifier against the same end point without the date, which * is the request a port that forgets the date makes. The end point * answers with the key in force at the moment of the request, ten days - * after the identifier was signed, the signature does not match that - * key, and a genuine identifier reads as a forgery. + * after the identifier was signed, and states a span for it that does + * not include the identifier's date. Nothing verifies under a key the + * creator says was not in force then, so the key is reported as + * unavailable rather than a genuine identifier as a forgery. */ @Test void undatedFetchLeavesAnEarlierWeeksIdentifierUnverified() @@ -202,10 +204,11 @@ void undatedFetchLeavesAnEarlierWeeksIdentifierUnverified() KeyEndPoint endPoint = endPoint(KeyEndPoint.Answer.SCHEDULE); String undated = endPoint.base() + "/owid/api/v3/public-key?format=pkcs"; - assertEquals(OwidSignatureStatus.SIGNATURE_INVALID, + assertEquals(OwidSignatureStatus.KEY_UNAVAILABLE, statusAt(owid, undated), "an undated request gets the key in force at the request, " - + "which did not sign it"); + + "which the creator says was not in force when the " + + "identifier was signed"); assertEquals(Collections.singletonList((String) null), endPoint.dates(), "the request carried no date"); @@ -743,7 +746,9 @@ void aKeyIsNeverServedForAMinuteOutsideItsConfirmedSpan() * A minute within the clock drift allowance of now, or later, is asked * about every time and never held, because a creator whose clock differs * from this one's may have read it as its present rather than as the - * minute named. A minute beyond the allowance is held as usual. Live identifiers therefore cost one request per minute per creator and older ones cost none. + * minute named. A minute beyond the allowance is held as usual. Live + * identifiers from a creator that states no span therefore cost one + * request per minute and older ones cost none. */ @Test void aMinuteWithinTheDriftAllowanceIsNotHeld() throws Exception { @@ -942,7 +947,8 @@ void aSignatureFailingNearTheEdgeOfASpanIsCheckedAgainstTheNeighbour() assertEquals(OwidSignatureStatus.SIGNATURE_INVALID, statusOf(far, creator), "well inside the later key's span"); assertEquals(2, requests.size(), - "the neighbouring minutes lie inside the spans held"); + "the identifier is further from every edge than clocks may " + + "differ"); Owid genuine = signedAt("creator.test", rotation.plus(Duration.ofDays(3)), second); assertEquals(OwidSignatureStatus.SIGNATURE_VALID, @@ -954,6 +960,139 @@ void aSignatureFailingNearTheEdgeOfASpanIsCheckedAgainstTheNeighbour() "signed with a key not in force at its date"); } + /** The date parameter of the key URL, or null where it names none. */ + private static String dateOf(String url) { + int at = url.indexOf("date="); + if (at < 0) { + return null; + } + int end = url.indexOf('&', at); + return url.substring(at + 5, end < 0 ? url.length() : end); + } + + /** + * A stand in creator answering from the schedule through the library's + * own server side helper, recording the date each request asked for. + */ + private static PublicKeyTransport creatorServing( + final PublicKeySchedule schedule, final List asked) { + return (url, domain) -> { + String date = dateOf(url); + asked.add(date); + try { + Endpoints.Response response = Endpoints.publicKeyResponseAt( + schedule, "pkcs", date, Instant.now()); + if (response.getStatus() != 200) { + CompletableFuture refused = + new CompletableFuture(); + refused.completeExceptionally(new PublicKeyFetchException( + "no key for the date asked about", + OwidSignatureStatus.KEY_UNAVAILABLE, domain, + response.getStatus(), null)); + return refused; + } + return CompletableFuture.completedFuture(response.getBody()); + } catch (OwidException e) { + throw new IllegalStateException(e); + } + }; + } + + /** + * The neighbouring key is asked for by the minute just beyond the edge of + * the span the creator stated, not by a minute a fixed distance from the + * identifier, so a key in force for less than the drift allowance is + * still the one tried. + */ + @Test + void theNeighbourIsAskedForByTheMinuteJustBeyondTheEdge() + throws OwidException { + Crypto first = Crypto.generate(); + Crypto second = Crypto.generate(); + Instant rotation = Instant.parse("2026-08-31T00:00:00Z"); + Duration week = Duration.ofDays(7); + PublicKeySchedule schedule = PublicKeySchedule.of(Arrays.asList( + DatedPublicKey.of(rotation.minus(week), + first.subjectPublicKeyInfo()), + DatedPublicKey.of(rotation, second.subjectPublicKeyInfo()), + DatedPublicKey.of(rotation.plus(week), + Crypto.generate().subjectPublicKeyInfo()))); + List asked = new ArrayList(); + Owid late = signedAt("creator.test", + rotation.plus(Duration.ofMinutes(5)), first); + assertEquals(OwidSignatureStatus.SIGNATURE_VALID, + statusOf(late, creatorServing(schedule, asked)), + "signed with the earlier key just after the rotation"); + long minute = Io.minutesSinceBase(rotation); + assertEquals(Arrays.asList(Long.toString(minute + 5), + Long.toString(minute - 1)), asked, + "the identifier's own minute and then the minute just before " + + "the span started"); + } + + /** + * A key the creator states a start for and no end is in force until + * further notice as far as the creator has said, so a live identifier + * dated just after that start which does not verify under it is checked + * against the key before it, even though the cache holds the key only up + * to the drift allowance behind now. + */ + @Test + void aKeyStatedWithoutAnEndHasNoLaterEdge() throws OwidException { + Crypto first = Crypto.generate(); + Crypto second = Crypto.generate(); + Instant rotation = Io.baseDate().plus(Duration.ofMinutes( + Io.minutesSinceBase(Instant.now()) - 5)); + PublicKeySchedule schedule = PublicKeySchedule.of(Arrays.asList( + DatedPublicKey.of(rotation.minus(Duration.ofDays(7)), + first.subjectPublicKeyInfo()), + DatedPublicKey.of(rotation, second.subjectPublicKeyInfo()))); + List asked = new ArrayList(); + Owid live = signedAt("creator.test", + rotation.plus(Duration.ofMinutes(2)), first); + assertEquals(OwidSignatureStatus.SIGNATURE_VALID, + statusOf(live, creatorServing(schedule, asked)), + "a live identifier signed with the key before the current " + + "one verifies"); + assertEquals(2, asked.size(), + "the current key and then the key before it were asked for"); + } + + /** + * A creator whose own statement puts the identifier's date outside the + * span of the key it answered with has said that key did not sign at + * that date, so nothing verifying under it leaves the key unavailable + * rather than the signature not matching. A forgery dated inside the + * span is still reported as not matching. + */ + @Test + void aKeyTheCreatorSaysWasNotInForceLeavesTheSignatureUnjudged() + throws OwidException { + Crypto first = Crypto.generate(); + Crypto second = Crypto.generate(); + Crypto stranger = Crypto.generate(); + Instant rotation = Instant.parse("2026-08-31T00:00:00Z"); + Instant end = rotation.plus(Duration.ofDays(7)); + final String answer = Endpoints.publicKeyAnswer( + second.subjectPublicKeyInfo(), rotation, end, null); + // A creator that ignores the date asked about and answers with the + // current key and its span whatever the request. + PublicKeyTransport current = (url, domain) -> + CompletableFuture.completedFuture(answer); + Owid earlier = signedAt("creator.test", + rotation.minus(Duration.ofDays(3)), first); + assertEquals(OwidSignatureStatus.KEY_UNAVAILABLE, + statusOf(earlier, current), + "the key answered with was not in force at the identifier's " + + "date"); + Owid forged = signedAt("creator.test", + rotation.plus(Duration.ofDays(3)), stranger); + assertEquals(OwidSignatureStatus.SIGNATURE_INVALID, + statusOf(forged, current), + "a signature failing under the key in force at its date does " + + "not match"); + } + /** The status of an OWID checked through the transport given. */ private static OwidSignatureStatus statusOf(Owid owid, PublicKeyTransport transport) throws OwidException { From 133d1974344953c2878b7c4c570076c828ab3d4a Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 7 Sep 2026 16:16:17 +0100 Subject: [PATCH 7/9] Drop the creator end point The specification no longer has a creator end point. It repeated the key the public-key end point serves and added fields no verifier read. The end point helpers serve the public-key end point alone, the JSON writing that only the creator answer used goes with it, and the tests that exercised both end points exercise the one that remains. --- README.md | 2 - .../com/swancommunity/owid/Endpoints.java | 90 +------------------ .../com/swancommunity/owid/EndpointsTest.java | 20 ----- 3 files changed, 4 insertions(+), 108 deletions(-) diff --git a/README.md b/README.md index 0dab28a..f6cec34 100644 --- a/README.md +++ b/README.md @@ -476,8 +476,6 @@ domain, a null payload, or a field that cannot be serialized. no generation moment, so nothing can select by one. - `Endpoints` provides framework agnostic helpers for the well known end points. - - `creatorResponse` returns JSON with the fields `domain`, `name`, - `publicKeySPKI`, and `contractURL`. The path is `/owid/api/v{n}/creator`. - `publicKeyResponse` returns the JSON body of the public key end point for a creator with one key and no schedule. The path is `/owid/api/v{n}/public-key` with a `format` parameter of `spki` or `pkcs`. diff --git a/src/main/java/com/swancommunity/owid/Endpoints.java b/src/main/java/com/swancommunity/owid/Endpoints.java index 99a92ef..c4bcfea 100644 --- a/src/main/java/com/swancommunity/owid/Endpoints.java +++ b/src/main/java/com/swancommunity/owid/Endpoints.java @@ -23,33 +23,16 @@ * specification. These are framework agnostic. They return the path and body * so that any HTTP server can serve them. * - *

The mandatory end points are:

- * - *
    - *
  • {@code /owid/api/v{version}/creator} returning JSON with the domain, - * common name, and public key of the creator.
    • - *
  • {@code /owid/api/v{version}/public-key} returning a JSON object - * carrying the public key as {@code publicKeySPKI} together with the - * moments it is valid from and to. The {@code format} query parameter - * must be {@code spki} or {@code pkcs}.
    • - *
+ *

The mandatory end point is {@code /owid/api/v{version}/public-key}, + * returning a JSON object carrying the public key as {@code publicKeySPKI} + * together with the moments it is valid from and to. The {@code format} + * query parameter must be {@code spki} or {@code pkcs}.

*/ public final class Endpoints { private Endpoints() { } - /** - * Returns the path of the creator end point for the version provided. For - * example {@code /owid/api/v3/creator}. - * - * @param version the OWID version - * @return the creator path - */ - public static String creatorPath(Version version) { - return "/owid/api/v" + (version.asByte() & 0xFF) + "/creator"; - } - /** * Returns the path of the public key end point for the version provided. * For example {@code /owid/api/v3/public-key}. @@ -61,33 +44,6 @@ public static String publicKeyPath(Version version) { return "/owid/api/v" + (version.asByte() & 0xFF) + "/public-key"; } - /** - * Returns the JSON body for the creator end point. The JSON has the - * fields domain, name, publicKeySPKI, and contractURL named exactly as - * required by the specification. - * - * @param creator the creator - * @param name the common name of the creator - * @param contractUrl the URL with the terms associated with the data - * @return the JSON body - * @throws OwidException if the public key cannot be exported - */ - public static String creatorResponse(Creator creator, String name, - String contractUrl) throws OwidException { - String spki = creator.crypto().subjectPublicKeyInfo(); - StringBuilder json = new StringBuilder(); - json.append('{'); - appendField(json, "domain", creator.domain()); - json.append(','); - appendField(json, "name", name); - json.append(','); - appendField(json, "publicKeySPKI", spki); - json.append(','); - appendField(json, "contractURL", contractUrl); - json.append('}'); - return json.toString(); - } - /** * Returns the JSON body for the public key end point of a creator with * one key and no schedule. The key is stated as {@code publicKeySPKI} and @@ -214,42 +170,4 @@ public static Response publicKeyResponseAt(PublicKeySchedule schedule, return new Response(200, publicKeyAnswer(key.getPublicKeyPem(), key.getStartsAt(), schedule.nextStartAfter(key), asked)); } - - private static void appendField(StringBuilder json, String name, - String value) { - json.append('"').append(name).append("\":\"") - .append(escape(value)).append('"'); - } - - /** Escapes a string for inclusion in a JSON string literal. */ - private static String escape(String value) { - StringBuilder builder = new StringBuilder(value.length()); - for (int i = 0; i < value.length(); i++) { - char c = value.charAt(i); - switch (c) { - case '"': - builder.append("\\\""); - break; - case '\\': - builder.append("\\\\"); - break; - case '\n': - builder.append("\\n"); - break; - case '\r': - builder.append("\\r"); - break; - case '\t': - builder.append("\\t"); - break; - default: - if (c < 0x20) { - builder.append(String.format("\\u%04x", (int) c)); - } else { - builder.append(c); - } - } - } - return builder.toString(); - } } diff --git a/src/test/java/com/swancommunity/owid/EndpointsTest.java b/src/test/java/com/swancommunity/owid/EndpointsTest.java index da77510..dc99d67 100644 --- a/src/test/java/com/swancommunity/owid/EndpointsTest.java +++ b/src/test/java/com/swancommunity/owid/EndpointsTest.java @@ -32,31 +32,11 @@ private static Creator newCreator() throws OwidException { @Test void paths() { - assertEquals("/owid/api/v3/creator", - Endpoints.creatorPath(Version.VERSION3), - "should match the creator path"); assertEquals("/owid/api/v3/public-key", Endpoints.publicKeyPath(Version.VERSION3), "should match the public key path"); } - @Test - void creatorResponseFields() throws OwidException { - Creator creator = newCreator(); - String body = Endpoints.creatorResponse(creator, "Example Org", - "https://example.com/terms"); - assertTrue(body.contains("\"domain\":\"example.com\""), - "should contain the domain"); - assertTrue(body.contains("\"name\":\"Example Org\""), - "should contain the name"); - assertTrue(body.contains("publicKeySPKI"), - "should use the specification field names"); - assertTrue(body.contains("BEGIN PUBLIC KEY"), - "should embed the public key PEM"); - assertTrue(body.contains("\"contractURL\":\"https://example.com/terms\""), - "should contain the contract URL"); - } - @Test void publicKeyResponseFormats() throws OwidException { Creator creator = newCreator(); From d122cae40997b6dced92f81e7d5d4b40c9328908 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 7 Sep 2026 16:21:54 +0100 Subject: [PATCH 8/9] Answer with publicKey and format The public-key answer carries the key as publicKey and the encoding it is in as format. The one format served is spki, which a request without the parameter receives, and any other value is answered 400 rather than in an encoding the caller did not ask for. The answer is checked for its format before its key and its span, so a creator never sends an encoding it does not itself read. The client asks for spki by name, reads publicKey, and refuses an answer that states another format as a key it cannot read. The stand in creator in the tests honours the format the request asks for, so the client is exercised against a creator that refuses the way the specification requires. --- README.md | 21 +++-- .../com/swancommunity/owid/Endpoints.java | 56 +++++++----- .../swancommunity/owid/PublicKeyFetch.java | 18 ++-- .../swancommunity/owid/PublicKeyResponse.java | 91 +++++++++++++------ .../swancommunity/owid/DatedKeyFetchTest.java | 47 +++++++--- .../com/swancommunity/owid/EndpointsTest.java | 49 +++++++++- .../com/swancommunity/owid/KeyEndPoint.java | 15 +-- 7 files changed, 204 insertions(+), 93 deletions(-) diff --git a/README.md b/README.md index f6cec34..bdd337e 100644 --- a/README.md +++ b/README.md @@ -154,7 +154,7 @@ anything older than a few days means asking for the key that was in force on the date the identifier carries. `PublicKeyFetch` asks the creator for that key. The request is -`/owid/api/v{n}/public-key?date={minutes}&format=pkcs`, where the version in +`/owid/api/v{n}/public-key?date={minutes}&format=spki`, where the version in the path is the version byte of the identifier being checked and the minutes are counted from 2020-01-01 in the same way the identifier stores its date. A creator that ignores the parameter returns its current key, so every @@ -448,12 +448,16 @@ domain, a null payload, or a field that cannot be serialized. `HttpUrlConnectionTransport` on its shared pool where none is given. - `clearCache` empties the keys already fetched. - `Endpoints.publicKeyResponse` and `Endpoints.publicKeyResponseAt` return the - JSON body of the public key end point, the key as `publicKeySPKI` with - `validFrom` and `validTo`, the UTC moments the key came into force and the - next key starts, and `Endpoints.publicKeyAnswer` builds and checks any such - answer so a key that cannot be read or a schedule that contradicts itself is - refused before it is sent. `PublicKeyResponse` reads and writes the body. The - PEM alone as text is no longer a valid answer. + JSON body of the public key end point, the key as `publicKey`, the encoding + it is in as `format`, and `validFrom` and `validTo`, the UTC moments the key + came into force and the next key starts. The one format defined is `spki`, + a Subject Public Key Info PEM. It is what a request without a `format` + receives, and a request for any other value is answered 400 rather than in + an encoding the caller did not ask for. `Endpoints.publicKeyAnswer` builds + and checks any such answer so a key that cannot be read or a schedule that + contradicts itself is refused before it is sent. `PublicKeyResponse` reads + and writes the body, and refuses an answer that states another format. The + PEM alone as text is not a valid answer. - `PublicKeyTransport` makes the request and answers with a `CompletableFuture` of the body, so a transport over any HTTP client can be supplied. It must never follow a redirect and must request the URL @@ -478,7 +482,8 @@ domain, a null payload, or a field that cannot be serialized. points. - `publicKeyResponse` returns the JSON body of the public key end point for a creator with one key and no schedule. The path is - `/owid/api/v{n}/public-key` with a `format` parameter of `spki` or `pkcs`. + `/owid/api/v{n}/public-key` with an optional `format` parameter whose one + defined value is `spki`. ## Data structure notes diff --git a/src/main/java/com/swancommunity/owid/Endpoints.java b/src/main/java/com/swancommunity/owid/Endpoints.java index c4bcfea..8ead5fc 100644 --- a/src/main/java/com/swancommunity/owid/Endpoints.java +++ b/src/main/java/com/swancommunity/owid/Endpoints.java @@ -24,9 +24,11 @@ * so that any HTTP server can serve them. * *

The mandatory end point is {@code /owid/api/v{version}/public-key}, - * returning a JSON object carrying the public key as {@code publicKeySPKI} - * together with the moments it is valid from and to. The {@code format} - * query parameter must be {@code spki} or {@code pkcs}.

+ * returning a JSON object carrying the public key as {@code publicKey}, the + * encoding it is in as {@code format}, and the moments it is valid from and + * to. The one format defined is {@code spki}, which a request without the + * parameter receives, and a request for any other value is answered + * 400.

*/ public final class Endpoints { @@ -46,33 +48,41 @@ public static String publicKeyPath(Version version) { /** * Returns the JSON body for the public key end point of a creator with - * one key and no schedule. The key is stated as {@code publicKeySPKI} and - * both {@code validFrom} and {@code validTo} are null, because the - * creator knows nothing about when the key started or will stop. - * - *

The specification allows the key to be requested in SPKI or PKCS - * form. This implementation returns the SPKI PEM for both values because - * the importers accept it.

+ * one key and no schedule. The key is stated as {@code publicKey} in the + * {@code spki} format and both {@code validFrom} and {@code validTo} are + * null, because the creator knows nothing about when the key started or + * will stop. * * @param creator the creator - * @param format the format parameter, {@code spki} or {@code pkcs} + * @param format the format parameter, {@code spki} or null where the + * request has none * @return the JSON body - * @throws OwidException if the format is not valid, or the public key - * cannot be exported or read back + * @throws OwidException if the format is one this library does not + * serve, which a creator answers 400, or the public + * key cannot be exported or read back */ public static String publicKeyResponse(Creator creator, String format) throws OwidException { - if ("spki".equals(format) == false && "pkcs".equals(format) == false) { + if (served(format) == false) { // The value is not repeated back, because it arrives on a query // string from whoever called the end point and a refusal is often // logged. - throw new OwidException( - "format parameter 'spki' or 'pkcs' must be provided"); + throw new OwidException("the only format served is " + + PublicKeyResponse.SPKI_FORMAT); } return publicKeyAnswer(creator.crypto().subjectPublicKeyInfo(), null, null, null); } + /** + * Whether the format parameter asks for the one encoding this library + * serves, which a request without the parameter is taken to ask for. + */ + private static boolean served(String format) { + return format == null || format.isEmpty() + || PublicKeyResponse.SPKI_FORMAT.equals(format); + } + /** * Returns the JSON body of the public key end point for the key and the * span it covers, checked with {@link PublicKeyResponse#validate(Instant)} @@ -131,21 +141,21 @@ public String getBody() { * {@link #publicKeyAnswer}, stating the key and the moments it is valid * from and to, 404 with an empty body where no key is in force at the * date, and 400 with an empty body where the date is not a count of - * minutes.

+ * minutes or the format is one this creator does not serve.

* * @param schedule the published schedule - * @param format the format parameter, {@code spki} or {@code pkcs} + * @param format the format parameter, {@code spki} or null where the + * request has none * @param date the date parameter, or null where the request has none * @param now the moment of the request * @return the status and body - * @throws OwidException if the format is not valid, or the answer would - * fail its check, which is a fault in the schedule + * @throws OwidException if the answer would fail its check, which is a + * fault in the schedule */ public static Response publicKeyResponseAt(PublicKeySchedule schedule, String format, String date, Instant now) throws OwidException { - if ("spki".equals(format) == false && "pkcs".equals(format) == false) { - throw new OwidException( - "format parameter 'spki' or 'pkcs' must be provided"); + if (served(format) == false) { + return new Response(400, ""); } Instant asked = now; if (date != null && date.isEmpty() == false) { diff --git a/src/main/java/com/swancommunity/owid/PublicKeyFetch.java b/src/main/java/com/swancommunity/owid/PublicKeyFetch.java index bb66658..31f2cf0 100644 --- a/src/main/java/com/swancommunity/owid/PublicKeyFetch.java +++ b/src/main/java/com/swancommunity/owid/PublicKeyFetch.java @@ -30,7 +30,7 @@ * date the OWID carries. * *

The end point is - * {@code /owid/api/v{n}/public-key?date={minutes}&format=pkcs}, where the + * {@code /owid/api/v{n}/public-key?date={minutes}&format=spki}, where the * version in the path is the version byte of the OWID being checked rather * than a constant, and the minutes are counted from 2020-01-01 in the same * way the OWID stores the date. Creators rotate weekly, so without the date @@ -219,7 +219,8 @@ private PublicKeyFetch() { * rotates its key returns the key that was in force when this OWID was * signed. The parameter is left out where the date cannot be counted, * which no OWID this library reads can be, because the wire format - * cannot hold such a date.

+ * cannot hold such a date. The key is asked for by name in the one + * format this library reads, {@link PublicKeyResponse#SPKI_FORMAT}.

* * @param owid the OWID whose creator key is wanted * @param scheme the scheme to use, normally {@code https} @@ -246,7 +247,7 @@ public static String publicKeyUrl(Owid owid, String scheme) if (minutes >= 0) { url.append("date=").append(minutes).append('&'); } - url.append("format=pkcs"); + url.append("format=").append(PublicKeyResponse.SPKI_FORMAT); return url.toString(); } @@ -474,7 +475,8 @@ private static CompletableFuture neighbourVerifies( if (already) { return CompletableFuture.completedFuture(true); } - return keyAtUrl(endPoint + "?date=" + at + "&format=pkcs", + return keyAtUrl(endPoint + "?date=" + at + "&format=" + + PublicKeyResponse.SPKI_FORMAT, owid.getDomain(), transport) .handle((neighbour, failure) -> failure == null && neighbour.pem.equals(tried.pem) == false @@ -587,9 +589,9 @@ static CompletableFuture keyAtUrl(final String url, * Reads a public key answer and holds the key it carries against the * span it states, or against the minute asked about where it states * none. An answer that is not the JSON form the specification requires, - * the PEM alone among the other forms, or that fails the checks a - * creator applies before sending it, is reported as a key that cannot be - * read. + * the PEM alone among the other forms, that states a format this library + * does not read, or that fails the checks a creator applies before + * sending it, is reported as a key that cannot be read. */ private static KeyAnswer readAnswer(String body, String domain, String endPoint, String url) throws PublicKeyFetchException { @@ -604,7 +606,7 @@ private static KeyAnswer readAnswer(String body, String domain, OwidSignatureStatus.INVALID_KEY, domain, 0, e); } synchronized (LOCK) { - return hold(endPoint, url, answer.getPublicKeySpki(), + return hold(endPoint, url, answer.getPublicKey(), minutesOrNull(answer.getValidFrom()), minutesOrNull(answer.getValidTo())); } diff --git a/src/main/java/com/swancommunity/owid/PublicKeyResponse.java b/src/main/java/com/swancommunity/owid/PublicKeyResponse.java index 1d8ed7f..1d66987 100644 --- a/src/main/java/com/swancommunity/owid/PublicKeyResponse.java +++ b/src/main/java/com/swancommunity/owid/PublicKeyResponse.java @@ -22,9 +22,10 @@ import java.util.Map; /** - * The JSON body of the public key end point. It carries the key together with - * the moments it is valid from and to, in UTC, so a client holds the key for - * the whole span from one answer rather than asking again for every minute. + * The JSON body of the public key end point. It carries the key, the + * encoding the key is in, and the moments it is valid from and to, in UTC, + * so a client holds the key for the whole span from one answer rather than + * asking again for every minute. * *

{@code validFrom} is null where the creator has a single key and no * schedule, and {@code validTo} is null where no later key has been @@ -34,39 +35,57 @@ * a client then has to refuse.

* *

Only the JDK is used, so the library keeps its promise of no runtime - * dependencies. The answer is a flat object of three fields, each a string or + * dependencies. The answer is a flat object of four fields, each a string or * null, which is all the reading and writing here supports.

*/ public final class PublicKeyResponse { - private final String publicKeySpki; + /** + * The one encoding of the key this library reads and writes, a Subject + * Public Key Info PEM. A request that asks for no format is answered in + * this one. + */ + public static final String SPKI_FORMAT = "spki"; + + private final String format; + private final String publicKey; private final Instant validFrom; private final Instant validTo; - private PublicKeyResponse(String publicKeySpki, Instant validFrom, - Instant validTo) { - this.publicKeySpki = publicKeySpki; + private PublicKeyResponse(String format, String publicKey, + Instant validFrom, Instant validTo) { + this.format = format; + this.publicKey = publicKey; this.validFrom = validFrom; this.validTo = validTo; } /** - * An answer for the key and the moments it is valid from and to, either - * of which may be null. + * An answer for the key in the one format this library writes and the + * moments it is valid from and to, either of which may be null. * - * @param publicKeySpki the key in PEM form - * @param validFrom the UTC moment the key came into force, or null - * @param validTo the UTC moment the next key starts, or null + * @param publicKey the key as a Subject Public Key Info PEM + * @param validFrom the UTC moment the key came into force, or null + * @param validTo the UTC moment the next key starts, or null * @return the answer, not yet checked */ - public static PublicKeyResponse of(String publicKeySpki, Instant validFrom, + public static PublicKeyResponse of(String publicKey, Instant validFrom, Instant validTo) { - return new PublicKeyResponse(publicKeySpki, validFrom, validTo); + return new PublicKeyResponse(SPKI_FORMAT, publicKey, validFrom, + validTo); } - /** The key in PEM form. */ - public String getPublicKeySpki() { - return publicKeySpki; + /** + * The encoding of the key, which is {@link #SPKI_FORMAT} for any answer + * this library can read. + */ + public String getFormat() { + return format; + } + + /** The public key in the encoding {@link #getFormat()} names. */ + public String getPublicKey() { + return publicKey; } /** The UTC moment the key came into force, or null where not known. */ @@ -81,20 +100,25 @@ public Instant getValidTo() { /** * Checks the answer the way both the creator that sends it and the client - * that reads it must. The key must be a public key this library can read, - * a key valid to a moment must be valid from an earlier one, and where the - * moment asked about is known the key must have come into force by then - * and, if it has an end, not have ended. + * that reads it must. The format must be the one this library reads and + * the key must be a public key in it, a key valid to a moment must be + * valid from an earlier one, and where the moment asked about is known + * the key must have come into force by then and, if it has an end, not + * have ended. * * @param asked the moment asked about, or null where it is not known * @throws OwidException if the answer is not valid */ public void validate(Instant asked) throws OwidException { - if (publicKeySpki == null || publicKeySpki.trim().isEmpty()) { + if (SPKI_FORMAT.equals(format) == false) { + throw new OwidException("the public key answer states a format " + + "this library does not read"); + } + if (publicKey == null || publicKey.trim().isEmpty()) { throw new OwidException("the public key answer holds no key"); } try { - Crypto.newVerifyOnly(publicKeySpki); + Crypto.newVerifyOnly(publicKey); } catch (OwidException e) { throw new OwidException( "the public key answer holds a key that cannot be read"); @@ -128,8 +152,10 @@ public void validate(Instant asked) throws OwidException { * @return the JSON body */ public String toJson() { - StringBuilder json = new StringBuilder("{\"publicKeySPKI\":"); - appendString(json, publicKeySpki); + StringBuilder json = new StringBuilder("{\"format\":"); + appendString(json, format); + json.append(",\"publicKey\":"); + appendString(json, publicKey); json.append(",\"validFrom\":"); appendMoment(json, validFrom); json.append(",\"validTo\":"); @@ -138,17 +164,22 @@ public String toJson() { } /** - * Reads an answer from its JSON body. + * Reads an answer from its JSON body. An answer that names no format is + * read as {@link #SPKI_FORMAT}, the encoding a request that asks for none + * receives. * * @param json the body * @return the answer, not yet checked with {@link #validate(Instant)} - * @throws OwidException if the body is not a JSON object of the three + * @throws OwidException if the body is not a JSON object of the four * fields, each a string or null */ public static PublicKeyResponse parse(String json) throws OwidException { Map fields = readFlatObject(json); return new PublicKeyResponse( - fields.get("publicKeySPKI"), + fields.containsKey("format") + ? fields.get("format") + : SPKI_FORMAT, + fields.get("publicKey"), moment(fields.get("validFrom"), "validFrom"), moment(fields.get("validTo"), "validTo")); } @@ -331,6 +362,6 @@ private static String readString(String json, int[] at) private static OwidException notJson() { return new OwidException("the public key answer is not the JSON " - + "object of three fields the specification requires"); + + "object of four fields the specification requires"); } } diff --git a/src/test/java/com/swancommunity/owid/DatedKeyFetchTest.java b/src/test/java/com/swancommunity/owid/DatedKeyFetchTest.java index 9b68e2f..6dfd0d8 100644 --- a/src/test/java/com/swancommunity/owid/DatedKeyFetchTest.java +++ b/src/test/java/com/swancommunity/owid/DatedKeyFetchTest.java @@ -128,7 +128,7 @@ private static T failureOf( void urlNamesTheMinuteTheIdentifierWasCreated() throws OwidException { assertEquals( "https://51d.es/owid/api/v3/public-key?date=" - + KeyFixtures.IDENTIFIER_MINUTES + "&format=pkcs", + + KeyFixtures.IDENTIFIER_MINUTES + "&format=spki", PublicKeyFetch.publicKeyUrl(KeyFixtures.identifier(), "https"), "should ask 51d.es for the key in force on 4 September 2026"); @@ -147,7 +147,7 @@ void urlUsesTheVersionTheIdentifierCarries() throws OwidException { "the crafted identifier is version 2"); assertEquals( "https://example.com/owid/api/v2/public-key?date=" - + KeyFixtures.IDENTIFIER_MINUTES + "&format=pkcs", + + KeyFixtures.IDENTIFIER_MINUTES + "&format=spki", PublicKeyFetch.publicKeyUrl(version2, "https"), "should ask the version 2 end point"); } @@ -160,7 +160,7 @@ void urlOfANewlySignedOwidNamesItsOwnMinute() throws OwidException { assertEquals( "https://example.com/owid/api/v3/public-key?date=" + Io.minutesSinceBase(owid.getDate()) - + "&format=pkcs", + + "&format=spki", PublicKeyFetch.publicKeyUrl(owid, "https"), "should name the minute the OWID was signed"); } @@ -203,7 +203,7 @@ void undatedFetchLeavesAnEarlierWeeksIdentifierUnverified() Owid owid = KeyFixtures.identifier(); KeyEndPoint endPoint = endPoint(KeyEndPoint.Answer.SCHEDULE); String undated = endPoint.base() - + "/owid/api/v3/public-key?format=pkcs"; + + "/owid/api/v3/public-key?format=spki"; assertEquals(OwidSignatureStatus.KEY_UNAVAILABLE, statusAt(owid, undated), "an undated request gets the key in force at the request, " @@ -228,7 +228,7 @@ void aKeyTheEndPointCannotServeIsKeyUnavailable() Instant before = KeyFixtures.scheduledKeys().get(0).startsAt() .minus(Duration.ofDays(14)); String url = endPoint.base() + "/owid/api/v3/public-key?date=" - + Io.minutesSinceBase(before) + "&format=pkcs"; + + Io.minutesSinceBase(before) + "&format=spki"; assertEquals(OwidSignatureStatus.KEY_UNAVAILABLE, statusAt(owid, url), "no key means the signature was never examined"); @@ -240,7 +240,7 @@ void aRefusedRequestCarriesTheStatusAndTheCode() throws IOException, OwidException { KeyEndPoint endPoint = endPoint(KeyEndPoint.Answer.SCHEDULE); String url = endPoint.base() + "/owid/api/v3/public-key?date=0" - + "&format=pkcs"; + + "&format=spki"; PublicKeyFetchException failure = failureOf( PublicKeyFetch.publicKeyPemAtUrl(url, "51d.es", HTTP), PublicKeyFetchException.class, @@ -434,7 +434,7 @@ void theRequestRunsOnTheExecutorGiven() "the thread that asked is not the one that fetches"); assertEquals(OwidSignatureStatus.SIGNATURE_VALID, owid.verify(PublicKeyResponse.parse(fetch.join()) - .getPublicKeySpki(), ALONE).getStatus(), + .getPublicKey(), ALONE).getStatus(), "the key fetched on the executor verifies the identifier"); } @@ -619,7 +619,7 @@ private static long minutes(String moment) { /** A key URL on the end point for the minute given. */ private static String urlFor(KeyEndPoint endPoint, long minute) { return endPoint.base() + "/owid/api/v3/public-key?date=" + minute - + "&format=pkcs"; + + "&format=spki"; } /** The PEM the published schedule says was in force at the minute. */ @@ -763,7 +763,7 @@ void aMinuteWithinTheDriftAllowanceIsNotHeld() throws Exception { pemAt(urlFor(endPoint, recent), KeyFixtures.IDENTIFIER_DOMAIN); pemAt(urlFor(endPoint, started + 7 * 24 * 60), KeyFixtures.IDENTIFIER_DOMAIN); - pemAt(endPoint.base() + "/owid/api/v3/public-key?format=pkcs", + pemAt(endPoint.base() + "/owid/api/v3/public-key?format=spki", KeyFixtures.IDENTIFIER_DOMAIN); long old = started - allowance - 1; pemAt(urlFor(endPoint, old), KeyFixtures.IDENTIFIER_DOMAIN); @@ -808,7 +808,7 @@ void theCacheIsBounded() throws Exception { for (int i = 0; i <= maximum; i++) { PublicKeyFetch.publicKeyPemAtUrl( "https://example.invalid/owid/api/v3/public-key?date=" + i - + "&format=pkcs", + + "&format=spki", "example.invalid", distinct).join(); } assertEquals(maximum + 1, requests.get(), @@ -923,7 +923,7 @@ void aSignatureFailingNearTheEdgeOfASpanIsCheckedAgainstTheNeighbour() } try { Endpoints.Response response = Endpoints.publicKeyResponseAt( - schedule, "pkcs", date, Instant.now()); + schedule, "spki", date, Instant.now()); return CompletableFuture.completedFuture(response.getBody()); } catch (OwidException e) { throw new IllegalStateException(e); @@ -981,7 +981,7 @@ private static PublicKeyTransport creatorServing( asked.add(date); try { Endpoints.Response response = Endpoints.publicKeyResponseAt( - schedule, "pkcs", date, Instant.now()); + schedule, "spki", date, Instant.now()); if (response.getStatus() != 200) { CompletableFuture refused = new CompletableFuture(); @@ -1121,6 +1121,29 @@ void anAnswerThatIsNotTheJsonFormIsAKeyThatCannotBeRead() statusOf(owid, contradictory)); } + /** + * An answer that states a format other than the one this library reads + * is a key that cannot be read, whatever the key field holds, because + * the key is not in the encoding the request asked for. + */ + @Test + void anAnswerInAnotherFormatIsAKeyThatCannotBeRead() + throws OwidException { + Owid owid = KeyFixtures.identifier(); + final String pem = KeyFixtures.schedule().keyFor(owid) + .getPublicKeyPem(); + final String other = PublicKeyResponse.of(pem, null, null).toJson() + .replace("\"spki\"", "\"pkcs\""); + assertEquals(OwidSignatureStatus.INVALID_KEY, statusOf(owid, + (url, domain) -> CompletableFuture.completedFuture(other)), + "a format this library does not read leaves the signature " + + "unjudged"); + assertEquals(OwidSignatureStatus.SIGNATURE_VALID, statusOf(owid, + (url, domain) -> CompletableFuture.completedFuture( + PublicKeyResponse.of(pem, null, null).toJson())), + "the same key in the format asked for verifies"); + } + /** * Threads verifying the same OWID at the same moment make one request for * its key between them, and every one of them gets the answer. The stand diff --git a/src/test/java/com/swancommunity/owid/EndpointsTest.java b/src/test/java/com/swancommunity/owid/EndpointsTest.java index dc99d67..b6133b2 100644 --- a/src/test/java/com/swancommunity/owid/EndpointsTest.java +++ b/src/test/java/com/swancommunity/owid/EndpointsTest.java @@ -21,6 +21,8 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.time.Instant; +import java.util.Collections; import org.junit.jupiter.api.Test; /** Unit tests for the well known end point helpers. */ @@ -37,19 +39,56 @@ void paths() { "should match the public key path"); } + /** + * The format parameter names the encoding of the key in the answer. The + * one encoding defined is answered whether or not it is asked for by + * name, the answer echoes it, and any other value is refused rather than + * answered in an encoding the caller did not ask for. + */ @Test void publicKeyResponseFormats() throws OwidException { Creator creator = newCreator(); - for (String format : new String[] {"spki", "pkcs"}) { + for (String format : new String[] {"spki", null, ""}) { String body = Endpoints.publicKeyResponse(creator, format); PublicKeyResponse answer = PublicKeyResponse.parse(body); - assertTrue(answer.getPublicKeySpki().contains("BEGIN PUBLIC KEY"), + assertEquals("spki", answer.getFormat(), + "the answer names the encoding of the key"); + assertTrue(answer.getPublicKey().contains("BEGIN PUBLIC KEY"), "should return the PEM for format " + format); assertNull(answer.getValidFrom(), "a single key has no schedule"); assertNull(answer.getValidTo()); } - assertThrows(OwidException.class, - () -> Endpoints.publicKeyResponse(creator, "other"), - "should reject an unknown format"); + for (String format : new String[] {"pkcs", "other"}) { + assertThrows(OwidException.class, + () -> Endpoints.publicKeyResponse(creator, format), + "should refuse format " + format); + } + } + + /** + * The scheduled form answers 400 to a format it does not serve, the way + * the specification requires of a creator, and answers the one format + * defined whether or not the request names it. + */ + @Test + void publicKeyResponseAtRefusesAnotherFormat() throws OwidException { + Instant now = Instant.parse("2026-09-07T12:00:00Z"); + PublicKeySchedule schedule = PublicKeySchedule.of( + Collections.singletonList(DatedPublicKey.of( + Instant.parse("2026-08-31T00:00:00Z"), + Crypto.generate().subjectPublicKeyInfo()))); + Endpoints.Response refused = Endpoints.publicKeyResponseAt(schedule, + "pkcs", null, now); + assertEquals(400, refused.getStatus(), + "a format this creator does not serve is a bad request"); + assertEquals("", refused.getBody()); + for (String format : new String[] {"spki", null}) { + Endpoints.Response served = Endpoints.publicKeyResponseAt( + schedule, format, null, now); + assertEquals(200, served.getStatus()); + assertEquals("spki", + PublicKeyResponse.parse(served.getBody()).getFormat(), + "the answer echoes the one format defined"); + } } } diff --git a/src/test/java/com/swancommunity/owid/KeyEndPoint.java b/src/test/java/com/swancommunity/owid/KeyEndPoint.java index cdad305..7722d64 100644 --- a/src/test/java/com/swancommunity/owid/KeyEndPoint.java +++ b/src/test/java/com/swancommunity/owid/KeyEndPoint.java @@ -119,8 +119,9 @@ static KeyEndPoint start(final Answer answer, final String redirectTo) server.createContext("/", new HttpHandler() { @Override public void handle(HttpExchange exchange) throws IOException { - String date = parameter( - exchange.getRequestURI().getRawQuery(), "date"); + String query = exchange.getRequestURI().getRawQuery(); + String date = parameter(query, "date"); + String format = parameter(query, "format"); endPoint.dates.add(date); if (answer == Answer.REDIRECT) { exchange.getResponseHeaders().set("Location", redirectTo); @@ -130,7 +131,7 @@ public void handle(HttpExchange exchange) throws IOException { } Endpoints.Response response; try { - response = body(schedule, answer, date); + response = body(schedule, answer, date, format); } catch (OwidException fault) { exchange.sendResponseHeaders(500, -1); exchange.close(); @@ -187,15 +188,15 @@ List dates() { } } - /** The body to serve, or null where the end point has no key. */ /** * The answer for the request, built by the library's own server side * helper so the client is tested against what a creator built on it - * sends. A creator stating no moments, and the key alone as text, are + * sends, honouring the format the request asks for the way the cloud + * does. A creator stating no moments, and the key alone as text, are * built here for the tests that need them. */ private static Endpoints.Response body(PublicKeySchedule schedule, - Answer answer, String date) throws OwidException { + Answer answer, String date, String format) throws OwidException { if (answer == Answer.BROKEN_KEY) { // Shaped like a PEM, with a body no key can be read out of, sent // as the JSON form without the check a creator applies, because @@ -207,7 +208,7 @@ private static Endpoints.Response body(PublicKeySchedule schedule, .toJson()); } if (answer == Answer.SCHEDULE) { - return Endpoints.publicKeyResponseAt(schedule, "pkcs", date, + return Endpoints.publicKeyResponseAt(schedule, format, date, REQUEST_MOMENT); } Instant asked = REQUEST_MOMENT; From 792b3e33159c3e08cf1403914ba71d2aa37bff72 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 7 Sep 2026 16:30:04 +0100 Subject: [PATCH 9/9] Sign over the OWID alone A signature covers the OWID's own bytes without the signature field and nothing else. Signing and verifying over other OWIDs is gone from Creator, from every verification surface and from the tests, along with the chained interop fixtures, because nothing in production signs that way and an undocumented signing input is a liability for anyone implementing from the specification. --- README.md | 39 +++-------- .../java/com/swancommunity/owid/Creator.java | 52 +++----------- .../java/com/swancommunity/owid/Owid.java | 68 ++++++------------ .../owid/OwidSignatureStatus.java | 4 +- .../swancommunity/owid/PublicKeyFetch.java | 29 +++----- .../swancommunity/owid/PublicKeySchedule.java | 9 +-- .../ConstructionBoundaryTest.java | 16 ++--- .../owidconsumer/ReadmeExampleTest.java | 26 +------ .../com/swancommunity/owid/CreatorTest.java | 28 ++------ .../swancommunity/owid/DatedKeyFetchTest.java | 25 +++---- .../swancommunity/owid/DomainLengthTest.java | 8 +-- .../com/swancommunity/owid/FixturesTest.java | 69 ++++--------------- .../swancommunity/owid/ParseContractTest.java | 3 +- .../swancommunity/owid/PayloadLengthTest.java | 3 +- .../owid/PublicKeyScheduleTest.java | 19 +++-- .../owid/SignatureStatusTest.java | 34 ++++----- 16 files changed, 121 insertions(+), 311 deletions(-) diff --git a/README.md b/README.md index bdd337e..713cbaf 100644 --- a/README.md +++ b/README.md @@ -8,10 +8,9 @@ pure Java with no external runtime dependencies. ## Overview An OWID records that the entity operating a domain captured or generated a -payload at a date and time, with an ECDSA signature over the OWID and any -other OWIDs it was signed together with. OWIDs chain together to form -verifiable trees. The cryptography is ECDSA on the NIST P-256 curve (also -known as secp256r1 or prime256v1) with the SHA-256 hash. +payload at a date and time, with an ECDSA signature over the OWID. The +cryptography is ECDSA on the NIST P-256 curve (also known as secp256r1 or +prime256v1) with the SHA-256 hash. Read the [OWID project](https://github.com/SWAN-community/owid) to learn more about the concepts before looking into this implementation. This library @@ -106,8 +105,6 @@ import com.swancommunity.owid.Crypto; import com.swancommunity.owid.Owid; import com.swancommunity.owid.OwidParseResult; -import java.util.Collections; - // The creator operates a domain and holds the signing keys. Crypto crypto = Crypto.generate(); Creator creator = Creator.create("example.com", crypto); @@ -125,26 +122,13 @@ OwidParseResult result = Owid.parse(encoded); if (result.isSuccess()) { Owid copy = result.getValue(); String publicPem = crypto.publicKeyPem(); - boolean valid = copy.verifyWithPublicKey( - publicPem, Collections.emptyList()); + boolean valid = copy.verifyWithPublicKey(publicPem); } else { // result.getStatus() names which of the expected problems it was, and // result.getValue() is null. } ``` -Chaining covers other OWIDs with the same signature. The same others, in the -same order, must be supplied when verifying as were supplied when signing. - -```java -Owid root = creator.createString("root"); -Owid party = creator.createString("party", Collections.singletonList(root)); - -// Verifies with the root as the single other, fails without it. -party.verifyWithCrypto(crypto, Collections.singletonList(root)); // true -party.verifyWithCrypto(crypto, Collections.emptyList()); // false -``` - ## Verifying an identifier signed in an earlier week Creators rotate their signing key, weekly in the case of the 51Degrees cloud, @@ -305,7 +289,7 @@ as the outage it is. | `INVALID_SIGNATURE_LENGTH` | A signature field of the wrong length reached the check. A consumer cannot produce one, because reading and creation both settle the signature at 64 bytes. | | `KEY_UNAVAILABLE` | No key was supplied, or the one supplied cannot verify. | | `INVALID_KEY` | Key material arrived and cannot be decoded or used. | -| `IMPLEMENTATION_CAPACITY_EXCEEDED` | More work than this runtime can hold, which needs an OWID and its chain to approach the two gigabyte limit of a Java array. | +| `IMPLEMENTATION_CAPACITY_EXCEEDED` | More work than this runtime can hold, which needs an OWID whose payload approaches the two gigabyte limit of a Java array. | | `VERIFICATION_ERROR` | The check could not be completed for a reason that is not the identifier's fault. | ## Reading one OWID out of something longer @@ -386,7 +370,7 @@ copies, because a Java byte array is mutable. | `new Owid()`, then `setPayload`, then `creator.sign(owid)` | `creator.createBytes(payload)` | | `creator.signString(value)` | `creator.createString(value)` | | `creator.signBytes(value)` | `creator.createBytes(value)` | -| `new Owid()`, then `creator.signWithOthers(owid, others)` | `creator.createBytes(payload, others)` | +| `new Owid()`, then `creator.signWithOthers(owid, others)` | no replacement, a signature covers the OWID alone | | `owid.setVersion`, `setDomain`, `setDate`, `setPayload` | no replacement, the state is read only | | `Version.fromByte(b)` | no replacement, an unknown version byte is `UNSUPPORTED_VERSION` from a read, and version zero is `ABSENT_NODE` | @@ -412,8 +396,8 @@ domain, a null payload, or a field that cannot be serialized. returns zero padded lower case hexadecimal with no separator. `payloadAsBase64` returns the payload as base 64. `getPayloadLength` reports the payload size without copying it. - - `verifyWithCrypto` and `verifyWithPublicKey` return whether the signature, - covering this OWID and any others provided, is valid. + - `verifyWithCrypto` and `verifyWithPublicKey` return whether the signature + is valid. - `verify`, taking either the `Crypto` or the public key PEM, answers the same question with a status, keeping a key that could not be used apart from a signature that does not match. @@ -431,8 +415,7 @@ domain, a null payload, or a field that cannot be serialized. - `Creator` binds a domain to a signing `Crypto`. - `createString` and `createBytes` create a complete signed OWID, setting the domain to the creator domain, the date to the current time and the - version to the current version. Both take an optional list of other OWIDs - to cover with the same signature. + version to the current version. - `PublicKeyFetch` obtains the key of another creator from the well known end point on the domain the OWID carries. - `publicKeyUrl` builds the request, naming the version of the OWID and the @@ -534,8 +517,8 @@ mvn test ``` The tests round trip the canonical wire format vectors byte for byte, verify -cross language signed fixtures including the chained case, confirm that a -flipped signature byte fails verification, and cover the binary write +cross language signed fixtures, confirm that a flipped signature byte fails +verification, and cover the binary write helpers, the crypto, the creator, and the end point helpers. They also cover the parse contract, being every status the reading surfaces report together with a run of malformed buffers that must never throw, the framed read and diff --git a/src/main/java/com/swancommunity/owid/Creator.java b/src/main/java/com/swancommunity/owid/Creator.java index 6295796..4a1bbe5 100644 --- a/src/main/java/com/swancommunity/owid/Creator.java +++ b/src/main/java/com/swancommunity/owid/Creator.java @@ -19,8 +19,6 @@ import java.nio.charset.StandardCharsets; import java.time.Instant; import java.time.temporal.ChronoUnit; -import java.util.Collections; -import java.util.List; /** * Needed to create new OWIDs. @@ -123,44 +121,15 @@ public Crypto crypto() { * encoded, or the signing operation fails */ public Owid createString(String value) throws OwidException { - return createString(value, Collections.emptyList()); - } - - /** - * Creates a new signed OWID for this creator carrying the bytes as the - * payload. - * - * @param value the payload bytes - * @return the signed OWID - * @throws OwidException if the payload is null, a field cannot be - * encoded, or the signing operation fails - */ - public Owid createBytes(byte[] value) throws OwidException { - return createBytes(value, Collections.emptyList()); - } - - /** - * Creates a new signed OWID carrying the string as the UTF-8 payload, - * with the other OWIDs covered by the same signature so that a tree can - * be verified as a whole. The same others, in the same order, must be - * passed when verifying. - * - * @param value the payload string - * @param others the other OWIDs to cover with the signature - * @return the signed OWID - * @throws OwidException see {@link #createString(String)} - */ - public Owid createString(String value, List others) - throws OwidException { if (value == null) { throw new OwidException("payload is null"); } - return createBytes(value.getBytes(StandardCharsets.UTF_8), others); + return createBytes(value.getBytes(StandardCharsets.UTF_8)); } /** - * Creates a new signed OWID carrying the bytes as the payload, with the - * other OWIDs covered by the same signature. + * Creates a new signed OWID for this creator carrying the bytes as the + * payload. The signature covers the fields of the OWID and nothing else. * *

This is one of only two ways an OWID reaches calling code, the other * being a successful read of a complete serialized one. The creator owns @@ -168,24 +137,19 @@ public Owid createString(String value, List others) * supplies the payload and nothing else, so there is no moment at which a * partly built OWID exists for anyone to hold or pass on.

* - * @param value the payload bytes - * @param others the other OWIDs to cover with the signature + * @param value the payload bytes * @return the signed OWID - * @throws OwidException see {@link #createBytes(byte[])} + * @throws OwidException if the payload is null, a field cannot be + * encoded, or the signing operation fails */ - public Owid createBytes(byte[] value, List others) - throws OwidException { + public Owid createBytes(byte[] value) throws OwidException { if (value == null) { throw new OwidException("payload is null"); } - if (others == null) { - throw new OwidException("others is null"); - } Version version = Version.current(); Instant date = Instant.now().truncatedTo(ChronoUnit.MINUTES); byte[] payload = value.clone(); - byte[] data = Owid.dataForCrypto( - version, domain, date, payload, others); + byte[] data = Owid.dataForCrypto(version, domain, date, payload); byte[] signature = crypto.signByteArray(data); if (signature.length != Owid.SIGNATURE_LENGTH) { throw Io.invalidSignatureLength(signature.length); diff --git a/src/main/java/com/swancommunity/owid/Owid.java b/src/main/java/com/swancommunity/owid/Owid.java index 51f06cd..7f0b09a 100644 --- a/src/main/java/com/swancommunity/owid/Owid.java +++ b/src/main/java/com/swancommunity/owid/Owid.java @@ -24,14 +24,12 @@ import java.time.Instant; import java.util.Arrays; import java.util.Base64; -import java.util.List; /** * OWID structure which can be used as a node in a tree. * *

An OWID records that the processor operating the domain handled the - * payload, and any other OWIDs covered by the signature, at the date and time - * given.

+ * payload at the date and time given.

* *

An OWID is only worth anything because it is signed, so a caller cannot * build one. An instance reaches calling code by one of two routes, being @@ -109,7 +107,7 @@ public final class Owid { * *

A successful read says the bytes are a structurally valid OWID. It * says nothing about whether the signature is genuine, which is a - * separate question answered by {@link #verify(Crypto, List)}.

+ * separate question answered by {@link #verify(Crypto)}.

* * @param value the base 64 encoded OWID, which may be null * @return the OWID and {@link OwidParseStatus#PARSED}, or no value and @@ -281,12 +279,11 @@ private static void writeNoSignature(ByteArrayOutputStream buffer, } /** - * Builds the byte array used for signing and verification. Contains the - * fields of this OWID without the signature, followed by the complete byte - * form of each of the others in the order provided. + * Builds the byte array used for signing and verification, being the + * fields of this OWID without the signature and nothing else. */ - byte[] dataForCrypto(List others) throws OwidException { - return dataForCrypto(version, domain, date, payload, others); + byte[] dataForCrypto() throws OwidException { + return dataForCrypto(version, domain, date, payload); } /** @@ -296,17 +293,10 @@ byte[] dataForCrypto(List others) throws OwidException { * then builds the finished OWID in one step. */ static byte[] dataForCrypto(Version version, String domain, Instant date, - byte[] payload, List others) throws OwidException { - int length = byteCount(version, domain, payload, null, false); - for (Owid other : others) { - length = addLength(length, other.byteCount(true)); - } - ExactByteArrayOutputStream buffer = - new ExactByteArrayOutputStream(length); + byte[] payload) throws OwidException { + ExactByteArrayOutputStream buffer = new ExactByteArrayOutputStream( + byteCount(version, domain, payload, null, false)); writeNoSignature(buffer, version, domain, date, payload); - for (Owid other : others) { - other.toBuffer(buffer); - } return buffer.toExactByteArray(); } @@ -428,38 +418,27 @@ public long ageMinutes() { } /** - * Verifies this OWID, and any others that were included when it was - * signed, using the crypto instance provided. Pass an empty list for the - * others when the OWID was signed on its own. + * Verifies this OWID using the crypto instance provided. * * @param crypto the crypto instance holding the public key - * @param others the other OWIDs that were signed together with this one, - * in the same order as when signed * @return true if the signature verifies, false otherwise * @throws OwidException if the crypto instance cannot verify, or a field * cannot be encoded */ - public boolean verifyWithCrypto(Crypto crypto, List others) - throws OwidException { - byte[] data = dataForCrypto(others); - return crypto.verifyByteArray(data, signature); + public boolean verifyWithCrypto(Crypto crypto) throws OwidException { + return crypto.verifyByteArray(dataForCrypto(), signature); } /** - * Verifies this OWID, and any others that were included when it was - * signed, using the public key in SPKI PEM form provided. + * Verifies this OWID using the public key in SPKI PEM form provided. * * @param publicPem the public key in SPKI PEM form - * @param others the other OWIDs that were signed together with this - * one, in the same order as when signed * @return true if the signature verifies, false otherwise * @throws OwidException if the PEM is not a valid public key, or a field * cannot be encoded */ - public boolean verifyWithPublicKey(String publicPem, List others) - throws OwidException { - Crypto crypto = Crypto.newVerifyOnly(publicPem); - return verifyWithCrypto(crypto, others); + public boolean verifyWithPublicKey(String publicPem) throws OwidException { + return verifyWithCrypto(Crypto.newVerifyOnly(publicPem)); } /** @@ -472,11 +451,9 @@ public boolean verifyWithPublicKey(String publicPem, List others) * * @param crypto the crypto instance holding the public key, which may be * null when no key could be obtained - * @param others the other OWIDs that were signed together with this one, - * in the same order as when signed * @return the outcome of the check */ - public OwidVerificationResult verify(Crypto crypto, List others) { + public OwidVerificationResult verify(Crypto crypto) { if (crypto == null || crypto.canVerify() == false) { return OwidVerificationResult.of( OwidSignatureStatus.KEY_UNAVAILABLE); @@ -487,7 +464,7 @@ public OwidVerificationResult verify(Crypto crypto, List others) { } byte[] data; try { - data = dataForCrypto(others); + data = dataForCrypto(); } catch (CapacityException e) { return OwidVerificationResult.of( OwidSignatureStatus.IMPLEMENTATION_CAPACITY_EXCEEDED); @@ -508,8 +485,8 @@ public OwidVerificationResult verify(Crypto crypto, List others) { } /** - * The same question as {@link #verify(Crypto, List)}, starting from the - * public key in SPKI PEM form. + * The same question as {@link #verify(Crypto)}, starting from the public + * key in SPKI PEM form. * *

Key material that cannot be decoded reports * {@link OwidSignatureStatus#INVALID_KEY}, because the fault is in the @@ -517,12 +494,9 @@ public OwidVerificationResult verify(Crypto crypto, List others) { * * @param publicPem the public key in SPKI PEM form, which may be null * when no key could be obtained - * @param others the other OWIDs that were signed together with this - * one, in the same order as when signed * @return the outcome of the check */ - public OwidVerificationResult verify(String publicPem, - List others) { + public OwidVerificationResult verify(String publicPem) { if (publicPem == null || publicPem.trim().isEmpty()) { return OwidVerificationResult.of( OwidSignatureStatus.KEY_UNAVAILABLE); @@ -534,7 +508,7 @@ public OwidVerificationResult verify(String publicPem, return OwidVerificationResult.of( OwidSignatureStatus.INVALID_KEY); } - return verify(crypto, others); + return verify(crypto); } /** diff --git a/src/main/java/com/swancommunity/owid/OwidSignatureStatus.java b/src/main/java/com/swancommunity/owid/OwidSignatureStatus.java index e24153e..976baec 100644 --- a/src/main/java/com/swancommunity/owid/OwidSignatureStatus.java +++ b/src/main/java/com/swancommunity/owid/OwidSignatureStatus.java @@ -75,8 +75,8 @@ public enum OwidSignatureStatus { /** * The work required is more than this runtime can hold. * - *

Not covered by a test, because reaching it needs an OWID and its - * chain to approach the two gigabyte limit of a Java array, which cannot + *

Not covered by a test, because reaching it needs an OWID whose + * payload approaches the two gigabyte limit of a Java array, which cannot * be built in a test suite that has to run on an ordinary machine. The * path to it is real, being the overflow guard on the serialized length, * which raises a distinct exception so this status does not have to be diff --git a/src/main/java/com/swancommunity/owid/PublicKeyFetch.java b/src/main/java/com/swancommunity/owid/PublicKeyFetch.java index 31f2cf0..53da3e1 100644 --- a/src/main/java/com/swancommunity/owid/PublicKeyFetch.java +++ b/src/main/java/com/swancommunity/owid/PublicKeyFetch.java @@ -317,14 +317,11 @@ public static CompletableFuture publicKeyPem(Owid owid, * * @param owid the OWID to check * @param scheme the scheme to use, normally {@code https} - * @param others the other OWIDs that were signed together with this one, - * in the same order as when signed - * * @return the outcome of the check, through a future */ public static CompletableFuture verify(Owid owid, - String scheme, List others) { - return verify(owid, scheme, others, DEFAULT_TRANSPORT); + String scheme) { + return verify(owid, scheme, DEFAULT_TRANSPORT); } /** @@ -346,14 +343,11 @@ public static CompletableFuture verify(Owid owid, * * @param owid the OWID to check * @param scheme the scheme to use, normally {@code https} - * @param others the other OWIDs that were signed together with this - * one, in the same order as when signed - * * @param transport the transport to make the request with * @return the outcome of the check, through a future */ public static CompletableFuture verify(Owid owid, - String scheme, List others, PublicKeyTransport transport) { + String scheme, PublicKeyTransport transport) { String url; try { url = publicKeyUrl(owid, scheme); @@ -362,7 +356,7 @@ public static CompletableFuture verify(Owid owid, OwidVerificationResult.of( OwidSignatureStatus.KEY_UNAVAILABLE)); } - return verifyAtUrl(owid, url, others, transport); + return verifyAtUrl(owid, url, transport); } /** @@ -390,8 +384,8 @@ static int cachedKeyCount() { } /** - * The work {@link #verify(Owid, String, List, PublicKeyTransport)} does - * once the URL is known, kept apart so that the tests drive the real + * The work {@link #verify(Owid, String, PublicKeyTransport)} does once + * the URL is known, kept apart so that the tests drive the real * fetch against a key end point the tests can stand up locally rather * than against a near copy of the fetch. * @@ -404,7 +398,7 @@ static int cachedKeyCount() { * signature does not match.

*/ static CompletableFuture verifyAtUrl( - final Owid owid, final String url, final List others, + final Owid owid, final String url, final PublicKeyTransport transport) { final long minute = Io.minutesSinceBase(owid.getDate()); return keyAtUrl(url, owid.getDomain(), transport) @@ -413,15 +407,14 @@ static CompletableFuture verifyAtUrl( return CompletableFuture.completedFuture( OwidVerificationResult.of(statusOf(failure))); } - OwidVerificationResult result = owid.verify(answer.pem, - others); + OwidVerificationResult result = owid.verify(answer.pem); if (result.getStatus() != OwidSignatureStatus.SIGNATURE_INVALID || minute < 0) { return CompletableFuture.completedFuture(result); } return neighbourVerifies(owid, minute, url, answer, - others, transport).thenApply(verified -> { + transport).thenApply(verified -> { if (verified) { return OwidVerificationResult.of( OwidSignatureStatus.SIGNATURE_VALID); @@ -456,7 +449,7 @@ static CompletableFuture verifyAtUrl( */ private static CompletableFuture neighbourVerifies( final Owid owid, long minute, String url, final KeyAnswer tried, - final List others, PublicKeyTransport transport) { + PublicKeyTransport transport) { if (tried.known == false) { return CompletableFuture.completedFuture(false); } @@ -480,7 +473,7 @@ private static CompletableFuture neighbourVerifies( owid.getDomain(), transport) .handle((neighbour, failure) -> failure == null && neighbour.pem.equals(tried.pem) == false - && owid.verify(neighbour.pem, others).getStatus() + && owid.verify(neighbour.pem).getStatus() == OwidSignatureStatus.SIGNATURE_VALID); }); } diff --git a/src/main/java/com/swancommunity/owid/PublicKeySchedule.java b/src/main/java/com/swancommunity/owid/PublicKeySchedule.java index 1bf0a38..b82c3a8 100644 --- a/src/main/java/com/swancommunity/owid/PublicKeySchedule.java +++ b/src/main/java/com/swancommunity/owid/PublicKeySchedule.java @@ -200,15 +200,12 @@ public DatedPublicKey keyFor(Owid owid) { * Asks whether the signature on the OWID is genuine, using the key that * was in force when the OWID was signed. * - * @param owid the OWID to check - * @param others the other OWIDs that were signed together with this one, - * in the same order as when signed - * + * @param owid the OWID to check * @return the outcome of the check, which is * {@link OwidSignatureStatus#KEY_UNAVAILABLE} where the schedule * holds no key for the date */ - public OwidVerificationResult verify(Owid owid, List others) { + public OwidVerificationResult verify(Owid owid) { if (owid == null) { return OwidVerificationResult.of( OwidSignatureStatus.KEY_UNAVAILABLE); @@ -218,6 +215,6 @@ public OwidVerificationResult verify(Owid owid, List others) { return OwidVerificationResult.of( OwidSignatureStatus.KEY_UNAVAILABLE); } - return owid.verify(key.getPublicKeyPem(), others); + return owid.verify(key.getPublicKeyPem()); } } diff --git a/src/test/java/com/example/owidconsumer/ConstructionBoundaryTest.java b/src/test/java/com/example/owidconsumer/ConstructionBoundaryTest.java index 0a40473..fb4fd5c 100644 --- a/src/test/java/com/example/owidconsumer/ConstructionBoundaryTest.java +++ b/src/test/java/com/example/owidconsumer/ConstructionBoundaryTest.java @@ -34,7 +34,6 @@ import java.lang.reflect.Method; import java.lang.reflect.Modifier; import java.nio.charset.StandardCharsets; -import java.util.Collections; import org.junit.jupiter.api.Test; /** @@ -142,24 +141,22 @@ void writingIntoReturnedArraysDoesNotAlterTheOwid() throws OwidException { "writing into the copy should not reach the OWID"); assertArrayEquals(encoded, owid.asByteArray(), "the OWID should serialise to the same bytes"); - assertTrue(owid.verifyWithCrypto(crypto, Collections.emptyList()), + assertTrue(owid.verifyWithCrypto(crypto), "the OWID should still verify"); } /** * A library user can still do everything the old surface allowed, by the - * new route. Creating, chaining, serialising, reading back and verifying - * all work without ever naming a constructor. + * new route. Creating, serialising, reading back and verifying all work + * without ever naming a constructor. */ @Test void aLibraryUserCanStillDoEverything() throws OwidException { Crypto crypto = Crypto.generate(); Creator creator = Creator.create("example.com", crypto); - Owid root = creator.createString("root"); Owid party = creator.createBytes( - "party".getBytes(StandardCharsets.UTF_8), - Collections.singletonList(root)); + "party".getBytes(StandardCharsets.UTF_8)); OwidParseResult result = Owid.parse(party.asBase64()); assertEquals(OwidParseStatus.PARSED, result.getStatus(), @@ -167,9 +164,8 @@ void aLibraryUserCanStillDoEverything() throws OwidException { Owid copy = result.getValue(); assertEquals(party, copy, "should read back an equal OWID"); - assertTrue(copy.verifyWithPublicKey(crypto.publicKeyPem(), - Collections.singletonList(root)), - "should verify with the same others"); + assertTrue(copy.verifyWithPublicKey(crypto.publicKeyPem()), + "should verify with the creator's public key"); assertEquals("party", copy.payloadAsString(), "should carry the payload it was created with"); } diff --git a/src/test/java/com/example/owidconsumer/ReadmeExampleTest.java b/src/test/java/com/example/owidconsumer/ReadmeExampleTest.java index cab7c3e..593eebe 100644 --- a/src/test/java/com/example/owidconsumer/ReadmeExampleTest.java +++ b/src/test/java/com/example/owidconsumer/ReadmeExampleTest.java @@ -34,7 +34,6 @@ import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.concurrent.CompletableFuture; import org.junit.jupiter.api.Test; @@ -73,7 +72,7 @@ void createSerializeReadBackAndVerify() throws OwidException { Owid copy = result.getValue(); String publicPem = crypto.publicKeyPem(); boolean valid = copy.verifyWithPublicKey( - publicPem, Collections.emptyList()); + publicPem); assertTrue(valid, "the OWID read back should verify"); assertEquals("Hello World", copy.payloadAsString(), @@ -121,25 +120,6 @@ void readingOneOwidAfterAnother() throws OwidException { "the two OWIDs should account for every byte"); } - @Test - void chainingCoversTheOtherOwids() throws OwidException { - Crypto crypto = Crypto.generate(); - Creator creator = Creator.create("example.com", crypto); - - Owid root = creator.createString("root"); - Owid party = creator.createString( - "party", Collections.singletonList(root)); - - // Verifies with the root as the single other, fails without it. - assertTrue( - party.verifyWithCrypto( - crypto, Collections.singletonList(root)), - "should verify with the same others"); - assertFalse( - party.verifyWithCrypto(crypto, Collections.emptyList()), - "should fail to verify without the others"); - } - /** * The schedule example from the README, choosing between two weekly keys * by the date the identifier carries. @@ -159,7 +139,7 @@ void aScheduleChoosesTheKeyThatWasInForce() throws OwidException { DatedPublicKey.of( Instant.parse("2026-08-31T00:00:00Z"), thisWeekPem))); OwidVerificationResult result = schedule.verify( - owid, Collections.emptyList()); + owid); assertEquals(OwidSignatureStatus.SIGNATURE_VALID, result.getStatus(), "should choose the key that was in force and verify"); @@ -182,7 +162,7 @@ void fetchingTheKeyFromTheCreatorDomain() throws OwidException { CompletableFuture pending = PublicKeyFetch.verify( - owid, "https", Collections.emptyList()); + owid, "https"); // The call returns at once and the request runs on a background // thread. Continue from the future, or join it where waiting is // acceptable, as it is here. diff --git a/src/test/java/com/swancommunity/owid/CreatorTest.java b/src/test/java/com/swancommunity/owid/CreatorTest.java index 2662a91..862b342 100644 --- a/src/test/java/com/swancommunity/owid/CreatorTest.java +++ b/src/test/java/com/swancommunity/owid/CreatorTest.java @@ -22,7 +22,6 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import java.util.Collections; import org.junit.jupiter.api.Test; /** Unit tests for the creator signing behaviour. */ @@ -56,7 +55,7 @@ void signSetsDomainVersionAndVerifies() throws OwidException { "should set the current version"); assertEquals(Owid.SIGNATURE_LENGTH, owid.getSignature().length, "should produce a 64 byte signature"); - assertTrue(owid.verifyWithCrypto(crypto, Collections.emptyList()), + assertTrue(owid.verifyWithCrypto(crypto), "the signed OWID should verify"); } @@ -67,8 +66,8 @@ void signAndSelfVerifyThroughPem() throws OwidException { Owid owid = creator.createString("payload"); String encoded = owid.asBase64(); Owid copy = ParseAssert.parsed(Owid.parse(encoded)); - assertTrue(copy.verifyWithPublicKey(crypto.publicKeyPem(), - Collections.emptyList()), "the decoded OWID should verify"); + assertTrue(copy.verifyWithPublicKey(crypto.publicKeyPem()), + "the decoded OWID should verify"); } @Test @@ -79,25 +78,10 @@ void tamperedSignedOwidFails() throws OwidException { byte[] bytes = owid.asByteArray(); bytes[bytes.length - 1] ^= 0x01; Owid tampered = ParseAssert.parsed(Owid.parse(bytes)); - assertFalse(tampered.verifyWithCrypto(crypto, Collections.emptyList()), + assertFalse(tampered.verifyWithCrypto(crypto), "a tampered signature should not verify"); } - @Test - void createWithOthersRoundTrips() throws OwidException { - Crypto crypto = Crypto.generate(); - Creator creator = Creator.create("example.com", crypto); - Owid root = creator.createString("root"); - Owid party = creator.createString( - "party", Collections.singletonList(root)); - assertTrue( - party.verifyWithCrypto( - crypto, Collections.singletonList(root)), - "should verify with the same others"); - assertFalse(party.verifyWithCrypto(crypto, Collections.emptyList()), - "should fail to verify without the others"); - } - /** * A creator refuses a null payload rather than producing an OWID with * nothing in it. This is a caller mistake in code rather than data that @@ -130,7 +114,7 @@ void payloadHandedToCreatorIsCopied() throws OwidException { assertArrayEquals(new byte[] {1, 2, 3}, owid.getPayload(), "the OWID should keep the bytes it was signed over"); - assertTrue(owid.verifyWithCrypto(crypto, Collections.emptyList()), + assertTrue(owid.verifyWithCrypto(crypto), "the OWID should still verify"); } @@ -140,7 +124,7 @@ void fromPrivatePemCreatesWorkingCreator() throws OwidException { Creator creator = Creator.fromPrivatePem("example.com", crypto.privateKeyPem()); Owid owid = creator.createString("data"); - assertTrue(owid.verifyWithCrypto(crypto, Collections.emptyList()), + assertTrue(owid.verifyWithCrypto(crypto), "should sign with the imported key"); } } diff --git a/src/test/java/com/swancommunity/owid/DatedKeyFetchTest.java b/src/test/java/com/swancommunity/owid/DatedKeyFetchTest.java index 6dfd0d8..7305ac3 100644 --- a/src/test/java/com/swancommunity/owid/DatedKeyFetchTest.java +++ b/src/test/java/com/swancommunity/owid/DatedKeyFetchTest.java @@ -62,9 +62,6 @@ */ class DatedKeyFetchTest { - /** No other OWIDs were covered by the signature on the fixture. */ - private static final List ALONE = Collections.emptyList(); - /** The transport a caller gets without naming one. */ private static final PublicKeyTransport HTTP = new HttpUrlConnectionTransport(); @@ -98,7 +95,7 @@ private KeyEndPoint endPoint(KeyEndPoint.Answer answer) /** The status a fetch through the default transport ends with. */ private static OwidSignatureStatus statusAt(Owid owid, String url) { - return PublicKeyFetch.verifyAtUrl(owid, url, ALONE, HTTP).join() + return PublicKeyFetch.verifyAtUrl(owid, url, HTTP).join() .getStatus(); } @@ -370,7 +367,7 @@ void twoRequestsInFlightForOneKeyMakeOneRequest() assertEquals(first.join(), second.join(), "both callers get the one key that was fetched"); assertEquals(OwidSignatureStatus.SIGNATURE_VALID, - PublicKeyFetch.verifyAtUrl(owid, url, ALONE, held).join() + PublicKeyFetch.verifyAtUrl(owid, url, held).join() .getStatus(), "the key that arrived verifies the identifier"); assertEquals(1, held.requests.get(), @@ -434,7 +431,7 @@ void theRequestRunsOnTheExecutorGiven() "the thread that asked is not the one that fetches"); assertEquals(OwidSignatureStatus.SIGNATURE_VALID, owid.verify(PublicKeyResponse.parse(fetch.join()) - .getPublicKey(), ALONE).getStatus(), + .getPublicKey()).getStatus(), "the key fetched on the executor verifies the identifier"); } @@ -462,8 +459,7 @@ void aRequestTheExecutorRefusesIsKeyUnavailable() assertEquals(owid.getDomain(), failure.getDomain(), "the domain asked of is carried"); assertEquals(OwidSignatureStatus.KEY_UNAVAILABLE, - PublicKeyFetch.verifyAtUrl(owid, endPoint.urlFor(owid), - ALONE, transport).join().getStatus(), + PublicKeyFetch.verifyAtUrl(owid, endPoint.urlFor(owid), transport).join().getStatus(), "a check through the refusing executor is unjudged"); assertTrue(endPoint.dates().isEmpty(), "the end point was never reached"); @@ -478,7 +474,7 @@ void aMissingTransportIsRefused() OwidException.class, "the key cannot be fetched with no transport"); assertEquals(OwidSignatureStatus.KEY_UNAVAILABLE, - PublicKeyFetch.verify(owid, "https", ALONE, null).join() + PublicKeyFetch.verify(owid, "https", null).join() .getStatus(), "a check with no transport is unjudged"); assertThrows(IllegalArgumentException.class, @@ -530,7 +526,7 @@ void aDomainThatIsNotADomainNameIsRefused() throws OwidException { OwidException.class, "a URL that cannot be built fails the fetch"); assertEquals(OwidSignatureStatus.KEY_UNAVAILABLE, - PublicKeyFetch.verify(owid, "https", ALONE).join() + PublicKeyFetch.verify(owid, "https").join() .getStatus(), "a URL that cannot be built leaves the signature unjudged"); } @@ -550,7 +546,7 @@ void aDomainThatIsNotADomainNameIsRefused() throws OwidException { void aSchemeThatIsNotHttpIsKeyUnavailable() throws OwidException { assertEquals(OwidSignatureStatus.KEY_UNAVAILABLE, PublicKeyFetch.verify( - KeyFixtures.identifier(), "mailto", ALONE).join() + KeyFixtures.identifier(), "mailto").join() .getStatus(), "a scheme that fetches no key leaves the signature unjudged"); } @@ -832,7 +828,7 @@ private static Owid signedAt(String domain, Instant moment, Crypto crypto) throws OwidException { byte[] payload = "payload".getBytes(StandardCharsets.UTF_8); byte[] data = Owid.dataForCrypto(Version.VERSION3, domain, moment, - payload, ALONE); + payload); return new Owid(Version.VERSION3, domain, moment, payload, crypto.signByteArray(data)); } @@ -1097,7 +1093,7 @@ void aKeyTheCreatorSaysWasNotInForceLeavesTheSignatureUnjudged() private static OwidSignatureStatus statusOf(Owid owid, PublicKeyTransport transport) throws OwidException { return PublicKeyFetch.verifyAtUrl(owid, - PublicKeyFetch.publicKeyUrl(owid, "https"), ALONE, transport) + PublicKeyFetch.publicKeyUrl(owid, "https"), transport) .join().getStatus(); } @@ -1172,8 +1168,7 @@ void manyThreadsVerifyingOneOwidTogetherMakeOneRequest() Thread thread = new Thread(() -> { try { start.await(); - statuses.add(PublicKeyFetch.verifyAtUrl(owid, url, ALONE, - transport).join().getStatus()); + statuses.add(PublicKeyFetch.verifyAtUrl(owid, url, transport).join().getStatus()); } catch (Exception e) { throw new IllegalStateException(e); } diff --git a/src/test/java/com/swancommunity/owid/DomainLengthTest.java b/src/test/java/com/swancommunity/owid/DomainLengthTest.java index f25ab6e..9f45c40 100644 --- a/src/test/java/com/swancommunity/owid/DomainLengthTest.java +++ b/src/test/java/com/swancommunity/owid/DomainLengthTest.java @@ -27,7 +27,6 @@ import java.lang.management.ThreadMXBean; import java.nio.charset.StandardCharsets; import java.util.Arrays; -import java.util.Collections; import org.junit.jupiter.api.Test; /** @@ -221,7 +220,7 @@ void maximumLengthDomainWritten() throws OwidException { assertEquals(domain, parsed.getDomain(), "should round trip the domain the creator holds"); assertEquals(signed, parsed, "should parse to an equal OWID"); - assertTrue(parsed.verifyWithCrypto(crypto, Collections.emptyList()), + assertTrue(parsed.verifyWithCrypto(crypto), "the parsed OWID should still verify"); } @@ -272,8 +271,7 @@ void overMaximumLengthDomainRefusedWhenAssemblingDataToSign() { OwidException thrown = assertThrows(OwidException.class, () -> Owid.dataForCrypto(Version.current(), domain, - Io.baseDate(), PAYLOAD, - Collections.emptyList()), + Io.baseDate(), PAYLOAD), "should refuse to assemble the bytes that would be signed"); assertNamesMaximum(thrown); @@ -320,7 +318,7 @@ void libraryOutputParses() throws OwidException { assertEquals("51d.es", parsed.getDomain(), "should read the domain the library wrote"); assertEquals(original, parsed, "should parse to an equal OWID"); - assertTrue(parsed.verifyWithCrypto(crypto, Collections.emptyList()), + assertTrue(parsed.verifyWithCrypto(crypto), "the parsed OWID should still verify"); } } diff --git a/src/test/java/com/swancommunity/owid/FixturesTest.java b/src/test/java/com/swancommunity/owid/FixturesTest.java index 9b94cb5..2ac1133 100644 --- a/src/test/java/com/swancommunity/owid/FixturesTest.java +++ b/src/test/java/com/swancommunity/owid/FixturesTest.java @@ -20,15 +20,13 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import java.util.Base64; -import java.util.Collections; -import java.util.List; import org.junit.jupiter.api.Test; /** * Cross language signed fixtures. Each set of OWIDs was produced by a separate * implementation and signed with the matching public key. The test verifies - * the real signatures, the chain relationship, and that flipping the last - * signature byte breaks verification. + * the real signatures and that flipping the last signature byte breaks + * verification. */ class FixturesTest { @@ -38,16 +36,11 @@ private static final class Fixtures { private final String spki; private final String simple; private final String utf8; - private final String chainParty; - private final String chainRoot; - Fixtures(String spki, String simple, String utf8, - String chainParty, String chainRoot) { + Fixtures(String spki, String simple, String utf8) { this.spki = spki; this.simple = simple; this.utf8 = utf8; - this.chainParty = chainParty; - this.chainRoot = chainRoot; } String spki() { @@ -61,14 +54,6 @@ String simple() { String utf8() { return utf8; } - - String chainParty() { - return chainParty; - } - - String chainRoot() { - return chainRoot; - } } private static final String UTF8_TEXT = "Zürich ❤ OWID £€"; @@ -82,11 +67,7 @@ String chainRoot() { + "Rk8U4fYacm0Ck4aOxoRDJPK/QrKavqZqCf7cCKbNuJ0aA7GhVeuy4ojeSzNX56Qn", "A2dvLnN3YW4tZGVtby51awA/vTMAFgAAAFrDvHJpY2gg4p2kIE9XSUQgwqPigqzx" + "Y+4QgUGt84xC9HxHmHXDt+wcB0Y9a6E+Txm2F147Qacbp0CtrF8x7QCWZfkcKCKN" - + "GSM8hYZEfYjJtViG+tA+", - "A2dvLnN3YW4tZGVtby51awA/vTMABQAAAHBhcnR5l7NyNmFw2lxqc4DKJWoq0UVd" - + "5ujGV/+fvVxqYTRlwCFxaSuwvnhLQQHjX5spxWb4O08IeuiuGCat1WFB/Wqlyw==", - "A2dvLnN3YW4tZGVtby51awA/vTMABAAAAHJvb3R/bEqzG8gAy9yTF1UMEtOlYXBB" - + "mn3a20jxXq5NmxIC8iuZvduOXKMf+K8VoAapkWwfpoDKQHS09IhljasZqC0k"); + + "GSM8hYZEfYjJtViG+tA+"); private static final Fixtures DOTNET = new Fixtures( "-----BEGIN PUBLIC KEY-----\n" @@ -98,13 +79,7 @@ String chainRoot() { + "Wzyh0w==", "A2RvdG5ldC5zd2FuLWRlbW8udWsAPb0zABYAAABaw7xyaWNoIOKdpCBPV0lEIMKj" + "4oKsVuaeaDUej0sF+cHfYj/icDBmlBLOviC6ZE28am8EtY+IGuesFcg2rKMybcsA" - + "xMmnrDtF2xsk1cJvHgoIYpSJJQ==", - "A2RvdG5ldC5zd2FuLWRlbW8udWsAPb0zAAUAAABwYXJ0eXtD6H4R7GbvRyFU+bCK" - + "gjMAZFFm8KHln80XPwQOBb/Ub9EZfE4Ml3ueRkKX51+MD98RFgTSmjbqrAnzFkLl" - + "ilA=", - "A2RvdG5ldC5zd2FuLWRlbW8udWsAPb0zAAQAAAByb290fErj2LccPYCduWUW8vY2" - + "aBjrecDfnTpVpv3+SESJMFW5pcuPKEQik2rC0fWEoB5Vr6e0k5inrhUGiF2c2Y2Y" - + "Dw=="); + + "xMmnrDtF2xsk1cJvHgoIYpSJJQ=="); private static final Fixtures RUST = new Fixtures( "-----BEGIN PUBLIC KEY-----\n" @@ -116,11 +91,7 @@ String chainRoot() { + "5CI=", "A3J1c3Quc3dhbi1kZW1vLnVrAD69MwAWAAAAWsO8cmljaCDinaQgT1dJRCDCo+KC" + "rDHenDds+W587AzXpBb94gmLOloeBJTlHnjCkez4Dz2yAPtjcoQ6M/ZUWDIobtJH" - + "E5n9a81pTsn/Kvi74Azzx4s=", - "A3J1c3Quc3dhbi1kZW1vLnVrAD69MwAFAAAAcGFydHmJ7qaxWgIZUHmGOQb2xC+R" - + "uZNwrkMmo1SA9/MfI4SoEpRYdnteXAKUQXxTOK3lmQ3Qz3UwBB6gBb3Q8hi1Wx0R", - "A3J1c3Quc3dhbi1kZW1vLnVrAD69MwAEAAAAcm9vdFd0+QLaBLGPyBrQO+VNunBI" - + "QZzw8/lhEiDOKTx36Dc93A0n0fzPDMt/C+BdWMqhnL4nVvyurb3IHR7DUAmgmO0="); + + "E5n9a81pTsn/Kvi74Azzx4s="); /** Returns a copy of the bytes with the final byte flipped. */ private static byte[] flipLastByte(byte[] bytes) { @@ -131,42 +102,26 @@ private static byte[] flipLastByte(byte[] bytes) { private void runFixtures(Fixtures fixtures) throws OwidException { Crypto crypto = Crypto.newVerifyOnly(fixtures.spki()); - List none = Collections.emptyList(); Owid simple = ParseAssert.parsed(Owid.parse(fixtures.simple())); - assertTrue(simple.verifyWithCrypto(crypto, none), + assertTrue(simple.verifyWithCrypto(crypto), "simple should verify"); - assertTrue(simple.verifyWithPublicKey(fixtures.spki(), none), + assertTrue(simple.verifyWithPublicKey(fixtures.spki()), "simple should verify by public key PEM"); Owid utf8 = ParseAssert.parsed(Owid.parse(fixtures.utf8())); - assertTrue(utf8.verifyWithCrypto(crypto, none), "utf8 should verify"); + assertTrue(utf8.verifyWithCrypto(crypto), "utf8 should verify"); org.junit.jupiter.api.Assertions.assertEquals(UTF8_TEXT, utf8.payloadAsString(), "utf8 payload text should match"); - Owid root = ParseAssert.parsed(Owid.parse(fixtures.chainRoot())); - assertTrue(root.verifyWithCrypto(crypto, none), - "chain root should verify alone"); - - Owid party = ParseAssert.parsed(Owid.parse(fixtures.chainParty())); - assertTrue(party.verifyWithCrypto(crypto, Collections.singletonList(root)), - "chain party should verify with the root as the other"); - assertFalse(party.verifyWithCrypto(crypto, none), - "chain party should fail with no others"); - // Each fixture with its last signature byte flipped must fail. - for (String encoded : new String[] {fixtures.simple(), fixtures.utf8(), - fixtures.chainRoot()}) { + for (String encoded : new String[] {fixtures.simple(), + fixtures.utf8()}) { byte[] tampered = flipLastByte(Base64.getMimeDecoder().decode(encoded)); Owid owid = ParseAssert.parsed(Owid.parse(tampered)); - assertFalse(owid.verifyWithCrypto(crypto, none), + assertFalse(owid.verifyWithCrypto(crypto), "a flipped signature byte should break verification"); } - byte[] tamperedParty = - flipLastByte(Base64.getMimeDecoder().decode(fixtures.chainParty())); - Owid party2 = ParseAssert.parsed(Owid.parse(tamperedParty)); - assertFalse(party2.verifyWithCrypto(crypto, Collections.singletonList(root)), - "a flipped party signature byte should break verification"); } @Test diff --git a/src/test/java/com/swancommunity/owid/ParseContractTest.java b/src/test/java/com/swancommunity/owid/ParseContractTest.java index 930cec3..b0fcc88 100644 --- a/src/test/java/com/swancommunity/owid/ParseContractTest.java +++ b/src/test/java/com/swancommunity/owid/ParseContractTest.java @@ -27,7 +27,6 @@ import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Base64; -import java.util.Collections; import java.util.Random; import org.junit.jupiter.api.Test; @@ -273,7 +272,7 @@ void structurallyValidWithWrongSignatureParsesThenFailsVerification() Owid owid = ParseAssert.parsed(Owid.parse(bytes)); OwidVerificationResult verification = - owid.verify(crypto, Collections.emptyList()); + owid.verify(crypto); assertEquals(OwidSignatureStatus.SIGNATURE_INVALID, verification.getStatus(), "the signature should be reported as not matching"); diff --git a/src/test/java/com/swancommunity/owid/PayloadLengthTest.java b/src/test/java/com/swancommunity/owid/PayloadLengthTest.java index e6f9b70..54306ba 100644 --- a/src/test/java/com/swancommunity/owid/PayloadLengthTest.java +++ b/src/test/java/com/swancommunity/owid/PayloadLengthTest.java @@ -26,7 +26,6 @@ import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.Arrays; -import java.util.Collections; import org.junit.jupiter.api.Test; /** @@ -151,7 +150,7 @@ void libraryOutputParses() throws OwidException { assertArrayEquals(PAYLOAD, parsed.getPayload(), "should read the payload the library wrote"); assertEquals(original, parsed, "should parse to an equal OWID"); - assertTrue(parsed.verifyWithCrypto(crypto, Collections.emptyList()), + assertTrue(parsed.verifyWithCrypto(crypto), "the parsed OWID should still verify"); } diff --git a/src/test/java/com/swancommunity/owid/PublicKeyScheduleTest.java b/src/test/java/com/swancommunity/owid/PublicKeyScheduleTest.java index b9be284..6d8705b 100644 --- a/src/test/java/com/swancommunity/owid/PublicKeyScheduleTest.java +++ b/src/test/java/com/swancommunity/owid/PublicKeyScheduleTest.java @@ -45,9 +45,6 @@ class PublicKeyScheduleTest { private static final Instant WEEK_OF_THE_IDENTIFIER = Instant.parse("2026-08-31T00:00:00Z"); - /** No other OWIDs were covered by the signature on the fixture. */ - private static final List ALONE = Collections.emptyList(); - /** * The genuine identifier verifies against the key the published schedule * says was in force on the day the identifier was signed. This is the @@ -67,7 +64,7 @@ void genuineIdentifierVerifiesAgainstTheKeyInForceOnItsDate() assertEquals(WEEK_OF_THE_IDENTIFIER, key.getStartsAt(), "the week beginning 31 August covers 4 September"); assertEquals(OwidSignatureStatus.SIGNATURE_VALID, - owid.verify(key.getPublicKeyPem(), ALONE).getStatus(), + owid.verify(key.getPublicKeyPem()).getStatus(), "should verify against the key that signed it"); } @@ -79,7 +76,7 @@ void genuineIdentifierVerifiesAgainstTheKeyInForceOnItsDate() void scheduleVerifiesTheGenuineIdentifier() throws OwidException { assertEquals(OwidSignatureStatus.SIGNATURE_VALID, KeyFixtures.schedule() - .verify(KeyFixtures.identifier(), ALONE).getStatus(), + .verify(KeyFixtures.identifier()).getStatus(), "should pick the signing key and verify in one call"); } @@ -100,7 +97,7 @@ void aLaterWeeksKeyDoesNotVerifyAnEarlierWeeksIdentifier() "the key in force in the following week starts after the identifier " + "was signed"); assertEquals(OwidSignatureStatus.SIGNATURE_INVALID, - owid.verify(later.getPublicKeyPem(), ALONE).getStatus(), + owid.verify(later.getPublicKeyPem()).getStatus(), "a later week's key should not verify an earlier week's " + "identifier"); } @@ -170,7 +167,7 @@ void selectionIgnoresTheMomentTheKeysWereGenerated() throws OwidException { "the newest generated key had not started when the " + "identifier was signed"); assertEquals(OwidSignatureStatus.SIGNATURE_INVALID, - owid.verify(newestGenerated.pem(), ALONE).getStatus(), + owid.verify(newestGenerated.pem()).getStatus(), "selecting on the generation moment reports a genuine " + "identifier as not matching"); @@ -179,7 +176,7 @@ void selectionIgnoresTheMomentTheKeysWereGenerated() throws OwidException { assertEquals(WEEK_OF_THE_IDENTIFIER, chosen.getStartsAt(), "selecting on the start picks the week that was running"); assertEquals(OwidSignatureStatus.SIGNATURE_VALID, - owid.verify(chosen.getPublicKeyPem(), ALONE).getStatus(), + owid.verify(chosen.getPublicKeyPem()).getStatus(), "selecting on the start verifies the genuine identifier"); } @@ -225,7 +222,7 @@ void keysMayArriveInAnyOrder() throws OwidException { "the keys are held oldest start first"); } assertEquals(OwidSignatureStatus.SIGNATURE_VALID, - reversed.verify(KeyFixtures.identifier(), ALONE).getStatus(), + reversed.verify(KeyFixtures.identifier()).getStatus(), "the order the keys arrived in changes nothing"); } @@ -255,7 +252,7 @@ void anEmptyScheduleHasNoKey() throws OwidException { assertNull(schedule.keyInForce(Instant.now()), "no key was in force"); assertEquals(OwidSignatureStatus.KEY_UNAVAILABLE, - schedule.verify(KeyFixtures.identifier(), ALONE).getStatus(), + schedule.verify(KeyFixtures.identifier()).getStatus(), "no key means the signature was never examined"); } @@ -263,7 +260,7 @@ void anEmptyScheduleHasNoKey() throws OwidException { @Test void aMissingOwidIsKeyUnavailable() throws OwidException { assertEquals(OwidSignatureStatus.KEY_UNAVAILABLE, - KeyFixtures.schedule().verify(null, ALONE).getStatus(), + KeyFixtures.schedule().verify(null).getStatus(), "there is nothing to find a key for"); } diff --git a/src/test/java/com/swancommunity/owid/SignatureStatusTest.java b/src/test/java/com/swancommunity/owid/SignatureStatusTest.java index 581442b..e09cef2 100644 --- a/src/test/java/com/swancommunity/owid/SignatureStatusTest.java +++ b/src/test/java/com/swancommunity/owid/SignatureStatusTest.java @@ -20,8 +20,6 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; -import java.util.Collections; -import java.util.List; import org.junit.jupiter.api.Test; /** @@ -35,14 +33,12 @@ * *

Every member of {@link OwidSignatureStatus} is exercised here except * {@link OwidSignatureStatus#IMPLEMENTATION_CAPACITY_EXCEEDED}, which needs - * an OWID and its chain to approach the two gigabyte limit of a Java array + * an OWID whose payload approaches the two gigabyte limit of a Java array * and so cannot be built in a suite that has to run on an ordinary machine. * The reason is recorded on the member itself as well.

*/ class SignatureStatusTest { - private static final List NONE = Collections.emptyList(); - private static Crypto crypto() throws OwidException { return Crypto.generate(); } @@ -54,7 +50,7 @@ void genuineSignatureIsValid() throws OwidException { Owid owid = Creator.create("example.com", crypto) .createString("payload"); - OwidVerificationResult result = owid.verify(crypto, NONE); + OwidVerificationResult result = owid.verify(crypto); assertTrue(result.isValid(), "a genuine signature should be valid"); assertEquals(OwidSignatureStatus.SIGNATURE_VALID, result.getStatus(), @@ -69,7 +65,7 @@ void genuineSignatureIsValidThroughPem() throws OwidException { .createString("payload"); OwidVerificationResult result = owid.verify( - crypto.publicKeyPem(), NONE); + crypto.publicKeyPem()); assertEquals(OwidSignatureStatus.SIGNATURE_VALID, result.getStatus(), "should report the signature as valid"); @@ -85,7 +81,7 @@ void wrongKeyIsSignatureInvalid() throws OwidException { .createString("payload"); OwidVerificationResult result = - owid.verify(crypto(), NONE); + owid.verify(crypto()); assertFalse(result.isValid(), "the signature should not be valid"); assertEquals(OwidSignatureStatus.SIGNATURE_INVALID, result.getStatus(), @@ -102,13 +98,13 @@ void noKeyIsKeyUnavailable() throws OwidException { .createString("payload"); assertEquals(OwidSignatureStatus.KEY_UNAVAILABLE, - owid.verify((Crypto) null, NONE).getStatus(), + owid.verify((Crypto) null).getStatus(), "a missing crypto instance should not judge the signature"); assertEquals(OwidSignatureStatus.KEY_UNAVAILABLE, - owid.verify((String) null, NONE).getStatus(), + owid.verify((String) null).getStatus(), "a missing PEM should not judge the signature"); assertEquals(OwidSignatureStatus.KEY_UNAVAILABLE, - owid.verify(" ", NONE).getStatus(), + owid.verify(" ").getStatus(), "an empty PEM should not judge the signature"); } @@ -125,13 +121,13 @@ void undecodableKeyIsInvalidKey() throws OwidException { .createString("payload"); assertEquals(OwidSignatureStatus.INVALID_KEY, - owid.verify("not a PEM", NONE) + owid.verify("not a PEM") .getStatus(), "material that is not a key should be reported as the key"); assertEquals(OwidSignatureStatus.INVALID_KEY, owid.verify( "-----BEGIN PUBLIC KEY-----\nAAAA\n" - + "-----END PUBLIC KEY-----\n", NONE) + + "-----END PUBLIC KEY-----\n") .getStatus(), "a PEM whose body is not a key should be reported as the key"); } @@ -157,10 +153,10 @@ void wrongLengthSignatureIsInvalidSignatureLength() throws OwidException { Envelope.filled(Owid.SIGNATURE_LENGTH - 1, (byte) 1)); assertEquals(OwidSignatureStatus.INVALID_SIGNATURE_LENGTH, - noSignature.verify(crypto(), NONE).getStatus(), + noSignature.verify(crypto()).getStatus(), "no signature is not the same as a signature that is wrong"); assertEquals(OwidSignatureStatus.INVALID_SIGNATURE_LENGTH, - shortSignature.verify(crypto(), NONE).getStatus(), + shortSignature.verify(crypto()).getStatus(), "a 63 byte signature is not a signature that is wrong"); } @@ -181,7 +177,7 @@ void unencodableFieldIsVerificationError() throws OwidException { Envelope.filled(Owid.SIGNATURE_LENGTH, (byte) 1)); assertEquals(OwidSignatureStatus.VERIFICATION_ERROR, - owid.verify(crypto(), NONE).getStatus(), + owid.verify(crypto()).getStatus(), "a field that cannot be encoded is not an invalid signature"); } @@ -196,11 +192,11 @@ void booleanSurfacesKeepTheirBehaviour() throws OwidException { Owid owid = Creator.create("example.com", crypto) .createString("payload"); - assertTrue(owid.verifyWithCrypto(crypto, NONE), + assertTrue(owid.verifyWithCrypto(crypto), "a genuine signature should verify"); - assertTrue(owid.verifyWithPublicKey(crypto.publicKeyPem(), NONE), + assertTrue(owid.verifyWithPublicKey(crypto.publicKeyPem()), "a genuine signature should verify through the PEM"); - assertFalse(owid.verifyWithCrypto(crypto(), NONE), + assertFalse(owid.verifyWithCrypto(crypto()), "a signature checked against another key should not verify"); } }