From 01b5102953dbd1f23838d042f105937950d0ffc0 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sun, 10 Mar 2024 17:48:52 +0100 Subject: [PATCH 001/156] Minor javadoc fixes --- src/main/java/org/dataloader/BatchLoader.java | 2 +- .../dataloader/BatchLoaderEnvironment.java | 2 +- src/main/java/org/dataloader/CacheMap.java | 2 +- src/main/java/org/dataloader/DataLoader.java | 14 +++++----- .../org/dataloader/DataLoaderFactory.java | 8 +++--- .../java/org/dataloader/DataLoaderHelper.java | 14 +++++----- .../org/dataloader/DataLoaderOptions.java | 8 +++--- .../org/dataloader/DataLoaderRegistry.java | 4 +-- .../org/dataloader/MappedBatchLoader.java | 3 +-- .../MappedBatchLoaderWithContext.java | 1 - src/main/java/org/dataloader/Try.java | 6 ++--- src/main/java/org/dataloader/ValueCache.java | 14 +++++----- .../org/dataloader/ValueCacheOptions.java | 2 +- .../annotations/ExperimentalApi.java | 2 +- .../org/dataloader/annotations/Internal.java | 2 +- .../org/dataloader/annotations/PublicSpi.java | 2 +- .../annotations/VisibleForTesting.java | 2 +- .../org/dataloader/impl/PromisedValues.java | 25 +++++++++--------- .../dataloader/impl/PromisedValuesImpl.java | 2 +- .../registries/DispatchPredicate.java | 2 +- .../ScheduledDataLoaderRegistry.java | 2 +- .../scheduler/BatchLoaderScheduler.java | 4 +-- .../stats/DelegatingStatisticsCollector.java | 2 +- .../java/org/dataloader/stats/Statistics.java | 2 +- .../dataloader/stats/StatisticsCollector.java | 2 +- .../dataloader/DataLoaderIfPresentTest.java | 4 +-- .../org/dataloader/DataLoaderStatsTest.java | 2 +- .../java/org/dataloader/DataLoaderTest.java | 2 +- .../dataloader/DataLoaderValueCacheTest.java | 6 ++--- src/test/java/org/dataloader/TryTest.java | 26 +++++++++---------- .../java/org/dataloader/fixtures/TestKit.java | 4 +-- .../impl/PromisedValuesImplTest.java | 2 +- 32 files changed, 85 insertions(+), 90 deletions(-) diff --git a/src/main/java/org/dataloader/BatchLoader.java b/src/main/java/org/dataloader/BatchLoader.java index fed2baf5..c1916e3e 100644 --- a/src/main/java/org/dataloader/BatchLoader.java +++ b/src/main/java/org/dataloader/BatchLoader.java @@ -54,7 +54,7 @@ * The back-end service returned results in a different order than we requested, likely because it was more efficient for it to * do so. Also, it omitted a result for key 6, which we may interpret as no value existing for that key. *

- * To uphold the constraints of the batch function, it must return an List of values the same length as + * To uphold the constraints of the batch function, it must return a List of values the same length as * the List of keys, and re-order them to ensure each index aligns with the original keys [ 2, 9, 6, 1 ]: * *

diff --git a/src/main/java/org/dataloader/BatchLoaderEnvironment.java b/src/main/java/org/dataloader/BatchLoaderEnvironment.java
index dd2572b3..cd995ecd 100644
--- a/src/main/java/org/dataloader/BatchLoaderEnvironment.java
+++ b/src/main/java/org/dataloader/BatchLoaderEnvironment.java
@@ -54,7 +54,7 @@ public Map getKeyContexts() {
      * {@link org.dataloader.DataLoader#loadMany(java.util.List, java.util.List)} can be given
      * a context object when it is invoked.  A list of them is present by this method.
      *
-     * @return a list of key context objects in the order they where encountered
+     * @return a list of key context objects in the order they were encountered
      */
     public List getKeyContextsList() {
         return keyContextsList;
diff --git a/src/main/java/org/dataloader/CacheMap.java b/src/main/java/org/dataloader/CacheMap.java
index 6d27a474..48d0f41c 100644
--- a/src/main/java/org/dataloader/CacheMap.java
+++ b/src/main/java/org/dataloader/CacheMap.java
@@ -65,7 +65,7 @@ static  CacheMap simpleMap() {
     /**
      * Gets the specified key from the cache map.
      * 

- * May throw an exception if the key does not exists, depending on the cache map implementation that is used, + * May throw an exception if the key does not exist, depending on the cache map implementation that is used, * so be sure to check {@link CacheMap#containsKey(Object)} first. * * @param key the key to retrieve diff --git a/src/main/java/org/dataloader/DataLoader.java b/src/main/java/org/dataloader/DataLoader.java index e800a55e..91d8791f 100644 --- a/src/main/java/org/dataloader/DataLoader.java +++ b/src/main/java/org/dataloader/DataLoader.java @@ -108,7 +108,7 @@ public static DataLoader newDataLoader(BatchLoader batchLoadF * (batching, caching and unlimited batch size) where the batch loader function returns a list of * {@link org.dataloader.Try} objects. *

- * If its important you to know the exact status of each item in a batch call and whether it threw exceptions then + * If it's important you to know the exact status of each item in a batch call and whether it threw exceptions then * you can use this form to create the data loader. *

* Using Try objects allows you to capture a value returned or an exception that might @@ -186,7 +186,7 @@ public static DataLoader newDataLoader(BatchLoaderWithContext * (batching, caching and unlimited batch size) where the batch loader function returns a list of * {@link org.dataloader.Try} objects. *

- * If its important you to know the exact status of each item in a batch call and whether it threw exceptions then + * If it's important you to know the exact status of each item in a batch call and whether it threw exceptions then * you can use this form to create the data loader. *

* Using Try objects allows you to capture a value returned or an exception that might @@ -264,7 +264,7 @@ public static DataLoader newMappedDataLoader(MappedBatchLoader - * If its important you to know the exact status of each item in a batch call and whether it threw exceptions then + * If it's important you to know the exact status of each item in a batch call and whether it threw exceptions then * you can use this form to create the data loader. *

* Using Try objects allows you to capture a value returned or an exception that might @@ -343,7 +343,7 @@ public static DataLoader newMappedDataLoader(MappedBatchLoaderWithC * (batching, caching and unlimited batch size) where the batch loader function returns a list of * {@link org.dataloader.Try} objects. *

- * If its important you to know the exact status of each item in a batch call and whether it threw exceptions then + * If it's important you to know the exact status of each item in a batch call and whether it threw exceptions then * you can use this form to create the data loader. *

* Using Try objects allows you to capture a value returned or an exception that might @@ -471,11 +471,11 @@ public CompletableFuture load(K key) { * This will return an optional promise to a value previously loaded via a {@link #load(Object)} call or empty if not call has been made for that key. *

* If you do get a present CompletableFuture it does not mean it has been dispatched and completed yet. It just means - * its at least pending and in cache. + * it's at least pending and in cache. *

* If caching is disabled there will never be a present Optional returned. *

- * NOTE : This will NOT cause a data load to happen. You must called {@link #load(Object)} for that to happen. + * NOTE : This will NOT cause a data load to happen. You must call {@link #load(Object)} for that to happen. * * @param key the key to check * @@ -494,7 +494,7 @@ public Optional> getIfPresent(K key) { *

* If caching is disabled there will never be a present Optional returned. *

- * NOTE : This will NOT cause a data load to happen. You must called {@link #load(Object)} for that to happen. + * NOTE : This will NOT cause a data load to happen. You must call {@link #load(Object)} for that to happen. * * @param key the key to check * diff --git a/src/main/java/org/dataloader/DataLoaderFactory.java b/src/main/java/org/dataloader/DataLoaderFactory.java index 0f910c1b..013f473e 100644 --- a/src/main/java/org/dataloader/DataLoaderFactory.java +++ b/src/main/java/org/dataloader/DataLoaderFactory.java @@ -42,7 +42,7 @@ public static DataLoader newDataLoader(BatchLoader batchLoadF * (batching, caching and unlimited batch size) where the batch loader function returns a list of * {@link org.dataloader.Try} objects. *

- * If its important you to know the exact status of each item in a batch call and whether it threw exceptions then + * If it's important you to know the exact status of each item in a batch call and whether it threw exceptions then * you can use this form to create the data loader. *

* Using Try objects allows you to capture a value returned or an exception that might @@ -109,7 +109,7 @@ public static DataLoader newDataLoader(BatchLoaderWithContext * (batching, caching and unlimited batch size) where the batch loader function returns a list of * {@link org.dataloader.Try} objects. *

- * If its important you to know the exact status of each item in a batch call and whether it threw exceptions then + * If it's important you to know the exact status of each item in a batch call and whether it threw exceptions then * you can use this form to create the data loader. *

* Using Try objects allows you to capture a value returned or an exception that might @@ -176,7 +176,7 @@ public static DataLoader newMappedDataLoader(MappedBatchLoader - * If its important you to know the exact status of each item in a batch call and whether it threw exceptions then + * If it's important you to know the exact status of each item in a batch call and whether it threw exceptions then * you can use this form to create the data loader. *

* Using Try objects allows you to capture a value returned or an exception that might @@ -244,7 +244,7 @@ public static DataLoader newMappedDataLoader(MappedBatchLoaderWithC * (batching, caching and unlimited batch size) where the batch loader function returns a list of * {@link org.dataloader.Try} objects. *

- * If its important you to know the exact status of each item in a batch call and whether it threw exceptions then + * If it's important you to know the exact status of each item in a batch call and whether it threw exceptions then * you can use this form to create the data loader. *

* Using Try objects allows you to capture a value returned or an exception that might diff --git a/src/main/java/org/dataloader/DataLoaderHelper.java b/src/main/java/org/dataloader/DataLoaderHelper.java index 066214cf..9b03dc30 100644 --- a/src/main/java/org/dataloader/DataLoaderHelper.java +++ b/src/main/java/org/dataloader/DataLoaderHelper.java @@ -34,8 +34,8 @@ import static org.dataloader.impl.Assertions.nonNull; /** - * This helps break up the large DataLoader class functionality and it contains the logic to dispatch the - * promises on behalf of its peer dataloader + * This helps break up the large DataLoader class functionality, and it contains the logic to dispatch the + * promises on behalf of its peer dataloader. * * @param the type of keys * @param the type of values @@ -148,13 +148,11 @@ CompletableFuture load(K key, Object loadContext) { } } - @SuppressWarnings("unchecked") Object getCacheKey(K key) { return loaderOptions.cacheKeyFunction().isPresent() ? loaderOptions.cacheKeyFunction().get().getKey(key) : key; } - @SuppressWarnings("unchecked") Object getCacheKeyWithContext(K key, Object context) { return loaderOptions.cacheKeyFunction().isPresent() ? loaderOptions.cacheKeyFunction().get().getKeyWithContext(key, context) : key; @@ -296,9 +294,9 @@ private void possiblyClearCacheEntriesOnExceptions(List keys) { if (keys.isEmpty()) { return; } - // by default we don't clear the cached view of this entry to avoid - // frequently loading the same error. This works for short lived request caches - // but might work against long lived caches. Hence we have an option that allows + // by default, we don't clear the cached view of this entry to avoid + // frequently loading the same error. This works for short-lived request caches + // but might work against long-lived caches. Hence, we have an option that allows // it to be cleared if (!loaderOptions.cachingExceptionsEnabled()) { keys.forEach(dataLoader::clear); @@ -384,7 +382,7 @@ CompletableFuture> invokeLoader(List keys, List keyContexts, return completedFuture(assembledValues); } else { // - // we missed some of the keys from cache, so send them to the batch loader + // we missed some keys from cache, so send them to the batch loader // and then fill in their values // CompletableFuture> batchLoad = invokeLoader(missedKeys, missedKeyContexts); diff --git a/src/main/java/org/dataloader/DataLoaderOptions.java b/src/main/java/org/dataloader/DataLoaderOptions.java index bac94761..b96e7857 100644 --- a/src/main/java/org/dataloader/DataLoaderOptions.java +++ b/src/main/java/org/dataloader/DataLoaderOptions.java @@ -135,8 +135,8 @@ public DataLoaderOptions setCachingEnabled(boolean cachingEnabled) { /** * Option that determines whether to cache exceptional values (the default), or not. * - * For short lived caches (that is request caches) it makes sense to cache exceptions since - * its likely the key is still poisoned. However if you have long lived caches, then it may make + * For short-lived caches (that is request caches) it makes sense to cache exceptions since + * it's likely the key is still poisoned. However, if you have long-lived caches, then it may make * sense to set this to false since the downstream system may have recovered from its failure * mode. * @@ -147,7 +147,7 @@ public boolean cachingExceptionsEnabled() { } /** - * Sets the option that determines whether exceptional values are cachedis enabled. + * Sets the option that determines whether exceptional values are cache enabled. * * @param cachingExceptionsEnabled {@code true} to enable caching exceptional values, {@code false} otherwise * @@ -236,7 +236,7 @@ public StatisticsCollector getStatisticsCollector() { /** * Sets the statistics collector supplier that will be used with these data loader options. Since it uses - * the supplier pattern, you can create a new statistics collector on each call or you can reuse + * the supplier pattern, you can create a new statistics collector on each call, or you can reuse * a common value * * @param statisticsCollector the statistics collector to use diff --git a/src/main/java/org/dataloader/DataLoaderRegistry.java b/src/main/java/org/dataloader/DataLoaderRegistry.java index 0bc54cbf..5a3f90f1 100644 --- a/src/main/java/org/dataloader/DataLoaderRegistry.java +++ b/src/main/java/org/dataloader/DataLoaderRegistry.java @@ -127,7 +127,7 @@ public Set getKeys() { } /** - * This will called {@link org.dataloader.DataLoader#dispatch()} on each of the registered + * This will be called {@link org.dataloader.DataLoader#dispatch()} on each of the registered * {@link org.dataloader.DataLoader}s */ public void dispatchAll() { @@ -197,7 +197,7 @@ public Builder register(String key, DataLoader dataLoader) { } /** - * This will combine together the data loaders in this builder with the ones + * This will combine the data loaders in this builder with the ones * from a previous {@link DataLoaderRegistry} * * @param otherRegistry the previous {@link DataLoaderRegistry} diff --git a/src/main/java/org/dataloader/MappedBatchLoader.java b/src/main/java/org/dataloader/MappedBatchLoader.java index 4b489fac..5a7a1a63 100644 --- a/src/main/java/org/dataloader/MappedBatchLoader.java +++ b/src/main/java/org/dataloader/MappedBatchLoader.java @@ -16,13 +16,12 @@ package org.dataloader; -import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.CompletionStage; /** - * A function that is invoked for batch loading a map of of data values indicated by the provided set of keys. The + * A function that is invoked for batch loading a map of data values indicated by the provided set of keys. The * function returns a promise of a map of results of individual load requests. *

* There are a few constraints that must be upheld: diff --git a/src/main/java/org/dataloader/MappedBatchLoaderWithContext.java b/src/main/java/org/dataloader/MappedBatchLoaderWithContext.java index 6e3a2f0b..7438d204 100644 --- a/src/main/java/org/dataloader/MappedBatchLoaderWithContext.java +++ b/src/main/java/org/dataloader/MappedBatchLoaderWithContext.java @@ -16,7 +16,6 @@ package org.dataloader; -import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.CompletionStage; diff --git a/src/main/java/org/dataloader/Try.java b/src/main/java/org/dataloader/Try.java index c7eac678..cd33afd6 100644 --- a/src/main/java/org/dataloader/Try.java +++ b/src/main/java/org/dataloader/Try.java @@ -15,7 +15,7 @@ /** * Try is class that allows you to hold the result of computation or the throwable it produced. * - * This class is useful in {@link org.dataloader.BatchLoader}s so you can mix a batch of calls where some of + * This class is useful in {@link org.dataloader.BatchLoader}s so you can mix a batch of calls where some * the calls succeeded and some of them failed. You would make your batch loader declaration like : * *

@@ -89,8 +89,8 @@ public static  Try failed(Throwable throwable) {
     }
 
     /**
-     * This returns a Try that has always failed with an consistent exception.  Use this when
-     * yiu dont care about the exception but only that the Try failed.
+     * This returns a Try that has always failed with a consistent exception.  Use this when
+     * you don't care about the exception but only that the Try failed.
      *
      * @param  the type of value
      *
diff --git a/src/main/java/org/dataloader/ValueCache.java b/src/main/java/org/dataloader/ValueCache.java
index 551dc5d5..200b1672 100644
--- a/src/main/java/org/dataloader/ValueCache.java
+++ b/src/main/java/org/dataloader/ValueCache.java
@@ -15,11 +15,11 @@
  * 

* It differs from {@link CacheMap} which is in fact a cache of promised values aka {@link CompletableFuture}<V>'s. *

- * {@link ValueCache} is more suited to be a wrapper of a long-lived or externallly cached values. {@link CompletableFuture}s cant + * {@link ValueCache} is more suited to be a wrapper of a long-lived or externally cached values. {@link CompletableFuture}s can't * be easily placed in an external cache outside the JVM say, hence the need for the {@link ValueCache}. *

* {@link DataLoader}s use a two stage cache strategy if caching is enabled. If the {@link CacheMap} already has the promise to a value - * that is used. If not then the {@link ValueCache} is asked for a value, if it has one then that is returned (and cached as a promise in the {@link CacheMap}. + * that is used. If not then the {@link ValueCache} is asked for a value, if it has one then that is returned (and cached as a promise in the {@link CacheMap}). *

* If there is no value then the key is queued and loaded via the {@link BatchLoader} calls. The returned values will then be stored in * the {@link ValueCache} and the promises to those values are also stored in the {@link CacheMap}. @@ -29,7 +29,7 @@ * out of the box. *

* The API signature uses {@link CompletableFuture}s because the backing implementation MAY be a remote external cache - * and hence exceptions may happen in retrieving values and they may take time to complete. + * and hence exceptions may happen in retrieving values, and they may take time to complete. * * @param the type of cache keys * @param the type of cache values @@ -67,8 +67,8 @@ static ValueCache defaultValueCache() { CompletableFuture get(K key); /** - * Gets the specified keys from the value cache, in a batch call. If your underlying cache cant do batch caching retrieval - * then do not implement this method and it will delegate back to {@link #get(Object)} for you + * Gets the specified keys from the value cache, in a batch call. If your underlying cache cannot do batch caching retrieval + * then do not implement this method, and it will delegate back to {@link #get(Object)} for you *

* Each item in the returned list of values is a {@link Try}. If the key could not be found then a failed Try just be returned otherwise * a successful Try contain the cached value is returned. @@ -104,8 +104,8 @@ default CompletableFuture>> getValues(List keys) throws ValueCach CompletableFuture set(K key, V value); /** - * Stores the value with the specified keys, or updates it if the keys if they already exist. If your underlying cache cant do batch caching setting - * then do not implement this method and it will delegate back to {@link #set(Object, Object)} for you + * Stores the value with the specified keys, or updates it if the keys if they already exist. If your underlying cache can't do batch caching setting + * then do not implement this method, and it will delegate back to {@link #set(Object, Object)} for you * * @param keys the keys to store * @param values the values to store diff --git a/src/main/java/org/dataloader/ValueCacheOptions.java b/src/main/java/org/dataloader/ValueCacheOptions.java index 1a0c1a12..7e2f0255 100644 --- a/src/main/java/org/dataloader/ValueCacheOptions.java +++ b/src/main/java/org/dataloader/ValueCacheOptions.java @@ -22,7 +22,7 @@ public static ValueCacheOptions newOptions() { /** * This controls whether the {@link DataLoader} will wait for the {@link ValueCache#set(Object, Object)} call - * to complete before it completes the returned value. By default this is false and hence + * to complete before it completes the returned value. By default, this is false and hence * the {@link ValueCache#set(Object, Object)} call may complete some time AFTER the data loader * value has been returned. * diff --git a/src/main/java/org/dataloader/annotations/ExperimentalApi.java b/src/main/java/org/dataloader/annotations/ExperimentalApi.java index 6be889e0..782998ea 100644 --- a/src/main/java/org/dataloader/annotations/ExperimentalApi.java +++ b/src/main/java/org/dataloader/annotations/ExperimentalApi.java @@ -14,7 +14,7 @@ * This represents code that the graphql-java project considers experimental API and while our intention is that it will * progress to be {@link PublicApi}, its existence, signature of behavior may change between releases. * - * In general unnecessary changes will be avoided but you should not depend on experimental classes being stable + * In general unnecessary changes will be avoided, but you should not depend on experimental classes being stable */ @Retention(RetentionPolicy.RUNTIME) @Target(value = {CONSTRUCTOR, METHOD, TYPE, FIELD}) diff --git a/src/main/java/org/dataloader/annotations/Internal.java b/src/main/java/org/dataloader/annotations/Internal.java index 4ad04cd8..51cfef23 100644 --- a/src/main/java/org/dataloader/annotations/Internal.java +++ b/src/main/java/org/dataloader/annotations/Internal.java @@ -13,7 +13,7 @@ * This represents code that the java-dataloader project considers internal code that MAY not be stable within * major releases. * - * In general unnecessary changes will be avoided but you should not depend on internal classes being stable + * In general unnecessary changes will be avoided, but you should not depend on internal classes being stable */ @Retention(RetentionPolicy.RUNTIME) @Target(value = {CONSTRUCTOR, METHOD, TYPE, FIELD}) diff --git a/src/main/java/org/dataloader/annotations/PublicSpi.java b/src/main/java/org/dataloader/annotations/PublicSpi.java index 5f385b74..7384fa9c 100644 --- a/src/main/java/org/dataloader/annotations/PublicSpi.java +++ b/src/main/java/org/dataloader/annotations/PublicSpi.java @@ -15,7 +15,7 @@ * * The guarantee is for callers of code with this annotation as well as derivations that inherit / implement this code. * - * New methods will not be added (without using default methods say) that would nominally breaks SPI implementations + * New methods will not be added (without using default methods say) that would nominally break SPI implementations * within a major release. */ @Retention(RetentionPolicy.RUNTIME) diff --git a/src/main/java/org/dataloader/annotations/VisibleForTesting.java b/src/main/java/org/dataloader/annotations/VisibleForTesting.java index 99f97f04..a3911136 100644 --- a/src/main/java/org/dataloader/annotations/VisibleForTesting.java +++ b/src/main/java/org/dataloader/annotations/VisibleForTesting.java @@ -9,7 +9,7 @@ import static java.lang.annotation.ElementType.METHOD; /** - * Marks fields, methods etc as more visible than actually needed for testing purposes. + * Marks fields, methods etc. as more visible than actually needed for testing purposes. */ @Retention(RetentionPolicy.RUNTIME) @Target(value = {CONSTRUCTOR, METHOD, FIELD}) diff --git a/src/main/java/org/dataloader/impl/PromisedValues.java b/src/main/java/org/dataloader/impl/PromisedValues.java index 89ae8def..ab750442 100644 --- a/src/main/java/org/dataloader/impl/PromisedValues.java +++ b/src/main/java/org/dataloader/impl/PromisedValues.java @@ -12,11 +12,11 @@ import static java.util.Arrays.asList; /** - * This allows multiple {@link CompletionStage}s to be combined together and completed + * This allows multiple {@link CompletionStage}s to be combined and completed * as one and should something go wrong, instead of throwing {@link CompletionException}s it captures the cause and returns null for that - * data value, other wise it allows you to access them as a list of values. + * data value, otherwise it allows you to access them as a list of values. *

- * This class really encapsulate a list of promised values. It is considered finished when all of the underlying futures + * This class really encapsulate a list of promised values. It is considered finished when all the underlying futures * are finished. *

* You can get that list of values via {@link #toList()}. You can also compose a {@link CompletableFuture} of that @@ -28,7 +28,7 @@ public interface PromisedValues { /** - * Returns a new {@link PromisedValues} that is completed when all of + * Returns a new {@link PromisedValues} that is completed when all * the given {@link CompletionStage}s complete. If any of the given * {@link CompletionStage}s complete exceptionally, then the returned * {@link PromisedValues} also does so. @@ -43,7 +43,7 @@ static PromisedValues allOf(List> cfs) { } /** - * Returns a new {@link PromisedValues} that is completed when all of + * Returns a new {@link PromisedValues} that is completed when all * the given {@link CompletionStage}s complete. If any of the given * {@link CompletionStage}s complete exceptionally, then the returned * {@link PromisedValues} also does so. @@ -59,7 +59,7 @@ static PromisedValues allOf(CompletionStage f1, CompletionStage f2) } /** - * Returns a new {@link PromisedValues} that is completed when all of + * Returns a new {@link PromisedValues} that is completed when all * the given {@link CompletionStage}s complete. If any of the given * {@link CompletionStage}s complete exceptionally, then the returned * {@link PromisedValues} also does so. @@ -77,7 +77,7 @@ static PromisedValues allOf(CompletionStage f1, CompletionStage f2, /** - * Returns a new {@link PromisedValues} that is completed when all of + * Returns a new {@link PromisedValues} that is completed when all * the given {@link CompletionStage}s complete. If any of the given * {@link CompletionStage}s complete exceptionally, then the returned * {@link PromisedValues} also does so. @@ -96,7 +96,7 @@ static PromisedValues allOf(CompletionStage f1, CompletionStage f2, /** - * Returns a new {@link PromisedValues} that is completed when all of + * Returns a new {@link PromisedValues} that is completed when all * the given {@link PromisedValues}s complete. If any of the given * {@link PromisedValues}s complete exceptionally, then the returned * {@link PromisedValues} also does so. @@ -111,7 +111,7 @@ static PromisedValues allPromisedValues(List> cfs) { } /** - * Returns a new {@link PromisedValues} that is completed when all of + * Returns a new {@link PromisedValues} that is completed when all * the given {@link PromisedValues}s complete. If any of the given * {@link PromisedValues}s complete exceptionally, then the returned * {@link PromisedValues} also does so. @@ -127,7 +127,7 @@ static PromisedValues allPromisedValues(PromisedValues pv1, PromisedVa } /** - * Returns a new {@link PromisedValues} that is completed when all of + * Returns a new {@link PromisedValues} that is completed when all * the given {@link PromisedValues}s complete. If any of the given * {@link PromisedValues}s complete exceptionally, then the returned * {@link PromisedValues} also does so. @@ -144,7 +144,7 @@ static PromisedValues allPromisedValues(PromisedValues pv1, PromisedVa } /** - * Returns a new {@link PromisedValues} that is completed when all of + * Returns a new {@link PromisedValues} that is completed when all * the given {@link PromisedValues}s complete. If any of the given * {@link PromisedValues}s complete exceptionally, then the returned * {@link PromisedValues} also does so. @@ -177,7 +177,7 @@ static PromisedValues allPromisedValues(PromisedValues pv1, PromisedVa boolean succeeded(); /** - * @return true if any of the the futures completed unsuccessfully + * @return true if any of the futures completed unsuccessfully */ boolean failed(); @@ -220,7 +220,6 @@ static PromisedValues allPromisedValues(PromisedValues pv1, PromisedVa * * @return the value of the future */ - @SuppressWarnings("unchecked") T get(int index); /** diff --git a/src/main/java/org/dataloader/impl/PromisedValuesImpl.java b/src/main/java/org/dataloader/impl/PromisedValuesImpl.java index 2ba592b4..ddaba818 100644 --- a/src/main/java/org/dataloader/impl/PromisedValuesImpl.java +++ b/src/main/java/org/dataloader/impl/PromisedValuesImpl.java @@ -25,7 +25,7 @@ public class PromisedValuesImpl implements PromisedValues { private PromisedValuesImpl(List> cs) { this.futures = nonNull(cs); this.cause = new AtomicReference<>(); - CompletableFuture[] futuresArray = cs.stream().map(CompletionStage::toCompletableFuture).toArray(CompletableFuture[]::new); + CompletableFuture[] futuresArray = cs.stream().map(CompletionStage::toCompletableFuture).toArray(CompletableFuture[]::new); this.controller = CompletableFuture.allOf(futuresArray).handle((result, throwable) -> { setCause(throwable); return null; diff --git a/src/main/java/org/dataloader/registries/DispatchPredicate.java b/src/main/java/org/dataloader/registries/DispatchPredicate.java index 247a51ab..677f484b 100644 --- a/src/main/java/org/dataloader/registries/DispatchPredicate.java +++ b/src/main/java/org/dataloader/registries/DispatchPredicate.java @@ -73,7 +73,7 @@ default DispatchPredicate or(DispatchPredicate other) { } /** - * This predicate will return true if the {@link DataLoader} has not be dispatched + * This predicate will return true if the {@link DataLoader} has not been dispatched * for at least the duration length of time. * * @param duration the length of time to check diff --git a/src/main/java/org/dataloader/registries/ScheduledDataLoaderRegistry.java b/src/main/java/org/dataloader/registries/ScheduledDataLoaderRegistry.java index 28b13e04..2eb98433 100644 --- a/src/main/java/org/dataloader/registries/ScheduledDataLoaderRegistry.java +++ b/src/main/java/org/dataloader/registries/ScheduledDataLoaderRegistry.java @@ -198,7 +198,7 @@ public int dispatchAllWithCountImmediately() { /** * This will schedule a task to check the predicate and dispatch if true right now. It will not do - * a pre check of the preodicate like {@link #dispatchAll()} would + * a pre-check of the predicate like {@link #dispatchAll()} would */ public void rescheduleNow() { dataLoaders.forEach(this::reschedule); diff --git a/src/main/java/org/dataloader/scheduler/BatchLoaderScheduler.java b/src/main/java/org/dataloader/scheduler/BatchLoaderScheduler.java index bcebfa04..7cddd540 100644 --- a/src/main/java/org/dataloader/scheduler/BatchLoaderScheduler.java +++ b/src/main/java/org/dataloader/scheduler/BatchLoaderScheduler.java @@ -47,7 +47,7 @@ interface ScheduledMappedBatchLoaderCall { * * @param scheduledCall the callback that needs to be invoked to allow the {@link BatchLoader} to proceed. * @param keys this is the list of keys that will be passed to the {@link BatchLoader}. - * This is provided only for informative reasons and you cant change the keys that are used + * This is provided only for informative reasons, and you can't change the keys that are used * @param environment this is the {@link BatchLoaderEnvironment} in place, * which can be null if it's a simple {@link BatchLoader} call * @param the key type @@ -62,7 +62,7 @@ interface ScheduledMappedBatchLoaderCall { * * @param scheduledCall the callback that needs to be invoked to allow the {@link MappedBatchLoader} to proceed. * @param keys this is the list of keys that will be passed to the {@link MappedBatchLoader}. - * This is provided only for informative reasons and you cant change the keys that are used + * This is provided only for informative reasons and, you can't change the keys that are used * @param environment this is the {@link BatchLoaderEnvironment} in place, * which can be null if it's a simple {@link MappedBatchLoader} call * @param the key type diff --git a/src/main/java/org/dataloader/stats/DelegatingStatisticsCollector.java b/src/main/java/org/dataloader/stats/DelegatingStatisticsCollector.java index f964b293..563d37b5 100644 --- a/src/main/java/org/dataloader/stats/DelegatingStatisticsCollector.java +++ b/src/main/java/org/dataloader/stats/DelegatingStatisticsCollector.java @@ -86,7 +86,7 @@ public long incrementCacheHitCount() { } /** - * @return the statistics of the this collector (and not its delegate) + * @return the statistics of the collector (and not its delegate) */ @Override public Statistics getStatistics() { diff --git a/src/main/java/org/dataloader/stats/Statistics.java b/src/main/java/org/dataloader/stats/Statistics.java index 4bc9c692..f5b5e74c 100644 --- a/src/main/java/org/dataloader/stats/Statistics.java +++ b/src/main/java/org/dataloader/stats/Statistics.java @@ -54,7 +54,7 @@ public long getLoadCount() { } /** - * @return the number of times the {@link org.dataloader.DataLoader} batch loader function return an specific object that was in error + * @return the number of times the {@link org.dataloader.DataLoader} batch loader function return a specific object that was in error */ public long getLoadErrorCount() { return loadErrorCount; diff --git a/src/main/java/org/dataloader/stats/StatisticsCollector.java b/src/main/java/org/dataloader/stats/StatisticsCollector.java index b32d17e4..33e417fd 100644 --- a/src/main/java/org/dataloader/stats/StatisticsCollector.java +++ b/src/main/java/org/dataloader/stats/StatisticsCollector.java @@ -122,7 +122,7 @@ default long incrementCacheHitCount(IncrementCacheHitCountStatisticsContext< long incrementCacheHitCount(); /** - * @return the statistics that have been gathered up to this point in time + * @return the statistics that have been gathered to this point in time */ Statistics getStatistics(); } diff --git a/src/test/java/org/dataloader/DataLoaderIfPresentTest.java b/src/test/java/org/dataloader/DataLoaderIfPresentTest.java index 916fdefc..1d897f2d 100644 --- a/src/test/java/org/dataloader/DataLoaderIfPresentTest.java +++ b/src/test/java/org/dataloader/DataLoaderIfPresentTest.java @@ -34,10 +34,10 @@ public void should_detect_if_present_cf() { assertThat(cachedPromise.get(), sameInstance(future1)); - // but its not done! + // but it's not done! assertThat(cachedPromise.get().isDone(), equalTo(false)); // - // and hence it cant be loaded as complete + // and hence it can't be loaded as complete cachedPromise = dataLoader.getIfCompleted(1); assertThat(cachedPromise.isPresent(), equalTo(false)); } diff --git a/src/test/java/org/dataloader/DataLoaderStatsTest.java b/src/test/java/org/dataloader/DataLoaderStatsTest.java index a76d0f7a..c2faa504 100644 --- a/src/test/java/org/dataloader/DataLoaderStatsTest.java +++ b/src/test/java/org/dataloader/DataLoaderStatsTest.java @@ -68,7 +68,7 @@ public void stats_are_collected_by_default() { @Test public void stats_are_collected_with_specified_collector() { - // lets prime it with some numbers so we know its ours + // let's prime it with some numbers, so we know it's ours StatisticsCollector collector = new SimpleStatisticsCollector(); collector.incrementLoadCount(new IncrementLoadCountStatisticsContext<>(1, null)); collector.incrementBatchLoadCountBy(1, new IncrementBatchLoadCountByStatisticsContext<>(1, null)); diff --git a/src/test/java/org/dataloader/DataLoaderTest.java b/src/test/java/org/dataloader/DataLoaderTest.java index 63a834da..18dd6f89 100644 --- a/src/test/java/org/dataloader/DataLoaderTest.java +++ b/src/test/java/org/dataloader/DataLoaderTest.java @@ -144,7 +144,7 @@ public void should_Return_number_of_batched_entries() { DispatchResult dispatchResult = identityLoader.dispatchWithCounts(); await().until(() -> future1.isDone() && future2.isDone()); - assertThat(dispatchResult.getKeysCount(), equalTo(2)); // its two because its the number dispatched (by key) not the load calls + assertThat(dispatchResult.getKeysCount(), equalTo(2)); // its two because it's the number dispatched (by key) not the load calls assertThat(dispatchResult.getPromisedResults().isDone(), equalTo(true)); } diff --git a/src/test/java/org/dataloader/DataLoaderValueCacheTest.java b/src/test/java/org/dataloader/DataLoaderValueCacheTest.java index 38cfe779..2716fae1 100644 --- a/src/test/java/org/dataloader/DataLoaderValueCacheTest.java +++ b/src/test/java/org/dataloader/DataLoaderValueCacheTest.java @@ -174,7 +174,7 @@ public CompletableFuture get(String key) { assertThat(fA.join(), equalTo("a")); assertThat(fB.join(), equalTo("From Cache")); - // a was not in cache (according to get) and hence needed to be loaded + // "a" was not in cache (according to get) and hence needed to be loaded assertThat(loadCalls, equalTo(singletonList(singletonList("a")))); } @@ -201,7 +201,7 @@ public CompletableFuture set(String key, Object value) { assertThat(fA.join(), equalTo("a")); assertThat(fB.join(), equalTo("b")); - // a was not in cache (according to get) and hence needed to be loaded + // "a" was not in cache (according to get) and hence needed to be loaded assertThat(loadCalls, equalTo(singletonList(asList("a", "b")))); assertArrayEquals(customValueCache.store.keySet().toArray(), singletonList("b").toArray()); } @@ -288,7 +288,7 @@ public CompletableFuture>> getValues(List keys) { assertThat(loadCalls, equalTo(singletonList(asList("missC", "missD")))); List values = new ArrayList<>(customValueCache.asMap().values()); - // it will only set back in values that are missed - it wont set in values that successfully + // it will only set back in values that are missed - it won't set in values that successfully // came out of the cache assertThat(values, equalTo(asList("missC", "missD"))); } diff --git a/src/test/java/org/dataloader/TryTest.java b/src/test/java/org/dataloader/TryTest.java index 1fdd286e..4da7bcab 100644 --- a/src/test/java/org/dataloader/TryTest.java +++ b/src/test/java/org/dataloader/TryTest.java @@ -15,7 +15,6 @@ import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; -@SuppressWarnings("ConstantConditions") public class TryTest { interface RunThatCanThrow { @@ -33,6 +32,7 @@ private void expectThrowable(RunThatCanThrow runnable, Class sTry, String expectedString) { assertThat(sTry.isSuccess(), equalTo(false)); assertThat(sTry.isFailure(), equalTo(true)); @@ -51,21 +51,21 @@ private void assertSuccess(Try sTry, String expectedStr) { } @Test - public void tryFailed() throws Exception { + public void tryFailed() { Try sTry = Try.failed(new RuntimeException("Goodbye Cruel World")); assertFailure(sTry, "Goodbye Cruel World"); } @Test - public void trySucceeded() throws Exception { + public void trySucceeded() { Try sTry = Try.succeeded("Hello World"); assertSuccess(sTry, "Hello World"); } @Test - public void tryCallable() throws Exception { + public void tryCallable() { Try sTry = Try.tryCall(() -> "Hello World"); assertSuccess(sTry, "Hello World"); @@ -78,7 +78,7 @@ public void tryCallable() throws Exception { } @Test - public void triedStage() throws Exception { + public void triedStage() { CompletionStage> sTry = Try.tryStage(CompletableFuture.completedFuture("Hello World")); sTry.thenAccept(stageTry -> assertSuccess(stageTry, "Hello World")); @@ -93,7 +93,7 @@ public void triedStage() throws Exception { } @Test - public void map() throws Exception { + public void map() { Try iTry = Try.succeeded(666); Try sTry = iTry.map(Object::toString); @@ -106,7 +106,7 @@ public void map() throws Exception { } @Test - public void flatMap() throws Exception { + public void flatMap() { Function> intToStringFunc = i -> Try.succeeded(i.toString()); Try iTry = Try.succeeded(666); @@ -123,7 +123,7 @@ public void flatMap() throws Exception { } @Test - public void toOptional() throws Exception { + public void toOptional() { Try iTry = Try.succeeded(666); Optional optional = iTry.toOptional(); assertThat(optional.isPresent(), equalTo(true)); @@ -135,7 +135,7 @@ public void toOptional() throws Exception { } @Test - public void orElse() throws Exception { + public void orElse() { Try sTry = Try.tryCall(() -> "Hello World"); String result = sTry.orElse("other"); @@ -147,7 +147,7 @@ public void orElse() throws Exception { } @Test - public void orElseGet() throws Exception { + public void orElseGet() { Try sTry = Try.tryCall(() -> "Hello World"); String result = sTry.orElseGet(() -> "other"); @@ -159,7 +159,7 @@ public void orElseGet() throws Exception { } @Test - public void reThrow() throws Exception { + public void reThrow() { Try sTry = Try.failed(new RuntimeException("Goodbye Cruel World")); expectThrowable(sTry::reThrow, RuntimeException.class); @@ -169,7 +169,7 @@ public void reThrow() throws Exception { } @Test - public void forEach() throws Exception { + public void forEach() { AtomicReference sRef = new AtomicReference<>(); Try sTry = Try.tryCall(() -> "Hello World"); sTry.forEach(sRef::set); @@ -183,7 +183,7 @@ public void forEach() throws Exception { } @Test - public void recover() throws Exception { + public void recover() { Try sTry = Try.failed(new RuntimeException("Goodbye Cruel World")); sTry = sTry.recover(t -> "Hello World"); diff --git a/src/test/java/org/dataloader/fixtures/TestKit.java b/src/test/java/org/dataloader/fixtures/TestKit.java index ac29a0cf..adffb069 100644 --- a/src/test/java/org/dataloader/fixtures/TestKit.java +++ b/src/test/java/org/dataloader/fixtures/TestKit.java @@ -15,7 +15,6 @@ import java.util.LinkedHashSet; import java.util.HashMap; import java.util.List; -import java.util.Set; import java.util.Map; import java.util.Set; import java.util.concurrent.CompletableFuture; @@ -34,7 +33,7 @@ public static BatchLoaderWithContext keysAsValuesWithContext() { } public static MappedBatchLoader keysAsMapOfValues() { - return keys -> mapOfKeys(keys); + return TestKit::mapOfKeys; } public static MappedBatchLoaderWithContext keysAsMapOfValuesWithContext() { @@ -124,6 +123,7 @@ public static List sort(Collection collection) { return collection.stream().sorted().collect(toList()); } + @SafeVarargs public static Set asSet(T... elements) { return new LinkedHashSet<>(Arrays.asList(elements)); } diff --git a/src/test/java/org/dataloader/impl/PromisedValuesImplTest.java b/src/test/java/org/dataloader/impl/PromisedValuesImplTest.java index cbf8cc8a..3c9ce65a 100644 --- a/src/test/java/org/dataloader/impl/PromisedValuesImplTest.java +++ b/src/test/java/org/dataloader/impl/PromisedValuesImplTest.java @@ -186,7 +186,7 @@ public void exceptions_are_captured_and_reported() throws Exception { @Test public void type_generics_compile_as_expected() throws Exception { - PromisedValues pvList = PromisedValues.allOf(Collections.singletonList(new CompletableFuture())); + PromisedValues pvList = PromisedValues.allOf(Collections.singletonList(new CompletableFuture<>())); PromisedValues pvList2 = PromisedValues.allOf(Collections.>singletonList(new CompletableFuture<>())); assertThat(pvList, notNullValue()); From 58ad6584364677ac3140088c252df249c7f6cb85 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Sun, 17 Mar 2024 17:29:40 +0100 Subject: [PATCH 002/156] Pre-size resulting list --- src/main/java/org/dataloader/BatchLoaderEnvironment.java | 2 +- src/main/java/org/dataloader/DataLoader.java | 2 +- src/main/java/org/dataloader/DataLoaderHelper.java | 4 ++-- src/main/java/org/dataloader/ValueCache.java | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/dataloader/BatchLoaderEnvironment.java b/src/main/java/org/dataloader/BatchLoaderEnvironment.java index dd2572b3..ef74d13a 100644 --- a/src/main/java/org/dataloader/BatchLoaderEnvironment.java +++ b/src/main/java/org/dataloader/BatchLoaderEnvironment.java @@ -83,7 +83,7 @@ public Builder keyContexts(List keys, List keyContexts) { Assertions.nonNull(keyContexts); Map map = new HashMap<>(); - List list = new ArrayList<>(); + List list = new ArrayList<>(keys.size()); for (int i = 0; i < keys.size(); i++) { K key = keys.get(i); Object keyContext = null; diff --git a/src/main/java/org/dataloader/DataLoader.java b/src/main/java/org/dataloader/DataLoader.java index e800a55e..0f7cdff4 100644 --- a/src/main/java/org/dataloader/DataLoader.java +++ b/src/main/java/org/dataloader/DataLoader.java @@ -561,7 +561,7 @@ public CompletableFuture> loadMany(List keys, List keyContext nonNull(keyContexts); synchronized (this) { - List> collect = new ArrayList<>(); + List> collect = new ArrayList<>(keys.size()); for (int i = 0; i < keys.size(); i++) { K key = keys.get(i); Object keyContext = null; diff --git a/src/main/java/org/dataloader/DataLoaderHelper.java b/src/main/java/org/dataloader/DataLoaderHelper.java index 066214cf..cf491cca 100644 --- a/src/main/java/org/dataloader/DataLoaderHelper.java +++ b/src/main/java/org/dataloader/DataLoaderHelper.java @@ -213,9 +213,9 @@ DispatchResult dispatch() { private CompletableFuture> sliceIntoBatchesOfBatches(List keys, List> queuedFutures, List callContexts, int maxBatchSize) { // the number of keys is > than what the batch loader function can accept // so make multiple calls to the loader - List>> allBatches = new ArrayList<>(); int len = keys.size(); int batchCount = (int) Math.ceil(len / (double) maxBatchSize); + List>> allBatches = new ArrayList<>(batchCount); for (int i = 0; i < batchCount; i++) { int fromIndex = i * maxBatchSize; @@ -477,7 +477,7 @@ private CompletableFuture> invokeMapBatchLoader(List keys, BatchLoade } CompletableFuture> mapBatchLoad = nonNull(loadResult, () -> "Your batch loader function MUST return a non null CompletionStage").toCompletableFuture(); return mapBatchLoad.thenApply(map -> { - List values = new ArrayList<>(); + List values = new ArrayList<>(keys.size()); for (K key : keys) { V value = map.get(key); values.add(value); diff --git a/src/main/java/org/dataloader/ValueCache.java b/src/main/java/org/dataloader/ValueCache.java index 551dc5d5..9a0ab6e0 100644 --- a/src/main/java/org/dataloader/ValueCache.java +++ b/src/main/java/org/dataloader/ValueCache.java @@ -115,7 +115,7 @@ default CompletableFuture>> getValues(List keys) throws ValueCach * @throws ValueCachingNotSupported if this cache wants to short-circuit this method completely */ default CompletableFuture> setValues(List keys, List values) throws ValueCachingNotSupported { - List> cacheSets = new ArrayList<>(); + List> cacheSets = new ArrayList<>(keys.size()); for (int i = 0; i < keys.size(); i++) { K k = keys.get(i); V v = values.get(i); From 6c87d75510499ae38e400cbe42c1c7d0b9d79766 Mon Sep 17 00:00:00 2001 From: Brad Baker Date: Sat, 13 Apr 2024 12:32:27 +1000 Subject: [PATCH 003/156] Shutsdown executor if its was auto added by us --- .../ScheduledDataLoaderRegistry.java | 23 ++++++++++++++++ .../ScheduledDataLoaderRegistryTest.java | 26 +++++++++++++++++-- 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/dataloader/registries/ScheduledDataLoaderRegistry.java b/src/main/java/org/dataloader/registries/ScheduledDataLoaderRegistry.java index 2eb98433..74ec3054 100644 --- a/src/main/java/org/dataloader/registries/ScheduledDataLoaderRegistry.java +++ b/src/main/java/org/dataloader/registries/ScheduledDataLoaderRegistry.java @@ -58,6 +58,8 @@ public class ScheduledDataLoaderRegistry extends DataLoaderRegistry implements A private final Map, DispatchPredicate> dataLoaderPredicates = new ConcurrentHashMap<>(); private final DispatchPredicate dispatchPredicate; private final ScheduledExecutorService scheduledExecutorService; + private final boolean defaultExecutorUsed; + private final Duration schedule; private final boolean tickerMode; private volatile boolean closed; @@ -66,6 +68,7 @@ private ScheduledDataLoaderRegistry(Builder builder) { super(); this.dataLoaders.putAll(builder.dataLoaders); this.scheduledExecutorService = builder.scheduledExecutorService; + this.defaultExecutorUsed = builder.defaultExecutorUsed; this.schedule = builder.schedule; this.tickerMode = builder.tickerMode; this.closed = false; @@ -79,6 +82,16 @@ private ScheduledDataLoaderRegistry(Builder builder) { @Override public void close() { closed = true; + if (defaultExecutorUsed) { + scheduledExecutorService.shutdown(); + } + } + + /** + * @return executor being used by this registry + */ + public ScheduledExecutorService getScheduledExecutorService() { + return scheduledExecutorService; } /** @@ -258,9 +271,18 @@ public static class Builder { private final Map, DispatchPredicate> dataLoaderPredicates = new LinkedHashMap<>(); private DispatchPredicate dispatchPredicate = DispatchPredicate.DISPATCH_ALWAYS; private ScheduledExecutorService scheduledExecutorService; + private boolean defaultExecutorUsed = false; private Duration schedule = Duration.ofMillis(10); private boolean tickerMode = false; + /** + * If you provide a {@link ScheduledExecutorService} then it will NOT be shutdown when + * {@link ScheduledDataLoaderRegistry#close()} is called. This is left to the code that made this setup code + * + * @param executorService the executor service to run the ticker on + * + * @return this builder for a fluent pattern + */ public Builder scheduledExecutorService(ScheduledExecutorService executorService) { this.scheduledExecutorService = nonNull(executorService); return this; @@ -350,6 +372,7 @@ public Builder tickerMode(boolean tickerMode) { public ScheduledDataLoaderRegistry build() { if (scheduledExecutorService == null) { scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); + defaultExecutorUsed = true; } return new ScheduledDataLoaderRegistry(this); } diff --git a/src/test/java/org/dataloader/registries/ScheduledDataLoaderRegistryTest.java b/src/test/java/org/dataloader/registries/ScheduledDataLoaderRegistryTest.java index 5e0cd9a2..146c1861 100644 --- a/src/test/java/org/dataloader/registries/ScheduledDataLoaderRegistryTest.java +++ b/src/test/java/org/dataloader/registries/ScheduledDataLoaderRegistryTest.java @@ -12,6 +12,7 @@ import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -285,7 +286,7 @@ public void test_can_tick_after_first_dispatch_for_chain_data_loaders() { assertThat(registry.isTickerMode(), equalTo(true)); int count = registry.dispatchAllWithCount(); - assertThat(count,equalTo(1)); + assertThat(count, equalTo(1)); await().atMost(TWO_SECONDS).untilAtomic(done, is(true)); @@ -314,7 +315,7 @@ public void test_chain_data_loaders_will_hang_if_not_in_ticker_mode() { assertThat(registry.isTickerMode(), equalTo(false)); int count = registry.dispatchAllWithCount(); - assertThat(count,equalTo(1)); + assertThat(count, equalTo(1)); try { await().atMost(TWO_SECONDS).untilAtomic(done, is(true)); @@ -323,4 +324,25 @@ public void test_chain_data_loaders_will_hang_if_not_in_ticker_mode() { } registry.close(); } + + public void test_executors_are_shutdown() { + ScheduledDataLoaderRegistry registry = ScheduledDataLoaderRegistry.newScheduledRegistry().build(); + + ScheduledExecutorService executorService = registry.getScheduledExecutorService(); + assertThat(executorService.isShutdown(), equalTo(false)); + registry.close(); + assertThat(executorService.isShutdown(), equalTo(true)); + + executorService = Executors.newSingleThreadScheduledExecutor(); + registry = ScheduledDataLoaderRegistry.newScheduledRegistry() + .scheduledExecutorService(executorService).build(); + + executorService = registry.getScheduledExecutorService(); + assertThat(executorService.isShutdown(), equalTo(false)); + registry.close(); + // if they provide the executor, we don't close it down + assertThat(executorService.isShutdown(), equalTo(false)); + + + } } \ No newline at end of file From 69efc4e323f0d5ec0f12c236ce6caf13152538e8 Mon Sep 17 00:00:00 2001 From: Brad Baker Date: Sat, 13 Apr 2024 12:35:19 +1000 Subject: [PATCH 004/156] Shutsdown executor if its was auto added by us - PR tweak --- .../org/dataloader/registries/ScheduledDataLoaderRegistry.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/org/dataloader/registries/ScheduledDataLoaderRegistry.java b/src/main/java/org/dataloader/registries/ScheduledDataLoaderRegistry.java index 74ec3054..7e54fab7 100644 --- a/src/main/java/org/dataloader/registries/ScheduledDataLoaderRegistry.java +++ b/src/main/java/org/dataloader/registries/ScheduledDataLoaderRegistry.java @@ -59,7 +59,6 @@ public class ScheduledDataLoaderRegistry extends DataLoaderRegistry implements A private final DispatchPredicate dispatchPredicate; private final ScheduledExecutorService scheduledExecutorService; private final boolean defaultExecutorUsed; - private final Duration schedule; private final boolean tickerMode; private volatile boolean closed; From 432733b0f4f26b96035a182ec03984c2bae7de02 Mon Sep 17 00:00:00 2001 From: Brad Baker Date: Sat, 13 Apr 2024 16:46:24 +1000 Subject: [PATCH 005/156] ig there is a specific predicate for a dataloader - its is the final say --- .../ScheduledDataLoaderRegistry.java | 8 ++--- ...eduledDataLoaderRegistryPredicateTest.java | 32 +++++++++++-------- 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/src/main/java/org/dataloader/registries/ScheduledDataLoaderRegistry.java b/src/main/java/org/dataloader/registries/ScheduledDataLoaderRegistry.java index 7e54fab7..6ea9425f 100644 --- a/src/main/java/org/dataloader/registries/ScheduledDataLoaderRegistry.java +++ b/src/main/java/org/dataloader/registries/ScheduledDataLoaderRegistry.java @@ -217,8 +217,8 @@ public void rescheduleNow() { } /** - * Returns true if the dataloader has a predicate which returned true, OR the overall - * registry predicate returned true. + * If a specific {@link DispatchPredicate} is registered for this dataloader then it uses it values + * otherwise the overall registry predicate is used. * * @param dataLoaderKey the key in the dataloader map * @param dataLoader the dataloader @@ -228,9 +228,7 @@ public void rescheduleNow() { private boolean shouldDispatch(String dataLoaderKey, DataLoader dataLoader) { DispatchPredicate dispatchPredicate = dataLoaderPredicates.get(dataLoader); if (dispatchPredicate != null) { - if (dispatchPredicate.test(dataLoaderKey, dataLoader)) { - return true; - } + return dispatchPredicate.test(dataLoaderKey, dataLoader); } return this.dispatchPredicate.test(dataLoaderKey, dataLoader); } diff --git a/src/test/java/org/dataloader/registries/ScheduledDataLoaderRegistryPredicateTest.java b/src/test/java/org/dataloader/registries/ScheduledDataLoaderRegistryPredicateTest.java index 43da82fc..4eab5643 100644 --- a/src/test/java/org/dataloader/registries/ScheduledDataLoaderRegistryPredicateTest.java +++ b/src/test/java/org/dataloader/registries/ScheduledDataLoaderRegistryPredicateTest.java @@ -139,13 +139,13 @@ public void test_the_registry_overall_predicate_firing_works() { DataLoader dlB = newDataLoader(identityBatchLoader); DataLoader dlC = newDataLoader(identityBatchLoader); - DispatchPredicate predicateOnSix = new CountingDispatchPredicate(6); + DispatchPredicate predicateOnThree = new CountingDispatchPredicate(3); ScheduledDataLoaderRegistry registry = ScheduledDataLoaderRegistry.newScheduledRegistry() - .register("a", dlA, DISPATCH_NEVER) - .register("b", dlB, DISPATCH_NEVER) - .register("c", dlC, DISPATCH_NEVER) - .dispatchPredicate(predicateOnSix) + .register("a", dlA, new CountingDispatchPredicate(99)) + .register("b", dlB, new CountingDispatchPredicate(99)) + .register("c", dlC) // has none + .dispatchPredicate(predicateOnThree) .schedule(Duration.ofHours(1000)) .build(); @@ -160,16 +160,22 @@ public void test_the_registry_overall_predicate_firing_works() { assertThat(cfB.isDone(), equalTo(false)); assertThat(cfC.isDone(), equalTo(false)); - count = registry.dispatchAllWithCount(); // second firing but the overall been asked 6 times already + count = registry.dispatchAllWithCount(); // second firing assertThat(count, equalTo(0)); assertThat(cfA.isDone(), equalTo(false)); assertThat(cfB.isDone(), equalTo(false)); assertThat(cfC.isDone(), equalTo(false)); - count = registry.dispatchAllWithCount(); // third firing but the overall been asked 9 times already - assertThat(count, equalTo(3)); - assertThat(cfA.isDone(), equalTo(true)); - assertThat(cfB.isDone(), equalTo(true)); + count = registry.dispatchAllWithCount(); // third firing + assertThat(count, equalTo(0)); + assertThat(cfA.isDone(), equalTo(false)); + assertThat(cfB.isDone(), equalTo(false)); + assertThat(cfC.isDone(), equalTo(false)); + + count = registry.dispatchAllWithCount(); // fourth firing + assertThat(count, equalTo(1)); + assertThat(cfA.isDone(), equalTo(false)); + assertThat(cfB.isDone(), equalTo(false)); // they wont ever finish until 99 calls assertThat(cfC.isDone(), equalTo(true)); } @@ -217,9 +223,9 @@ public void test_the_registry_overall_predicate_firing_works_when_on_schedule() DispatchPredicate predicateOnTwenty = new CountingDispatchPredicate(20); ScheduledDataLoaderRegistry registry = ScheduledDataLoaderRegistry.newScheduledRegistry() - .register("a", dlA, DISPATCH_NEVER) - .register("b", dlB, DISPATCH_NEVER) - .register("c", dlC, DISPATCH_NEVER) + .register("a", dlA) + .register("b", dlB) + .register("c", dlC) .dispatchPredicate(predicateOnTwenty) .schedule(Duration.ofMillis(5)) .build(); From 12a73d31495b71e4f003bc260796718399034909 Mon Sep 17 00:00:00 2001 From: Alexandre Carlton Date: Mon, 29 Apr 2024 20:10:57 +1000 Subject: [PATCH 006/156] Remove CacheMap#containsKey before #get Currently, when accessing values from the `DataLoader`'s `CacheMap`, we first check `#containsKey` before invoking `#get`. This is problematic for two reasons: - the underlying cache's metrics are skewed (it will have a 100% hit rate). - if the cache has an automatic expiration policy, it is possible a value will expire between the `containsKey` and `get` invocations leading to an incorrect result. To ameliorate this, we always invoke `#get` and check if it is `null` to deterine whether the key is present. We wrap the invocation in a `try`/`catch` as the `#get` method is allowed to through per its documentation. --- src/main/java/org/dataloader/CacheMap.java | 3 +-- .../java/org/dataloader/DataLoaderHelper.java | 22 +++++++++++++------ 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/dataloader/CacheMap.java b/src/main/java/org/dataloader/CacheMap.java index 48d0f41c..1a4a4551 100644 --- a/src/main/java/org/dataloader/CacheMap.java +++ b/src/main/java/org/dataloader/CacheMap.java @@ -65,8 +65,7 @@ static CacheMap simpleMap() { /** * Gets the specified key from the cache map. *

- * May throw an exception if the key does not exist, depending on the cache map implementation that is used, - * so be sure to check {@link CacheMap#containsKey(Object)} first. + * May throw an exception if the key does not exist, depending on the cache map implementation that is used. * * @param key the key to retrieve * diff --git a/src/main/java/org/dataloader/DataLoaderHelper.java b/src/main/java/org/dataloader/DataLoaderHelper.java index 1e48a94d..d934de23 100644 --- a/src/main/java/org/dataloader/DataLoaderHelper.java +++ b/src/main/java/org/dataloader/DataLoaderHelper.java @@ -110,9 +110,13 @@ Optional> getIfPresent(K key) { boolean cachingEnabled = loaderOptions.cachingEnabled(); if (cachingEnabled) { Object cacheKey = getCacheKey(nonNull(key)); - if (futureCache.containsKey(cacheKey)) { - stats.incrementCacheHitCount(new IncrementCacheHitCountStatisticsContext<>(key)); - return Optional.of(futureCache.get(cacheKey)); + try { + CompletableFuture cacheValue = futureCache.get(cacheKey); + if (cacheValue != null) { + stats.incrementCacheHitCount(new IncrementCacheHitCountStatisticsContext<>(key)); + return Optional.of(cacheValue); + } + } catch (Exception ignored) { } } } @@ -307,10 +311,14 @@ private void possiblyClearCacheEntriesOnExceptions(List keys) { private CompletableFuture loadFromCache(K key, Object loadContext, boolean batchingEnabled) { final Object cacheKey = loadContext == null ? getCacheKey(key) : getCacheKeyWithContext(key, loadContext); - if (futureCache.containsKey(cacheKey)) { - // We already have a promise for this key, no need to check value cache or queue up load - stats.incrementCacheHitCount(new IncrementCacheHitCountStatisticsContext<>(key, loadContext)); - return futureCache.get(cacheKey); + try { + CompletableFuture cacheValue = futureCache.get(cacheKey); + if (cacheValue != null) { + // We already have a promise for this key, no need to check value cache or queue up load + stats.incrementCacheHitCount(new IncrementCacheHitCountStatisticsContext<>(key, loadContext)); + return cacheValue; + } + } catch (Exception ignored) { } CompletableFuture loadCallFuture = queueOrInvokeLoader(key, loadContext, batchingEnabled, true); From 42cab409b8f5d3b7b44d7e690a0bef7e17efdc85 Mon Sep 17 00:00:00 2001 From: Alexandre Carlton Date: Tue, 30 Apr 2024 20:46:50 +1000 Subject: [PATCH 007/156] Verify a throwing CacheMap#get does not break DataLoader As a fast follow to #146, we add a regression test to ensure that we still handle a `CacheMap` whose `#get` throws an exception (by falling back to the underlying batch loader). --- .../java/org/dataloader/DataLoaderTest.java | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/src/test/java/org/dataloader/DataLoaderTest.java b/src/test/java/org/dataloader/DataLoaderTest.java index 18dd6f89..bc9ecda3 100644 --- a/src/test/java/org/dataloader/DataLoaderTest.java +++ b/src/test/java/org/dataloader/DataLoaderTest.java @@ -27,6 +27,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; import java.util.concurrent.ExecutionException; @@ -826,6 +827,20 @@ public void should_Accept_a_custom_cache_map_implementation() throws ExecutionEx assertArrayEquals(customMap.stash.keySet().toArray(), emptyList().toArray()); } + @Test + public void should_degrade_gracefully_if_cache_get_throws() { + CacheMap cache = new ThrowingCacheMap(); + DataLoaderOptions options = newOptions().setCachingEnabled(true).setCacheMap(cache); + List> loadCalls = new ArrayList<>(); + DataLoader identityLoader = idLoader(options, loadCalls); + + assertThat(identityLoader.getIfPresent("a"), equalTo(Optional.empty())); + + CompletableFuture future = identityLoader.load("a"); + identityLoader.dispatch(); + assertThat(future.join(), equalTo("a")); + } + @Test public void batching_disabled_should_dispatch_immediately() { List> loadCalls = new ArrayList<>(); @@ -1097,10 +1112,15 @@ private static DataLoader idLoaderOddEvenExceptions( }, options); } - private BatchLoader keysAsValues() { return CompletableFuture::completedFuture; } + private static class ThrowingCacheMap extends CustomCacheMap { + @Override + public CompletableFuture get(String key) { + throw new RuntimeException("Cache implementation failed."); + } + } } From 6e302206f43a40b44982ff2e2747a6dff579d5ff Mon Sep 17 00:00:00 2001 From: Alexandre Carlton Date: Sun, 12 May 2024 17:56:56 +1000 Subject: [PATCH 008/156] Add a proof-of-concept for "Observer-like" batch loading **Note**: This commit, as-is, is not (yet) intended for merge. It is created to provide a proof-of-concept and gauge interest as polishing/testing this requires a non-trivial amount of effort. Motivation ========== The current DataLoader mechanism completes the corresponding `CompletableFuture` for a given key when the corresponding value is returned. However, DataLoader's `BatchLoader` assumes that the underlying batch function can only return all of its requested items at once (as an example, a SQL database query). However, the batch function may be a service that can return items progressively using a subscription-like architecture. Some examples include: - Project Reactor's [Subscriber](https://www.reactive-streams.org/reactive-streams-1.0.4-javadoc/org/reactivestreams/Subscriber.html). - gRPC's [StreamObserver](https://grpc.github.io/grpc-java/javadoc/io/grpc/stub/StreamObserver.html). - RX Java's [Flowable](https://reactivex.io/RxJava/3.x/javadoc/io/reactivex/rxjava3/core/Flowable.html). Streaming results in this fashion offers several advantages: - Certain values may be returned earlier than others (for example, the batch function may have cached values it can return early). - Memory load is lessened on the batch function (which may be an external service), as it does not need to keep hold of the retrieved values before it can send them out at once. - We are able to save the need to stream individual error values by providing an `onError` function to terminate the stream early. Proposal ======== We provide two new `BatchLoader`s and support for them in `java-dataloader`: - `ObserverBatchLoader`, with a load function that accepts: - a list of keys. - a `BatchObserver` intended as a delegate for publisher-like structures found in Project Reactor and Rx Java. This obviates the need to depend on external libraries. - `MappedObserverBatchLoader`, similar to `ObserverBatchLoader` but with an `onNext` that accepts a key _and_ value (to allow for early termination of streams without needing to process `null`s). - `*WithContext` variants for the above. The key value-add is that the implementation of `BatchObserver` (provided to the load functions) will immediately complete the queued future for a given key when `onNext` is called with a value. This means that if we have a batch function that can deliver values progressively, we can continue evaluating the query as the values arrive. As an arbitrary example, let's have a batch function that serves both the reporter and project fields on a Jira issue: ```graphql query { issue { project { issueTypes { ... } } reporter { ... } } } ``` If the batch function can return a `project` immediately but is delayed in when it can `reporter`, then our batch loader can return `project` and start evaluating the `issueTypes` immediately while we load the `reporter` in parallel. This would provide a more performant query evaluation. As mentioned above, this is not in a state to be merged - this is intended to gauge whether this is something the maintainers would be interested in owning. Should this be the case, the author is willing to test/polish this pull request so that it may be merged. --- .../java/org/dataloader/BatchObserver.java | 33 +++ .../java/org/dataloader/DataLoaderHelper.java | 267 +++++++++++++++++- .../org/dataloader/MappedBatchObserver.java | 34 +++ .../dataloader/MappedObserverBatchLoader.java | 17 ++ .../MappedObserverBatchLoaderWithContext.java | 10 + .../org/dataloader/ObserverBatchLoader.java | 19 ++ .../ObserverBatchLoaderWithContext.java | 10 + .../scheduler/BatchLoaderScheduler.java | 21 ++ src/test/java/ReadmeExamples.java | 6 + ...taLoaderMappedObserverBatchLoaderTest.java | 106 +++++++ .../DataLoaderObserverBatchLoaderTest.java | 108 +++++++ .../scheduler/BatchLoaderSchedulerTest.java | 20 ++ 12 files changed, 644 insertions(+), 7 deletions(-) create mode 100644 src/main/java/org/dataloader/BatchObserver.java create mode 100644 src/main/java/org/dataloader/MappedBatchObserver.java create mode 100644 src/main/java/org/dataloader/MappedObserverBatchLoader.java create mode 100644 src/main/java/org/dataloader/MappedObserverBatchLoaderWithContext.java create mode 100644 src/main/java/org/dataloader/ObserverBatchLoader.java create mode 100644 src/main/java/org/dataloader/ObserverBatchLoaderWithContext.java create mode 100644 src/test/java/org/dataloader/DataLoaderMappedObserverBatchLoaderTest.java create mode 100644 src/test/java/org/dataloader/DataLoaderObserverBatchLoaderTest.java diff --git a/src/main/java/org/dataloader/BatchObserver.java b/src/main/java/org/dataloader/BatchObserver.java new file mode 100644 index 00000000..14ef0519 --- /dev/null +++ b/src/main/java/org/dataloader/BatchObserver.java @@ -0,0 +1,33 @@ +package org.dataloader; + +/** + * A interface intended as a delegate for other Observer-like classes used in other libraries, to be invoked by the calling + * {@link ObserverBatchLoader}. + *

+ * Some examples include: + *

+ * @param the value type of the {@link ObserverBatchLoader} + */ +public interface BatchObserver { + + /** + * To be called by the {@link ObserverBatchLoader} to load a new value. + */ + void onNext(V value); + + /** + * To be called by the {@link ObserverBatchLoader} to indicate all values have been successfully processed. + * This {@link BatchObserver} should not have any method invoked after this is called. + */ + void onCompleted(); + + /** + * To be called by the {@link ObserverBatchLoader} to indicate an unrecoverable error has been encountered. + * This {@link BatchObserver} should not have any method invoked after this is called. + */ + void onError(Throwable e); +} diff --git a/src/main/java/org/dataloader/DataLoaderHelper.java b/src/main/java/org/dataloader/DataLoaderHelper.java index d934de23..47d2d350 100644 --- a/src/main/java/org/dataloader/DataLoaderHelper.java +++ b/src/main/java/org/dataloader/DataLoaderHelper.java @@ -15,6 +15,7 @@ import java.time.Instant; import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; @@ -241,10 +242,14 @@ private CompletableFuture> sliceIntoBatchesOfBatches(List keys, List< @SuppressWarnings("unchecked") private CompletableFuture> dispatchQueueBatch(List keys, List callContexts, List> queuedFutures) { stats.incrementBatchLoadCountBy(keys.size(), new IncrementBatchLoadCountByStatisticsContext<>(keys, callContexts)); - CompletableFuture> batchLoad = invokeLoader(keys, callContexts, loaderOptions.cachingEnabled()); + CompletableFuture> batchLoad = invokeLoader(keys, callContexts, queuedFutures, loaderOptions.cachingEnabled()); return batchLoad .thenApply(values -> { assertResultSize(keys, values); + if (isObserverLoader() || isMapObserverLoader()) { + // We have already completed the queued futures by the time the overall batchLoad future has completed. + return values; + } List clearCacheKeys = new ArrayList<>(); for (int idx = 0; idx < queuedFutures.size(); idx++) { @@ -342,14 +347,15 @@ private CompletableFuture queueOrInvokeLoader(K key, Object loadContext, bool CompletableFuture invokeLoaderImmediately(K key, Object keyContext, boolean cachingEnabled) { List keys = singletonList(key); List keyContexts = singletonList(keyContext); - return invokeLoader(keys, keyContexts, cachingEnabled) + List> queuedFutures = singletonList(new CompletableFuture<>()); + return invokeLoader(keys, keyContexts, queuedFutures, cachingEnabled) .thenApply(list -> list.get(0)) .toCompletableFuture(); } - CompletableFuture> invokeLoader(List keys, List keyContexts, boolean cachingEnabled) { + CompletableFuture> invokeLoader(List keys, List keyContexts, List> queuedFutures, boolean cachingEnabled) { if (!cachingEnabled) { - return invokeLoader(keys, keyContexts); + return invokeLoader(keys, keyContexts, queuedFutures); } CompletableFuture>> cacheCallCF = getFromValueCache(keys); return cacheCallCF.thenCompose(cachedValues -> { @@ -360,6 +366,7 @@ CompletableFuture> invokeLoader(List keys, List keyContexts, List missedKeyIndexes = new ArrayList<>(); List missedKeys = new ArrayList<>(); List missedKeyContexts = new ArrayList<>(); + List> missedQueuedFutures = new ArrayList<>(); // if they return a ValueCachingNotSupported exception then we insert this special marker value, and it // means it's a total miss, we need to get all these keys via the batch loader @@ -369,6 +376,7 @@ CompletableFuture> invokeLoader(List keys, List keyContexts, missedKeyIndexes.add(i); missedKeys.add(keys.get(i)); missedKeyContexts.add(keyContexts.get(i)); + missedQueuedFutures.add(queuedFutures.get(i)); } } else { assertState(keys.size() == cachedValues.size(), () -> "The size of the cached values MUST be the same size as the key list"); @@ -393,7 +401,7 @@ CompletableFuture> invokeLoader(List keys, List keyContexts, // we missed some keys from cache, so send them to the batch loader // and then fill in their values // - CompletableFuture> batchLoad = invokeLoader(missedKeys, missedKeyContexts); + CompletableFuture> batchLoad = invokeLoader(missedKeys, missedKeyContexts, missedQueuedFutures); return batchLoad.thenCompose(missedValues -> { assertResultSize(missedKeys, missedValues); @@ -412,8 +420,7 @@ CompletableFuture> invokeLoader(List keys, List keyContexts, }); } - - CompletableFuture> invokeLoader(List keys, List keyContexts) { + CompletableFuture> invokeLoader(List keys, List keyContexts, List> queuedFutures) { CompletableFuture> batchLoad; try { Object context = loaderOptions.getBatchLoaderContextProvider().getContext(); @@ -421,6 +428,10 @@ CompletableFuture> invokeLoader(List keys, List keyContexts) .context(context).keyContexts(keys, keyContexts).build(); if (isMapLoader()) { batchLoad = invokeMapBatchLoader(keys, environment); + } else if (isObserverLoader()) { + batchLoad = invokeObserverBatchLoader(keys, keyContexts, queuedFutures, environment); + } else if (isMapObserverLoader()) { + batchLoad = invokeMappedObserverBatchLoader(keys, keyContexts, queuedFutures, environment); } else { batchLoad = invokeListBatchLoader(keys, environment); } @@ -492,10 +503,68 @@ private CompletableFuture> invokeMapBatchLoader(List keys, BatchLoade }); } + private CompletableFuture> invokeObserverBatchLoader(List keys, List keyContexts, List> queuedFutures, BatchLoaderEnvironment environment) { + CompletableFuture> loadResult = new CompletableFuture<>(); + BatchObserver observer = new BatchObserverImpl(loadResult, keys, keyContexts, queuedFutures); + + BatchLoaderScheduler batchLoaderScheduler = loaderOptions.getBatchLoaderScheduler(); + if (batchLoadFunction instanceof ObserverBatchLoaderWithContext) { + ObserverBatchLoaderWithContext loadFunction = (ObserverBatchLoaderWithContext) batchLoadFunction; + if (batchLoaderScheduler != null) { + BatchLoaderScheduler.ScheduledObserverBatchLoaderCall loadCall = () -> loadFunction.load(keys, observer, environment); + batchLoaderScheduler.scheduleObserverBatchLoader(loadCall, keys, environment); + } else { + loadFunction.load(keys, observer, environment); + } + } else { + ObserverBatchLoader loadFunction = (ObserverBatchLoader) batchLoadFunction; + if (batchLoaderScheduler != null) { + BatchLoaderScheduler.ScheduledObserverBatchLoaderCall loadCall = () -> loadFunction.load(keys, observer); + batchLoaderScheduler.scheduleObserverBatchLoader(loadCall, keys, null); + } else { + loadFunction.load(keys, observer); + } + } + return loadResult; + } + + private CompletableFuture> invokeMappedObserverBatchLoader(List keys, List keyContexts, List> queuedFutures, BatchLoaderEnvironment environment) { + CompletableFuture> loadResult = new CompletableFuture<>(); + MappedBatchObserver observer = new MappedBatchObserverImpl(loadResult, keys, keyContexts, queuedFutures); + + BatchLoaderScheduler batchLoaderScheduler = loaderOptions.getBatchLoaderScheduler(); + if (batchLoadFunction instanceof MappedObserverBatchLoaderWithContext) { + MappedObserverBatchLoaderWithContext loadFunction = (MappedObserverBatchLoaderWithContext) batchLoadFunction; + if (batchLoaderScheduler != null) { + BatchLoaderScheduler.ScheduledObserverBatchLoaderCall loadCall = () -> loadFunction.load(keys, observer, environment); + batchLoaderScheduler.scheduleObserverBatchLoader(loadCall, keys, environment); + } else { + loadFunction.load(keys, observer, environment); + } + } else { + MappedObserverBatchLoader loadFunction = (MappedObserverBatchLoader) batchLoadFunction; + if (batchLoaderScheduler != null) { + BatchLoaderScheduler.ScheduledObserverBatchLoaderCall loadCall = () -> loadFunction.load(keys, observer); + batchLoaderScheduler.scheduleObserverBatchLoader(loadCall, keys, null); + } else { + loadFunction.load(keys, observer); + } + } + return loadResult; + } + private boolean isMapLoader() { return batchLoadFunction instanceof MappedBatchLoader || batchLoadFunction instanceof MappedBatchLoaderWithContext; } + private boolean isObserverLoader() { + return batchLoadFunction instanceof ObserverBatchLoader; + } + + private boolean isMapObserverLoader() { + return batchLoadFunction instanceof MappedObserverBatchLoader; + } + int dispatchDepth() { synchronized (dataLoader) { return loaderQueue.size(); @@ -546,4 +615,188 @@ private CompletableFuture> setToValueCache(List assembledValues, List private static DispatchResult emptyDispatchResult() { return (DispatchResult) EMPTY_DISPATCH_RESULT; } + + private class BatchObserverImpl implements BatchObserver { + private final CompletableFuture> valuesFuture; + private final List keys; + private final List callContexts; + private final List> queuedFutures; + + private final List clearCacheKeys = new ArrayList<>(); + private final List completedValues = new ArrayList<>(); + private int idx = 0; + private boolean onErrorCalled = false; + private boolean onCompletedCalled = false; + + private BatchObserverImpl( + CompletableFuture> valuesFuture, + List keys, + List callContexts, + List> queuedFutures + ) { + this.valuesFuture = valuesFuture; + this.keys = keys; + this.callContexts = callContexts; + this.queuedFutures = queuedFutures; + } + + @Override + public void onNext(V value) { + assert !onErrorCalled && !onCompletedCalled; + + K key = keys.get(idx); + Object callContext = callContexts.get(idx); + CompletableFuture future = queuedFutures.get(idx); + if (value instanceof Throwable) { + stats.incrementLoadErrorCount(new IncrementLoadErrorCountStatisticsContext<>(key, callContext)); + future.completeExceptionally((Throwable) value); + clearCacheKeys.add(keys.get(idx)); + } else if (value instanceof Try) { + // we allow the batch loader to return a Try so we can better represent a computation + // that might have worked or not. + Try tryValue = (Try) value; + if (tryValue.isSuccess()) { + future.complete(tryValue.get()); + } else { + stats.incrementLoadErrorCount(new IncrementLoadErrorCountStatisticsContext<>(key, callContext)); + future.completeExceptionally(tryValue.getThrowable()); + clearCacheKeys.add(keys.get(idx)); + } + } else { + future.complete(value); + } + + completedValues.add(value); + idx++; + } + + @Override + public void onCompleted() { + assert !onErrorCalled; + onCompletedCalled = true; + + assertResultSize(keys, completedValues); + + possiblyClearCacheEntriesOnExceptions(clearCacheKeys); + valuesFuture.complete(completedValues); + } + + @Override + public void onError(Throwable ex) { + assert !onCompletedCalled; + onErrorCalled = true; + + stats.incrementBatchLoadExceptionCount(new IncrementBatchLoadExceptionCountStatisticsContext<>(keys, callContexts)); + if (ex instanceof CompletionException) { + ex = ex.getCause(); + } + // Set the remaining keys to the exception. + for (int i = idx; i < queuedFutures.size(); i++) { + K key = keys.get(i); + CompletableFuture future = queuedFutures.get(i); + future.completeExceptionally(ex); + // clear any cached view of this key because they all failed + dataLoader.clear(key); + } + } + } + + private class MappedBatchObserverImpl implements MappedBatchObserver { + private final CompletableFuture> valuesFuture; + private final List keys; + private final List callContexts; + private final List> queuedFutures; + private final Map callContextByKey; + private final Map> queuedFutureByKey; + + private final List clearCacheKeys = new ArrayList<>(); + private final Map completedValuesByKey = new HashMap<>(); + private boolean onErrorCalled = false; + private boolean onCompletedCalled = false; + + private MappedBatchObserverImpl( + CompletableFuture> valuesFuture, + List keys, + List callContexts, + List> queuedFutures + ) { + this.valuesFuture = valuesFuture; + this.keys = keys; + this.callContexts = callContexts; + this.queuedFutures = queuedFutures; + + this.callContextByKey = new HashMap<>(); + this.queuedFutureByKey = new HashMap<>(); + for (int idx = 0; idx < queuedFutures.size(); idx++) { + K key = keys.get(idx); + Object callContext = callContexts.get(idx); + CompletableFuture queuedFuture = queuedFutures.get(idx); + callContextByKey.put(key, callContext); + queuedFutureByKey.put(key, queuedFuture); + } + } + + @Override + public void onNext(K key, V value) { + assert !onErrorCalled && !onCompletedCalled; + + Object callContext = callContextByKey.get(key); + CompletableFuture future = queuedFutureByKey.get(key); + if (value instanceof Throwable) { + stats.incrementLoadErrorCount(new IncrementLoadErrorCountStatisticsContext<>(key, callContext)); + future.completeExceptionally((Throwable) value); + clearCacheKeys.add(key); + } else if (value instanceof Try) { + // we allow the batch loader to return a Try so we can better represent a computation + // that might have worked or not. + Try tryValue = (Try) value; + if (tryValue.isSuccess()) { + future.complete(tryValue.get()); + } else { + stats.incrementLoadErrorCount(new IncrementLoadErrorCountStatisticsContext<>(key, callContext)); + future.completeExceptionally(tryValue.getThrowable()); + clearCacheKeys.add(key); + } + } else { + future.complete(value); + } + + completedValuesByKey.put(key, value); + } + + @Override + public void onCompleted() { + assert !onErrorCalled; + onCompletedCalled = true; + + possiblyClearCacheEntriesOnExceptions(clearCacheKeys); + List values = new ArrayList<>(keys.size()); + for (K key : keys) { + V value = completedValuesByKey.get(key); + values.add(value); + } + valuesFuture.complete(values); + } + + @Override + public void onError(Throwable ex) { + assert !onCompletedCalled; + onErrorCalled = true; + + stats.incrementBatchLoadExceptionCount(new IncrementBatchLoadExceptionCountStatisticsContext<>(keys, callContexts)); + if (ex instanceof CompletionException) { + ex = ex.getCause(); + } + // Complete the futures for the remaining keys with the exception. + for (int idx = 0; idx < queuedFutures.size(); idx++) { + K key = keys.get(idx); + CompletableFuture future = queuedFutureByKey.get(key); + if (!completedValuesByKey.containsKey(key)) { + future.completeExceptionally(ex); + // clear any cached view of this key because they all failed + dataLoader.clear(key); + } + } + } + } } diff --git a/src/main/java/org/dataloader/MappedBatchObserver.java b/src/main/java/org/dataloader/MappedBatchObserver.java new file mode 100644 index 00000000..59a0f73a --- /dev/null +++ b/src/main/java/org/dataloader/MappedBatchObserver.java @@ -0,0 +1,34 @@ +package org.dataloader; + +/** + * A interface intended as a delegate for other Observer-like classes used in other libraries, to be invoked by the calling + * {@link MappedObserverBatchLoader}. + *

+ * Some examples include: + *

+ * @param the key type of the calling {@link MappedObserverBatchLoader}. + * @param the value type of the calling {@link MappedObserverBatchLoader}. + */ +public interface MappedBatchObserver { + + /** + * To be called by the {@link MappedObserverBatchLoader} to process a new key/value pair. + */ + void onNext(K key, V value); + + /** + * To be called by the {@link MappedObserverBatchLoader} to indicate all values have been successfully processed. + * This {@link MappedBatchObserver} should not have any method invoked after this method is called. + */ + void onCompleted(); + + /** + * To be called by the {@link MappedObserverBatchLoader} to indicate an unrecoverable error has been encountered. + * This {@link MappedBatchObserver} should not have any method invoked after this method is called. + */ + void onError(Throwable e); +} diff --git a/src/main/java/org/dataloader/MappedObserverBatchLoader.java b/src/main/java/org/dataloader/MappedObserverBatchLoader.java new file mode 100644 index 00000000..d82ec75e --- /dev/null +++ b/src/main/java/org/dataloader/MappedObserverBatchLoader.java @@ -0,0 +1,17 @@ +package org.dataloader; + +import java.util.List; + +/** + * A function that is invoked for batch loading a stream of data values indicated by the provided list of keys. + *

+ * The function will call the provided {@link MappedBatchObserver} to process the key/value pairs it has retrieved to allow + * the future returned by {@link DataLoader#load(Object)} to complete as soon as the individual value is available + * (rather than when all values have been retrieved). + * + * @param type parameter indicating the type of keys to use for data load requests. + * @param type parameter indicating the type of values returned + */ +public interface MappedObserverBatchLoader { + void load(List keys, MappedBatchObserver observer); +} diff --git a/src/main/java/org/dataloader/MappedObserverBatchLoaderWithContext.java b/src/main/java/org/dataloader/MappedObserverBatchLoaderWithContext.java new file mode 100644 index 00000000..66191982 --- /dev/null +++ b/src/main/java/org/dataloader/MappedObserverBatchLoaderWithContext.java @@ -0,0 +1,10 @@ +package org.dataloader; + +import java.util.List; + +/** + * A {@link MappedObserverBatchLoader} with a {@link BatchLoaderEnvironment} provided as an extra parameter to {@link #load}. + */ +public interface MappedObserverBatchLoaderWithContext { + void load(List keys, MappedBatchObserver observer, BatchLoaderEnvironment environment); +} diff --git a/src/main/java/org/dataloader/ObserverBatchLoader.java b/src/main/java/org/dataloader/ObserverBatchLoader.java new file mode 100644 index 00000000..0c481f95 --- /dev/null +++ b/src/main/java/org/dataloader/ObserverBatchLoader.java @@ -0,0 +1,19 @@ +package org.dataloader; + +import java.util.List; + +/** + * A function that is invoked for batch loading a stream of data values indicated by the provided list of keys. + *

+ * The function will call the provided {@link BatchObserver} to process the values it has retrieved to allow + * the future returned by {@link DataLoader#load(Object)} to complete as soon as the individual value is available + * (rather than when all values have been retrieved). + *

+ * It is required that values be returned in the same order as the keys provided. + * + * @param type parameter indicating the type of keys to use for data load requests. + * @param type parameter indicating the type of values returned + */ +public interface ObserverBatchLoader { + void load(List keys, BatchObserver observer); +} diff --git a/src/main/java/org/dataloader/ObserverBatchLoaderWithContext.java b/src/main/java/org/dataloader/ObserverBatchLoaderWithContext.java new file mode 100644 index 00000000..14a3dd15 --- /dev/null +++ b/src/main/java/org/dataloader/ObserverBatchLoaderWithContext.java @@ -0,0 +1,10 @@ +package org.dataloader; + +import java.util.List; + +/** + * An {@link ObserverBatchLoader} with a {@link BatchLoaderEnvironment} provided as an extra parameter to {@link #load}. + */ +public interface ObserverBatchLoaderWithContext { + void load(List keys, BatchObserver observer, BatchLoaderEnvironment environment); +} diff --git a/src/main/java/org/dataloader/scheduler/BatchLoaderScheduler.java b/src/main/java/org/dataloader/scheduler/BatchLoaderScheduler.java index 7cddd540..5b88d2ce 100644 --- a/src/main/java/org/dataloader/scheduler/BatchLoaderScheduler.java +++ b/src/main/java/org/dataloader/scheduler/BatchLoaderScheduler.java @@ -5,6 +5,8 @@ import org.dataloader.DataLoader; import org.dataloader.DataLoaderOptions; import org.dataloader.MappedBatchLoader; +import org.dataloader.MappedObserverBatchLoader; +import org.dataloader.ObserverBatchLoader; import java.util.List; import java.util.Map; @@ -42,6 +44,13 @@ interface ScheduledMappedBatchLoaderCall { CompletionStage> invoke(); } + /** + * This represents a callback that will invoke a {@link ObserverBatchLoader} or {@link MappedObserverBatchLoader} function under the covers + */ + interface ScheduledObserverBatchLoaderCall { + void invoke(); + } + /** * This is called to schedule a {@link BatchLoader} call. * @@ -71,4 +80,16 @@ interface ScheduledMappedBatchLoaderCall { * @return a promise to the values that come from the {@link BatchLoader} */ CompletionStage> scheduleMappedBatchLoader(ScheduledMappedBatchLoaderCall scheduledCall, List keys, BatchLoaderEnvironment environment); + + /** + * This is called to schedule a {@link ObserverBatchLoader} call. + * + * @param scheduledCall the callback that needs to be invoked to allow the {@link ObserverBatchLoader} to proceed. + * @param keys this is the list of keys that will be passed to the {@link ObserverBatchLoader}. + * This is provided only for informative reasons and, you can't change the keys that are used + * @param environment this is the {@link BatchLoaderEnvironment} in place, + * which can be null if it's a simple {@link ObserverBatchLoader} call + * @param the key type + */ + void scheduleObserverBatchLoader(ScheduledObserverBatchLoaderCall scheduledCall, List keys, BatchLoaderEnvironment environment); } diff --git a/src/test/java/ReadmeExamples.java b/src/test/java/ReadmeExamples.java index d25dfa77..df733ed5 100644 --- a/src/test/java/ReadmeExamples.java +++ b/src/test/java/ReadmeExamples.java @@ -304,6 +304,12 @@ public CompletionStage> scheduleMappedBatchLoader(ScheduledMapp return scheduledCall.invoke(); }).thenCompose(Function.identity()); } + + @Override + public void scheduleObserverBatchLoader(ScheduledObserverBatchLoaderCall scheduledCall, List keys, BatchLoaderEnvironment environment) { + snooze(10); + scheduledCall.invoke(); + } }; } diff --git a/src/test/java/org/dataloader/DataLoaderMappedObserverBatchLoaderTest.java b/src/test/java/org/dataloader/DataLoaderMappedObserverBatchLoaderTest.java new file mode 100644 index 00000000..e6f11685 --- /dev/null +++ b/src/test/java/org/dataloader/DataLoaderMappedObserverBatchLoaderTest.java @@ -0,0 +1,106 @@ +package org.dataloader; + +import org.junit.Test; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicBoolean; + +import static java.util.Arrays.asList; +import static java.util.function.Function.identity; +import static java.util.stream.Collectors.toMap; +import static org.awaitility.Awaitility.await; +import static org.dataloader.DataLoaderFactory.mkDataLoader; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThat; + +public class DataLoaderMappedObserverBatchLoaderTest { + + @Test + public void should_Build_a_really_really_simple_data_loader() { + AtomicBoolean success = new AtomicBoolean(); + DataLoader identityLoader = mkDataLoader(keysAsValues(), DataLoaderOptions.newOptions()); + + CompletionStage future1 = identityLoader.load(1); + + future1.thenAccept(value -> { + assertThat(value, equalTo(1)); + success.set(true); + }); + identityLoader.dispatch(); + await().untilAtomic(success, is(true)); + } + + @Test + public void should_Support_loading_multiple_keys_in_one_call() { + AtomicBoolean success = new AtomicBoolean(); + DataLoader identityLoader = mkDataLoader(keysAsValues(), DataLoaderOptions.newOptions()); + + CompletionStage> futureAll = identityLoader.loadMany(asList(1, 2)); + futureAll.thenAccept(promisedValues -> { + assertThat(promisedValues.size(), is(2)); + success.set(true); + }); + identityLoader.dispatch(); + await().untilAtomic(success, is(true)); + assertThat(futureAll.toCompletableFuture().join(), equalTo(asList(1, 2))); + } + + @Test + public void simple_dataloader() { + DataLoader loader = mkDataLoader(keysAsValues(), DataLoaderOptions.newOptions()); + + loader.load("A"); + loader.load("B"); + loader.loadMany(asList("C", "D")); + + List results = loader.dispatchAndJoin(); + + assertThat(results.size(), equalTo(4)); + assertThat(results, equalTo(asList("A", "B", "C", "D"))); + } + + @Test + public void should_observer_batch_multiple_requests() throws ExecutionException, InterruptedException { + DataLoader identityLoader = mkDataLoader(keysAsValues(), new DataLoaderOptions()); + + CompletableFuture future1 = identityLoader.load(1); + CompletableFuture future2 = identityLoader.load(2); + identityLoader.dispatch(); + + await().until(() -> future1.isDone() && future2.isDone()); + assertThat(future1.get(), equalTo(1)); + assertThat(future2.get(), equalTo(2)); + } + + // A simple wrapper class intended as a proof external libraries can leverage this. + private static class Publisher { + + private final MappedBatchObserver delegate; + private Publisher(MappedBatchObserver delegate) { this.delegate = delegate; } + void onNext(Map.Entry entry) { delegate.onNext(entry.getKey(), entry.getValue()); } + void onCompleted() { delegate.onCompleted(); } + void onError(Throwable e) { delegate.onError(e); } + // Mock 'subscribe' methods to simulate what would happen in the real thing. + void subscribe(Map valueByKey) { + valueByKey.entrySet().forEach(this::onNext); + this.onCompleted(); + } + void subscribe(Map valueByKey, Throwable e) { + valueByKey.entrySet().forEach(this::onNext); + this.onError(e); + } + } + + private static MappedObserverBatchLoader keysAsValues() { + return (keys, observer) -> { + Publisher publisher = new Publisher<>(observer); + Map valueByKey = keys.stream().collect(toMap(identity(), identity())); + publisher.subscribe(valueByKey); + }; + } +} diff --git a/src/test/java/org/dataloader/DataLoaderObserverBatchLoaderTest.java b/src/test/java/org/dataloader/DataLoaderObserverBatchLoaderTest.java new file mode 100644 index 00000000..eaeef8f3 --- /dev/null +++ b/src/test/java/org/dataloader/DataLoaderObserverBatchLoaderTest.java @@ -0,0 +1,108 @@ +package org.dataloader; + +import org.junit.Test; + +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicBoolean; + +import static java.util.Arrays.asList; +import static org.awaitility.Awaitility.await; +import static org.dataloader.DataLoaderFactory.mkDataLoader; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThat; + +public class DataLoaderObserverBatchLoaderTest { + + @Test + public void should_Build_a_really_really_simple_data_loader() { + AtomicBoolean success = new AtomicBoolean(); + DataLoader identityLoader = mkDataLoader(keysAsValues(), DataLoaderOptions.newOptions()); + + CompletionStage future1 = identityLoader.load(1); + + future1.thenAccept(value -> { + assertThat(value, equalTo(1)); + success.set(true); + }); + identityLoader.dispatch(); + await().untilAtomic(success, is(true)); + } + + @Test + public void should_Support_loading_multiple_keys_in_one_call() { + AtomicBoolean success = new AtomicBoolean(); + DataLoader identityLoader = mkDataLoader(keysAsValues(), DataLoaderOptions.newOptions()); + + CompletionStage> futureAll = identityLoader.loadMany(asList(1, 2)); + futureAll.thenAccept(promisedValues -> { + assertThat(promisedValues.size(), is(2)); + success.set(true); + }); + identityLoader.dispatch(); + await().untilAtomic(success, is(true)); + assertThat(futureAll.toCompletableFuture().join(), equalTo(asList(1, 2))); + } + + @Test + public void simple_dataloader() { + DataLoader loader = mkDataLoader(keysAsValues(), DataLoaderOptions.newOptions()); + + loader.load("A"); + loader.load("B"); + loader.loadMany(asList("C", "D")); + + List results = loader.dispatchAndJoin(); + + assertThat(results.size(), equalTo(4)); + assertThat(results, equalTo(asList("A", "B", "C", "D"))); + } + + @Test + public void should_observer_batch_multiple_requests() throws ExecutionException, InterruptedException { + DataLoader identityLoader = mkDataLoader(keysAsValues(), new DataLoaderOptions()); + + CompletableFuture future1 = identityLoader.load(1); + CompletableFuture future2 = identityLoader.load(2); + identityLoader.dispatch(); + + await().until(() -> future1.isDone() && future2.isDone()); + assertThat(future1.get(), equalTo(1)); + assertThat(future2.get(), equalTo(2)); + } + + // A simple wrapper class intended as a proof external libraries can leverage this. + private static class Publisher { + private final BatchObserver delegate; + private Publisher(BatchObserver delegate) { this.delegate = delegate; } + void onNext(V value) { delegate.onNext(value); } + void onCompleted() { delegate.onCompleted(); } + void onError(Throwable e) { delegate.onError(e); } + // Mock 'subscribe' methods to simulate what would happen in the real thing. + void subscribe(List values) { + values.forEach(this::onNext); + this.onCompleted(); + } + void subscribe(List values, Throwable e) { + values.forEach(this::onNext); + this.onError(e); + } + } + + private static ObserverBatchLoader keysAsValues() { + return (keys, observer) -> { + Publisher publisher = new Publisher<>(observer); + publisher.subscribe(keys); + }; + } + + private static ObserverBatchLoader keysWithValuesAndException(List values, Throwable e) { + return (keys, observer) -> { + Publisher publisher = new Publisher<>(observer); + publisher.subscribe(values, e); + }; + } +} diff --git a/src/test/java/org/dataloader/scheduler/BatchLoaderSchedulerTest.java b/src/test/java/org/dataloader/scheduler/BatchLoaderSchedulerTest.java index beb7c183..b77026c8 100644 --- a/src/test/java/org/dataloader/scheduler/BatchLoaderSchedulerTest.java +++ b/src/test/java/org/dataloader/scheduler/BatchLoaderSchedulerTest.java @@ -36,6 +36,11 @@ public CompletionStage> scheduleBatchLoader(ScheduledBatchLoaderC public CompletionStage> scheduleMappedBatchLoader(ScheduledMappedBatchLoaderCall scheduledCall, List keys, BatchLoaderEnvironment environment) { return scheduledCall.invoke(); } + + @Override + public void scheduleObserverBatchLoader(ScheduledObserverBatchLoaderCall scheduledCall, List keys, BatchLoaderEnvironment environment) { + scheduledCall.invoke(); + } }; private BatchLoaderScheduler delayedScheduling(int ms) { @@ -56,6 +61,12 @@ public CompletionStage> scheduleMappedBatchLoader(ScheduledMapp return scheduledCall.invoke(); }).thenCompose(Function.identity()); } + + @Override + public void scheduleObserverBatchLoader(ScheduledObserverBatchLoaderCall scheduledCall, List keys, BatchLoaderEnvironment environment) { + snooze(ms); + scheduledCall.invoke(); + } }; } @@ -139,6 +150,15 @@ public CompletionStage> scheduleMappedBatchLoader(ScheduledMapp return scheduledCall.invoke(); }).thenCompose(Function.identity()); } + + @Override + public void scheduleObserverBatchLoader(ScheduledObserverBatchLoaderCall scheduledCall, List keys, BatchLoaderEnvironment environment) { + CompletableFuture.supplyAsync(() -> { + snooze(10); + scheduledCall.invoke(); + return null; + }); + } }; DataLoaderOptions options = DataLoaderOptions.newOptions().setBatchLoaderScheduler(funkyScheduler); From c36f637b5b9c22122549549f31626d553f745657 Mon Sep 17 00:00:00 2001 From: Alexandre Carlton Date: Thu, 16 May 2024 21:35:17 +1000 Subject: [PATCH 009/156] Bump to Java 11 `graphql-java` is already on Java 11 so we take the liberty of bumping this library to the same version to unlock some nice features. --- .github/workflows/master.yml | 4 ++-- .github/workflows/pull_request.yml | 4 ++-- .github/workflows/release.yml | 4 ++-- build.gradle | 20 ++++++++----------- .../java/org/dataloader/DataLoaderTest.java | 20 +++++++++---------- 5 files changed, 24 insertions(+), 28 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index bc9d5f82..6073cf37 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -15,9 +15,9 @@ jobs: steps: - uses: actions/checkout@v1 - uses: gradle/wrapper-validation-action@v1 - - name: Set up JDK 1.8 + - name: Set up JDK 11 uses: actions/setup-java@v1 with: - java-version: '8.0.282' + java-version: '11.0.23' - name: build test and publish run: ./gradlew assemble && ./gradlew check --info && ./gradlew publishToSonatype closeAndReleaseSonatypeStagingRepository -x check --info --stacktrace diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 13a366a9..5bd98e9c 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -13,9 +13,9 @@ jobs: steps: - uses: actions/checkout@v1 - uses: gradle/wrapper-validation-action@v1 - - name: Set up JDK 1.8 + - name: Set up JDK 11 uses: actions/setup-java@v1 with: - java-version: '8.0.282' + java-version: '11.0.23' - name: build and test run: ./gradlew assemble && ./gradlew check --info --stacktrace diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b61d755e..090fc88c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -19,9 +19,9 @@ jobs: steps: - uses: actions/checkout@v1 - uses: gradle/wrapper-validation-action@v1 - - name: Set up JDK 1.8 + - name: Set up JDK 11 uses: actions/setup-java@v1 with: - java-version: '8.0.282' + java-version: '11.0.23' - name: build test and publish run: ./gradlew assemble && ./gradlew check --info && ./gradlew publishToSonatype closeAndReleaseSonatypeStagingRepository -x check --info --stacktrace diff --git a/build.gradle b/build.gradle index f5064edc..6222df4e 100644 --- a/build.gradle +++ b/build.gradle @@ -5,10 +5,15 @@ plugins { id 'java-library' id 'maven-publish' id 'signing' - id "biz.aQute.bnd.builder" version "6.2.0" + id "biz.aQute.bnd.builder" version "6.2.0" id "io.github.gradle-nexus.publish-plugin" version "1.0.0" } +java { + toolchain { + languageVersion = JavaLanguageVersion.of(11) + } +} def getDevelopmentVersion() { def output = new StringBuilder() @@ -25,20 +30,11 @@ def getDevelopmentVersion() { version } -if (JavaVersion.current() != JavaVersion.VERSION_1_8) { - def msg = String.format("This build must be run with java 1.8 - you are running %s - gradle finds the JDK via JAVA_HOME=%s", - JavaVersion.current(), System.getenv("JAVA_HOME")) - throw new IllegalStateException(msg) -} - - -sourceCompatibility = 1.8 -targetCompatibility = 1.8 def slf4jVersion = '1.7.30' def releaseVersion = System.env.RELEASE_VERSION version = releaseVersion ? releaseVersion : getDevelopmentVersion() group = 'com.graphql-java' -description = 'A pure Java 8 port of Facebook Dataloader' +description = 'A pure Java 11 port of Facebook Dataloader' gradle.buildFinished { buildResult -> println "*******************************" @@ -117,7 +113,7 @@ publishing { asNode().children().last() + { resolveStrategy = Closure.DELEGATE_FIRST name 'java-dataloader' - description 'A pure Java 8 port of Facebook Dataloader' + description 'A pure Java 11 port of Facebook Dataloader' url 'https://github.com/graphql-java/java-dataloader' inceptionYear '2017' diff --git a/src/test/java/org/dataloader/DataLoaderTest.java b/src/test/java/org/dataloader/DataLoaderTest.java index bc9ecda3..cd9710e2 100644 --- a/src/test/java/org/dataloader/DataLoaderTest.java +++ b/src/test/java/org/dataloader/DataLoaderTest.java @@ -692,13 +692,13 @@ public void should_work_with_duplicate_keys_when_caching_enabled() throws Execut public void should_Accept_objects_with_a_complex_key() throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); DataLoaderOptions options = newOptions().setCacheKeyFunction(getJsonObjectCacheMapFn()); - DataLoader identityLoader = idLoader(options, loadCalls); + DataLoader identityLoader = idLoader(options, loadCalls); JsonObject key1 = new JsonObject().put("id", 123); JsonObject key2 = new JsonObject().put("id", 123); - CompletableFuture future1 = identityLoader.load(key1); - CompletableFuture future2 = identityLoader.load(key2); + CompletableFuture future1 = identityLoader.load(key1); + CompletableFuture future2 = identityLoader.load(key2); identityLoader.dispatch(); await().until(() -> future1.isDone() && future2.isDone()); @@ -713,18 +713,18 @@ public void should_Accept_objects_with_a_complex_key() throws ExecutionException public void should_Clear_objects_with_complex_key() throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); DataLoaderOptions options = newOptions().setCacheKeyFunction(getJsonObjectCacheMapFn()); - DataLoader identityLoader = idLoader(options, loadCalls); + DataLoader identityLoader = idLoader(options, loadCalls); JsonObject key1 = new JsonObject().put("id", 123); JsonObject key2 = new JsonObject().put("id", 123); - CompletableFuture future1 = identityLoader.load(key1); + CompletableFuture future1 = identityLoader.load(key1); identityLoader.dispatch(); await().until(future1::isDone); identityLoader.clear(key2); // clear equivalent object key - CompletableFuture future2 = identityLoader.load(key1); + CompletableFuture future2 = identityLoader.load(key1); identityLoader.dispatch(); await().until(future2::isDone); @@ -737,22 +737,22 @@ public void should_Clear_objects_with_complex_key() throws ExecutionException, I public void should_Accept_objects_with_different_order_of_keys() throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); DataLoaderOptions options = newOptions().setCacheKeyFunction(getJsonObjectCacheMapFn()); - DataLoader identityLoader = idLoader(options, loadCalls); + DataLoader identityLoader = idLoader(options, loadCalls); JsonObject key1 = new JsonObject().put("a", 123).put("b", 321); JsonObject key2 = new JsonObject().put("b", 321).put("a", 123); // Fetches as expected - CompletableFuture future1 = identityLoader.load(key1); - CompletableFuture future2 = identityLoader.load(key2); + CompletableFuture future1 = identityLoader.load(key1); + CompletableFuture future2 = identityLoader.load(key2); identityLoader.dispatch(); await().until(() -> future1.isDone() && future2.isDone()); assertThat(loadCalls, equalTo(singletonList(singletonList(key1)))); assertThat(loadCalls.size(), equalTo(1)); assertThat(future1.get(), equalTo(key1)); - assertThat(future2.get(), equalTo(key1)); + assertThat(future2.get(), equalTo(key2)); } @Test From 95540ffce41b851245f87f95c4b214f63753298c Mon Sep 17 00:00:00 2001 From: bbaker Date: Fri, 17 May 2024 21:13:04 +1000 Subject: [PATCH 010/156] reactive streams support branch --- build.gradle | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index f5064edc..459ad309 100644 --- a/build.gradle +++ b/build.gradle @@ -34,7 +34,7 @@ if (JavaVersion.current() != JavaVersion.VERSION_1_8) { sourceCompatibility = 1.8 targetCompatibility = 1.8 -def slf4jVersion = '1.7.30' + def releaseVersion = System.env.RELEASE_VERSION version = releaseVersion ? releaseVersion : getDevelopmentVersion() group = 'com.graphql-java' @@ -68,11 +68,17 @@ jar { } } +def slf4jVersion = '1.7.30' +def reactiveStreamsVersion = '1.0.3' + dependencies { api 'org.slf4j:slf4j-api:' + slf4jVersion + api 'org.reactivestreams:reactive-streams:' + reactiveStreamsVersion + testImplementation 'org.slf4j:slf4j-simple:' + slf4jVersion testImplementation 'junit:junit:4.12' testImplementation 'org.awaitility:awaitility:2.0.0' + testImplementation 'io.projectreactor:reactor-core:3.6.6' testImplementation 'com.github.ben-manes.caffeine:caffeine:2.9.0' } From 1d782557ab2198b5e381a1710e29c9a0cf594fbe Mon Sep 17 00:00:00 2001 From: bbaker Date: Fri, 17 May 2024 21:38:48 +1000 Subject: [PATCH 011/156] reactive streams support branch - merged master --- build.gradle | 9 --------- 1 file changed, 9 deletions(-) diff --git a/build.gradle b/build.gradle index 30198f8e..6dea10e2 100644 --- a/build.gradle +++ b/build.gradle @@ -30,15 +30,6 @@ def getDevelopmentVersion() { version } -if (JavaVersion.current() != JavaVersion.VERSION_1_8) { - def msg = String.format("This build must be run with java 1.8 - you are running %s - gradle finds the JDK via JAVA_HOME=%s", - JavaVersion.current(), System.getenv("JAVA_HOME")) - throw new IllegalStateException(msg) -} - - -sourceCompatibility = 1.8 -targetCompatibility = 1.8 def releaseVersion = System.env.RELEASE_VERSION version = releaseVersion ? releaseVersion : getDevelopmentVersion() From 6b5a732194b63a80f46dc4dc87214ac25fb860e0 Mon Sep 17 00:00:00 2001 From: Alexandre Carlton Date: Sat, 18 May 2024 22:32:23 +1000 Subject: [PATCH 012/156] Eliminate *BatchObserver in favour of Publisher `reactive-streams` has become the de-facto standard for reactive frameworks; we thus use this as a base to allow seamless interop (rather than prompt an extra adapter layer). --- build.gradle | 7 +- .../java/org/dataloader/BatchObserver.java | 33 -------- .../java/org/dataloader/DataLoaderHelper.java | 75 +++++++++++-------- .../org/dataloader/MappedBatchObserver.java | 34 --------- .../MappedObserverBatchLoaderWithContext.java | 10 --- ...r.java => MappedPublisherBatchLoader.java} | 9 ++- ...MappedPublisherBatchLoaderWithContext.java | 13 ++++ .../ObserverBatchLoaderWithContext.java | 10 --- ...hLoader.java => PublisherBatchLoader.java} | 8 +- .../PublisherBatchLoaderWithContext.java | 12 +++ .../scheduler/BatchLoaderScheduler.java | 14 ++-- ...LoaderMappedPublisherBatchLoaderTest.java} | 38 ++-------- ...> DataLoaderPublisherBatchLoaderTest.java} | 33 +------- 13 files changed, 105 insertions(+), 191 deletions(-) delete mode 100644 src/main/java/org/dataloader/BatchObserver.java delete mode 100644 src/main/java/org/dataloader/MappedBatchObserver.java delete mode 100644 src/main/java/org/dataloader/MappedObserverBatchLoaderWithContext.java rename src/main/java/org/dataloader/{MappedObserverBatchLoader.java => MappedPublisherBatchLoader.java} (63%) create mode 100644 src/main/java/org/dataloader/MappedPublisherBatchLoaderWithContext.java delete mode 100644 src/main/java/org/dataloader/ObserverBatchLoaderWithContext.java rename src/main/java/org/dataloader/{ObserverBatchLoader.java => PublisherBatchLoader.java} (70%) create mode 100644 src/main/java/org/dataloader/PublisherBatchLoaderWithContext.java rename src/test/java/org/dataloader/{DataLoaderObserverBatchLoaderTest.java => DataLoaderMappedPublisherBatchLoaderTest.java} (67%) rename src/test/java/org/dataloader/{DataLoaderMappedObserverBatchLoaderTest.java => DataLoaderPublisherBatchLoaderTest.java} (66%) diff --git a/build.gradle b/build.gradle index 6222df4e..a22fc177 100644 --- a/build.gradle +++ b/build.gradle @@ -30,7 +30,6 @@ def getDevelopmentVersion() { version } -def slf4jVersion = '1.7.30' def releaseVersion = System.env.RELEASE_VERSION version = releaseVersion ? releaseVersion : getDevelopmentVersion() group = 'com.graphql-java' @@ -64,12 +63,18 @@ jar { } } +def slf4jVersion = '1.7.30' +def reactiveStreamsVersion = '1.0.3' + dependencies { api 'org.slf4j:slf4j-api:' + slf4jVersion + api 'org.reactivestreams:reactive-streams:' + reactiveStreamsVersion + testImplementation 'org.slf4j:slf4j-simple:' + slf4jVersion testImplementation 'junit:junit:4.12' testImplementation 'org.awaitility:awaitility:2.0.0' testImplementation 'com.github.ben-manes.caffeine:caffeine:2.9.0' + testImplementation 'io.projectreactor:reactor-core:3.6.6' } task sourcesJar(type: Jar) { diff --git a/src/main/java/org/dataloader/BatchObserver.java b/src/main/java/org/dataloader/BatchObserver.java deleted file mode 100644 index 14ef0519..00000000 --- a/src/main/java/org/dataloader/BatchObserver.java +++ /dev/null @@ -1,33 +0,0 @@ -package org.dataloader; - -/** - * A interface intended as a delegate for other Observer-like classes used in other libraries, to be invoked by the calling - * {@link ObserverBatchLoader}. - *

- * Some examples include: - *

- * @param the value type of the {@link ObserverBatchLoader} - */ -public interface BatchObserver { - - /** - * To be called by the {@link ObserverBatchLoader} to load a new value. - */ - void onNext(V value); - - /** - * To be called by the {@link ObserverBatchLoader} to indicate all values have been successfully processed. - * This {@link BatchObserver} should not have any method invoked after this is called. - */ - void onCompleted(); - - /** - * To be called by the {@link ObserverBatchLoader} to indicate an unrecoverable error has been encountered. - * This {@link BatchObserver} should not have any method invoked after this is called. - */ - void onError(Throwable e); -} diff --git a/src/main/java/org/dataloader/DataLoaderHelper.java b/src/main/java/org/dataloader/DataLoaderHelper.java index 47d2d350..a7e10525 100644 --- a/src/main/java/org/dataloader/DataLoaderHelper.java +++ b/src/main/java/org/dataloader/DataLoaderHelper.java @@ -10,6 +10,8 @@ import org.dataloader.stats.context.IncrementCacheHitCountStatisticsContext; import org.dataloader.stats.context.IncrementLoadCountStatisticsContext; import org.dataloader.stats.context.IncrementLoadErrorCountStatisticsContext; +import org.reactivestreams.Subscriber; +import org.reactivestreams.Subscription; import java.time.Clock; import java.time.Instant; @@ -246,7 +248,7 @@ private CompletableFuture> dispatchQueueBatch(List keys, List return batchLoad .thenApply(values -> { assertResultSize(keys, values); - if (isObserverLoader() || isMapObserverLoader()) { + if (isPublisherLoader() || isMappedPublisherLoader()) { // We have already completed the queued futures by the time the overall batchLoad future has completed. return values; } @@ -428,10 +430,10 @@ CompletableFuture> invokeLoader(List keys, List keyContexts, .context(context).keyContexts(keys, keyContexts).build(); if (isMapLoader()) { batchLoad = invokeMapBatchLoader(keys, environment); - } else if (isObserverLoader()) { - batchLoad = invokeObserverBatchLoader(keys, keyContexts, queuedFutures, environment); - } else if (isMapObserverLoader()) { - batchLoad = invokeMappedObserverBatchLoader(keys, keyContexts, queuedFutures, environment); + } else if (isPublisherLoader()) { + batchLoad = invokePublisherBatchLoader(keys, keyContexts, queuedFutures, environment); + } else if (isMappedPublisherLoader()) { + batchLoad = invokeMappedPublisherBatchLoader(keys, keyContexts, queuedFutures, environment); } else { batchLoad = invokeListBatchLoader(keys, environment); } @@ -503,38 +505,38 @@ private CompletableFuture> invokeMapBatchLoader(List keys, BatchLoade }); } - private CompletableFuture> invokeObserverBatchLoader(List keys, List keyContexts, List> queuedFutures, BatchLoaderEnvironment environment) { + private CompletableFuture> invokePublisherBatchLoader(List keys, List keyContexts, List> queuedFutures, BatchLoaderEnvironment environment) { CompletableFuture> loadResult = new CompletableFuture<>(); - BatchObserver observer = new BatchObserverImpl(loadResult, keys, keyContexts, queuedFutures); + Subscriber subscriber = new DataLoaderSubscriber(loadResult, keys, keyContexts, queuedFutures); BatchLoaderScheduler batchLoaderScheduler = loaderOptions.getBatchLoaderScheduler(); - if (batchLoadFunction instanceof ObserverBatchLoaderWithContext) { - ObserverBatchLoaderWithContext loadFunction = (ObserverBatchLoaderWithContext) batchLoadFunction; + if (batchLoadFunction instanceof PublisherBatchLoaderWithContext) { + PublisherBatchLoaderWithContext loadFunction = (PublisherBatchLoaderWithContext) batchLoadFunction; if (batchLoaderScheduler != null) { - BatchLoaderScheduler.ScheduledObserverBatchLoaderCall loadCall = () -> loadFunction.load(keys, observer, environment); + BatchLoaderScheduler.ScheduledObserverBatchLoaderCall loadCall = () -> loadFunction.load(keys, subscriber, environment); batchLoaderScheduler.scheduleObserverBatchLoader(loadCall, keys, environment); } else { - loadFunction.load(keys, observer, environment); + loadFunction.load(keys, subscriber, environment); } } else { - ObserverBatchLoader loadFunction = (ObserverBatchLoader) batchLoadFunction; + PublisherBatchLoader loadFunction = (PublisherBatchLoader) batchLoadFunction; if (batchLoaderScheduler != null) { - BatchLoaderScheduler.ScheduledObserverBatchLoaderCall loadCall = () -> loadFunction.load(keys, observer); + BatchLoaderScheduler.ScheduledObserverBatchLoaderCall loadCall = () -> loadFunction.load(keys, subscriber); batchLoaderScheduler.scheduleObserverBatchLoader(loadCall, keys, null); } else { - loadFunction.load(keys, observer); + loadFunction.load(keys, subscriber); } } return loadResult; } - private CompletableFuture> invokeMappedObserverBatchLoader(List keys, List keyContexts, List> queuedFutures, BatchLoaderEnvironment environment) { + private CompletableFuture> invokeMappedPublisherBatchLoader(List keys, List keyContexts, List> queuedFutures, BatchLoaderEnvironment environment) { CompletableFuture> loadResult = new CompletableFuture<>(); - MappedBatchObserver observer = new MappedBatchObserverImpl(loadResult, keys, keyContexts, queuedFutures); + Subscriber> observer = new DataLoaderMapEntrySubscriber(loadResult, keys, keyContexts, queuedFutures); BatchLoaderScheduler batchLoaderScheduler = loaderOptions.getBatchLoaderScheduler(); - if (batchLoadFunction instanceof MappedObserverBatchLoaderWithContext) { - MappedObserverBatchLoaderWithContext loadFunction = (MappedObserverBatchLoaderWithContext) batchLoadFunction; + if (batchLoadFunction instanceof MappedPublisherBatchLoaderWithContext) { + MappedPublisherBatchLoaderWithContext loadFunction = (MappedPublisherBatchLoaderWithContext) batchLoadFunction; if (batchLoaderScheduler != null) { BatchLoaderScheduler.ScheduledObserverBatchLoaderCall loadCall = () -> loadFunction.load(keys, observer, environment); batchLoaderScheduler.scheduleObserverBatchLoader(loadCall, keys, environment); @@ -542,7 +544,7 @@ private CompletableFuture> invokeMappedObserverBatchLoader(List keys, loadFunction.load(keys, observer, environment); } } else { - MappedObserverBatchLoader loadFunction = (MappedObserverBatchLoader) batchLoadFunction; + MappedPublisherBatchLoader loadFunction = (MappedPublisherBatchLoader) batchLoadFunction; if (batchLoaderScheduler != null) { BatchLoaderScheduler.ScheduledObserverBatchLoaderCall loadCall = () -> loadFunction.load(keys, observer); batchLoaderScheduler.scheduleObserverBatchLoader(loadCall, keys, null); @@ -557,12 +559,12 @@ private boolean isMapLoader() { return batchLoadFunction instanceof MappedBatchLoader || batchLoadFunction instanceof MappedBatchLoaderWithContext; } - private boolean isObserverLoader() { - return batchLoadFunction instanceof ObserverBatchLoader; + private boolean isPublisherLoader() { + return batchLoadFunction instanceof PublisherBatchLoader; } - private boolean isMapObserverLoader() { - return batchLoadFunction instanceof MappedObserverBatchLoader; + private boolean isMappedPublisherLoader() { + return batchLoadFunction instanceof MappedPublisherBatchLoader; } int dispatchDepth() { @@ -616,7 +618,8 @@ private static DispatchResult emptyDispatchResult() { return (DispatchResult) EMPTY_DISPATCH_RESULT; } - private class BatchObserverImpl implements BatchObserver { + private class DataLoaderSubscriber implements Subscriber { + private final CompletableFuture> valuesFuture; private final List keys; private final List callContexts; @@ -628,7 +631,7 @@ private class BatchObserverImpl implements BatchObserver { private boolean onErrorCalled = false; private boolean onCompletedCalled = false; - private BatchObserverImpl( + private DataLoaderSubscriber( CompletableFuture> valuesFuture, List keys, List callContexts, @@ -640,6 +643,11 @@ private BatchObserverImpl( this.queuedFutures = queuedFutures; } + @Override + public void onSubscribe(Subscription subscription) { + subscription.request(keys.size()); + } + @Override public void onNext(V value) { assert !onErrorCalled && !onCompletedCalled; @@ -671,7 +679,7 @@ public void onNext(V value) { } @Override - public void onCompleted() { + public void onComplete() { assert !onErrorCalled; onCompletedCalled = true; @@ -701,7 +709,7 @@ public void onError(Throwable ex) { } } - private class MappedBatchObserverImpl implements MappedBatchObserver { + private class DataLoaderMapEntrySubscriber implements Subscriber> { private final CompletableFuture> valuesFuture; private final List keys; private final List callContexts; @@ -714,7 +722,7 @@ private class MappedBatchObserverImpl implements MappedBatchObserver { private boolean onErrorCalled = false; private boolean onCompletedCalled = false; - private MappedBatchObserverImpl( + private DataLoaderMapEntrySubscriber( CompletableFuture> valuesFuture, List keys, List callContexts, @@ -737,8 +745,15 @@ private MappedBatchObserverImpl( } @Override - public void onNext(K key, V value) { + public void onSubscribe(Subscription subscription) { + subscription.request(keys.size()); + } + + @Override + public void onNext(Map.Entry entry) { assert !onErrorCalled && !onCompletedCalled; + K key = entry.getKey(); + V value = entry.getValue(); Object callContext = callContextByKey.get(key); CompletableFuture future = queuedFutureByKey.get(key); @@ -765,7 +780,7 @@ public void onNext(K key, V value) { } @Override - public void onCompleted() { + public void onComplete() { assert !onErrorCalled; onCompletedCalled = true; diff --git a/src/main/java/org/dataloader/MappedBatchObserver.java b/src/main/java/org/dataloader/MappedBatchObserver.java deleted file mode 100644 index 59a0f73a..00000000 --- a/src/main/java/org/dataloader/MappedBatchObserver.java +++ /dev/null @@ -1,34 +0,0 @@ -package org.dataloader; - -/** - * A interface intended as a delegate for other Observer-like classes used in other libraries, to be invoked by the calling - * {@link MappedObserverBatchLoader}. - *

- * Some examples include: - *

- * @param the key type of the calling {@link MappedObserverBatchLoader}. - * @param the value type of the calling {@link MappedObserverBatchLoader}. - */ -public interface MappedBatchObserver { - - /** - * To be called by the {@link MappedObserverBatchLoader} to process a new key/value pair. - */ - void onNext(K key, V value); - - /** - * To be called by the {@link MappedObserverBatchLoader} to indicate all values have been successfully processed. - * This {@link MappedBatchObserver} should not have any method invoked after this method is called. - */ - void onCompleted(); - - /** - * To be called by the {@link MappedObserverBatchLoader} to indicate an unrecoverable error has been encountered. - * This {@link MappedBatchObserver} should not have any method invoked after this method is called. - */ - void onError(Throwable e); -} diff --git a/src/main/java/org/dataloader/MappedObserverBatchLoaderWithContext.java b/src/main/java/org/dataloader/MappedObserverBatchLoaderWithContext.java deleted file mode 100644 index 66191982..00000000 --- a/src/main/java/org/dataloader/MappedObserverBatchLoaderWithContext.java +++ /dev/null @@ -1,10 +0,0 @@ -package org.dataloader; - -import java.util.List; - -/** - * A {@link MappedObserverBatchLoader} with a {@link BatchLoaderEnvironment} provided as an extra parameter to {@link #load}. - */ -public interface MappedObserverBatchLoaderWithContext { - void load(List keys, MappedBatchObserver observer, BatchLoaderEnvironment environment); -} diff --git a/src/main/java/org/dataloader/MappedObserverBatchLoader.java b/src/main/java/org/dataloader/MappedPublisherBatchLoader.java similarity index 63% rename from src/main/java/org/dataloader/MappedObserverBatchLoader.java rename to src/main/java/org/dataloader/MappedPublisherBatchLoader.java index d82ec75e..9c7430aa 100644 --- a/src/main/java/org/dataloader/MappedObserverBatchLoader.java +++ b/src/main/java/org/dataloader/MappedPublisherBatchLoader.java @@ -1,17 +1,20 @@ package org.dataloader; +import org.reactivestreams.Subscriber; + import java.util.List; +import java.util.Map; /** * A function that is invoked for batch loading a stream of data values indicated by the provided list of keys. *

- * The function will call the provided {@link MappedBatchObserver} to process the key/value pairs it has retrieved to allow + * The function will call the provided {@link Subscriber} to process the key/value pairs it has retrieved to allow * the future returned by {@link DataLoader#load(Object)} to complete as soon as the individual value is available * (rather than when all values have been retrieved). * * @param type parameter indicating the type of keys to use for data load requests. * @param type parameter indicating the type of values returned */ -public interface MappedObserverBatchLoader { - void load(List keys, MappedBatchObserver observer); +public interface MappedPublisherBatchLoader { + void load(List keys, Subscriber> subscriber); } diff --git a/src/main/java/org/dataloader/MappedPublisherBatchLoaderWithContext.java b/src/main/java/org/dataloader/MappedPublisherBatchLoaderWithContext.java new file mode 100644 index 00000000..a752abcc --- /dev/null +++ b/src/main/java/org/dataloader/MappedPublisherBatchLoaderWithContext.java @@ -0,0 +1,13 @@ +package org.dataloader; + +import org.reactivestreams.Subscriber; + +import java.util.List; +import java.util.Map; + +/** + * A {@link MappedPublisherBatchLoader} with a {@link BatchLoaderEnvironment} provided as an extra parameter to {@link #load}. + */ +public interface MappedPublisherBatchLoaderWithContext { + void load(List keys, Subscriber> subscriber, BatchLoaderEnvironment environment); +} diff --git a/src/main/java/org/dataloader/ObserverBatchLoaderWithContext.java b/src/main/java/org/dataloader/ObserverBatchLoaderWithContext.java deleted file mode 100644 index 14a3dd15..00000000 --- a/src/main/java/org/dataloader/ObserverBatchLoaderWithContext.java +++ /dev/null @@ -1,10 +0,0 @@ -package org.dataloader; - -import java.util.List; - -/** - * An {@link ObserverBatchLoader} with a {@link BatchLoaderEnvironment} provided as an extra parameter to {@link #load}. - */ -public interface ObserverBatchLoaderWithContext { - void load(List keys, BatchObserver observer, BatchLoaderEnvironment environment); -} diff --git a/src/main/java/org/dataloader/ObserverBatchLoader.java b/src/main/java/org/dataloader/PublisherBatchLoader.java similarity index 70% rename from src/main/java/org/dataloader/ObserverBatchLoader.java rename to src/main/java/org/dataloader/PublisherBatchLoader.java index 0c481f95..2dcdf1e3 100644 --- a/src/main/java/org/dataloader/ObserverBatchLoader.java +++ b/src/main/java/org/dataloader/PublisherBatchLoader.java @@ -1,11 +1,13 @@ package org.dataloader; +import org.reactivestreams.Subscriber; + import java.util.List; /** * A function that is invoked for batch loading a stream of data values indicated by the provided list of keys. *

- * The function will call the provided {@link BatchObserver} to process the values it has retrieved to allow + * The function will call the provided {@link Subscriber} to process the values it has retrieved to allow * the future returned by {@link DataLoader#load(Object)} to complete as soon as the individual value is available * (rather than when all values have been retrieved). *

@@ -14,6 +16,6 @@ * @param type parameter indicating the type of keys to use for data load requests. * @param type parameter indicating the type of values returned */ -public interface ObserverBatchLoader { - void load(List keys, BatchObserver observer); +public interface PublisherBatchLoader { + void load(List keys, Subscriber subscriber); } diff --git a/src/main/java/org/dataloader/PublisherBatchLoaderWithContext.java b/src/main/java/org/dataloader/PublisherBatchLoaderWithContext.java new file mode 100644 index 00000000..45ea36d6 --- /dev/null +++ b/src/main/java/org/dataloader/PublisherBatchLoaderWithContext.java @@ -0,0 +1,12 @@ +package org.dataloader; + +import org.reactivestreams.Subscriber; + +import java.util.List; + +/** + * An {@link PublisherBatchLoader} with a {@link BatchLoaderEnvironment} provided as an extra parameter to {@link #load}. + */ +public interface PublisherBatchLoaderWithContext { + void load(List keys, Subscriber subscriber, BatchLoaderEnvironment environment); +} diff --git a/src/main/java/org/dataloader/scheduler/BatchLoaderScheduler.java b/src/main/java/org/dataloader/scheduler/BatchLoaderScheduler.java index 5b88d2ce..2e82efff 100644 --- a/src/main/java/org/dataloader/scheduler/BatchLoaderScheduler.java +++ b/src/main/java/org/dataloader/scheduler/BatchLoaderScheduler.java @@ -5,8 +5,8 @@ import org.dataloader.DataLoader; import org.dataloader.DataLoaderOptions; import org.dataloader.MappedBatchLoader; -import org.dataloader.MappedObserverBatchLoader; -import org.dataloader.ObserverBatchLoader; +import org.dataloader.MappedPublisherBatchLoader; +import org.dataloader.PublisherBatchLoader; import java.util.List; import java.util.Map; @@ -45,7 +45,7 @@ interface ScheduledMappedBatchLoaderCall { } /** - * This represents a callback that will invoke a {@link ObserverBatchLoader} or {@link MappedObserverBatchLoader} function under the covers + * This represents a callback that will invoke a {@link PublisherBatchLoader} or {@link MappedPublisherBatchLoader} function under the covers */ interface ScheduledObserverBatchLoaderCall { void invoke(); @@ -82,13 +82,13 @@ interface ScheduledObserverBatchLoaderCall { CompletionStage> scheduleMappedBatchLoader(ScheduledMappedBatchLoaderCall scheduledCall, List keys, BatchLoaderEnvironment environment); /** - * This is called to schedule a {@link ObserverBatchLoader} call. + * This is called to schedule a {@link PublisherBatchLoader} call. * - * @param scheduledCall the callback that needs to be invoked to allow the {@link ObserverBatchLoader} to proceed. - * @param keys this is the list of keys that will be passed to the {@link ObserverBatchLoader}. + * @param scheduledCall the callback that needs to be invoked to allow the {@link PublisherBatchLoader} to proceed. + * @param keys this is the list of keys that will be passed to the {@link PublisherBatchLoader}. * This is provided only for informative reasons and, you can't change the keys that are used * @param environment this is the {@link BatchLoaderEnvironment} in place, - * which can be null if it's a simple {@link ObserverBatchLoader} call + * which can be null if it's a simple {@link PublisherBatchLoader} call * @param the key type */ void scheduleObserverBatchLoader(ScheduledObserverBatchLoaderCall scheduledCall, List keys, BatchLoaderEnvironment environment); diff --git a/src/test/java/org/dataloader/DataLoaderObserverBatchLoaderTest.java b/src/test/java/org/dataloader/DataLoaderMappedPublisherBatchLoaderTest.java similarity index 67% rename from src/test/java/org/dataloader/DataLoaderObserverBatchLoaderTest.java rename to src/test/java/org/dataloader/DataLoaderMappedPublisherBatchLoaderTest.java index eaeef8f3..82d6c298 100644 --- a/src/test/java/org/dataloader/DataLoaderObserverBatchLoaderTest.java +++ b/src/test/java/org/dataloader/DataLoaderMappedPublisherBatchLoaderTest.java @@ -1,8 +1,10 @@ package org.dataloader; import org.junit.Test; +import reactor.core.publisher.Flux; import java.util.List; +import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; import java.util.concurrent.ExecutionException; @@ -15,7 +17,7 @@ import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertThat; -public class DataLoaderObserverBatchLoaderTest { +public class DataLoaderMappedPublisherBatchLoaderTest { @Test public void should_Build_a_really_really_simple_data_loader() { @@ -74,35 +76,9 @@ public void should_observer_batch_multiple_requests() throws ExecutionException, assertThat(future2.get(), equalTo(2)); } - // A simple wrapper class intended as a proof external libraries can leverage this. - private static class Publisher { - private final BatchObserver delegate; - private Publisher(BatchObserver delegate) { this.delegate = delegate; } - void onNext(V value) { delegate.onNext(value); } - void onCompleted() { delegate.onCompleted(); } - void onError(Throwable e) { delegate.onError(e); } - // Mock 'subscribe' methods to simulate what would happen in the real thing. - void subscribe(List values) { - values.forEach(this::onNext); - this.onCompleted(); - } - void subscribe(List values, Throwable e) { - values.forEach(this::onNext); - this.onError(e); - } - } - - private static ObserverBatchLoader keysAsValues() { - return (keys, observer) -> { - Publisher publisher = new Publisher<>(observer); - publisher.subscribe(keys); - }; - } - - private static ObserverBatchLoader keysWithValuesAndException(List values, Throwable e) { - return (keys, observer) -> { - Publisher publisher = new Publisher<>(observer); - publisher.subscribe(values, e); - }; + private static MappedPublisherBatchLoader keysAsValues() { + return (keys, subscriber) -> Flux + .fromStream(keys.stream().map(k -> Map.entry(k, k))) + .subscribe(subscriber); } } diff --git a/src/test/java/org/dataloader/DataLoaderMappedObserverBatchLoaderTest.java b/src/test/java/org/dataloader/DataLoaderPublisherBatchLoaderTest.java similarity index 66% rename from src/test/java/org/dataloader/DataLoaderMappedObserverBatchLoaderTest.java rename to src/test/java/org/dataloader/DataLoaderPublisherBatchLoaderTest.java index e6f11685..a286ac87 100644 --- a/src/test/java/org/dataloader/DataLoaderMappedObserverBatchLoaderTest.java +++ b/src/test/java/org/dataloader/DataLoaderPublisherBatchLoaderTest.java @@ -1,24 +1,22 @@ package org.dataloader; import org.junit.Test; +import reactor.core.publisher.Flux; import java.util.List; -import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicBoolean; import static java.util.Arrays.asList; -import static java.util.function.Function.identity; -import static java.util.stream.Collectors.toMap; import static org.awaitility.Awaitility.await; import static org.dataloader.DataLoaderFactory.mkDataLoader; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertThat; -public class DataLoaderMappedObserverBatchLoaderTest { +public class DataLoaderPublisherBatchLoaderTest { @Test public void should_Build_a_really_really_simple_data_loader() { @@ -77,30 +75,7 @@ public void should_observer_batch_multiple_requests() throws ExecutionException, assertThat(future2.get(), equalTo(2)); } - // A simple wrapper class intended as a proof external libraries can leverage this. - private static class Publisher { - - private final MappedBatchObserver delegate; - private Publisher(MappedBatchObserver delegate) { this.delegate = delegate; } - void onNext(Map.Entry entry) { delegate.onNext(entry.getKey(), entry.getValue()); } - void onCompleted() { delegate.onCompleted(); } - void onError(Throwable e) { delegate.onError(e); } - // Mock 'subscribe' methods to simulate what would happen in the real thing. - void subscribe(Map valueByKey) { - valueByKey.entrySet().forEach(this::onNext); - this.onCompleted(); - } - void subscribe(Map valueByKey, Throwable e) { - valueByKey.entrySet().forEach(this::onNext); - this.onError(e); - } - } - - private static MappedObserverBatchLoader keysAsValues() { - return (keys, observer) -> { - Publisher publisher = new Publisher<>(observer); - Map valueByKey = keys.stream().collect(toMap(identity(), identity())); - publisher.subscribe(valueByKey); - }; + private static PublisherBatchLoader keysAsValues() { + return (keys, subscriber) -> Flux.fromIterable(keys).subscribe(subscriber); } } From 68d7f54984fe567f20e71f9d64da428a137323d7 Mon Sep 17 00:00:00 2001 From: Alexandre Carlton Date: Sat, 18 May 2024 22:44:45 +1000 Subject: [PATCH 013/156] Use internal Assertions over Java's raw assert This gives us more workable exceptions. --- .../java/org/dataloader/DataLoaderHelper.java | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/dataloader/DataLoaderHelper.java b/src/main/java/org/dataloader/DataLoaderHelper.java index a7e10525..df13e0ec 100644 --- a/src/main/java/org/dataloader/DataLoaderHelper.java +++ b/src/main/java/org/dataloader/DataLoaderHelper.java @@ -629,7 +629,7 @@ private class DataLoaderSubscriber implements Subscriber { private final List completedValues = new ArrayList<>(); private int idx = 0; private boolean onErrorCalled = false; - private boolean onCompletedCalled = false; + private boolean onCompleteCalled = false; private DataLoaderSubscriber( CompletableFuture> valuesFuture, @@ -650,7 +650,8 @@ public void onSubscribe(Subscription subscription) { @Override public void onNext(V value) { - assert !onErrorCalled && !onCompletedCalled; + assertState(!onErrorCalled, () -> "onError has already been called; onNext may not be invoked."); + assertState(!onCompleteCalled, () -> "onComplete has already been called; onNext may not be invoked."); K key = keys.get(idx); Object callContext = callContexts.get(idx); @@ -680,8 +681,8 @@ public void onNext(V value) { @Override public void onComplete() { - assert !onErrorCalled; - onCompletedCalled = true; + assertState(!onErrorCalled, () -> "onError has already been called; onComplete may not be invoked."); + onCompleteCalled = true; assertResultSize(keys, completedValues); @@ -691,7 +692,7 @@ public void onComplete() { @Override public void onError(Throwable ex) { - assert !onCompletedCalled; + assertState(!onCompleteCalled, () -> "onComplete has already been called; onError may not be invoked."); onErrorCalled = true; stats.incrementBatchLoadExceptionCount(new IncrementBatchLoadExceptionCountStatisticsContext<>(keys, callContexts)); @@ -720,7 +721,7 @@ private class DataLoaderMapEntrySubscriber implements Subscriber private final List clearCacheKeys = new ArrayList<>(); private final Map completedValuesByKey = new HashMap<>(); private boolean onErrorCalled = false; - private boolean onCompletedCalled = false; + private boolean onCompleteCalled = false; private DataLoaderMapEntrySubscriber( CompletableFuture> valuesFuture, @@ -751,7 +752,8 @@ public void onSubscribe(Subscription subscription) { @Override public void onNext(Map.Entry entry) { - assert !onErrorCalled && !onCompletedCalled; + assertState(!onErrorCalled, () -> "onError has already been called; onNext may not be invoked."); + assertState(!onCompleteCalled, () -> "onComplete has already been called; onNext may not be invoked."); K key = entry.getKey(); V value = entry.getValue(); @@ -781,8 +783,8 @@ public void onNext(Map.Entry entry) { @Override public void onComplete() { - assert !onErrorCalled; - onCompletedCalled = true; + assertState(!onErrorCalled, () -> "onError has already been called; onComplete may not be invoked."); + onCompleteCalled = true; possiblyClearCacheEntriesOnExceptions(clearCacheKeys); List values = new ArrayList<>(keys.size()); @@ -795,7 +797,7 @@ public void onComplete() { @Override public void onError(Throwable ex) { - assert !onCompletedCalled; + assertState(!onCompleteCalled, () -> "onComplete has already been called; onError may not be invoked."); onErrorCalled = true; stats.incrementBatchLoadExceptionCount(new IncrementBatchLoadExceptionCountStatisticsContext<>(keys, callContexts)); From a3132b71631fe7aeaa358825fee518a05dfea357 Mon Sep 17 00:00:00 2001 From: Alexandre Carlton Date: Sat, 18 May 2024 22:52:34 +1000 Subject: [PATCH 014/156] Remove handling of Throwable passed into onNext Passing an exception into `onNext` is not typically done in reactive-land - we would instead call `onError(Throwable)`. We can thus avoid handling this case. --- src/main/java/org/dataloader/DataLoaderHelper.java | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/dataloader/DataLoaderHelper.java b/src/main/java/org/dataloader/DataLoaderHelper.java index df13e0ec..12817e16 100644 --- a/src/main/java/org/dataloader/DataLoaderHelper.java +++ b/src/main/java/org/dataloader/DataLoaderHelper.java @@ -656,11 +656,7 @@ public void onNext(V value) { K key = keys.get(idx); Object callContext = callContexts.get(idx); CompletableFuture future = queuedFutures.get(idx); - if (value instanceof Throwable) { - stats.incrementLoadErrorCount(new IncrementLoadErrorCountStatisticsContext<>(key, callContext)); - future.completeExceptionally((Throwable) value); - clearCacheKeys.add(keys.get(idx)); - } else if (value instanceof Try) { + if (value instanceof Try) { // we allow the batch loader to return a Try so we can better represent a computation // that might have worked or not. Try tryValue = (Try) value; @@ -759,11 +755,7 @@ public void onNext(Map.Entry entry) { Object callContext = callContextByKey.get(key); CompletableFuture future = queuedFutureByKey.get(key); - if (value instanceof Throwable) { - stats.incrementLoadErrorCount(new IncrementLoadErrorCountStatisticsContext<>(key, callContext)); - future.completeExceptionally((Throwable) value); - clearCacheKeys.add(key); - } else if (value instanceof Try) { + if (value instanceof Try) { // we allow the batch loader to return a Try so we can better represent a computation // that might have worked or not. Try tryValue = (Try) value; From fbeffae774d965e35af7b1b83be88d7e629171ed Mon Sep 17 00:00:00 2001 From: Alexandre Carlton Date: Sat, 18 May 2024 23:24:40 +1000 Subject: [PATCH 015/156] Expose `new*DataLoader` methods for *PublisherBatchLoader This is keeping in line with the other methods found in `DataLoaderFactory`. --- .../org/dataloader/DataLoaderFactory.java | 268 ++++++++++++++++++ ...aLoaderMappedPublisherBatchLoaderTest.java | 10 +- .../DataLoaderPublisherBatchLoaderTest.java | 10 +- 3 files changed, 278 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/dataloader/DataLoaderFactory.java b/src/main/java/org/dataloader/DataLoaderFactory.java index 013f473e..5b50874b 100644 --- a/src/main/java/org/dataloader/DataLoaderFactory.java +++ b/src/main/java/org/dataloader/DataLoaderFactory.java @@ -278,6 +278,274 @@ public static DataLoader newMappedDataLoaderWithTry(MappedBatchLoad return mkDataLoader(batchLoadFunction, options); } + /** + * Creates new DataLoader with the specified batch loader function and default options + * (batching, caching and unlimited batch size). + * + * @param batchLoadFunction the batch load function to use + * @param the key type + * @param the value type + * + * @return a new DataLoader + */ + public static DataLoader newPublisherDataLoader(PublisherBatchLoader batchLoadFunction) { + return newPublisherDataLoader(batchLoadFunction, null); + } + + /** + * Creates new DataLoader with the specified batch loader function with the provided options + * + * @param batchLoadFunction the batch load function to use + * @param options the options to use + * @param the key type + * @param the value type + * + * @return a new DataLoader + */ + public static DataLoader newPublisherDataLoader(PublisherBatchLoader batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(batchLoadFunction, options); + } + + /** + * Creates new DataLoader with the specified batch loader function and default options + * (batching, caching and unlimited batch size) where the batch loader function returns a list of + * {@link org.dataloader.Try} objects. + *

+ * If it's important you to know the exact status of each item in a batch call and whether it threw exceptions then + * you can use this form to create the data loader. + *

+ * Using Try objects allows you to capture a value returned or an exception that might + * have occurred trying to get a value. . + * + * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects + * @param the key type + * @param the value type + * + * @return a new DataLoader + */ + public static DataLoader newPublisherDataLoaderWithTry(PublisherBatchLoader> batchLoadFunction) { + return newPublisherDataLoaderWithTry(batchLoadFunction, null); + } + + /** + * Creates new DataLoader with the specified batch loader function and with the provided options + * where the batch loader function returns a list of + * {@link org.dataloader.Try} objects. + * + * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects + * @param options the options to use + * @param the key type + * @param the value type + * + * @return a new DataLoader + * + * @see #newDataLoaderWithTry(BatchLoader) + */ + public static DataLoader newPublisherDataLoaderWithTry(PublisherBatchLoader> batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(batchLoadFunction, options); + } + + /** + * Creates new DataLoader with the specified batch loader function and default options + * (batching, caching and unlimited batch size). + * + * @param batchLoadFunction the batch load function to use + * @param the key type + * @param the value type + * + * @return a new DataLoader + */ + public static DataLoader newPublisherDataLoader(PublisherBatchLoaderWithContext batchLoadFunction) { + return newPublisherDataLoader(batchLoadFunction, null); + } + + /** + * Creates new DataLoader with the specified batch loader function with the provided options + * + * @param batchLoadFunction the batch load function to use + * @param options the options to use + * @param the key type + * @param the value type + * + * @return a new DataLoader + */ + public static DataLoader newPublisherDataLoader(PublisherBatchLoaderWithContext batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(batchLoadFunction, options); + } + + /** + * Creates new DataLoader with the specified batch loader function and default options + * (batching, caching and unlimited batch size) where the batch loader function returns a list of + * {@link org.dataloader.Try} objects. + *

+ * If it's important you to know the exact status of each item in a batch call and whether it threw exceptions then + * you can use this form to create the data loader. + *

+ * Using Try objects allows you to capture a value returned or an exception that might + * have occurred trying to get a value. . + * + * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects + * @param the key type + * @param the value type + * + * @return a new DataLoader + */ + public static DataLoader newPublisherDataLoaderWithTry(PublisherBatchLoaderWithContext> batchLoadFunction) { + return newPublisherDataLoaderWithTry(batchLoadFunction, null); + } + + /** + * Creates new DataLoader with the specified batch loader function and with the provided options + * where the batch loader function returns a list of + * {@link org.dataloader.Try} objects. + * + * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects + * @param options the options to use + * @param the key type + * @param the value type + * + * @return a new DataLoader + * + * @see #newPublisherDataLoaderWithTry(PublisherBatchLoader) + */ + public static DataLoader newPublisherDataLoaderWithTry(PublisherBatchLoaderWithContext> batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(batchLoadFunction, options); + } + + /** + * Creates new DataLoader with the specified batch loader function and default options + * (batching, caching and unlimited batch size). + * + * @param batchLoadFunction the batch load function to use + * @param the key type + * @param the value type + * + * @return a new DataLoader + */ + public static DataLoader newMappedPublisherDataLoader(MappedPublisherBatchLoader batchLoadFunction) { + return newMappedPublisherDataLoader(batchLoadFunction, null); + } + + /** + * Creates new DataLoader with the specified batch loader function with the provided options + * + * @param batchLoadFunction the batch load function to use + * @param options the options to use + * @param the key type + * @param the value type + * + * @return a new DataLoader + */ + public static DataLoader newMappedPublisherDataLoader(MappedPublisherBatchLoader batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(batchLoadFunction, options); + } + + /** + * Creates new DataLoader with the specified batch loader function and default options + * (batching, caching and unlimited batch size) where the batch loader function returns a list of + * {@link org.dataloader.Try} objects. + *

+ * If it's important you to know the exact status of each item in a batch call and whether it threw exceptions then + * you can use this form to create the data loader. + *

+ * Using Try objects allows you to capture a value returned or an exception that might + * have occurred trying to get a value. . + * + * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects + * @param the key type + * @param the value type + * + * @return a new DataLoader + */ + public static DataLoader newMappedPublisherDataLoaderWithTry(MappedPublisherBatchLoader> batchLoadFunction) { + return newMappedPublisherDataLoaderWithTry(batchLoadFunction, null); + } + + /** + * Creates new DataLoader with the specified batch loader function and with the provided options + * where the batch loader function returns a list of + * {@link org.dataloader.Try} objects. + * + * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects + * @param options the options to use + * @param the key type + * @param the value type + * + * @return a new DataLoader + * + * @see #newDataLoaderWithTry(BatchLoader) + */ + public static DataLoader newMappedPublisherDataLoaderWithTry(MappedPublisherBatchLoader> batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(batchLoadFunction, options); + } + + /** + * Creates new DataLoader with the specified batch loader function and default options + * (batching, caching and unlimited batch size). + * + * @param batchLoadFunction the batch load function to use + * @param the key type + * @param the value type + * + * @return a new DataLoader + */ + public static DataLoader newMappedPublisherDataLoader(MappedPublisherBatchLoaderWithContext batchLoadFunction) { + return newMappedPublisherDataLoader(batchLoadFunction, null); + } + + /** + * Creates new DataLoader with the specified batch loader function with the provided options + * + * @param batchLoadFunction the batch load function to use + * @param options the options to use + * @param the key type + * @param the value type + * + * @return a new DataLoader + */ + public static DataLoader newMappedPublisherDataLoader(MappedPublisherBatchLoaderWithContext batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(batchLoadFunction, options); + } + + /** + * Creates new DataLoader with the specified batch loader function and default options + * (batching, caching and unlimited batch size) where the batch loader function returns a list of + * {@link org.dataloader.Try} objects. + *

+ * If it's important you to know the exact status of each item in a batch call and whether it threw exceptions then + * you can use this form to create the data loader. + *

+ * Using Try objects allows you to capture a value returned or an exception that might + * have occurred trying to get a value. . + * + * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects + * @param the key type + * @param the value type + * + * @return a new DataLoader + */ + public static DataLoader newMappedPublisherDataLoaderWithTry(MappedPublisherBatchLoaderWithContext> batchLoadFunction) { + return newMappedPublisherDataLoaderWithTry(batchLoadFunction, null); + } + + /** + * Creates new DataLoader with the specified batch loader function and with the provided options + * where the batch loader function returns a list of + * {@link org.dataloader.Try} objects. + * + * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects + * @param options the options to use + * @param the key type + * @param the value type + * + * @return a new DataLoader + * + * @see #newMappedPublisherDataLoaderWithTry(MappedPublisherBatchLoader) + */ + public static DataLoader newMappedPublisherDataLoaderWithTry(MappedPublisherBatchLoaderWithContext> batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(batchLoadFunction, options); + } + static DataLoader mkDataLoader(Object batchLoadFunction, DataLoaderOptions options) { return new DataLoader<>(batchLoadFunction, options); } diff --git a/src/test/java/org/dataloader/DataLoaderMappedPublisherBatchLoaderTest.java b/src/test/java/org/dataloader/DataLoaderMappedPublisherBatchLoaderTest.java index 82d6c298..757abb71 100644 --- a/src/test/java/org/dataloader/DataLoaderMappedPublisherBatchLoaderTest.java +++ b/src/test/java/org/dataloader/DataLoaderMappedPublisherBatchLoaderTest.java @@ -12,7 +12,7 @@ import static java.util.Arrays.asList; import static org.awaitility.Awaitility.await; -import static org.dataloader.DataLoaderFactory.mkDataLoader; +import static org.dataloader.DataLoaderFactory.newMappedPublisherDataLoader; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertThat; @@ -22,7 +22,7 @@ public class DataLoaderMappedPublisherBatchLoaderTest { @Test public void should_Build_a_really_really_simple_data_loader() { AtomicBoolean success = new AtomicBoolean(); - DataLoader identityLoader = mkDataLoader(keysAsValues(), DataLoaderOptions.newOptions()); + DataLoader identityLoader = newMappedPublisherDataLoader(keysAsValues(), DataLoaderOptions.newOptions()); CompletionStage future1 = identityLoader.load(1); @@ -37,7 +37,7 @@ public void should_Build_a_really_really_simple_data_loader() { @Test public void should_Support_loading_multiple_keys_in_one_call() { AtomicBoolean success = new AtomicBoolean(); - DataLoader identityLoader = mkDataLoader(keysAsValues(), DataLoaderOptions.newOptions()); + DataLoader identityLoader = newMappedPublisherDataLoader(keysAsValues(), DataLoaderOptions.newOptions()); CompletionStage> futureAll = identityLoader.loadMany(asList(1, 2)); futureAll.thenAccept(promisedValues -> { @@ -51,7 +51,7 @@ public void should_Support_loading_multiple_keys_in_one_call() { @Test public void simple_dataloader() { - DataLoader loader = mkDataLoader(keysAsValues(), DataLoaderOptions.newOptions()); + DataLoader loader = newMappedPublisherDataLoader(keysAsValues(), DataLoaderOptions.newOptions()); loader.load("A"); loader.load("B"); @@ -65,7 +65,7 @@ public void simple_dataloader() { @Test public void should_observer_batch_multiple_requests() throws ExecutionException, InterruptedException { - DataLoader identityLoader = mkDataLoader(keysAsValues(), new DataLoaderOptions()); + DataLoader identityLoader = newMappedPublisherDataLoader(keysAsValues(), new DataLoaderOptions()); CompletableFuture future1 = identityLoader.load(1); CompletableFuture future2 = identityLoader.load(2); diff --git a/src/test/java/org/dataloader/DataLoaderPublisherBatchLoaderTest.java b/src/test/java/org/dataloader/DataLoaderPublisherBatchLoaderTest.java index a286ac87..4e5d3e15 100644 --- a/src/test/java/org/dataloader/DataLoaderPublisherBatchLoaderTest.java +++ b/src/test/java/org/dataloader/DataLoaderPublisherBatchLoaderTest.java @@ -11,7 +11,7 @@ import static java.util.Arrays.asList; import static org.awaitility.Awaitility.await; -import static org.dataloader.DataLoaderFactory.mkDataLoader; +import static org.dataloader.DataLoaderFactory.newPublisherDataLoader; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertThat; @@ -21,7 +21,7 @@ public class DataLoaderPublisherBatchLoaderTest { @Test public void should_Build_a_really_really_simple_data_loader() { AtomicBoolean success = new AtomicBoolean(); - DataLoader identityLoader = mkDataLoader(keysAsValues(), DataLoaderOptions.newOptions()); + DataLoader identityLoader = newPublisherDataLoader(keysAsValues(), DataLoaderOptions.newOptions()); CompletionStage future1 = identityLoader.load(1); @@ -36,7 +36,7 @@ public void should_Build_a_really_really_simple_data_loader() { @Test public void should_Support_loading_multiple_keys_in_one_call() { AtomicBoolean success = new AtomicBoolean(); - DataLoader identityLoader = mkDataLoader(keysAsValues(), DataLoaderOptions.newOptions()); + DataLoader identityLoader = newPublisherDataLoader(keysAsValues(), DataLoaderOptions.newOptions()); CompletionStage> futureAll = identityLoader.loadMany(asList(1, 2)); futureAll.thenAccept(promisedValues -> { @@ -50,7 +50,7 @@ public void should_Support_loading_multiple_keys_in_one_call() { @Test public void simple_dataloader() { - DataLoader loader = mkDataLoader(keysAsValues(), DataLoaderOptions.newOptions()); + DataLoader loader = newPublisherDataLoader(keysAsValues(), DataLoaderOptions.newOptions()); loader.load("A"); loader.load("B"); @@ -64,7 +64,7 @@ public void simple_dataloader() { @Test public void should_observer_batch_multiple_requests() throws ExecutionException, InterruptedException { - DataLoader identityLoader = mkDataLoader(keysAsValues(), new DataLoaderOptions()); + DataLoader identityLoader = newPublisherDataLoader(keysAsValues(), new DataLoaderOptions()); CompletableFuture future1 = identityLoader.load(1); CompletableFuture future2 = identityLoader.load(2); From b2a662da67dcd89285f8fae83c11ac90f38805bb Mon Sep 17 00:00:00 2001 From: Alexandre Carlton Date: Sun, 19 May 2024 00:33:35 +1000 Subject: [PATCH 016/156] Copy/tweak original/ DataLoader tests for publisher equivalents Given the large number of existing tests, we copy across this existing set for our publisher tests. What this really indicates is that we should invest in parameterised testing, but this is a bit painful in JUnit 4 - so we'll bump to JUnit 5 independently and parameterise when we have this available. This is important because re-using the existing test suite reveals a failure that we'll need to address. --- ...aLoaderMappedPublisherBatchLoaderTest.java | 161 ++- .../DataLoaderPublisherBatchLoaderTest.java | 1041 ++++++++++++++++- 2 files changed, 1154 insertions(+), 48 deletions(-) diff --git a/src/test/java/org/dataloader/DataLoaderMappedPublisherBatchLoaderTest.java b/src/test/java/org/dataloader/DataLoaderMappedPublisherBatchLoaderTest.java index 757abb71..8e33300a 100644 --- a/src/test/java/org/dataloader/DataLoaderMappedPublisherBatchLoaderTest.java +++ b/src/test/java/org/dataloader/DataLoaderMappedPublisherBatchLoaderTest.java @@ -3,55 +3,65 @@ import org.junit.Test; import reactor.core.publisher.Flux; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionStage; import java.util.concurrent.ExecutionException; -import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import static java.util.Arrays.asList; +import static java.util.Collections.singletonList; import static org.awaitility.Awaitility.await; import static org.dataloader.DataLoaderFactory.newMappedPublisherDataLoader; +import static org.dataloader.DataLoaderOptions.newOptions; +import static org.dataloader.fixtures.TestKit.listFrom; +import static org.dataloader.impl.CompletableFutureKit.cause; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertThat; public class DataLoaderMappedPublisherBatchLoaderTest { - @Test - public void should_Build_a_really_really_simple_data_loader() { - AtomicBoolean success = new AtomicBoolean(); - DataLoader identityLoader = newMappedPublisherDataLoader(keysAsValues(), DataLoaderOptions.newOptions()); - - CompletionStage future1 = identityLoader.load(1); + MappedPublisherBatchLoader evensOnlyMappedBatchLoader = (keys, subscriber) -> { + Map mapOfResults = new HashMap<>(); - future1.thenAccept(value -> { - assertThat(value, equalTo(1)); - success.set(true); + AtomicInteger index = new AtomicInteger(); + keys.forEach(k -> { + int i = index.getAndIncrement(); + if (i % 2 == 0) { + mapOfResults.put(k, k); + } }); - identityLoader.dispatch(); - await().untilAtomic(success, is(true)); + Flux.fromIterable(mapOfResults.entrySet()).subscribe(subscriber); + }; + + private static DataLoader idMapLoader(DataLoaderOptions options, List> loadCalls) { + MappedPublisherBatchLoader kvBatchLoader = (keys, subscriber) -> { + loadCalls.add(new ArrayList<>(keys)); + Map map = new HashMap<>(); + //noinspection unchecked + keys.forEach(k -> map.put(k, (V) k)); + Flux.fromIterable(map.entrySet()).subscribe(subscriber); + }; + return DataLoaderFactory.newMappedPublisherDataLoader(kvBatchLoader, options); } - @Test - public void should_Support_loading_multiple_keys_in_one_call() { - AtomicBoolean success = new AtomicBoolean(); - DataLoader identityLoader = newMappedPublisherDataLoader(keysAsValues(), DataLoaderOptions.newOptions()); - - CompletionStage> futureAll = identityLoader.loadMany(asList(1, 2)); - futureAll.thenAccept(promisedValues -> { - assertThat(promisedValues.size(), is(2)); - success.set(true); - }); - identityLoader.dispatch(); - await().untilAtomic(success, is(true)); - assertThat(futureAll.toCompletableFuture().join(), equalTo(asList(1, 2))); + private static DataLoader idMapLoaderBlowsUps( + DataLoaderOptions options, List> loadCalls) { + return newMappedPublisherDataLoader((MappedPublisherBatchLoader) (keys, subscriber) -> { + loadCalls.add(new ArrayList<>(keys)); + Flux.>error(new IllegalStateException("Error")).subscribe(subscriber); + }, options); } + @Test - public void simple_dataloader() { - DataLoader loader = newMappedPublisherDataLoader(keysAsValues(), DataLoaderOptions.newOptions()); + public void basic_map_batch_loading() { + DataLoader loader = DataLoaderFactory.newMappedPublisherDataLoader(evensOnlyMappedBatchLoader); loader.load("A"); loader.load("B"); @@ -60,12 +70,13 @@ public void simple_dataloader() { List results = loader.dispatchAndJoin(); assertThat(results.size(), equalTo(4)); - assertThat(results, equalTo(asList("A", "B", "C", "D"))); + assertThat(results, equalTo(asList("A", null, "C", null))); } @Test - public void should_observer_batch_multiple_requests() throws ExecutionException, InterruptedException { - DataLoader identityLoader = newMappedPublisherDataLoader(keysAsValues(), new DataLoaderOptions()); + public void should_map_Batch_multiple_requests() throws ExecutionException, InterruptedException { + List> loadCalls = new ArrayList<>(); + DataLoader identityLoader = idMapLoader(new DataLoaderOptions(), loadCalls); CompletableFuture future1 = identityLoader.load(1); CompletableFuture future2 = identityLoader.load(2); @@ -74,11 +85,91 @@ public void should_observer_batch_multiple_requests() throws ExecutionException, await().until(() -> future1.isDone() && future2.isDone()); assertThat(future1.get(), equalTo(1)); assertThat(future2.get(), equalTo(2)); + assertThat(loadCalls, equalTo(singletonList(asList(1, 2)))); + } + + @Test + public void can_split_max_batch_sizes_correctly() { + List> loadCalls = new ArrayList<>(); + DataLoader identityLoader = idMapLoader(newOptions().setMaxBatchSize(5), loadCalls); + + for (int i = 0; i < 21; i++) { + identityLoader.load(i); + } + List> expectedCalls = new ArrayList<>(); + expectedCalls.add(listFrom(0, 5)); + expectedCalls.add(listFrom(5, 10)); + expectedCalls.add(listFrom(10, 15)); + expectedCalls.add(listFrom(15, 20)); + expectedCalls.add(listFrom(20, 21)); + + List result = identityLoader.dispatch().join(); + + assertThat(result, equalTo(listFrom(0, 21))); + assertThat(loadCalls, equalTo(expectedCalls)); + } + + @Test + public void should_Propagate_error_to_all_loads() { + List> loadCalls = new ArrayList<>(); + DataLoader errorLoader = idMapLoaderBlowsUps(new DataLoaderOptions(), loadCalls); + + CompletableFuture future1 = errorLoader.load(1); + CompletableFuture future2 = errorLoader.load(2); + errorLoader.dispatch(); + + await().until(future1::isDone); + + assertThat(future1.isCompletedExceptionally(), is(true)); + Throwable cause = cause(future1); + assert cause != null; + assertThat(cause, instanceOf(IllegalStateException.class)); + assertThat(cause.getMessage(), equalTo("Error")); + + await().until(future2::isDone); + cause = cause(future2); + assert cause != null; + assertThat(cause.getMessage(), equalTo(cause.getMessage())); + + assertThat(loadCalls, equalTo(singletonList(asList(1, 2)))); + } + + @Test + public void should_work_with_duplicate_keys_when_caching_disabled() throws ExecutionException, InterruptedException { + List> loadCalls = new ArrayList<>(); + DataLoader identityLoader = + idMapLoader(newOptions().setCachingEnabled(false), loadCalls); + + CompletableFuture future1 = identityLoader.load("A"); + CompletableFuture future2 = identityLoader.load("B"); + CompletableFuture future3 = identityLoader.load("A"); + identityLoader.dispatch(); + + await().until(() -> future1.isDone() && future2.isDone() && future3.isDone()); + assertThat(future1.get(), equalTo("A")); + assertThat(future2.get(), equalTo("B")); + assertThat(future3.get(), equalTo("A")); + + // the map batch functions use a set of keys as input and hence remove duplicates unlike list variant + assertThat(loadCalls, equalTo(singletonList(asList("A", "B")))); } - private static MappedPublisherBatchLoader keysAsValues() { - return (keys, subscriber) -> Flux - .fromStream(keys.stream().map(k -> Map.entry(k, k))) - .subscribe(subscriber); + @Test + public void should_work_with_duplicate_keys_when_caching_enabled() throws ExecutionException, InterruptedException { + List> loadCalls = new ArrayList<>(); + DataLoader identityLoader = + idMapLoader(newOptions().setCachingEnabled(true), loadCalls); + + CompletableFuture future1 = identityLoader.load("A"); + CompletableFuture future2 = identityLoader.load("B"); + CompletableFuture future3 = identityLoader.load("A"); + identityLoader.dispatch(); + + await().until(() -> future1.isDone() && future2.isDone() && future3.isDone()); + assertThat(future1.get(), equalTo("A")); + assertThat(future2.get(), equalTo("B")); + assertThat(future3.get(), equalTo("A")); + assertThat(loadCalls, equalTo(singletonList(asList("A", "B")))); } + } diff --git a/src/test/java/org/dataloader/DataLoaderPublisherBatchLoaderTest.java b/src/test/java/org/dataloader/DataLoaderPublisherBatchLoaderTest.java index 4e5d3e15..508a031c 100644 --- a/src/test/java/org/dataloader/DataLoaderPublisherBatchLoaderTest.java +++ b/src/test/java/org/dataloader/DataLoaderPublisherBatchLoaderTest.java @@ -1,19 +1,40 @@ package org.dataloader; +import org.dataloader.fixtures.CustomCacheMap; +import org.dataloader.fixtures.JsonObject; +import org.dataloader.fixtures.User; +import org.dataloader.fixtures.UserManager; +import org.dataloader.impl.CompletableFutureKit; import org.junit.Test; import reactor.core.publisher.Flux; +import java.util.ArrayList; +import java.util.Collection; import java.util.List; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.Stream; import static java.util.Arrays.asList; +import static java.util.Collections.emptyList; +import static java.util.Collections.singletonList; import static org.awaitility.Awaitility.await; +import static org.dataloader.DataLoaderFactory.newDataLoader; import static org.dataloader.DataLoaderFactory.newPublisherDataLoader; +import static org.dataloader.DataLoaderFactory.newPublisherDataLoaderWithTry; +import static org.dataloader.DataLoaderOptions.newOptions; +import static org.dataloader.fixtures.TestKit.listFrom; +import static org.dataloader.impl.CompletableFutureKit.cause; +import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertThat; public class DataLoaderPublisherBatchLoaderTest { @@ -21,7 +42,7 @@ public class DataLoaderPublisherBatchLoaderTest { @Test public void should_Build_a_really_really_simple_data_loader() { AtomicBoolean success = new AtomicBoolean(); - DataLoader identityLoader = newPublisherDataLoader(keysAsValues(), DataLoaderOptions.newOptions()); + DataLoader identityLoader = newPublisherDataLoader(keysAsValues()); CompletionStage future1 = identityLoader.load(1); @@ -36,7 +57,7 @@ public void should_Build_a_really_really_simple_data_loader() { @Test public void should_Support_loading_multiple_keys_in_one_call() { AtomicBoolean success = new AtomicBoolean(); - DataLoader identityLoader = newPublisherDataLoader(keysAsValues(), DataLoaderOptions.newOptions()); + DataLoader identityLoader = newPublisherDataLoader(keysAsValues()); CompletionStage> futureAll = identityLoader.loadMany(asList(1, 2)); futureAll.thenAccept(promisedValues -> { @@ -49,33 +70,1027 @@ public void should_Support_loading_multiple_keys_in_one_call() { } @Test - public void simple_dataloader() { - DataLoader loader = newPublisherDataLoader(keysAsValues(), DataLoaderOptions.newOptions()); + public void should_Resolve_to_empty_list_when_no_keys_supplied() { + AtomicBoolean success = new AtomicBoolean(); + DataLoader identityLoader = newPublisherDataLoader(keysAsValues()); + CompletableFuture> futureEmpty = identityLoader.loadMany(emptyList()); + futureEmpty.thenAccept(promisedValues -> { + assertThat(promisedValues.size(), is(0)); + success.set(true); + }); + identityLoader.dispatch(); + await().untilAtomic(success, is(true)); + assertThat(futureEmpty.join(), empty()); + } - loader.load("A"); - loader.load("B"); - loader.loadMany(asList("C", "D")); + @Test + public void should_Return_zero_entries_dispatched_when_no_keys_supplied() { + AtomicBoolean success = new AtomicBoolean(); + DataLoader identityLoader = newPublisherDataLoader(keysAsValues()); + CompletableFuture> futureEmpty = identityLoader.loadMany(emptyList()); + futureEmpty.thenAccept(promisedValues -> { + assertThat(promisedValues.size(), is(0)); + success.set(true); + }); + DispatchResult dispatchResult = identityLoader.dispatchWithCounts(); + await().untilAtomic(success, is(true)); + assertThat(dispatchResult.getKeysCount(), equalTo(0)); + } - List results = loader.dispatchAndJoin(); + @Test + public void should_Batch_multiple_requests() throws ExecutionException, InterruptedException { + List> loadCalls = new ArrayList<>(); + DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); + + CompletableFuture future1 = identityLoader.load(1); + CompletableFuture future2 = identityLoader.load(2); + identityLoader.dispatch(); - assertThat(results.size(), equalTo(4)); - assertThat(results, equalTo(asList("A", "B", "C", "D"))); + await().until(() -> future1.isDone() && future2.isDone()); + assertThat(future1.get(), equalTo(1)); + assertThat(future2.get(), equalTo(2)); + assertThat(loadCalls, equalTo(singletonList(asList(1, 2)))); } @Test - public void should_observer_batch_multiple_requests() throws ExecutionException, InterruptedException { - DataLoader identityLoader = newPublisherDataLoader(keysAsValues(), new DataLoaderOptions()); + public void should_Return_number_of_batched_entries() { + List> loadCalls = new ArrayList<>(); + DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); CompletableFuture future1 = identityLoader.load(1); CompletableFuture future2 = identityLoader.load(2); + DispatchResult dispatchResult = identityLoader.dispatchWithCounts(); + + await().until(() -> future1.isDone() && future2.isDone()); + assertThat(dispatchResult.getKeysCount(), equalTo(2)); // its two because it's the number dispatched (by key) not the load calls + assertThat(dispatchResult.getPromisedResults().isDone(), equalTo(true)); + } + + @Test + public void should_Coalesce_identical_requests() throws ExecutionException, InterruptedException { + List> loadCalls = new ArrayList<>(); + DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); + + CompletableFuture future1a = identityLoader.load(1); + CompletableFuture future1b = identityLoader.load(1); + assertThat(future1a, equalTo(future1b)); + identityLoader.dispatch(); + + await().until(future1a::isDone); + assertThat(future1a.get(), equalTo(1)); + assertThat(future1b.get(), equalTo(1)); + assertThat(loadCalls, equalTo(singletonList(singletonList(1)))); + } + + @Test + public void should_Cache_repeated_requests() throws ExecutionException, InterruptedException { + List> loadCalls = new ArrayList<>(); + DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); + + CompletableFuture future1 = identityLoader.load("A"); + CompletableFuture future2 = identityLoader.load("B"); identityLoader.dispatch(); await().until(() -> future1.isDone() && future2.isDone()); - assertThat(future1.get(), equalTo(1)); + assertThat(future1.get(), equalTo("A")); + assertThat(future2.get(), equalTo("B")); + assertThat(loadCalls, equalTo(singletonList(asList("A", "B")))); + + CompletableFuture future1a = identityLoader.load("A"); + CompletableFuture future3 = identityLoader.load("C"); + identityLoader.dispatch(); + + await().until(() -> future1a.isDone() && future3.isDone()); + assertThat(future1a.get(), equalTo("A")); + assertThat(future3.get(), equalTo("C")); + assertThat(loadCalls, equalTo(asList(asList("A", "B"), singletonList("C")))); + + CompletableFuture future1b = identityLoader.load("A"); + CompletableFuture future2a = identityLoader.load("B"); + CompletableFuture future3a = identityLoader.load("C"); + identityLoader.dispatch(); + + await().until(() -> future1b.isDone() && future2a.isDone() && future3a.isDone()); + assertThat(future1b.get(), equalTo("A")); + assertThat(future2a.get(), equalTo("B")); + assertThat(future3a.get(), equalTo("C")); + assertThat(loadCalls, equalTo(asList(asList("A", "B"), singletonList("C")))); + } + + @Test + public void should_Not_redispatch_previous_load() throws ExecutionException, InterruptedException { + List> loadCalls = new ArrayList<>(); + DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); + + CompletableFuture future1 = identityLoader.load("A"); + identityLoader.dispatch(); + + CompletableFuture future2 = identityLoader.load("B"); + identityLoader.dispatch(); + + await().until(() -> future1.isDone() && future2.isDone()); + assertThat(future1.get(), equalTo("A")); + assertThat(future2.get(), equalTo("B")); + assertThat(loadCalls, equalTo(asList(singletonList("A"), singletonList("B")))); + } + + @Test + public void should_Cache_on_redispatch() throws ExecutionException, InterruptedException { + List> loadCalls = new ArrayList<>(); + DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); + + CompletableFuture future1 = identityLoader.load("A"); + identityLoader.dispatch(); + + CompletableFuture> future2 = identityLoader.loadMany(asList("A", "B")); + identityLoader.dispatch(); + + await().until(() -> future1.isDone() && future2.isDone()); + assertThat(future1.get(), equalTo("A")); + assertThat(future2.get(), equalTo(asList("A", "B"))); + assertThat(loadCalls, equalTo(asList(singletonList("A"), singletonList("B")))); + } + + @Test + public void should_Clear_single_value_in_loader() throws ExecutionException, InterruptedException { + List> loadCalls = new ArrayList<>(); + DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); + + CompletableFuture future1 = identityLoader.load("A"); + CompletableFuture future2 = identityLoader.load("B"); + identityLoader.dispatch(); + + await().until(() -> future1.isDone() && future2.isDone()); + assertThat(future1.get(), equalTo("A")); + assertThat(future2.get(), equalTo("B")); + assertThat(loadCalls, equalTo(singletonList(asList("A", "B")))); + + // fluency + DataLoader dl = identityLoader.clear("A"); + assertThat(dl, equalTo(identityLoader)); + + CompletableFuture future1a = identityLoader.load("A"); + CompletableFuture future2a = identityLoader.load("B"); + identityLoader.dispatch(); + + await().until(() -> future1a.isDone() && future2a.isDone()); + assertThat(future1a.get(), equalTo("A")); + assertThat(future2a.get(), equalTo("B")); + assertThat(loadCalls, equalTo(asList(asList("A", "B"), singletonList("A")))); + } + + @Test + public void should_Clear_all_values_in_loader() throws ExecutionException, InterruptedException { + List> loadCalls = new ArrayList<>(); + DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); + + CompletableFuture future1 = identityLoader.load("A"); + CompletableFuture future2 = identityLoader.load("B"); + identityLoader.dispatch(); + + await().until(() -> future1.isDone() && future2.isDone()); + assertThat(future1.get(), equalTo("A")); + assertThat(future2.get(), equalTo("B")); + assertThat(loadCalls, equalTo(singletonList(asList("A", "B")))); + + DataLoader dlFluent = identityLoader.clearAll(); + assertThat(dlFluent, equalTo(identityLoader)); // fluency + + CompletableFuture future1a = identityLoader.load("A"); + CompletableFuture future2a = identityLoader.load("B"); + identityLoader.dispatch(); + + await().until(() -> future1a.isDone() && future2a.isDone()); + assertThat(future1a.get(), equalTo("A")); + assertThat(future2a.get(), equalTo("B")); + assertThat(loadCalls, equalTo(asList(asList("A", "B"), asList("A", "B")))); + } + + @Test + public void should_Allow_priming_the_cache() throws ExecutionException, InterruptedException { + List> loadCalls = new ArrayList<>(); + DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); + + DataLoader dlFluency = identityLoader.prime("A", "A"); + assertThat(dlFluency, equalTo(identityLoader)); + + CompletableFuture future1 = identityLoader.load("A"); + CompletableFuture future2 = identityLoader.load("B"); + identityLoader.dispatch(); + + await().until(() -> future1.isDone() && future2.isDone()); + assertThat(future1.get(), equalTo("A")); + assertThat(future2.get(), equalTo("B")); + assertThat(loadCalls, equalTo(singletonList(singletonList("B")))); + } + + @Test + public void should_Not_prime_keys_that_already_exist() throws ExecutionException, InterruptedException { + List> loadCalls = new ArrayList<>(); + DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); + + identityLoader.prime("A", "X"); + + CompletableFuture future1 = identityLoader.load("A"); + CompletableFuture future2 = identityLoader.load("B"); + CompletableFuture> composite = identityLoader.dispatch(); + + await().until(composite::isDone); + assertThat(future1.get(), equalTo("X")); + assertThat(future2.get(), equalTo("B")); + + identityLoader.prime("A", "Y"); + identityLoader.prime("B", "Y"); + + CompletableFuture future1a = identityLoader.load("A"); + CompletableFuture future2a = identityLoader.load("B"); + CompletableFuture> composite2 = identityLoader.dispatch(); + + await().until(composite2::isDone); + assertThat(future1a.get(), equalTo("X")); + assertThat(future2a.get(), equalTo("B")); + assertThat(loadCalls, equalTo(singletonList(singletonList("B")))); + } + + @Test + public void should_Allow_to_forcefully_prime_the_cache() throws ExecutionException, InterruptedException { + List> loadCalls = new ArrayList<>(); + DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); + + identityLoader.prime("A", "X"); + + CompletableFuture future1 = identityLoader.load("A"); + CompletableFuture future2 = identityLoader.load("B"); + CompletableFuture> composite = identityLoader.dispatch(); + + await().until(composite::isDone); + assertThat(future1.get(), equalTo("X")); + assertThat(future2.get(), equalTo("B")); + + identityLoader.clear("A").prime("A", "Y"); + identityLoader.clear("B").prime("B", "Y"); + + CompletableFuture future1a = identityLoader.load("A"); + CompletableFuture future2a = identityLoader.load("B"); + CompletableFuture> composite2 = identityLoader.dispatch(); + + await().until(composite2::isDone); + assertThat(future1a.get(), equalTo("Y")); + assertThat(future2a.get(), equalTo("Y")); + assertThat(loadCalls, equalTo(singletonList(singletonList("B")))); + } + + @Test + public void should_Allow_priming_the_cache_with_a_future() throws ExecutionException, InterruptedException { + List> loadCalls = new ArrayList<>(); + DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); + + DataLoader dlFluency = identityLoader.prime("A", CompletableFuture.completedFuture("A")); + assertThat(dlFluency, equalTo(identityLoader)); + + CompletableFuture future1 = identityLoader.load("A"); + CompletableFuture future2 = identityLoader.load("B"); + identityLoader.dispatch(); + + await().until(() -> future1.isDone() && future2.isDone()); + assertThat(future1.get(), equalTo("A")); + assertThat(future2.get(), equalTo("B")); + assertThat(loadCalls, equalTo(singletonList(singletonList("B")))); + } + + @Test + public void should_not_Cache_failed_fetches_on_complete_failure() { + List> loadCalls = new ArrayList<>(); + DataLoader errorLoader = idLoaderBlowsUps(new DataLoaderOptions(), loadCalls); + + CompletableFuture future1 = errorLoader.load(1); + errorLoader.dispatch(); + + await().until(future1::isDone); + assertThat(future1.isCompletedExceptionally(), is(true)); + assertThat(cause(future1), instanceOf(IllegalStateException.class)); + + CompletableFuture future2 = errorLoader.load(1); + errorLoader.dispatch(); + + await().until(future2::isDone); + assertThat(future2.isCompletedExceptionally(), is(true)); + assertThat(cause(future2), instanceOf(IllegalStateException.class)); + assertThat(loadCalls, equalTo(asList(singletonList(1), singletonList(1)))); + } + + @Test + public void should_Resolve_to_error_to_indicate_failure() throws ExecutionException, InterruptedException { + List> loadCalls = new ArrayList<>(); + DataLoader evenLoader = idLoaderOddEvenExceptions(new DataLoaderOptions(), loadCalls); + + CompletableFuture future1 = evenLoader.load(1); + evenLoader.dispatch(); + + await().until(future1::isDone); + assertThat(future1.isCompletedExceptionally(), is(true)); + assertThat(cause(future1), instanceOf(IllegalStateException.class)); + + CompletableFuture future2 = evenLoader.load(2); + evenLoader.dispatch(); + + await().until(future2::isDone); assertThat(future2.get(), equalTo(2)); + assertThat(loadCalls, equalTo(asList(singletonList(1), singletonList(2)))); + } + + // Accept any kind of key. + + @Test + public void should_Represent_failures_and_successes_simultaneously() throws ExecutionException, InterruptedException { + AtomicBoolean success = new AtomicBoolean(); + List> loadCalls = new ArrayList<>(); + DataLoader evenLoader = idLoaderOddEvenExceptions(new DataLoaderOptions(), loadCalls); + + CompletableFuture future1 = evenLoader.load(1); + CompletableFuture future2 = evenLoader.load(2); + CompletableFuture future3 = evenLoader.load(3); + CompletableFuture future4 = evenLoader.load(4); + CompletableFuture> result = evenLoader.dispatch(); + result.thenAccept(promisedValues -> success.set(true)); + + await().untilAtomic(success, is(true)); + + assertThat(future1.isCompletedExceptionally(), is(true)); + assertThat(cause(future1), instanceOf(IllegalStateException.class)); + assertThat(future2.get(), equalTo(2)); + assertThat(future3.isCompletedExceptionally(), is(true)); + assertThat(future4.get(), equalTo(4)); + + assertThat(loadCalls, equalTo(singletonList(asList(1, 2, 3, 4)))); + } + + // Accepts options + + @Test + public void should_Cache_failed_fetches() { + List> loadCalls = new ArrayList<>(); + DataLoader errorLoader = idLoaderAllExceptions(new DataLoaderOptions(), loadCalls); + + CompletableFuture future1 = errorLoader.load(1); + errorLoader.dispatch(); + + await().until(future1::isDone); + assertThat(future1.isCompletedExceptionally(), is(true)); + assertThat(cause(future1), instanceOf(IllegalStateException.class)); + + CompletableFuture future2 = errorLoader.load(1); + errorLoader.dispatch(); + + await().until(future2::isDone); + assertThat(future2.isCompletedExceptionally(), is(true)); + assertThat(cause(future2), instanceOf(IllegalStateException.class)); + + assertThat(loadCalls, equalTo(singletonList(singletonList(1)))); + } + + @Test + public void should_NOT_Cache_failed_fetches_if_told_not_too() { + DataLoaderOptions options = DataLoaderOptions.newOptions().setCachingExceptionsEnabled(false); + List> loadCalls = new ArrayList<>(); + DataLoader errorLoader = idLoaderAllExceptions(options, loadCalls); + + CompletableFuture future1 = errorLoader.load(1); + errorLoader.dispatch(); + + await().until(future1::isDone); + assertThat(future1.isCompletedExceptionally(), is(true)); + assertThat(cause(future1), instanceOf(IllegalStateException.class)); + + CompletableFuture future2 = errorLoader.load(1); + errorLoader.dispatch(); + + await().until(future2::isDone); + assertThat(future2.isCompletedExceptionally(), is(true)); + assertThat(cause(future2), instanceOf(IllegalStateException.class)); + + assertThat(loadCalls, equalTo(asList(singletonList(1), singletonList(1)))); + } + + + // Accepts object key in custom cacheKey function + + @Test + public void should_Handle_priming_the_cache_with_an_error() { + List> loadCalls = new ArrayList<>(); + DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); + + identityLoader.prime(1, new IllegalStateException("Error")); + + CompletableFuture future1 = identityLoader.load(1); + identityLoader.dispatch(); + + await().until(future1::isDone); + assertThat(future1.isCompletedExceptionally(), is(true)); + assertThat(cause(future1), instanceOf(IllegalStateException.class)); + assertThat(loadCalls, equalTo(emptyList())); + } + + @Test + public void should_Clear_values_from_cache_after_errors() { + List> loadCalls = new ArrayList<>(); + DataLoader errorLoader = idLoaderBlowsUps(new DataLoaderOptions(), loadCalls); + + CompletableFuture future1 = errorLoader.load(1); + future1.handle((value, t) -> { + if (t != null) { + // Presumably determine if this error is transient, and only clear the cache in that case. + errorLoader.clear(1); + } + return null; + }); + errorLoader.dispatch(); + + await().until(future1::isDone); + assertThat(future1.isCompletedExceptionally(), is(true)); + assertThat(cause(future1), instanceOf(IllegalStateException.class)); + + CompletableFuture future2 = errorLoader.load(1); + future2.handle((value, t) -> { + if (t != null) { + // Again, only do this if you can determine the error is transient. + errorLoader.clear(1); + } + return null; + }); + errorLoader.dispatch(); + + await().until(future2::isDone); + assertThat(future2.isCompletedExceptionally(), is(true)); + assertThat(cause(future2), instanceOf(IllegalStateException.class)); + assertThat(loadCalls, equalTo(asList(singletonList(1), singletonList(1)))); + } + + @Test + public void should_Propagate_error_to_all_loads() { + List> loadCalls = new ArrayList<>(); + DataLoader errorLoader = idLoaderBlowsUps(new DataLoaderOptions(), loadCalls); + + CompletableFuture future1 = errorLoader.load(1); + CompletableFuture future2 = errorLoader.load(2); + errorLoader.dispatch(); + + await().until(future1::isDone); + assertThat(future1.isCompletedExceptionally(), is(true)); + Throwable cause = cause(future1); + assert cause != null; + assertThat(cause, instanceOf(IllegalStateException.class)); + assertThat(cause.getMessage(), equalTo("Error")); + + await().until(future2::isDone); + cause = cause(future2); + assert cause != null; + assertThat(cause.getMessage(), equalTo(cause.getMessage())); + assertThat(loadCalls, equalTo(singletonList(asList(1, 2)))); + } + + @Test + public void should_Accept_objects_as_keys() { + List> loadCalls = new ArrayList<>(); + DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); + + Object keyA = new Object(); + Object keyB = new Object(); + + // Fetches as expected + + identityLoader.load(keyA); + identityLoader.load(keyB); + + identityLoader.dispatch().thenAccept(promisedValues -> { + assertThat(promisedValues.get(0), equalTo(keyA)); + assertThat(promisedValues.get(1), equalTo(keyB)); + }); + + assertThat(loadCalls.size(), equalTo(1)); + assertThat(loadCalls.get(0).size(), equalTo(2)); + assertThat(loadCalls.get(0).toArray()[0], equalTo(keyA)); + assertThat(loadCalls.get(0).toArray()[1], equalTo(keyB)); + + // Caching + identityLoader.clear(keyA); + //noinspection SuspiciousMethodCalls + loadCalls.remove(keyA); + + identityLoader.load(keyA); + identityLoader.load(keyB); + + identityLoader.dispatch().thenAccept(promisedValues -> { + assertThat(promisedValues.get(0), equalTo(keyA)); + assertThat(identityLoader.getCacheKey(keyB), equalTo(keyB)); + }); + + assertThat(loadCalls.size(), equalTo(2)); + assertThat(loadCalls.get(1).size(), equalTo(1)); + assertThat(loadCalls.get(1).toArray()[0], equalTo(keyA)); + } + + @Test + public void should_Disable_caching() throws ExecutionException, InterruptedException { + List> loadCalls = new ArrayList<>(); + DataLoader identityLoader = + idLoader(newOptions().setCachingEnabled(false), loadCalls); + + CompletableFuture future1 = identityLoader.load("A"); + CompletableFuture future2 = identityLoader.load("B"); + identityLoader.dispatch(); + + await().until(() -> future1.isDone() && future2.isDone()); + assertThat(future1.get(), equalTo("A")); + assertThat(future2.get(), equalTo("B")); + assertThat(loadCalls, equalTo(singletonList(asList("A", "B")))); + + CompletableFuture future1a = identityLoader.load("A"); + CompletableFuture future3 = identityLoader.load("C"); + identityLoader.dispatch(); + + await().until(() -> future1a.isDone() && future3.isDone()); + assertThat(future1a.get(), equalTo("A")); + assertThat(future3.get(), equalTo("C")); + assertThat(loadCalls, equalTo(asList(asList("A", "B"), asList("A", "C")))); + + CompletableFuture future1b = identityLoader.load("A"); + CompletableFuture future2a = identityLoader.load("B"); + CompletableFuture future3a = identityLoader.load("C"); + identityLoader.dispatch(); + + await().until(() -> future1b.isDone() && future2a.isDone() && future3a.isDone()); + assertThat(future1b.get(), equalTo("A")); + assertThat(future2a.get(), equalTo("B")); + assertThat(future3a.get(), equalTo("C")); + assertThat(loadCalls, equalTo(asList(asList("A", "B"), + asList("A", "C"), asList("A", "B", "C")))); + } + + @Test + public void should_work_with_duplicate_keys_when_caching_disabled() throws ExecutionException, InterruptedException { + List> loadCalls = new ArrayList<>(); + DataLoader identityLoader = + idLoader(newOptions().setCachingEnabled(false), loadCalls); + + CompletableFuture future1 = identityLoader.load("A"); + CompletableFuture future2 = identityLoader.load("B"); + CompletableFuture future3 = identityLoader.load("A"); + identityLoader.dispatch(); + + await().until(() -> future1.isDone() && future2.isDone() && future3.isDone()); + assertThat(future1.get(), equalTo("A")); + assertThat(future2.get(), equalTo("B")); + assertThat(future3.get(), equalTo("A")); + assertThat(loadCalls, equalTo(singletonList(asList("A", "B", "A")))); + } + + @Test + public void should_work_with_duplicate_keys_when_caching_enabled() throws ExecutionException, InterruptedException { + List> loadCalls = new ArrayList<>(); + DataLoader identityLoader = + idLoader(newOptions().setCachingEnabled(true), loadCalls); + + CompletableFuture future1 = identityLoader.load("A"); + CompletableFuture future2 = identityLoader.load("B"); + CompletableFuture future3 = identityLoader.load("A"); + identityLoader.dispatch(); + + await().until(() -> future1.isDone() && future2.isDone() && future3.isDone()); + assertThat(future1.get(), equalTo("A")); + assertThat(future2.get(), equalTo("B")); + assertThat(future3.get(), equalTo("A")); + assertThat(loadCalls, equalTo(singletonList(asList("A", "B")))); + } + + // It is resilient to job queue ordering + + @Test + public void should_Accept_objects_with_a_complex_key() throws ExecutionException, InterruptedException { + List> loadCalls = new ArrayList<>(); + DataLoaderOptions options = newOptions().setCacheKeyFunction(getJsonObjectCacheMapFn()); + DataLoader identityLoader = idLoader(options, loadCalls); + + JsonObject key1 = new JsonObject().put("id", 123); + JsonObject key2 = new JsonObject().put("id", 123); + + CompletableFuture future1 = identityLoader.load(key1); + CompletableFuture future2 = identityLoader.load(key2); + identityLoader.dispatch(); + + await().until(() -> future1.isDone() && future2.isDone()); + assertThat(loadCalls, equalTo(singletonList(singletonList(key1)))); + assertThat(future1.get(), equalTo(key1)); + assertThat(future2.get(), equalTo(key1)); + } + + // Helper methods + + @Test + public void should_Clear_objects_with_complex_key() throws ExecutionException, InterruptedException { + List> loadCalls = new ArrayList<>(); + DataLoaderOptions options = newOptions().setCacheKeyFunction(getJsonObjectCacheMapFn()); + DataLoader identityLoader = idLoader(options, loadCalls); + + JsonObject key1 = new JsonObject().put("id", 123); + JsonObject key2 = new JsonObject().put("id", 123); + + CompletableFuture future1 = identityLoader.load(key1); + identityLoader.dispatch(); + + await().until(future1::isDone); + identityLoader.clear(key2); // clear equivalent object key + + CompletableFuture future2 = identityLoader.load(key1); + identityLoader.dispatch(); + + await().until(future2::isDone); + assertThat(loadCalls, equalTo(asList(singletonList(key1), singletonList(key1)))); + assertThat(future1.get(), equalTo(key1)); + assertThat(future2.get(), equalTo(key1)); + } + + @Test + public void should_Accept_objects_with_different_order_of_keys() throws ExecutionException, InterruptedException { + List> loadCalls = new ArrayList<>(); + DataLoaderOptions options = newOptions().setCacheKeyFunction(getJsonObjectCacheMapFn()); + DataLoader identityLoader = idLoader(options, loadCalls); + + JsonObject key1 = new JsonObject().put("a", 123).put("b", 321); + JsonObject key2 = new JsonObject().put("b", 321).put("a", 123); + + // Fetches as expected + + CompletableFuture future1 = identityLoader.load(key1); + CompletableFuture future2 = identityLoader.load(key2); + identityLoader.dispatch(); + + await().until(() -> future1.isDone() && future2.isDone()); + assertThat(loadCalls, equalTo(singletonList(singletonList(key1)))); + assertThat(loadCalls.size(), equalTo(1)); + assertThat(future1.get(), equalTo(key1)); + assertThat(future2.get(), equalTo(key2)); + } + + @Test + public void should_Allow_priming_the_cache_with_an_object_key() throws ExecutionException, InterruptedException { + List> loadCalls = new ArrayList<>(); + DataLoaderOptions options = newOptions().setCacheKeyFunction(getJsonObjectCacheMapFn()); + DataLoader identityLoader = idLoader(options, loadCalls); + + JsonObject key1 = new JsonObject().put("id", 123); + JsonObject key2 = new JsonObject().put("id", 123); + + identityLoader.prime(key1, key1); + + CompletableFuture future1 = identityLoader.load(key1); + CompletableFuture future2 = identityLoader.load(key2); + identityLoader.dispatch(); + + await().until(() -> future1.isDone() && future2.isDone()); + assertThat(loadCalls, equalTo(emptyList())); + assertThat(future1.get(), equalTo(key1)); + assertThat(future2.get(), equalTo(key1)); + } + + @Test + public void should_Accept_a_custom_cache_map_implementation() throws ExecutionException, InterruptedException { + CustomCacheMap customMap = new CustomCacheMap(); + List> loadCalls = new ArrayList<>(); + DataLoaderOptions options = newOptions().setCacheMap(customMap); + DataLoader identityLoader = idLoader(options, loadCalls); + + // Fetches as expected + + CompletableFuture future1 = identityLoader.load("a"); + CompletableFuture future2 = identityLoader.load("b"); + CompletableFuture> composite = identityLoader.dispatch(); + + await().until(composite::isDone); + assertThat(future1.get(), equalTo("a")); + assertThat(future2.get(), equalTo("b")); + + assertThat(loadCalls, equalTo(singletonList(asList("a", "b")))); + assertArrayEquals(customMap.stash.keySet().toArray(), asList("a", "b").toArray()); + + CompletableFuture future3 = identityLoader.load("c"); + CompletableFuture future2a = identityLoader.load("b"); + composite = identityLoader.dispatch(); + + await().until(composite::isDone); + assertThat(future3.get(), equalTo("c")); + assertThat(future2a.get(), equalTo("b")); + + assertThat(loadCalls, equalTo(asList(asList("a", "b"), singletonList("c")))); + assertArrayEquals(customMap.stash.keySet().toArray(), asList("a", "b", "c").toArray()); + + // Supports clear + + identityLoader.clear("b"); + assertArrayEquals(customMap.stash.keySet().toArray(), asList("a", "c").toArray()); + + CompletableFuture future2b = identityLoader.load("b"); + composite = identityLoader.dispatch(); + + await().until(composite::isDone); + assertThat(future2b.get(), equalTo("b")); + assertThat(loadCalls, equalTo(asList(asList("a", "b"), + singletonList("c"), singletonList("b")))); + assertArrayEquals(customMap.stash.keySet().toArray(), asList("a", "c", "b").toArray()); + + // Supports clear all + + identityLoader.clearAll(); + assertArrayEquals(customMap.stash.keySet().toArray(), emptyList().toArray()); + } + + @Test + public void should_degrade_gracefully_if_cache_get_throws() { + CacheMap cache = new ThrowingCacheMap(); + DataLoaderOptions options = newOptions().setCachingEnabled(true).setCacheMap(cache); + List> loadCalls = new ArrayList<>(); + DataLoader identityLoader = idLoader(options, loadCalls); + + assertThat(identityLoader.getIfPresent("a"), equalTo(Optional.empty())); + + CompletableFuture future = identityLoader.load("a"); + identityLoader.dispatch(); + assertThat(future.join(), equalTo("a")); + } + + @Test + public void batching_disabled_should_dispatch_immediately() { + List> loadCalls = new ArrayList<>(); + DataLoaderOptions options = newOptions().setBatchingEnabled(false); + DataLoader identityLoader = idLoader(options, loadCalls); + + CompletableFuture fa = identityLoader.load("A"); + CompletableFuture fb = identityLoader.load("B"); + + // caching is on still + CompletableFuture fa1 = identityLoader.load("A"); + CompletableFuture fb1 = identityLoader.load("B"); + + List values = CompletableFutureKit.allOf(asList(fa, fb, fa1, fb1)).join(); + + assertThat(fa.join(), equalTo("A")); + assertThat(fb.join(), equalTo("B")); + assertThat(fa1.join(), equalTo("A")); + assertThat(fb1.join(), equalTo("B")); + + assertThat(values, equalTo(asList("A", "B", "A", "B"))); + + assertThat(loadCalls, equalTo(asList( + singletonList("A"), + singletonList("B")))); + + } + + @Test + public void batching_disabled_and_caching_disabled_should_dispatch_immediately_and_forget() { + List> loadCalls = new ArrayList<>(); + DataLoaderOptions options = newOptions().setBatchingEnabled(false).setCachingEnabled(false); + DataLoader identityLoader = idLoader(options, loadCalls); + + CompletableFuture fa = identityLoader.load("A"); + CompletableFuture fb = identityLoader.load("B"); + + // caching is off + CompletableFuture fa1 = identityLoader.load("A"); + CompletableFuture fb1 = identityLoader.load("B"); + + List values = CompletableFutureKit.allOf(asList(fa, fb, fa1, fb1)).join(); + + assertThat(fa.join(), equalTo("A")); + assertThat(fb.join(), equalTo("B")); + assertThat(fa1.join(), equalTo("A")); + assertThat(fb1.join(), equalTo("B")); + + assertThat(values, equalTo(asList("A", "B", "A", "B"))); + + assertThat(loadCalls, equalTo(asList( + singletonList("A"), + singletonList("B"), + singletonList("A"), + singletonList("B") + ))); + + } + + @Test + public void batches_multiple_requests_with_max_batch_size() { + List> loadCalls = new ArrayList<>(); + DataLoader identityLoader = idLoader(newOptions().setMaxBatchSize(2), loadCalls); + + CompletableFuture f1 = identityLoader.load(1); + CompletableFuture f2 = identityLoader.load(2); + CompletableFuture f3 = identityLoader.load(3); + + identityLoader.dispatch(); + + CompletableFuture.allOf(f1, f2, f3).join(); + + assertThat(f1.join(), equalTo(1)); + assertThat(f2.join(), equalTo(2)); + assertThat(f3.join(), equalTo(3)); + + assertThat(loadCalls, equalTo(asList(asList(1, 2), singletonList(3)))); + + } + + @Test + public void can_split_max_batch_sizes_correctly() { + List> loadCalls = new ArrayList<>(); + DataLoader identityLoader = idLoader(newOptions().setMaxBatchSize(5), loadCalls); + + for (int i = 0; i < 21; i++) { + identityLoader.load(i); + } + List> expectedCalls = new ArrayList<>(); + expectedCalls.add(listFrom(0, 5)); + expectedCalls.add(listFrom(5, 10)); + expectedCalls.add(listFrom(10, 15)); + expectedCalls.add(listFrom(15, 20)); + expectedCalls.add(listFrom(20, 21)); + + List result = identityLoader.dispatch().join(); + + assertThat(result, equalTo(listFrom(0, 21))); + assertThat(loadCalls, equalTo(expectedCalls)); + + } + + @Test + public void should_Batch_loads_occurring_within_futures() { + List> loadCalls = new ArrayList<>(); + DataLoader identityLoader = idLoader(newOptions(), loadCalls); + + Supplier nullValue = () -> null; + + AtomicBoolean v4Called = new AtomicBoolean(); + + CompletableFuture.supplyAsync(nullValue).thenAccept(v1 -> { + identityLoader.load("a"); + CompletableFuture.supplyAsync(nullValue).thenAccept(v2 -> { + identityLoader.load("b"); + CompletableFuture.supplyAsync(nullValue).thenAccept(v3 -> { + identityLoader.load("c"); + CompletableFuture.supplyAsync(nullValue).thenAccept( + v4 -> { + identityLoader.load("d"); + v4Called.set(true); + }); + }); + }); + }); + + await().untilTrue(v4Called); + + identityLoader.dispatchAndJoin(); + + assertThat(loadCalls, equalTo( + singletonList(asList("a", "b", "c", "d")))); + } + + @Test + public void can_call_a_loader_from_a_loader() throws Exception { + List> deepLoadCalls = new ArrayList<>(); + DataLoader deepLoader = newDataLoader(keys -> { + deepLoadCalls.add(keys); + return CompletableFuture.completedFuture(keys); + }); + + List> aLoadCalls = new ArrayList<>(); + DataLoader aLoader = newDataLoader(keys -> { + aLoadCalls.add(keys); + return deepLoader.loadMany(keys); + }); + + List> bLoadCalls = new ArrayList<>(); + DataLoader bLoader = newDataLoader(keys -> { + bLoadCalls.add(keys); + return deepLoader.loadMany(keys); + }); + + CompletableFuture a1 = aLoader.load("A1"); + CompletableFuture a2 = aLoader.load("A2"); + CompletableFuture b1 = bLoader.load("B1"); + CompletableFuture b2 = bLoader.load("B2"); + + CompletableFuture.allOf( + aLoader.dispatch(), + deepLoader.dispatch(), + bLoader.dispatch(), + deepLoader.dispatch() + ).join(); + + assertThat(a1.get(), equalTo("A1")); + assertThat(a2.get(), equalTo("A2")); + assertThat(b1.get(), equalTo("B1")); + assertThat(b2.get(), equalTo("B2")); + + assertThat(aLoadCalls, equalTo( + singletonList(asList("A1", "A2")))); + + assertThat(bLoadCalls, equalTo( + singletonList(asList("B1", "B2")))); + + assertThat(deepLoadCalls, equalTo( + asList(asList("A1", "A2"), asList("B1", "B2")))); + } + + @Test + public void should_allow_composition_of_data_loader_calls() { + UserManager userManager = new UserManager(); + + BatchLoader userBatchLoader = userIds -> CompletableFuture + .supplyAsync(() -> userIds + .stream() + .map(userManager::loadUserById) + .collect(Collectors.toList())); + DataLoader userLoader = newDataLoader(userBatchLoader); + + AtomicBoolean gandalfCalled = new AtomicBoolean(false); + AtomicBoolean sarumanCalled = new AtomicBoolean(false); + + userLoader.load(1L) + .thenAccept(user -> userLoader.load(user.getInvitedByID()) + .thenAccept(invitedBy -> { + gandalfCalled.set(true); + assertThat(invitedBy.getName(), equalTo("Manwë")); + })); + + userLoader.load(2L) + .thenAccept(user -> userLoader.load(user.getInvitedByID()) + .thenAccept(invitedBy -> { + sarumanCalled.set(true); + assertThat(invitedBy.getName(), equalTo("Aulë")); + })); + + List allResults = userLoader.dispatchAndJoin(); + + await().untilTrue(gandalfCalled); + await().untilTrue(sarumanCalled); + + assertThat(allResults.size(), equalTo(4)); + } + + private static CacheKey getJsonObjectCacheMapFn() { + return key -> key.stream() + .map(entry -> entry.getKey() + ":" + entry.getValue()) + .sorted() + .collect(Collectors.joining()); + } + + private static DataLoader idLoader(DataLoaderOptions options, List> loadCalls) { + return newPublisherDataLoader((PublisherBatchLoader) (keys, subscriber) -> { + loadCalls.add(new ArrayList<>(keys)); + Flux.fromIterable(keys).subscribe(subscriber); + }, options); + } + + private static DataLoader idLoaderBlowsUps( + DataLoaderOptions options, List> loadCalls) { + return newPublisherDataLoader((PublisherBatchLoader) (keys, subscriber) -> { + loadCalls.add(new ArrayList<>(keys)); + Flux.error(new IllegalStateException("Error")).subscribe(subscriber); + }, options); + } + + private static DataLoader idLoaderAllExceptions( + DataLoaderOptions options, List> loadCalls) { + return newPublisherDataLoaderWithTry((PublisherBatchLoader>) (keys, subscriber) -> { + loadCalls.add(new ArrayList<>(keys)); + Stream> failures = keys.stream().map(k -> Try.failed(new IllegalStateException("Error"))); + Flux.fromStream(failures).subscribe(subscriber); + }, options); + } + + private static DataLoader idLoaderOddEvenExceptions( + DataLoaderOptions options, List> loadCalls) { + return newPublisherDataLoaderWithTry((PublisherBatchLoader>) (keys, subscriber) -> { + loadCalls.add(new ArrayList<>(keys)); + + List> errors = new ArrayList<>(); + for (Integer key : keys) { + if (key % 2 == 0) { + errors.add(Try.succeeded(key)); + } else { + errors.add(Try.failed(new IllegalStateException("Error"))); + } + } + Flux.fromIterable(errors).subscribe(subscriber); + }, options); } private static PublisherBatchLoader keysAsValues() { return (keys, subscriber) -> Flux.fromIterable(keys).subscribe(subscriber); } + + private static class ThrowingCacheMap extends CustomCacheMap { + @Override + public CompletableFuture get(String key) { + throw new RuntimeException("Cache implementation failed."); + } + } } From 0d0b2f8b9626dd1b967c164029a37f213c481f91 Mon Sep 17 00:00:00 2001 From: Alexandre Carlton Date: Sun, 19 May 2024 00:42:27 +1000 Subject: [PATCH 017/156] Rename '*PublisherBatchLoader' to 'BatchPublisher' This keeps in line with the original suggestion (because yours truly couldn't read, apparently). We also purge any remaining mention of 'observer', which was the first swing at this code. --- ...erBatchLoader.java => BatchPublisher.java} | 2 +- ...xt.java => BatchPublisherWithContext.java} | 4 +- .../org/dataloader/DataLoaderFactory.java | 36 ++++++------ .../java/org/dataloader/DataLoaderHelper.java | 56 +++++++++---------- ...hLoader.java => MappedBatchPublisher.java} | 2 +- ...a => MappedBatchPublisherWithContext.java} | 4 +- .../scheduler/BatchLoaderScheduler.java | 18 +++--- src/test/java/ReadmeExamples.java | 2 +- ...java => DataLoaderBatchPublisherTest.java} | 12 ++-- ...> DataLoaderMappedBatchPublisherTest.java} | 8 +-- .../scheduler/BatchLoaderSchedulerTest.java | 6 +- 11 files changed, 75 insertions(+), 75 deletions(-) rename src/main/java/org/dataloader/{PublisherBatchLoader.java => BatchPublisher.java} (94%) rename src/main/java/org/dataloader/{PublisherBatchLoaderWithContext.java => BatchPublisherWithContext.java} (51%) rename src/main/java/org/dataloader/{MappedPublisherBatchLoader.java => MappedBatchPublisher.java} (93%) rename src/main/java/org/dataloader/{MappedPublisherBatchLoaderWithContext.java => MappedBatchPublisherWithContext.java} (54%) rename src/test/java/org/dataloader/{DataLoaderPublisherBatchLoaderTest.java => DataLoaderBatchPublisherTest.java} (98%) rename src/test/java/org/dataloader/{DataLoaderMappedPublisherBatchLoaderTest.java => DataLoaderMappedBatchPublisherTest.java} (95%) diff --git a/src/main/java/org/dataloader/PublisherBatchLoader.java b/src/main/java/org/dataloader/BatchPublisher.java similarity index 94% rename from src/main/java/org/dataloader/PublisherBatchLoader.java rename to src/main/java/org/dataloader/BatchPublisher.java index 2dcdf1e3..9d3932ad 100644 --- a/src/main/java/org/dataloader/PublisherBatchLoader.java +++ b/src/main/java/org/dataloader/BatchPublisher.java @@ -16,6 +16,6 @@ * @param type parameter indicating the type of keys to use for data load requests. * @param type parameter indicating the type of values returned */ -public interface PublisherBatchLoader { +public interface BatchPublisher { void load(List keys, Subscriber subscriber); } diff --git a/src/main/java/org/dataloader/PublisherBatchLoaderWithContext.java b/src/main/java/org/dataloader/BatchPublisherWithContext.java similarity index 51% rename from src/main/java/org/dataloader/PublisherBatchLoaderWithContext.java rename to src/main/java/org/dataloader/BatchPublisherWithContext.java index 45ea36d6..effda90e 100644 --- a/src/main/java/org/dataloader/PublisherBatchLoaderWithContext.java +++ b/src/main/java/org/dataloader/BatchPublisherWithContext.java @@ -5,8 +5,8 @@ import java.util.List; /** - * An {@link PublisherBatchLoader} with a {@link BatchLoaderEnvironment} provided as an extra parameter to {@link #load}. + * An {@link BatchPublisher} with a {@link BatchLoaderEnvironment} provided as an extra parameter to {@link #load}. */ -public interface PublisherBatchLoaderWithContext { +public interface BatchPublisherWithContext { void load(List keys, Subscriber subscriber, BatchLoaderEnvironment environment); } diff --git a/src/main/java/org/dataloader/DataLoaderFactory.java b/src/main/java/org/dataloader/DataLoaderFactory.java index 5b50874b..db14f2e8 100644 --- a/src/main/java/org/dataloader/DataLoaderFactory.java +++ b/src/main/java/org/dataloader/DataLoaderFactory.java @@ -288,7 +288,7 @@ public static DataLoader newMappedDataLoaderWithTry(MappedBatchLoad * * @return a new DataLoader */ - public static DataLoader newPublisherDataLoader(PublisherBatchLoader batchLoadFunction) { + public static DataLoader newPublisherDataLoader(BatchPublisher batchLoadFunction) { return newPublisherDataLoader(batchLoadFunction, null); } @@ -302,7 +302,7 @@ public static DataLoader newPublisherDataLoader(PublisherBatchLoade * * @return a new DataLoader */ - public static DataLoader newPublisherDataLoader(PublisherBatchLoader batchLoadFunction, DataLoaderOptions options) { + public static DataLoader newPublisherDataLoader(BatchPublisher batchLoadFunction, DataLoaderOptions options) { return mkDataLoader(batchLoadFunction, options); } @@ -323,7 +323,7 @@ public static DataLoader newPublisherDataLoader(PublisherBatchLoade * * @return a new DataLoader */ - public static DataLoader newPublisherDataLoaderWithTry(PublisherBatchLoader> batchLoadFunction) { + public static DataLoader newPublisherDataLoaderWithTry(BatchPublisher> batchLoadFunction) { return newPublisherDataLoaderWithTry(batchLoadFunction, null); } @@ -341,7 +341,7 @@ public static DataLoader newPublisherDataLoaderWithTry(PublisherBat * * @see #newDataLoaderWithTry(BatchLoader) */ - public static DataLoader newPublisherDataLoaderWithTry(PublisherBatchLoader> batchLoadFunction, DataLoaderOptions options) { + public static DataLoader newPublisherDataLoaderWithTry(BatchPublisher> batchLoadFunction, DataLoaderOptions options) { return mkDataLoader(batchLoadFunction, options); } @@ -355,7 +355,7 @@ public static DataLoader newPublisherDataLoaderWithTry(PublisherBat * * @return a new DataLoader */ - public static DataLoader newPublisherDataLoader(PublisherBatchLoaderWithContext batchLoadFunction) { + public static DataLoader newPublisherDataLoader(BatchPublisherWithContext batchLoadFunction) { return newPublisherDataLoader(batchLoadFunction, null); } @@ -369,7 +369,7 @@ public static DataLoader newPublisherDataLoader(PublisherBatchLoade * * @return a new DataLoader */ - public static DataLoader newPublisherDataLoader(PublisherBatchLoaderWithContext batchLoadFunction, DataLoaderOptions options) { + public static DataLoader newPublisherDataLoader(BatchPublisherWithContext batchLoadFunction, DataLoaderOptions options) { return mkDataLoader(batchLoadFunction, options); } @@ -390,7 +390,7 @@ public static DataLoader newPublisherDataLoader(PublisherBatchLoade * * @return a new DataLoader */ - public static DataLoader newPublisherDataLoaderWithTry(PublisherBatchLoaderWithContext> batchLoadFunction) { + public static DataLoader newPublisherDataLoaderWithTry(BatchPublisherWithContext> batchLoadFunction) { return newPublisherDataLoaderWithTry(batchLoadFunction, null); } @@ -406,9 +406,9 @@ public static DataLoader newPublisherDataLoaderWithTry(PublisherBat * * @return a new DataLoader * - * @see #newPublisherDataLoaderWithTry(PublisherBatchLoader) + * @see #newPublisherDataLoaderWithTry(BatchPublisher) */ - public static DataLoader newPublisherDataLoaderWithTry(PublisherBatchLoaderWithContext> batchLoadFunction, DataLoaderOptions options) { + public static DataLoader newPublisherDataLoaderWithTry(BatchPublisherWithContext> batchLoadFunction, DataLoaderOptions options) { return mkDataLoader(batchLoadFunction, options); } @@ -422,7 +422,7 @@ public static DataLoader newPublisherDataLoaderWithTry(PublisherBat * * @return a new DataLoader */ - public static DataLoader newMappedPublisherDataLoader(MappedPublisherBatchLoader batchLoadFunction) { + public static DataLoader newMappedPublisherDataLoader(MappedBatchPublisher batchLoadFunction) { return newMappedPublisherDataLoader(batchLoadFunction, null); } @@ -436,7 +436,7 @@ public static DataLoader newMappedPublisherDataLoader(MappedPublish * * @return a new DataLoader */ - public static DataLoader newMappedPublisherDataLoader(MappedPublisherBatchLoader batchLoadFunction, DataLoaderOptions options) { + public static DataLoader newMappedPublisherDataLoader(MappedBatchPublisher batchLoadFunction, DataLoaderOptions options) { return mkDataLoader(batchLoadFunction, options); } @@ -457,7 +457,7 @@ public static DataLoader newMappedPublisherDataLoader(MappedPublish * * @return a new DataLoader */ - public static DataLoader newMappedPublisherDataLoaderWithTry(MappedPublisherBatchLoader> batchLoadFunction) { + public static DataLoader newMappedPublisherDataLoaderWithTry(MappedBatchPublisher> batchLoadFunction) { return newMappedPublisherDataLoaderWithTry(batchLoadFunction, null); } @@ -475,7 +475,7 @@ public static DataLoader newMappedPublisherDataLoaderWithTry(Mapped * * @see #newDataLoaderWithTry(BatchLoader) */ - public static DataLoader newMappedPublisherDataLoaderWithTry(MappedPublisherBatchLoader> batchLoadFunction, DataLoaderOptions options) { + public static DataLoader newMappedPublisherDataLoaderWithTry(MappedBatchPublisher> batchLoadFunction, DataLoaderOptions options) { return mkDataLoader(batchLoadFunction, options); } @@ -489,7 +489,7 @@ public static DataLoader newMappedPublisherDataLoaderWithTry(Mapped * * @return a new DataLoader */ - public static DataLoader newMappedPublisherDataLoader(MappedPublisherBatchLoaderWithContext batchLoadFunction) { + public static DataLoader newMappedPublisherDataLoader(MappedBatchPublisherWithContext batchLoadFunction) { return newMappedPublisherDataLoader(batchLoadFunction, null); } @@ -503,7 +503,7 @@ public static DataLoader newMappedPublisherDataLoader(MappedPublish * * @return a new DataLoader */ - public static DataLoader newMappedPublisherDataLoader(MappedPublisherBatchLoaderWithContext batchLoadFunction, DataLoaderOptions options) { + public static DataLoader newMappedPublisherDataLoader(MappedBatchPublisherWithContext batchLoadFunction, DataLoaderOptions options) { return mkDataLoader(batchLoadFunction, options); } @@ -524,7 +524,7 @@ public static DataLoader newMappedPublisherDataLoader(MappedPublish * * @return a new DataLoader */ - public static DataLoader newMappedPublisherDataLoaderWithTry(MappedPublisherBatchLoaderWithContext> batchLoadFunction) { + public static DataLoader newMappedPublisherDataLoaderWithTry(MappedBatchPublisherWithContext> batchLoadFunction) { return newMappedPublisherDataLoaderWithTry(batchLoadFunction, null); } @@ -540,9 +540,9 @@ public static DataLoader newMappedPublisherDataLoaderWithTry(Mapped * * @return a new DataLoader * - * @see #newMappedPublisherDataLoaderWithTry(MappedPublisherBatchLoader) + * @see #newMappedPublisherDataLoaderWithTry(MappedBatchPublisher) */ - public static DataLoader newMappedPublisherDataLoaderWithTry(MappedPublisherBatchLoaderWithContext> batchLoadFunction, DataLoaderOptions options) { + public static DataLoader newMappedPublisherDataLoaderWithTry(MappedBatchPublisherWithContext> batchLoadFunction, DataLoaderOptions options) { return mkDataLoader(batchLoadFunction, options); } diff --git a/src/main/java/org/dataloader/DataLoaderHelper.java b/src/main/java/org/dataloader/DataLoaderHelper.java index 12817e16..2cb6a7f5 100644 --- a/src/main/java/org/dataloader/DataLoaderHelper.java +++ b/src/main/java/org/dataloader/DataLoaderHelper.java @@ -248,7 +248,7 @@ private CompletableFuture> dispatchQueueBatch(List keys, List return batchLoad .thenApply(values -> { assertResultSize(keys, values); - if (isPublisherLoader() || isMappedPublisherLoader()) { + if (isPublisher() || isMappedPublisher()) { // We have already completed the queued futures by the time the overall batchLoad future has completed. return values; } @@ -430,10 +430,10 @@ CompletableFuture> invokeLoader(List keys, List keyContexts, .context(context).keyContexts(keys, keyContexts).build(); if (isMapLoader()) { batchLoad = invokeMapBatchLoader(keys, environment); - } else if (isPublisherLoader()) { - batchLoad = invokePublisherBatchLoader(keys, keyContexts, queuedFutures, environment); - } else if (isMappedPublisherLoader()) { - batchLoad = invokeMappedPublisherBatchLoader(keys, keyContexts, queuedFutures, environment); + } else if (isPublisher()) { + batchLoad = invokeBatchPublisher(keys, keyContexts, queuedFutures, environment); + } else if (isMappedPublisher()) { + batchLoad = invokeMappedBatchPublisher(keys, keyContexts, queuedFutures, environment); } else { batchLoad = invokeListBatchLoader(keys, environment); } @@ -505,24 +505,24 @@ private CompletableFuture> invokeMapBatchLoader(List keys, BatchLoade }); } - private CompletableFuture> invokePublisherBatchLoader(List keys, List keyContexts, List> queuedFutures, BatchLoaderEnvironment environment) { + private CompletableFuture> invokeBatchPublisher(List keys, List keyContexts, List> queuedFutures, BatchLoaderEnvironment environment) { CompletableFuture> loadResult = new CompletableFuture<>(); Subscriber subscriber = new DataLoaderSubscriber(loadResult, keys, keyContexts, queuedFutures); BatchLoaderScheduler batchLoaderScheduler = loaderOptions.getBatchLoaderScheduler(); - if (batchLoadFunction instanceof PublisherBatchLoaderWithContext) { - PublisherBatchLoaderWithContext loadFunction = (PublisherBatchLoaderWithContext) batchLoadFunction; + if (batchLoadFunction instanceof BatchPublisherWithContext) { + BatchPublisherWithContext loadFunction = (BatchPublisherWithContext) batchLoadFunction; if (batchLoaderScheduler != null) { - BatchLoaderScheduler.ScheduledObserverBatchLoaderCall loadCall = () -> loadFunction.load(keys, subscriber, environment); - batchLoaderScheduler.scheduleObserverBatchLoader(loadCall, keys, environment); + BatchLoaderScheduler.ScheduledBatchPublisherCall loadCall = () -> loadFunction.load(keys, subscriber, environment); + batchLoaderScheduler.scheduleBatchPublisher(loadCall, keys, environment); } else { loadFunction.load(keys, subscriber, environment); } } else { - PublisherBatchLoader loadFunction = (PublisherBatchLoader) batchLoadFunction; + BatchPublisher loadFunction = (BatchPublisher) batchLoadFunction; if (batchLoaderScheduler != null) { - BatchLoaderScheduler.ScheduledObserverBatchLoaderCall loadCall = () -> loadFunction.load(keys, subscriber); - batchLoaderScheduler.scheduleObserverBatchLoader(loadCall, keys, null); + BatchLoaderScheduler.ScheduledBatchPublisherCall loadCall = () -> loadFunction.load(keys, subscriber); + batchLoaderScheduler.scheduleBatchPublisher(loadCall, keys, null); } else { loadFunction.load(keys, subscriber); } @@ -530,26 +530,26 @@ private CompletableFuture> invokePublisherBatchLoader(List keys, List return loadResult; } - private CompletableFuture> invokeMappedPublisherBatchLoader(List keys, List keyContexts, List> queuedFutures, BatchLoaderEnvironment environment) { + private CompletableFuture> invokeMappedBatchPublisher(List keys, List keyContexts, List> queuedFutures, BatchLoaderEnvironment environment) { CompletableFuture> loadResult = new CompletableFuture<>(); - Subscriber> observer = new DataLoaderMapEntrySubscriber(loadResult, keys, keyContexts, queuedFutures); + Subscriber> subscriber = new DataLoaderMapEntrySubscriber(loadResult, keys, keyContexts, queuedFutures); BatchLoaderScheduler batchLoaderScheduler = loaderOptions.getBatchLoaderScheduler(); - if (batchLoadFunction instanceof MappedPublisherBatchLoaderWithContext) { - MappedPublisherBatchLoaderWithContext loadFunction = (MappedPublisherBatchLoaderWithContext) batchLoadFunction; + if (batchLoadFunction instanceof MappedBatchPublisherWithContext) { + MappedBatchPublisherWithContext loadFunction = (MappedBatchPublisherWithContext) batchLoadFunction; if (batchLoaderScheduler != null) { - BatchLoaderScheduler.ScheduledObserverBatchLoaderCall loadCall = () -> loadFunction.load(keys, observer, environment); - batchLoaderScheduler.scheduleObserverBatchLoader(loadCall, keys, environment); + BatchLoaderScheduler.ScheduledBatchPublisherCall loadCall = () -> loadFunction.load(keys, subscriber, environment); + batchLoaderScheduler.scheduleBatchPublisher(loadCall, keys, environment); } else { - loadFunction.load(keys, observer, environment); + loadFunction.load(keys, subscriber, environment); } } else { - MappedPublisherBatchLoader loadFunction = (MappedPublisherBatchLoader) batchLoadFunction; + MappedBatchPublisher loadFunction = (MappedBatchPublisher) batchLoadFunction; if (batchLoaderScheduler != null) { - BatchLoaderScheduler.ScheduledObserverBatchLoaderCall loadCall = () -> loadFunction.load(keys, observer); - batchLoaderScheduler.scheduleObserverBatchLoader(loadCall, keys, null); + BatchLoaderScheduler.ScheduledBatchPublisherCall loadCall = () -> loadFunction.load(keys, subscriber); + batchLoaderScheduler.scheduleBatchPublisher(loadCall, keys, null); } else { - loadFunction.load(keys, observer); + loadFunction.load(keys, subscriber); } } return loadResult; @@ -559,12 +559,12 @@ private boolean isMapLoader() { return batchLoadFunction instanceof MappedBatchLoader || batchLoadFunction instanceof MappedBatchLoaderWithContext; } - private boolean isPublisherLoader() { - return batchLoadFunction instanceof PublisherBatchLoader; + private boolean isPublisher() { + return batchLoadFunction instanceof BatchPublisher; } - private boolean isMappedPublisherLoader() { - return batchLoadFunction instanceof MappedPublisherBatchLoader; + private boolean isMappedPublisher() { + return batchLoadFunction instanceof MappedBatchPublisher; } int dispatchDepth() { diff --git a/src/main/java/org/dataloader/MappedPublisherBatchLoader.java b/src/main/java/org/dataloader/MappedBatchPublisher.java similarity index 93% rename from src/main/java/org/dataloader/MappedPublisherBatchLoader.java rename to src/main/java/org/dataloader/MappedBatchPublisher.java index 9c7430aa..9b3fcb95 100644 --- a/src/main/java/org/dataloader/MappedPublisherBatchLoader.java +++ b/src/main/java/org/dataloader/MappedBatchPublisher.java @@ -15,6 +15,6 @@ * @param type parameter indicating the type of keys to use for data load requests. * @param type parameter indicating the type of values returned */ -public interface MappedPublisherBatchLoader { +public interface MappedBatchPublisher { void load(List keys, Subscriber> subscriber); } diff --git a/src/main/java/org/dataloader/MappedPublisherBatchLoaderWithContext.java b/src/main/java/org/dataloader/MappedBatchPublisherWithContext.java similarity index 54% rename from src/main/java/org/dataloader/MappedPublisherBatchLoaderWithContext.java rename to src/main/java/org/dataloader/MappedBatchPublisherWithContext.java index a752abcc..4810111a 100644 --- a/src/main/java/org/dataloader/MappedPublisherBatchLoaderWithContext.java +++ b/src/main/java/org/dataloader/MappedBatchPublisherWithContext.java @@ -6,8 +6,8 @@ import java.util.Map; /** - * A {@link MappedPublisherBatchLoader} with a {@link BatchLoaderEnvironment} provided as an extra parameter to {@link #load}. + * A {@link MappedBatchPublisher} with a {@link BatchLoaderEnvironment} provided as an extra parameter to {@link #load}. */ -public interface MappedPublisherBatchLoaderWithContext { +public interface MappedBatchPublisherWithContext { void load(List keys, Subscriber> subscriber, BatchLoaderEnvironment environment); } diff --git a/src/main/java/org/dataloader/scheduler/BatchLoaderScheduler.java b/src/main/java/org/dataloader/scheduler/BatchLoaderScheduler.java index 2e82efff..e7e95d9e 100644 --- a/src/main/java/org/dataloader/scheduler/BatchLoaderScheduler.java +++ b/src/main/java/org/dataloader/scheduler/BatchLoaderScheduler.java @@ -5,8 +5,8 @@ import org.dataloader.DataLoader; import org.dataloader.DataLoaderOptions; import org.dataloader.MappedBatchLoader; -import org.dataloader.MappedPublisherBatchLoader; -import org.dataloader.PublisherBatchLoader; +import org.dataloader.MappedBatchPublisher; +import org.dataloader.BatchPublisher; import java.util.List; import java.util.Map; @@ -45,9 +45,9 @@ interface ScheduledMappedBatchLoaderCall { } /** - * This represents a callback that will invoke a {@link PublisherBatchLoader} or {@link MappedPublisherBatchLoader} function under the covers + * This represents a callback that will invoke a {@link BatchPublisher} or {@link MappedBatchPublisher} function under the covers */ - interface ScheduledObserverBatchLoaderCall { + interface ScheduledBatchPublisherCall { void invoke(); } @@ -82,14 +82,14 @@ interface ScheduledObserverBatchLoaderCall { CompletionStage> scheduleMappedBatchLoader(ScheduledMappedBatchLoaderCall scheduledCall, List keys, BatchLoaderEnvironment environment); /** - * This is called to schedule a {@link PublisherBatchLoader} call. + * This is called to schedule a {@link BatchPublisher} call. * - * @param scheduledCall the callback that needs to be invoked to allow the {@link PublisherBatchLoader} to proceed. - * @param keys this is the list of keys that will be passed to the {@link PublisherBatchLoader}. + * @param scheduledCall the callback that needs to be invoked to allow the {@link BatchPublisher} to proceed. + * @param keys this is the list of keys that will be passed to the {@link BatchPublisher}. * This is provided only for informative reasons and, you can't change the keys that are used * @param environment this is the {@link BatchLoaderEnvironment} in place, - * which can be null if it's a simple {@link PublisherBatchLoader} call + * which can be null if it's a simple {@link BatchPublisher} call * @param the key type */ - void scheduleObserverBatchLoader(ScheduledObserverBatchLoaderCall scheduledCall, List keys, BatchLoaderEnvironment environment); + void scheduleBatchPublisher(ScheduledBatchPublisherCall scheduledCall, List keys, BatchLoaderEnvironment environment); } diff --git a/src/test/java/ReadmeExamples.java b/src/test/java/ReadmeExamples.java index df733ed5..31354eaa 100644 --- a/src/test/java/ReadmeExamples.java +++ b/src/test/java/ReadmeExamples.java @@ -306,7 +306,7 @@ public CompletionStage> scheduleMappedBatchLoader(ScheduledMapp } @Override - public void scheduleObserverBatchLoader(ScheduledObserverBatchLoaderCall scheduledCall, List keys, BatchLoaderEnvironment environment) { + public void scheduleBatchPublisher(ScheduledBatchPublisherCall scheduledCall, List keys, BatchLoaderEnvironment environment) { snooze(10); scheduledCall.invoke(); } diff --git a/src/test/java/org/dataloader/DataLoaderPublisherBatchLoaderTest.java b/src/test/java/org/dataloader/DataLoaderBatchPublisherTest.java similarity index 98% rename from src/test/java/org/dataloader/DataLoaderPublisherBatchLoaderTest.java rename to src/test/java/org/dataloader/DataLoaderBatchPublisherTest.java index 508a031c..84a8b181 100644 --- a/src/test/java/org/dataloader/DataLoaderPublisherBatchLoaderTest.java +++ b/src/test/java/org/dataloader/DataLoaderBatchPublisherTest.java @@ -37,7 +37,7 @@ import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertThat; -public class DataLoaderPublisherBatchLoaderTest { +public class DataLoaderBatchPublisherTest { @Test public void should_Build_a_really_really_simple_data_loader() { @@ -1043,7 +1043,7 @@ private static CacheKey getJsonObjectCacheMapFn() { } private static DataLoader idLoader(DataLoaderOptions options, List> loadCalls) { - return newPublisherDataLoader((PublisherBatchLoader) (keys, subscriber) -> { + return newPublisherDataLoader((BatchPublisher) (keys, subscriber) -> { loadCalls.add(new ArrayList<>(keys)); Flux.fromIterable(keys).subscribe(subscriber); }, options); @@ -1051,7 +1051,7 @@ private static DataLoader idLoader(DataLoaderOptions options, List DataLoader idLoaderBlowsUps( DataLoaderOptions options, List> loadCalls) { - return newPublisherDataLoader((PublisherBatchLoader) (keys, subscriber) -> { + return newPublisherDataLoader((BatchPublisher) (keys, subscriber) -> { loadCalls.add(new ArrayList<>(keys)); Flux.error(new IllegalStateException("Error")).subscribe(subscriber); }, options); @@ -1059,7 +1059,7 @@ private static DataLoader idLoaderBlowsUps( private static DataLoader idLoaderAllExceptions( DataLoaderOptions options, List> loadCalls) { - return newPublisherDataLoaderWithTry((PublisherBatchLoader>) (keys, subscriber) -> { + return newPublisherDataLoaderWithTry((BatchPublisher>) (keys, subscriber) -> { loadCalls.add(new ArrayList<>(keys)); Stream> failures = keys.stream().map(k -> Try.failed(new IllegalStateException("Error"))); Flux.fromStream(failures).subscribe(subscriber); @@ -1068,7 +1068,7 @@ private static DataLoader idLoaderAllExceptions( private static DataLoader idLoaderOddEvenExceptions( DataLoaderOptions options, List> loadCalls) { - return newPublisherDataLoaderWithTry((PublisherBatchLoader>) (keys, subscriber) -> { + return newPublisherDataLoaderWithTry((BatchPublisher>) (keys, subscriber) -> { loadCalls.add(new ArrayList<>(keys)); List> errors = new ArrayList<>(); @@ -1083,7 +1083,7 @@ private static DataLoader idLoaderOddEvenExceptions( }, options); } - private static PublisherBatchLoader keysAsValues() { + private static BatchPublisher keysAsValues() { return (keys, subscriber) -> Flux.fromIterable(keys).subscribe(subscriber); } diff --git a/src/test/java/org/dataloader/DataLoaderMappedPublisherBatchLoaderTest.java b/src/test/java/org/dataloader/DataLoaderMappedBatchPublisherTest.java similarity index 95% rename from src/test/java/org/dataloader/DataLoaderMappedPublisherBatchLoaderTest.java rename to src/test/java/org/dataloader/DataLoaderMappedBatchPublisherTest.java index 8e33300a..c16c58f7 100644 --- a/src/test/java/org/dataloader/DataLoaderMappedPublisherBatchLoaderTest.java +++ b/src/test/java/org/dataloader/DataLoaderMappedBatchPublisherTest.java @@ -24,9 +24,9 @@ import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertThat; -public class DataLoaderMappedPublisherBatchLoaderTest { +public class DataLoaderMappedBatchPublisherTest { - MappedPublisherBatchLoader evensOnlyMappedBatchLoader = (keys, subscriber) -> { + MappedBatchPublisher evensOnlyMappedBatchLoader = (keys, subscriber) -> { Map mapOfResults = new HashMap<>(); AtomicInteger index = new AtomicInteger(); @@ -40,7 +40,7 @@ public class DataLoaderMappedPublisherBatchLoaderTest { }; private static DataLoader idMapLoader(DataLoaderOptions options, List> loadCalls) { - MappedPublisherBatchLoader kvBatchLoader = (keys, subscriber) -> { + MappedBatchPublisher kvBatchLoader = (keys, subscriber) -> { loadCalls.add(new ArrayList<>(keys)); Map map = new HashMap<>(); //noinspection unchecked @@ -52,7 +52,7 @@ private static DataLoader idMapLoader(DataLoaderOptions options, Li private static DataLoader idMapLoaderBlowsUps( DataLoaderOptions options, List> loadCalls) { - return newMappedPublisherDataLoader((MappedPublisherBatchLoader) (keys, subscriber) -> { + return newMappedPublisherDataLoader((MappedBatchPublisher) (keys, subscriber) -> { loadCalls.add(new ArrayList<>(keys)); Flux.>error(new IllegalStateException("Error")).subscribe(subscriber); }, options); diff --git a/src/test/java/org/dataloader/scheduler/BatchLoaderSchedulerTest.java b/src/test/java/org/dataloader/scheduler/BatchLoaderSchedulerTest.java index b77026c8..e9c43f8e 100644 --- a/src/test/java/org/dataloader/scheduler/BatchLoaderSchedulerTest.java +++ b/src/test/java/org/dataloader/scheduler/BatchLoaderSchedulerTest.java @@ -38,7 +38,7 @@ public CompletionStage> scheduleMappedBatchLoader(ScheduledMapp } @Override - public void scheduleObserverBatchLoader(ScheduledObserverBatchLoaderCall scheduledCall, List keys, BatchLoaderEnvironment environment) { + public void scheduleBatchPublisher(ScheduledBatchPublisherCall scheduledCall, List keys, BatchLoaderEnvironment environment) { scheduledCall.invoke(); } }; @@ -63,7 +63,7 @@ public CompletionStage> scheduleMappedBatchLoader(ScheduledMapp } @Override - public void scheduleObserverBatchLoader(ScheduledObserverBatchLoaderCall scheduledCall, List keys, BatchLoaderEnvironment environment) { + public void scheduleBatchPublisher(ScheduledBatchPublisherCall scheduledCall, List keys, BatchLoaderEnvironment environment) { snooze(ms); scheduledCall.invoke(); } @@ -152,7 +152,7 @@ public CompletionStage> scheduleMappedBatchLoader(ScheduledMapp } @Override - public void scheduleObserverBatchLoader(ScheduledObserverBatchLoaderCall scheduledCall, List keys, BatchLoaderEnvironment environment) { + public void scheduleBatchPublisher(ScheduledBatchPublisherCall scheduledCall, List keys, BatchLoaderEnvironment environment) { CompletableFuture.supplyAsync(() -> { snooze(10); scheduledCall.invoke(); From 14002f6097ef599529f301818b250883d5b0b817 Mon Sep 17 00:00:00 2001 From: Alexandre Carlton Date: Sun, 19 May 2024 00:51:47 +1000 Subject: [PATCH 018/156] Ensure DataLoaderSubscriber is only called by one thread Multiple threads may call `onNext` - we thus (lazily) chuck a `synchronized` to ensure correctness at the cost of speed. In future, we should examine how we should manage this concurrency better. --- src/main/java/org/dataloader/DataLoaderHelper.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/dataloader/DataLoaderHelper.java b/src/main/java/org/dataloader/DataLoaderHelper.java index 2cb6a7f5..ee8d78b7 100644 --- a/src/main/java/org/dataloader/DataLoaderHelper.java +++ b/src/main/java/org/dataloader/DataLoaderHelper.java @@ -648,8 +648,10 @@ public void onSubscribe(Subscription subscription) { subscription.request(keys.size()); } + // onNext may be called by multiple threads - for the time being, we pass 'synchronized' to guarantee + // correctness (at the cost of speed). @Override - public void onNext(V value) { + public synchronized void onNext(V value) { assertState(!onErrorCalled, () -> "onError has already been called; onNext may not be invoked."); assertState(!onCompleteCalled, () -> "onComplete has already been called; onNext may not be invoked."); From 0f303a83707f3443b49fbd0e281a8d395f032c6f Mon Sep 17 00:00:00 2001 From: Alexandre Carlton Date: Sun, 19 May 2024 00:55:17 +1000 Subject: [PATCH 019/156] Document Subscriber#onNext invocation order --- src/main/java/org/dataloader/BatchPublisher.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/dataloader/BatchPublisher.java b/src/main/java/org/dataloader/BatchPublisher.java index 9d3932ad..5ab41e11 100644 --- a/src/main/java/org/dataloader/BatchPublisher.java +++ b/src/main/java/org/dataloader/BatchPublisher.java @@ -11,7 +11,8 @@ * the future returned by {@link DataLoader#load(Object)} to complete as soon as the individual value is available * (rather than when all values have been retrieved). *

- * It is required that values be returned in the same order as the keys provided. + * NOTE: It is required that {@link Subscriber#onNext(V)} is invoked on each value in the same order as + * the provided keys. * * @param type parameter indicating the type of keys to use for data load requests. * @param type parameter indicating the type of values returned From b2584b7070a3d19f4c4530f9ffac5e308de40655 Mon Sep 17 00:00:00 2001 From: Alexandre Carlton Date: Sun, 19 May 2024 01:15:48 +1000 Subject: [PATCH 020/156] Bump to JUnit 5 (with vintage engine) This doesn't migrate the tests - this will happen in a separate commit. --- build.gradle | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 6222df4e..b8f9c7c0 100644 --- a/build.gradle +++ b/build.gradle @@ -67,9 +67,11 @@ jar { dependencies { api 'org.slf4j:slf4j-api:' + slf4jVersion testImplementation 'org.slf4j:slf4j-simple:' + slf4jVersion - testImplementation 'junit:junit:4.12' testImplementation 'org.awaitility:awaitility:2.0.0' testImplementation 'com.github.ben-manes.caffeine:caffeine:2.9.0' + + testImplementation platform('org.junit:junit-bom:5.10.2') + testImplementation 'org.junit.vintage:junit-vintage-engine' } task sourcesJar(type: Jar) { From 5bd1a760c2dcd2f32028740918376019b95182de Mon Sep 17 00:00:00 2001 From: Alexandre Carlton Date: Sun, 19 May 2024 01:34:18 +1000 Subject: [PATCH 021/156] Migrate to JUnit 5 Jupiter This change is being made to faciliate parameterised tests which should greatly aid in adding coverage for https://github.com/graphql-java/java-dataloader/pull/148. This was largely powered by the [OpenRewrite recipe](https://docs.openrewrite.org/recipes/java/testing/junit5/junit4to5migration) for JUnit 4.x to Jupiter migration, with a few extra tweaks: - imports optimised to be single-class (i.e. no `import foo.*;`). - removed `test_` prefix from legacy JUnit 3 methods. Notably, this pulls in `org.hamcrest` for `MatcherAssert.assertThat`, which is recommended by both the recipe (which handled this migration) and IntelliJ. --- build.gradle | 6 ++- .../DataLoaderBatchLoaderEnvironmentTest.java | 4 +- .../dataloader/DataLoaderCacheMapTest.java | 4 +- .../dataloader/DataLoaderIfPresentTest.java | 4 +- .../DataLoaderMapBatchLoaderTest.java | 4 +- .../dataloader/DataLoaderRegistryTest.java | 4 +- .../org/dataloader/DataLoaderStatsTest.java | 4 +- .../java/org/dataloader/DataLoaderTest.java | 6 +-- .../org/dataloader/DataLoaderTimeTest.java | 4 +- .../dataloader/DataLoaderValueCacheTest.java | 10 ++--- .../org/dataloader/DataLoaderWithTryTest.java | 4 +- src/test/java/org/dataloader/TryTest.java | 12 ++--- .../impl/PromisedValuesImplTest.java | 4 +- .../registries/DispatchPredicateTest.java | 6 +-- ...eduledDataLoaderRegistryPredicateTest.java | 5 +-- .../ScheduledDataLoaderRegistryTest.java | 45 ++++++++++++------- .../scheduler/BatchLoaderSchedulerTest.java | 4 +- .../stats/StatisticsCollectorTest.java | 4 +- .../org/dataloader/stats/StatisticsTest.java | 4 +- 19 files changed, 77 insertions(+), 61 deletions(-) diff --git a/build.gradle b/build.gradle index b8f9c7c0..89001a03 100644 --- a/build.gradle +++ b/build.gradle @@ -68,10 +68,11 @@ dependencies { api 'org.slf4j:slf4j-api:' + slf4jVersion testImplementation 'org.slf4j:slf4j-simple:' + slf4jVersion testImplementation 'org.awaitility:awaitility:2.0.0' + testImplementation "org.hamcrest:hamcrest:2.2" testImplementation 'com.github.ben-manes.caffeine:caffeine:2.9.0' - testImplementation platform('org.junit:junit-bom:5.10.2') - testImplementation 'org.junit.vintage:junit-vintage-engine' + testImplementation 'org.junit.jupiter:junit-jupiter-api' + testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine' } task sourcesJar(type: Jar) { @@ -98,6 +99,7 @@ test { testLogging { exceptionFormat = 'full' } + useJUnitPlatform() } publishing { diff --git a/src/test/java/org/dataloader/DataLoaderBatchLoaderEnvironmentTest.java b/src/test/java/org/dataloader/DataLoaderBatchLoaderEnvironmentTest.java index 36e0ed45..6d3010e0 100644 --- a/src/test/java/org/dataloader/DataLoaderBatchLoaderEnvironmentTest.java +++ b/src/test/java/org/dataloader/DataLoaderBatchLoaderEnvironmentTest.java @@ -1,6 +1,6 @@ package org.dataloader; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.util.ArrayList; import java.util.HashMap; @@ -14,8 +14,8 @@ import static java.util.Collections.singletonList; import static org.dataloader.DataLoaderFactory.newDataLoader; import static org.dataloader.DataLoaderFactory.newMappedDataLoader; +import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; -import static org.junit.Assert.assertThat; /** * Tests related to context. DataLoaderTest is getting to big and needs refactoring diff --git a/src/test/java/org/dataloader/DataLoaderCacheMapTest.java b/src/test/java/org/dataloader/DataLoaderCacheMapTest.java index abfc8d3d..a7b82b7c 100644 --- a/src/test/java/org/dataloader/DataLoaderCacheMapTest.java +++ b/src/test/java/org/dataloader/DataLoaderCacheMapTest.java @@ -1,6 +1,6 @@ package org.dataloader; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.util.ArrayList; import java.util.Collection; @@ -8,8 +8,8 @@ import java.util.concurrent.CompletableFuture; import static org.dataloader.DataLoaderFactory.newDataLoader; +import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; -import static org.junit.Assert.assertThat; /** * Tests for cacheMap functionality.. diff --git a/src/test/java/org/dataloader/DataLoaderIfPresentTest.java b/src/test/java/org/dataloader/DataLoaderIfPresentTest.java index 1d897f2d..f0a50d6c 100644 --- a/src/test/java/org/dataloader/DataLoaderIfPresentTest.java +++ b/src/test/java/org/dataloader/DataLoaderIfPresentTest.java @@ -1,15 +1,15 @@ package org.dataloader; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; import static org.dataloader.DataLoaderFactory.newDataLoader; +import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.sameInstance; -import static org.junit.Assert.assertThat; /** * Tests for IfPresent and IfCompleted functionality. diff --git a/src/test/java/org/dataloader/DataLoaderMapBatchLoaderTest.java b/src/test/java/org/dataloader/DataLoaderMapBatchLoaderTest.java index 0fced790..af5c3847 100644 --- a/src/test/java/org/dataloader/DataLoaderMapBatchLoaderTest.java +++ b/src/test/java/org/dataloader/DataLoaderMapBatchLoaderTest.java @@ -1,6 +1,6 @@ package org.dataloader; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.util.ArrayList; import java.util.Collection; @@ -19,10 +19,10 @@ import static org.dataloader.fixtures.TestKit.futureError; import static org.dataloader.fixtures.TestKit.listFrom; import static org.dataloader.impl.CompletableFutureKit.cause; +import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; -import static org.junit.Assert.assertThat; /** * Much of the tests that related to {@link MappedBatchLoader} also related to diff --git a/src/test/java/org/dataloader/DataLoaderRegistryTest.java b/src/test/java/org/dataloader/DataLoaderRegistryTest.java index aeaf6683..bd1534dd 100644 --- a/src/test/java/org/dataloader/DataLoaderRegistryTest.java +++ b/src/test/java/org/dataloader/DataLoaderRegistryTest.java @@ -2,16 +2,16 @@ import org.dataloader.stats.SimpleStatisticsCollector; import org.dataloader.stats.Statistics; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.util.concurrent.CompletableFuture; import static java.util.Arrays.asList; import static org.dataloader.DataLoaderFactory.newDataLoader; +import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasItems; import static org.hamcrest.Matchers.sameInstance; -import static org.junit.Assert.assertThat; public class DataLoaderRegistryTest { final BatchLoader identityBatchLoader = CompletableFuture::completedFuture; diff --git a/src/test/java/org/dataloader/DataLoaderStatsTest.java b/src/test/java/org/dataloader/DataLoaderStatsTest.java index c2faa504..06e8ae6b 100644 --- a/src/test/java/org/dataloader/DataLoaderStatsTest.java +++ b/src/test/java/org/dataloader/DataLoaderStatsTest.java @@ -9,7 +9,7 @@ import org.dataloader.stats.context.IncrementCacheHitCountStatisticsContext; import org.dataloader.stats.context.IncrementLoadCountStatisticsContext; import org.dataloader.stats.context.IncrementLoadErrorCountStatisticsContext; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.util.ArrayList; import java.util.List; @@ -19,9 +19,9 @@ import static java.util.Collections.singletonList; import static java.util.concurrent.CompletableFuture.completedFuture; import static org.dataloader.DataLoaderFactory.newDataLoader; +import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; -import static org.junit.Assert.assertThat; /** * Tests related to stats. DataLoaderTest is getting to big and needs refactoring diff --git a/src/test/java/org/dataloader/DataLoaderTest.java b/src/test/java/org/dataloader/DataLoaderTest.java index cd9710e2..4f8c7fa7 100644 --- a/src/test/java/org/dataloader/DataLoaderTest.java +++ b/src/test/java/org/dataloader/DataLoaderTest.java @@ -22,7 +22,7 @@ import org.dataloader.fixtures.User; import org.dataloader.fixtures.UserManager; import org.dataloader.impl.CompletableFutureKit; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.util.ArrayList; import java.util.Collection; @@ -43,12 +43,12 @@ import static org.dataloader.DataLoaderOptions.newOptions; import static org.dataloader.fixtures.TestKit.listFrom; import static org.dataloader.impl.CompletableFutureKit.cause; +import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertThat; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; /** * Tests for {@link DataLoader}. diff --git a/src/test/java/org/dataloader/DataLoaderTimeTest.java b/src/test/java/org/dataloader/DataLoaderTimeTest.java index ee73d854..b4d645c9 100644 --- a/src/test/java/org/dataloader/DataLoaderTimeTest.java +++ b/src/test/java/org/dataloader/DataLoaderTimeTest.java @@ -1,13 +1,13 @@ package org.dataloader; import org.dataloader.fixtures.TestingClock; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.time.Instant; import static org.dataloader.fixtures.TestKit.keysAsValues; +import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; -import static org.junit.Assert.assertThat; @SuppressWarnings("UnusedReturnValue") public class DataLoaderTimeTest { diff --git a/src/test/java/org/dataloader/DataLoaderValueCacheTest.java b/src/test/java/org/dataloader/DataLoaderValueCacheTest.java index 2716fae1..1fb5ea2b 100644 --- a/src/test/java/org/dataloader/DataLoaderValueCacheTest.java +++ b/src/test/java/org/dataloader/DataLoaderValueCacheTest.java @@ -5,7 +5,7 @@ import org.dataloader.fixtures.CaffeineValueCache; import org.dataloader.fixtures.CustomValueCache; import org.dataloader.impl.DataLoaderAssertionException; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.util.ArrayList; import java.util.List; @@ -22,11 +22,11 @@ import static org.dataloader.fixtures.TestKit.snooze; import static org.dataloader.fixtures.TestKit.sort; import static org.dataloader.impl.CompletableFutureKit.failedFuture; +import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; public class DataLoaderValueCacheTest { diff --git a/src/test/java/org/dataloader/DataLoaderWithTryTest.java b/src/test/java/org/dataloader/DataLoaderWithTryTest.java index e9e8538f..fda7bd42 100644 --- a/src/test/java/org/dataloader/DataLoaderWithTryTest.java +++ b/src/test/java/org/dataloader/DataLoaderWithTryTest.java @@ -1,7 +1,7 @@ package org.dataloader; import org.hamcrest.Matchers; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.util.ArrayList; import java.util.HashMap; @@ -12,9 +12,9 @@ import static java.util.Arrays.asList; import static java.util.Collections.singletonList; import static org.dataloader.DataLoaderFactory.*; +import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.nullValue; -import static org.junit.Assert.assertThat; public class DataLoaderWithTryTest { diff --git a/src/test/java/org/dataloader/TryTest.java b/src/test/java/org/dataloader/TryTest.java index 4da7bcab..1b237e24 100644 --- a/src/test/java/org/dataloader/TryTest.java +++ b/src/test/java/org/dataloader/TryTest.java @@ -1,7 +1,7 @@ package org.dataloader; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import java.util.Optional; import java.util.concurrent.CompletableFuture; @@ -9,11 +9,11 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; +import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.nullValue; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; public class TryTest { @@ -29,7 +29,7 @@ private void expectThrowable(RunThatCanThrow runnable, Class identityBatchLoader = CompletableFuture::completedFuture; diff --git a/src/test/java/org/dataloader/registries/ScheduledDataLoaderRegistryTest.java b/src/test/java/org/dataloader/registries/ScheduledDataLoaderRegistryTest.java index 146c1861..e82205d4 100644 --- a/src/test/java/org/dataloader/registries/ScheduledDataLoaderRegistryTest.java +++ b/src/test/java/org/dataloader/registries/ScheduledDataLoaderRegistryTest.java @@ -1,11 +1,11 @@ package org.dataloader.registries; -import junit.framework.TestCase; import org.awaitility.core.ConditionTimeoutException; import org.dataloader.DataLoader; import org.dataloader.DataLoaderFactory; import org.dataloader.DataLoaderRegistry; import org.dataloader.fixtures.TestKit; +import org.junit.jupiter.api.Test; import java.time.Duration; import java.util.ArrayList; @@ -22,17 +22,22 @@ import static org.awaitility.Duration.TWO_SECONDS; import static org.dataloader.fixtures.TestKit.keysAsValues; import static org.dataloader.fixtures.TestKit.snooze; +import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; -import static org.junit.Assert.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; -public class ScheduledDataLoaderRegistryTest extends TestCase { +public class ScheduledDataLoaderRegistryTest { DispatchPredicate alwaysDispatch = (key, dl) -> true; DispatchPredicate neverDispatch = (key, dl) -> false; - public void test_basic_setup_works_like_a_normal_dlr() { + @Test + public void basic_setup_works_like_a_normal_dlr() { List> aCalls = new ArrayList<>(); List> bCalls = new ArrayList<>(); @@ -63,7 +68,8 @@ public void test_basic_setup_works_like_a_normal_dlr() { assertThat(bCalls, equalTo(singletonList(asList("BK1", "BK2")))); } - public void test_predicate_always_false() { + @Test + public void predicate_always_false() { List> calls = new ArrayList<>(); DataLoader dlA = DataLoaderFactory.newDataLoader(keysAsValues(calls)); @@ -92,7 +98,8 @@ public void test_predicate_always_false() { assertThat(calls.size(), equalTo(0)); } - public void test_predicate_that_eventually_returns_true() { + @Test + public void predicate_that_eventually_returns_true() { AtomicInteger counter = new AtomicInteger(); @@ -123,7 +130,8 @@ public void test_predicate_that_eventually_returns_true() { assertTrue(p2.isDone()); } - public void test_dispatchAllWithCountImmediately() { + @Test + public void dispatchAllWithCountImmediately() { List> calls = new ArrayList<>(); DataLoader dlA = DataLoaderFactory.newDataLoader(keysAsValues(calls)); dlA.load("K1"); @@ -140,7 +148,8 @@ public void test_dispatchAllWithCountImmediately() { assertThat(calls, equalTo(singletonList(asList("K1", "K2")))); } - public void test_dispatchAllImmediately() { + @Test + public void dispatchAllImmediately() { List> calls = new ArrayList<>(); DataLoader dlA = DataLoaderFactory.newDataLoader(keysAsValues(calls)); dlA.load("K1"); @@ -156,7 +165,8 @@ public void test_dispatchAllImmediately() { assertThat(calls, equalTo(singletonList(asList("K1", "K2")))); } - public void test_rescheduleNow() { + @Test + public void rescheduleNow() { AtomicInteger i = new AtomicInteger(); DispatchPredicate countingPredicate = (dataLoaderKey, dataLoader) -> i.incrementAndGet() > 5; @@ -179,7 +189,8 @@ public void test_rescheduleNow() { assertThat(calls, equalTo(singletonList(asList("K1", "K2")))); } - public void test_it_will_take_out_the_schedule_once_it_dispatches() { + @Test + public void it_will_take_out_the_schedule_once_it_dispatches() { AtomicInteger counter = new AtomicInteger(); DispatchPredicate countingPredicate = (dataLoaderKey, dataLoader) -> counter.incrementAndGet() > 5; @@ -220,7 +231,8 @@ public void test_it_will_take_out_the_schedule_once_it_dispatches() { assertThat(calls, equalTo(asList(asList("K1", "K2"), asList("K3", "K4")))); } - public void test_close_is_a_one_way_door() { + @Test + public void close_is_a_one_way_door() { AtomicInteger counter = new AtomicInteger(); DispatchPredicate countingPredicate = (dataLoaderKey, dataLoader) -> { counter.incrementAndGet(); @@ -264,7 +276,8 @@ public void test_close_is_a_one_way_door() { assertEquals(counter.get(), countThen + 1); } - public void test_can_tick_after_first_dispatch_for_chain_data_loaders() { + @Test + public void can_tick_after_first_dispatch_for_chain_data_loaders() { // delays much bigger than the tick rate will mean multiple calls to dispatch DataLoader dlA = TestKit.idLoaderAsync(Duration.ofMillis(100)); @@ -293,7 +306,8 @@ public void test_can_tick_after_first_dispatch_for_chain_data_loaders() { registry.close(); } - public void test_chain_data_loaders_will_hang_if_not_in_ticker_mode() { + @Test + public void chain_data_loaders_will_hang_if_not_in_ticker_mode() { // delays much bigger than the tick rate will mean multiple calls to dispatch DataLoader dlA = TestKit.idLoaderAsync(Duration.ofMillis(100)); @@ -325,7 +339,8 @@ public void test_chain_data_loaders_will_hang_if_not_in_ticker_mode() { registry.close(); } - public void test_executors_are_shutdown() { + @Test + public void executors_are_shutdown() { ScheduledDataLoaderRegistry registry = ScheduledDataLoaderRegistry.newScheduledRegistry().build(); ScheduledExecutorService executorService = registry.getScheduledExecutorService(); @@ -345,4 +360,4 @@ public void test_executors_are_shutdown() { } -} \ No newline at end of file +} diff --git a/src/test/java/org/dataloader/scheduler/BatchLoaderSchedulerTest.java b/src/test/java/org/dataloader/scheduler/BatchLoaderSchedulerTest.java index beb7c183..274ed8ce 100644 --- a/src/test/java/org/dataloader/scheduler/BatchLoaderSchedulerTest.java +++ b/src/test/java/org/dataloader/scheduler/BatchLoaderSchedulerTest.java @@ -3,7 +3,7 @@ import org.dataloader.BatchLoaderEnvironment; import org.dataloader.DataLoader; import org.dataloader.DataLoaderOptions; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.util.List; import java.util.Map; @@ -20,8 +20,8 @@ import static org.dataloader.fixtures.TestKit.keysAsValues; import static org.dataloader.fixtures.TestKit.keysAsValuesWithContext; import static org.dataloader.fixtures.TestKit.snooze; +import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; -import static org.junit.Assert.assertThat; public class BatchLoaderSchedulerTest { diff --git a/src/test/java/org/dataloader/stats/StatisticsCollectorTest.java b/src/test/java/org/dataloader/stats/StatisticsCollectorTest.java index fbfd5e22..f1cc8d89 100644 --- a/src/test/java/org/dataloader/stats/StatisticsCollectorTest.java +++ b/src/test/java/org/dataloader/stats/StatisticsCollectorTest.java @@ -5,13 +5,13 @@ import org.dataloader.stats.context.IncrementCacheHitCountStatisticsContext; import org.dataloader.stats.context.IncrementLoadCountStatisticsContext; import org.dataloader.stats.context.IncrementLoadErrorCountStatisticsContext; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.util.concurrent.CompletableFuture; import static java.util.Collections.singletonList; +import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; -import static org.junit.Assert.assertThat; public class StatisticsCollectorTest { diff --git a/src/test/java/org/dataloader/stats/StatisticsTest.java b/src/test/java/org/dataloader/stats/StatisticsTest.java index b900807e..6c909079 100644 --- a/src/test/java/org/dataloader/stats/StatisticsTest.java +++ b/src/test/java/org/dataloader/stats/StatisticsTest.java @@ -1,11 +1,11 @@ package org.dataloader.stats; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.util.Map; +import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; -import static org.junit.Assert.assertThat; public class StatisticsTest { From d45186c4ab03b10b9738df835e55759dc2e10d95 Mon Sep 17 00:00:00 2001 From: Alexandre Carlton Date: Sun, 19 May 2024 14:44:11 +1000 Subject: [PATCH 022/156] Parameterise DataLoaderTest on DataLoader The existing `DataLoaderTest` covers a very large range of test cases which will be very useful to validate what is being added in https://github.com/graphql-java/java-dataloader/pull/148. To allow new `*Publisher` `DataLoader`s to leverage this without copy-pasting, we make `DataLoaderTest` a parameterised test, using two `DataLoader` variants: - the stock 'List' `DataLoader`. - the 'Mapped' `DataLoader`. Most of the tests in `DataLoaderTest` have been retrofitted for this parameterisation, resulting in: - deduplicated code (many of the `MappedDataLoader` tests have the same test cases, almost line-for-line). - increased coverage of the `MappedDataLoader`, bolstering confidence in subsequent changes (rather than relying on the author understanding how everything is put together internally). --- build.gradle | 1 + .../DataLoaderMapBatchLoaderTest.java | 184 ------- .../java/org/dataloader/DataLoaderTest.java | 452 ++++++++++++------ 3 files changed, 299 insertions(+), 338 deletions(-) delete mode 100644 src/test/java/org/dataloader/DataLoaderMapBatchLoaderTest.java diff --git a/build.gradle b/build.gradle index 89001a03..36a620ac 100644 --- a/build.gradle +++ b/build.gradle @@ -72,6 +72,7 @@ dependencies { testImplementation 'com.github.ben-manes.caffeine:caffeine:2.9.0' testImplementation platform('org.junit:junit-bom:5.10.2') testImplementation 'org.junit.jupiter:junit-jupiter-api' + testImplementation 'org.junit.jupiter:junit-jupiter-params' testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine' } diff --git a/src/test/java/org/dataloader/DataLoaderMapBatchLoaderTest.java b/src/test/java/org/dataloader/DataLoaderMapBatchLoaderTest.java deleted file mode 100644 index af5c3847..00000000 --- a/src/test/java/org/dataloader/DataLoaderMapBatchLoaderTest.java +++ /dev/null @@ -1,184 +0,0 @@ -package org.dataloader; - -import org.junit.jupiter.api.Test; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.atomic.AtomicInteger; - -import static java.util.Arrays.asList; -import static java.util.Collections.singletonList; -import static org.awaitility.Awaitility.await; -import static org.dataloader.DataLoaderFactory.newDataLoader; -import static org.dataloader.DataLoaderOptions.newOptions; -import static org.dataloader.fixtures.TestKit.futureError; -import static org.dataloader.fixtures.TestKit.listFrom; -import static org.dataloader.impl.CompletableFutureKit.cause; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.instanceOf; -import static org.hamcrest.Matchers.is; - -/** - * Much of the tests that related to {@link MappedBatchLoader} also related to - * {@link org.dataloader.BatchLoader}. This is white box testing somewhat because we could have repeated - * ALL the tests in {@link org.dataloader.DataLoaderTest} here as well but chose not to because we KNOW that - * DataLoader differs only a little in how it handles the 2 types of loader functions. We choose to grab some - * common functionality for repeat testing and otherwise rely on the very complete other tests. - */ -public class DataLoaderMapBatchLoaderTest { - - MappedBatchLoader evensOnlyMappedBatchLoader = (keys) -> { - Map mapOfResults = new HashMap<>(); - - AtomicInteger index = new AtomicInteger(); - keys.forEach(k -> { - int i = index.getAndIncrement(); - if (i % 2 == 0) { - mapOfResults.put(k, k); - } - }); - return CompletableFuture.completedFuture(mapOfResults); - }; - - private static DataLoader idMapLoader(DataLoaderOptions options, List> loadCalls) { - MappedBatchLoader kvBatchLoader = (keys) -> { - loadCalls.add(new ArrayList<>(keys)); - Map map = new HashMap<>(); - //noinspection unchecked - keys.forEach(k -> map.put(k, (V) k)); - return CompletableFuture.completedFuture(map); - }; - return DataLoaderFactory.newMappedDataLoader(kvBatchLoader, options); - } - - private static DataLoader idMapLoaderBlowsUps( - DataLoaderOptions options, List> loadCalls) { - return newDataLoader((keys) -> { - loadCalls.add(new ArrayList<>(keys)); - return futureError(); - }, options); - } - - - @Test - public void basic_map_batch_loading() { - DataLoader loader = DataLoaderFactory.newMappedDataLoader(evensOnlyMappedBatchLoader); - - loader.load("A"); - loader.load("B"); - loader.loadMany(asList("C", "D")); - - List results = loader.dispatchAndJoin(); - - assertThat(results.size(), equalTo(4)); - assertThat(results, equalTo(asList("A", null, "C", null))); - } - - - @Test - public void should_map_Batch_multiple_requests() throws ExecutionException, InterruptedException { - List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idMapLoader(new DataLoaderOptions(), loadCalls); - - CompletableFuture future1 = identityLoader.load(1); - CompletableFuture future2 = identityLoader.load(2); - identityLoader.dispatch(); - - await().until(() -> future1.isDone() && future2.isDone()); - assertThat(future1.get(), equalTo(1)); - assertThat(future2.get(), equalTo(2)); - assertThat(loadCalls, equalTo(singletonList(asList(1, 2)))); - } - - @Test - public void can_split_max_batch_sizes_correctly() { - List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idMapLoader(newOptions().setMaxBatchSize(5), loadCalls); - - for (int i = 0; i < 21; i++) { - identityLoader.load(i); - } - List> expectedCalls = new ArrayList<>(); - expectedCalls.add(listFrom(0, 5)); - expectedCalls.add(listFrom(5, 10)); - expectedCalls.add(listFrom(10, 15)); - expectedCalls.add(listFrom(15, 20)); - expectedCalls.add(listFrom(20, 21)); - - List result = identityLoader.dispatch().join(); - - assertThat(result, equalTo(listFrom(0, 21))); - assertThat(loadCalls, equalTo(expectedCalls)); - } - - @Test - public void should_Propagate_error_to_all_loads() { - List> loadCalls = new ArrayList<>(); - DataLoader errorLoader = idMapLoaderBlowsUps(new DataLoaderOptions(), loadCalls); - - CompletableFuture future1 = errorLoader.load(1); - CompletableFuture future2 = errorLoader.load(2); - errorLoader.dispatch(); - - await().until(future1::isDone); - - assertThat(future1.isCompletedExceptionally(), is(true)); - Throwable cause = cause(future1); - assert cause != null; - assertThat(cause, instanceOf(IllegalStateException.class)); - assertThat(cause.getMessage(), equalTo("Error")); - - await().until(future2::isDone); - cause = cause(future2); - assert cause != null; - assertThat(cause.getMessage(), equalTo(cause.getMessage())); - - assertThat(loadCalls, equalTo(singletonList(asList(1, 2)))); - } - - @Test - public void should_work_with_duplicate_keys_when_caching_disabled() throws ExecutionException, InterruptedException { - List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = - idMapLoader(newOptions().setCachingEnabled(false), loadCalls); - - CompletableFuture future1 = identityLoader.load("A"); - CompletableFuture future2 = identityLoader.load("B"); - CompletableFuture future3 = identityLoader.load("A"); - identityLoader.dispatch(); - - await().until(() -> future1.isDone() && future2.isDone() && future3.isDone()); - assertThat(future1.get(), equalTo("A")); - assertThat(future2.get(), equalTo("B")); - assertThat(future3.get(), equalTo("A")); - - // the map batch functions use a set of keys as input and hence remove duplicates unlike list variant - assertThat(loadCalls, equalTo(singletonList(asList("A", "B")))); - } - - @Test - public void should_work_with_duplicate_keys_when_caching_enabled() throws ExecutionException, InterruptedException { - List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = - idMapLoader(newOptions().setCachingEnabled(true), loadCalls); - - CompletableFuture future1 = identityLoader.load("A"); - CompletableFuture future2 = identityLoader.load("B"); - CompletableFuture future3 = identityLoader.load("A"); - identityLoader.dispatch(); - - await().until(() -> future1.isDone() && future2.isDone() && future3.isDone()); - assertThat(future1.get(), equalTo("A")); - assertThat(future2.get(), equalTo("B")); - assertThat(future3.get(), equalTo("A")); - assertThat(loadCalls, equalTo(singletonList(asList("A", "B")))); - } - - -} diff --git a/src/test/java/org/dataloader/DataLoaderTest.java b/src/test/java/org/dataloader/DataLoaderTest.java index 4f8c7fa7..db71c1e9 100644 --- a/src/test/java/org/dataloader/DataLoaderTest.java +++ b/src/test/java/org/dataloader/DataLoaderTest.java @@ -22,25 +22,35 @@ import org.dataloader.fixtures.User; import org.dataloader.fixtures.UserManager; import org.dataloader.impl.CompletableFutureKit; +import org.junit.jupiter.api.Named; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Supplier; import java.util.stream.Collectors; +import java.util.stream.Stream; import static java.util.Arrays.asList; import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; import static org.awaitility.Awaitility.await; import static org.dataloader.DataLoaderFactory.newDataLoader; +import static org.dataloader.DataLoaderFactory.newMappedDataLoader; import static org.dataloader.DataLoaderOptions.newOptions; +import static org.dataloader.fixtures.TestKit.futureError; import static org.dataloader.fixtures.TestKit.listFrom; import static org.dataloader.impl.CompletableFutureKit.cause; import static org.hamcrest.MatcherAssert.assertThat; @@ -65,7 +75,7 @@ public class DataLoaderTest { @Test public void should_Build_a_really_really_simple_data_loader() { AtomicBoolean success = new AtomicBoolean(); - DataLoader identityLoader = newDataLoader(keysAsValues()); + DataLoader identityLoader = newDataLoader(CompletableFuture::completedFuture); CompletionStage future1 = identityLoader.load(1); @@ -78,9 +88,36 @@ public void should_Build_a_really_really_simple_data_loader() { } @Test - public void should_Support_loading_multiple_keys_in_one_call() { + public void basic_map_batch_loading() { + MappedBatchLoader evensOnlyMappedBatchLoader = (keys) -> { + Map mapOfResults = new HashMap<>(); + + AtomicInteger index = new AtomicInteger(); + keys.forEach(k -> { + int i = index.getAndIncrement(); + if (i % 2 == 0) { + mapOfResults.put(k, k); + } + }); + return CompletableFuture.completedFuture(mapOfResults); + }; + DataLoader loader = DataLoaderFactory.newMappedDataLoader(evensOnlyMappedBatchLoader); + + loader.load("A"); + loader.load("B"); + loader.loadMany(asList("C", "D")); + + List results = loader.dispatchAndJoin(); + + assertThat(results.size(), equalTo(4)); + assertThat(results, equalTo(asList("A", null, "C", null))); + } + + @ParameterizedTest + @MethodSource("dataLoaderFactories") + public void should_Support_loading_multiple_keys_in_one_call(TestDataLoaderFactory factory) { AtomicBoolean success = new AtomicBoolean(); - DataLoader identityLoader = newDataLoader(keysAsValues()); + DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), new ArrayList<>()); CompletionStage> futureAll = identityLoader.loadMany(asList(1, 2)); futureAll.thenAccept(promisedValues -> { @@ -92,10 +129,11 @@ public void should_Support_loading_multiple_keys_in_one_call() { assertThat(futureAll.toCompletableFuture().join(), equalTo(asList(1, 2))); } - @Test - public void should_Resolve_to_empty_list_when_no_keys_supplied() { + @ParameterizedTest + @MethodSource("dataLoaderFactories") + public void should_Resolve_to_empty_list_when_no_keys_supplied(TestDataLoaderFactory factory) { AtomicBoolean success = new AtomicBoolean(); - DataLoader identityLoader = newDataLoader(keysAsValues()); + DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), new ArrayList<>()); CompletableFuture> futureEmpty = identityLoader.loadMany(emptyList()); futureEmpty.thenAccept(promisedValues -> { assertThat(promisedValues.size(), is(0)); @@ -106,10 +144,11 @@ public void should_Resolve_to_empty_list_when_no_keys_supplied() { assertThat(futureEmpty.join(), empty()); } - @Test - public void should_Return_zero_entries_dispatched_when_no_keys_supplied() { + @ParameterizedTest + @MethodSource("dataLoaderFactories") + public void should_Return_zero_entries_dispatched_when_no_keys_supplied(TestDataLoaderFactory factory) { AtomicBoolean success = new AtomicBoolean(); - DataLoader identityLoader = newDataLoader(keysAsValues()); + DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), new ArrayList<>()); CompletableFuture> futureEmpty = identityLoader.loadMany(emptyList()); futureEmpty.thenAccept(promisedValues -> { assertThat(promisedValues.size(), is(0)); @@ -120,10 +159,11 @@ public void should_Return_zero_entries_dispatched_when_no_keys_supplied() { assertThat(dispatchResult.getKeysCount(), equalTo(0)); } - @Test - public void should_Batch_multiple_requests() throws ExecutionException, InterruptedException { + @ParameterizedTest + @MethodSource("dataLoaderFactories") + public void should_Batch_multiple_requests(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); + DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), loadCalls); CompletableFuture future1 = identityLoader.load(1); CompletableFuture future2 = identityLoader.load(2); @@ -135,10 +175,11 @@ public void should_Batch_multiple_requests() throws ExecutionException, Interrup assertThat(loadCalls, equalTo(singletonList(asList(1, 2)))); } - @Test - public void should_Return_number_of_batched_entries() { + @ParameterizedTest + @MethodSource("dataLoaderFactories") + public void should_Return_number_of_batched_entries(TestDataLoaderFactory factory) { List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); + DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), loadCalls); CompletableFuture future1 = identityLoader.load(1); CompletableFuture future2 = identityLoader.load(2); @@ -149,10 +190,11 @@ public void should_Return_number_of_batched_entries() { assertThat(dispatchResult.getPromisedResults().isDone(), equalTo(true)); } - @Test - public void should_Coalesce_identical_requests() throws ExecutionException, InterruptedException { + @ParameterizedTest + @MethodSource("dataLoaderFactories") + public void should_Coalesce_identical_requests(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); + DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), loadCalls); CompletableFuture future1a = identityLoader.load(1); CompletableFuture future1b = identityLoader.load(1); @@ -165,10 +207,11 @@ public void should_Coalesce_identical_requests() throws ExecutionException, Inte assertThat(loadCalls, equalTo(singletonList(singletonList(1)))); } - @Test - public void should_Cache_repeated_requests() throws ExecutionException, InterruptedException { + @ParameterizedTest + @MethodSource("dataLoaderFactories") + public void should_Cache_repeated_requests(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); + DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), loadCalls); CompletableFuture future1 = identityLoader.load("A"); CompletableFuture future2 = identityLoader.load("B"); @@ -200,10 +243,11 @@ public void should_Cache_repeated_requests() throws ExecutionException, Interrup assertThat(loadCalls, equalTo(asList(asList("A", "B"), singletonList("C")))); } - @Test - public void should_Not_redispatch_previous_load() throws ExecutionException, InterruptedException { + @ParameterizedTest + @MethodSource("dataLoaderFactories") + public void should_Not_redispatch_previous_load(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); + DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), loadCalls); CompletableFuture future1 = identityLoader.load("A"); identityLoader.dispatch(); @@ -217,10 +261,11 @@ public void should_Not_redispatch_previous_load() throws ExecutionException, Int assertThat(loadCalls, equalTo(asList(singletonList("A"), singletonList("B")))); } - @Test - public void should_Cache_on_redispatch() throws ExecutionException, InterruptedException { + @ParameterizedTest + @MethodSource("dataLoaderFactories") + public void should_Cache_on_redispatch(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); + DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), loadCalls); CompletableFuture future1 = identityLoader.load("A"); identityLoader.dispatch(); @@ -234,10 +279,11 @@ public void should_Cache_on_redispatch() throws ExecutionException, InterruptedE assertThat(loadCalls, equalTo(asList(singletonList("A"), singletonList("B")))); } - @Test - public void should_Clear_single_value_in_loader() throws ExecutionException, InterruptedException { + @ParameterizedTest + @MethodSource("dataLoaderFactories") + public void should_Clear_single_value_in_loader(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); + DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), loadCalls); CompletableFuture future1 = identityLoader.load("A"); CompletableFuture future2 = identityLoader.load("B"); @@ -262,10 +308,11 @@ public void should_Clear_single_value_in_loader() throws ExecutionException, Int assertThat(loadCalls, equalTo(asList(asList("A", "B"), singletonList("A")))); } - @Test - public void should_Clear_all_values_in_loader() throws ExecutionException, InterruptedException { + @ParameterizedTest + @MethodSource("dataLoaderFactories") + public void should_Clear_all_values_in_loader(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); + DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), loadCalls); CompletableFuture future1 = identityLoader.load("A"); CompletableFuture future2 = identityLoader.load("B"); @@ -289,10 +336,11 @@ public void should_Clear_all_values_in_loader() throws ExecutionException, Inter assertThat(loadCalls, equalTo(asList(asList("A", "B"), asList("A", "B")))); } - @Test - public void should_Allow_priming_the_cache() throws ExecutionException, InterruptedException { + @ParameterizedTest + @MethodSource("dataLoaderFactories") + public void should_Allow_priming_the_cache(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); + DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), loadCalls); DataLoader dlFluency = identityLoader.prime("A", "A"); assertThat(dlFluency, equalTo(identityLoader)); @@ -307,10 +355,11 @@ public void should_Allow_priming_the_cache() throws ExecutionException, Interrup assertThat(loadCalls, equalTo(singletonList(singletonList("B")))); } - @Test - public void should_Not_prime_keys_that_already_exist() throws ExecutionException, InterruptedException { + @ParameterizedTest + @MethodSource("dataLoaderFactories") + public void should_Not_prime_keys_that_already_exist(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); + DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), loadCalls); identityLoader.prime("A", "X"); @@ -335,10 +384,11 @@ public void should_Not_prime_keys_that_already_exist() throws ExecutionException assertThat(loadCalls, equalTo(singletonList(singletonList("B")))); } - @Test - public void should_Allow_to_forcefully_prime_the_cache() throws ExecutionException, InterruptedException { + @ParameterizedTest + @MethodSource("dataLoaderFactories") + public void should_Allow_to_forcefully_prime_the_cache(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); + DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), loadCalls); identityLoader.prime("A", "X"); @@ -363,10 +413,11 @@ public void should_Allow_to_forcefully_prime_the_cache() throws ExecutionExcepti assertThat(loadCalls, equalTo(singletonList(singletonList("B")))); } - @Test - public void should_Allow_priming_the_cache_with_a_future() throws ExecutionException, InterruptedException { + @ParameterizedTest + @MethodSource("dataLoaderFactories") + public void should_Allow_priming_the_cache_with_a_future(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); + DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), loadCalls); DataLoader dlFluency = identityLoader.prime("A", CompletableFuture.completedFuture("A")); assertThat(dlFluency, equalTo(identityLoader)); @@ -381,10 +432,11 @@ public void should_Allow_priming_the_cache_with_a_future() throws ExecutionExcep assertThat(loadCalls, equalTo(singletonList(singletonList("B")))); } - @Test - public void should_not_Cache_failed_fetches_on_complete_failure() { + @ParameterizedTest + @MethodSource("dataLoaderFactories") + public void should_not_Cache_failed_fetches_on_complete_failure(TestDataLoaderFactory factory) { List> loadCalls = new ArrayList<>(); - DataLoader errorLoader = idLoaderBlowsUps(new DataLoaderOptions(), loadCalls); + DataLoader errorLoader = factory.idLoaderBlowsUps(new DataLoaderOptions(), loadCalls); CompletableFuture future1 = errorLoader.load(1); errorLoader.dispatch(); @@ -402,10 +454,11 @@ public void should_not_Cache_failed_fetches_on_complete_failure() { assertThat(loadCalls, equalTo(asList(singletonList(1), singletonList(1)))); } - @Test - public void should_Resolve_to_error_to_indicate_failure() throws ExecutionException, InterruptedException { + @ParameterizedTest + @MethodSource("dataLoaderFactories") + public void should_Resolve_to_error_to_indicate_failure(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); - DataLoader evenLoader = idLoaderOddEvenExceptions(new DataLoaderOptions(), loadCalls); + DataLoader evenLoader = factory.idLoaderOddEvenExceptions(new DataLoaderOptions(), loadCalls); CompletableFuture future1 = evenLoader.load(1); evenLoader.dispatch(); @@ -424,11 +477,12 @@ public void should_Resolve_to_error_to_indicate_failure() throws ExecutionExcept // Accept any kind of key. - @Test - public void should_Represent_failures_and_successes_simultaneously() throws ExecutionException, InterruptedException { + @ParameterizedTest + @MethodSource("dataLoaderFactories") + public void should_Represent_failures_and_successes_simultaneously(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { AtomicBoolean success = new AtomicBoolean(); List> loadCalls = new ArrayList<>(); - DataLoader evenLoader = idLoaderOddEvenExceptions(new DataLoaderOptions(), loadCalls); + DataLoader evenLoader = factory.idLoaderOddEvenExceptions(new DataLoaderOptions(), loadCalls); CompletableFuture future1 = evenLoader.load(1); CompletableFuture future2 = evenLoader.load(2); @@ -450,10 +504,11 @@ public void should_Represent_failures_and_successes_simultaneously() throws Exec // Accepts options - @Test - public void should_Cache_failed_fetches() { + @ParameterizedTest + @MethodSource("dataLoaderFactories") + public void should_Cache_failed_fetches(TestDataLoaderFactory factory) { List> loadCalls = new ArrayList<>(); - DataLoader errorLoader = idLoaderAllExceptions(new DataLoaderOptions(), loadCalls); + DataLoader errorLoader = factory.idLoaderAllExceptions(new DataLoaderOptions(), loadCalls); CompletableFuture future1 = errorLoader.load(1); errorLoader.dispatch(); @@ -472,11 +527,12 @@ public void should_Cache_failed_fetches() { assertThat(loadCalls, equalTo(singletonList(singletonList(1)))); } - @Test - public void should_NOT_Cache_failed_fetches_if_told_not_too() { + @ParameterizedTest + @MethodSource("dataLoaderFactories") + public void should_NOT_Cache_failed_fetches_if_told_not_too(TestDataLoaderFactory factory) { DataLoaderOptions options = DataLoaderOptions.newOptions().setCachingExceptionsEnabled(false); List> loadCalls = new ArrayList<>(); - DataLoader errorLoader = idLoaderAllExceptions(options, loadCalls); + DataLoader errorLoader = factory.idLoaderAllExceptions(options, loadCalls); CompletableFuture future1 = errorLoader.load(1); errorLoader.dispatch(); @@ -498,10 +554,11 @@ public void should_NOT_Cache_failed_fetches_if_told_not_too() { // Accepts object key in custom cacheKey function - @Test - public void should_Handle_priming_the_cache_with_an_error() { + @ParameterizedTest + @MethodSource("dataLoaderFactories") + public void should_Handle_priming_the_cache_with_an_error(TestDataLoaderFactory factory) { List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); + DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), loadCalls); identityLoader.prime(1, new IllegalStateException("Error")); @@ -514,10 +571,11 @@ public void should_Handle_priming_the_cache_with_an_error() { assertThat(loadCalls, equalTo(emptyList())); } - @Test - public void should_Clear_values_from_cache_after_errors() { + @ParameterizedTest + @MethodSource("dataLoaderFactories") + public void should_Clear_values_from_cache_after_errors(TestDataLoaderFactory factory) { List> loadCalls = new ArrayList<>(); - DataLoader errorLoader = idLoaderBlowsUps(new DataLoaderOptions(), loadCalls); + DataLoader errorLoader = factory.idLoaderBlowsUps(new DataLoaderOptions(), loadCalls); CompletableFuture future1 = errorLoader.load(1); future1.handle((value, t) -> { @@ -549,10 +607,11 @@ public void should_Clear_values_from_cache_after_errors() { assertThat(loadCalls, equalTo(asList(singletonList(1), singletonList(1)))); } - @Test - public void should_Propagate_error_to_all_loads() { + @ParameterizedTest + @MethodSource("dataLoaderFactories") + public void should_Propagate_error_to_all_loads(TestDataLoaderFactory factory) { List> loadCalls = new ArrayList<>(); - DataLoader errorLoader = idLoaderBlowsUps(new DataLoaderOptions(), loadCalls); + DataLoader errorLoader = factory.idLoaderBlowsUps(new DataLoaderOptions(), loadCalls); CompletableFuture future1 = errorLoader.load(1); CompletableFuture future2 = errorLoader.load(2); @@ -572,10 +631,11 @@ public void should_Propagate_error_to_all_loads() { assertThat(loadCalls, equalTo(singletonList(asList(1, 2)))); } - @Test - public void should_Accept_objects_as_keys() { + @ParameterizedTest + @MethodSource("dataLoaderFactories") + public void should_Accept_objects_as_keys(TestDataLoaderFactory factory) { List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); + DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), loadCalls); Object keyA = new Object(); Object keyB = new Object(); @@ -613,11 +673,12 @@ public void should_Accept_objects_as_keys() { assertThat(loadCalls.get(1).toArray()[0], equalTo(keyA)); } - @Test - public void should_Disable_caching() throws ExecutionException, InterruptedException { + @ParameterizedTest + @MethodSource("dataLoaderFactories") + public void should_Disable_caching(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); DataLoader identityLoader = - idLoader(newOptions().setCachingEnabled(false), loadCalls); + factory.idLoader(newOptions().setCachingEnabled(false), loadCalls); CompletableFuture future1 = identityLoader.load("A"); CompletableFuture future2 = identityLoader.load("B"); @@ -650,11 +711,12 @@ public void should_Disable_caching() throws ExecutionException, InterruptedExcep asList("A", "C"), asList("A", "B", "C")))); } - @Test - public void should_work_with_duplicate_keys_when_caching_disabled() throws ExecutionException, InterruptedException { + @ParameterizedTest + @MethodSource("dataLoaderFactories") + public void should_work_with_duplicate_keys_when_caching_disabled(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); DataLoader identityLoader = - idLoader(newOptions().setCachingEnabled(false), loadCalls); + factory.idLoader(newOptions().setCachingEnabled(false), loadCalls); CompletableFuture future1 = identityLoader.load("A"); CompletableFuture future2 = identityLoader.load("B"); @@ -665,14 +727,19 @@ public void should_work_with_duplicate_keys_when_caching_disabled() throws Execu assertThat(future1.get(), equalTo("A")); assertThat(future2.get(), equalTo("B")); assertThat(future3.get(), equalTo("A")); - assertThat(loadCalls, equalTo(singletonList(asList("A", "B", "A")))); + if (factory instanceof ListDataLoaderFactory) { + assertThat(loadCalls, equalTo(singletonList(asList("A", "B", "A")))); + } else { + assertThat(loadCalls, equalTo(singletonList(asList("A", "B")))); + } } - @Test - public void should_work_with_duplicate_keys_when_caching_enabled() throws ExecutionException, InterruptedException { + @ParameterizedTest + @MethodSource("dataLoaderFactories") + public void should_work_with_duplicate_keys_when_caching_enabled(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); DataLoader identityLoader = - idLoader(newOptions().setCachingEnabled(true), loadCalls); + factory.idLoader(newOptions().setCachingEnabled(true), loadCalls); CompletableFuture future1 = identityLoader.load("A"); CompletableFuture future2 = identityLoader.load("B"); @@ -688,11 +755,12 @@ public void should_work_with_duplicate_keys_when_caching_enabled() throws Execut // It is resilient to job queue ordering - @Test - public void should_Accept_objects_with_a_complex_key() throws ExecutionException, InterruptedException { + @ParameterizedTest + @MethodSource("dataLoaderFactories") + public void should_Accept_objects_with_a_complex_key(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); DataLoaderOptions options = newOptions().setCacheKeyFunction(getJsonObjectCacheMapFn()); - DataLoader identityLoader = idLoader(options, loadCalls); + DataLoader identityLoader = factory.idLoader(options, loadCalls); JsonObject key1 = new JsonObject().put("id", 123); JsonObject key2 = new JsonObject().put("id", 123); @@ -709,11 +777,12 @@ public void should_Accept_objects_with_a_complex_key() throws ExecutionException // Helper methods - @Test - public void should_Clear_objects_with_complex_key() throws ExecutionException, InterruptedException { + @ParameterizedTest + @MethodSource("dataLoaderFactories") + public void should_Clear_objects_with_complex_key(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); DataLoaderOptions options = newOptions().setCacheKeyFunction(getJsonObjectCacheMapFn()); - DataLoader identityLoader = idLoader(options, loadCalls); + DataLoader identityLoader = factory.idLoader(options, loadCalls); JsonObject key1 = new JsonObject().put("id", 123); JsonObject key2 = new JsonObject().put("id", 123); @@ -733,11 +802,12 @@ public void should_Clear_objects_with_complex_key() throws ExecutionException, I assertThat(future2.get(), equalTo(key1)); } - @Test - public void should_Accept_objects_with_different_order_of_keys() throws ExecutionException, InterruptedException { + @ParameterizedTest + @MethodSource("dataLoaderFactories") + public void should_Accept_objects_with_different_order_of_keys(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); DataLoaderOptions options = newOptions().setCacheKeyFunction(getJsonObjectCacheMapFn()); - DataLoader identityLoader = idLoader(options, loadCalls); + DataLoader identityLoader = factory.idLoader(options, loadCalls); JsonObject key1 = new JsonObject().put("a", 123).put("b", 321); JsonObject key2 = new JsonObject().put("b", 321).put("a", 123); @@ -755,11 +825,12 @@ public void should_Accept_objects_with_different_order_of_keys() throws Executio assertThat(future2.get(), equalTo(key2)); } - @Test - public void should_Allow_priming_the_cache_with_an_object_key() throws ExecutionException, InterruptedException { + @ParameterizedTest + @MethodSource("dataLoaderFactories") + public void should_Allow_priming_the_cache_with_an_object_key(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); DataLoaderOptions options = newOptions().setCacheKeyFunction(getJsonObjectCacheMapFn()); - DataLoader identityLoader = idLoader(options, loadCalls); + DataLoader identityLoader = factory.idLoader(options, loadCalls); JsonObject key1 = new JsonObject().put("id", 123); JsonObject key2 = new JsonObject().put("id", 123); @@ -776,12 +847,13 @@ public void should_Allow_priming_the_cache_with_an_object_key() throws Execution assertThat(future2.get(), equalTo(key1)); } - @Test - public void should_Accept_a_custom_cache_map_implementation() throws ExecutionException, InterruptedException { + @ParameterizedTest + @MethodSource("dataLoaderFactories") + public void should_Accept_a_custom_cache_map_implementation(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { CustomCacheMap customMap = new CustomCacheMap(); List> loadCalls = new ArrayList<>(); DataLoaderOptions options = newOptions().setCacheMap(customMap); - DataLoader identityLoader = idLoader(options, loadCalls); + DataLoader identityLoader = factory.idLoader(options, loadCalls); // Fetches as expected @@ -827,12 +899,13 @@ public void should_Accept_a_custom_cache_map_implementation() throws ExecutionEx assertArrayEquals(customMap.stash.keySet().toArray(), emptyList().toArray()); } - @Test - public void should_degrade_gracefully_if_cache_get_throws() { + @ParameterizedTest + @MethodSource("dataLoaderFactories") + public void should_degrade_gracefully_if_cache_get_throws(TestDataLoaderFactory factory) { CacheMap cache = new ThrowingCacheMap(); DataLoaderOptions options = newOptions().setCachingEnabled(true).setCacheMap(cache); List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idLoader(options, loadCalls); + DataLoader identityLoader = factory.idLoader(options, loadCalls); assertThat(identityLoader.getIfPresent("a"), equalTo(Optional.empty())); @@ -841,11 +914,12 @@ public void should_degrade_gracefully_if_cache_get_throws() { assertThat(future.join(), equalTo("a")); } - @Test - public void batching_disabled_should_dispatch_immediately() { + @ParameterizedTest + @MethodSource("dataLoaderFactories") + public void batching_disabled_should_dispatch_immediately(TestDataLoaderFactory factory) { List> loadCalls = new ArrayList<>(); DataLoaderOptions options = newOptions().setBatchingEnabled(false); - DataLoader identityLoader = idLoader(options, loadCalls); + DataLoader identityLoader = factory.idLoader(options, loadCalls); CompletableFuture fa = identityLoader.load("A"); CompletableFuture fb = identityLoader.load("B"); @@ -869,11 +943,12 @@ public void batching_disabled_should_dispatch_immediately() { } - @Test - public void batching_disabled_and_caching_disabled_should_dispatch_immediately_and_forget() { + @ParameterizedTest + @MethodSource("dataLoaderFactories") + public void batching_disabled_and_caching_disabled_should_dispatch_immediately_and_forget(TestDataLoaderFactory factory) { List> loadCalls = new ArrayList<>(); DataLoaderOptions options = newOptions().setBatchingEnabled(false).setCachingEnabled(false); - DataLoader identityLoader = idLoader(options, loadCalls); + DataLoader identityLoader = factory.idLoader(options, loadCalls); CompletableFuture fa = identityLoader.load("A"); CompletableFuture fb = identityLoader.load("B"); @@ -900,10 +975,11 @@ public void batching_disabled_and_caching_disabled_should_dispatch_immediately_a } - @Test - public void batches_multiple_requests_with_max_batch_size() { + @ParameterizedTest + @MethodSource("dataLoaderFactories") + public void batches_multiple_requests_with_max_batch_size(TestDataLoaderFactory factory) { List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idLoader(newOptions().setMaxBatchSize(2), loadCalls); + DataLoader identityLoader = factory.idLoader(newOptions().setMaxBatchSize(2), loadCalls); CompletableFuture f1 = identityLoader.load(1); CompletableFuture f2 = identityLoader.load(2); @@ -921,10 +997,11 @@ public void batches_multiple_requests_with_max_batch_size() { } - @Test - public void can_split_max_batch_sizes_correctly() { + @ParameterizedTest + @MethodSource("dataLoaderFactories") + public void can_split_max_batch_sizes_correctly(TestDataLoaderFactory factory) { List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idLoader(newOptions().setMaxBatchSize(5), loadCalls); + DataLoader identityLoader = factory.idLoader(newOptions().setMaxBatchSize(5), loadCalls); for (int i = 0; i < 21; i++) { identityLoader.load(i); @@ -943,10 +1020,11 @@ public void can_split_max_batch_sizes_correctly() { } - @Test - public void should_Batch_loads_occurring_within_futures() { + @ParameterizedTest + @MethodSource("dataLoaderFactories") + public void should_Batch_loads_occurring_within_futures(TestDataLoaderFactory factory) { List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idLoader(newOptions(), loadCalls); + DataLoader identityLoader = factory.idLoader(newOptions(), loadCalls); Supplier nullValue = () -> null; @@ -1066,54 +1144,120 @@ private static CacheKey getJsonObjectCacheMapFn() { .collect(Collectors.joining()); } - private static DataLoader idLoader(DataLoaderOptions options, List> loadCalls) { - return newDataLoader(keys -> { - loadCalls.add(new ArrayList<>(keys)); - @SuppressWarnings("unchecked") - List values = keys.stream() - .map(k -> (V) k) - .collect(Collectors.toList()); - return CompletableFuture.completedFuture(values); - }, options); + private static Stream dataLoaderFactories() { + return Stream.of( + Arguments.of(Named.of("List DataLoader", new ListDataLoaderFactory())), + Arguments.of(Named.of("Mapped DataLoader", new MappedDataLoaderFactory())) + ); } - private static DataLoader idLoaderBlowsUps( - DataLoaderOptions options, List> loadCalls) { - return newDataLoader(keys -> { - loadCalls.add(new ArrayList<>(keys)); - return TestKit.futureError(); - }, options); + public interface TestDataLoaderFactory { + DataLoader idLoader(DataLoaderOptions options, List> loadCalls); + DataLoader idLoaderBlowsUps(DataLoaderOptions options, List> loadCalls); + DataLoader idLoaderAllExceptions(DataLoaderOptions options, List> loadCalls); + DataLoader idLoaderOddEvenExceptions(DataLoaderOptions options, List> loadCalls); } - private static DataLoader idLoaderAllExceptions( - DataLoaderOptions options, List> loadCalls) { - return newDataLoader(keys -> { - loadCalls.add(new ArrayList<>(keys)); + private static class ListDataLoaderFactory implements TestDataLoaderFactory { + @Override + public DataLoader idLoader(DataLoaderOptions options, List> loadCalls) { + return newDataLoader(keys -> { + loadCalls.add(new ArrayList<>(keys)); + @SuppressWarnings("unchecked") + List values = keys.stream() + .map(k -> (V) k) + .collect(Collectors.toList()); + return CompletableFuture.completedFuture(values); + }, options); + } - List errors = keys.stream().map(k -> new IllegalStateException("Error")).collect(Collectors.toList()); - return CompletableFuture.completedFuture(errors); - }, options); - } + @Override + public DataLoader idLoaderBlowsUps( + DataLoaderOptions options, List> loadCalls) { + return newDataLoader(keys -> { + loadCalls.add(new ArrayList<>(keys)); + return TestKit.futureError(); + }, options); + } + + @Override + public DataLoader idLoaderAllExceptions(DataLoaderOptions options, List> loadCalls) { + return newDataLoader(keys -> { + loadCalls.add(new ArrayList<>(keys)); - private static DataLoader idLoaderOddEvenExceptions( - DataLoaderOptions options, List> loadCalls) { - return newDataLoader(keys -> { - loadCalls.add(new ArrayList<>(keys)); - - List errors = new ArrayList<>(); - for (Integer key : keys) { - if (key % 2 == 0) { - errors.add(key); - } else { - errors.add(new IllegalStateException("Error")); + List errors = keys.stream().map(k -> new IllegalStateException("Error")).collect(Collectors.toList()); + return CompletableFuture.completedFuture(errors); + }, options); + } + + @Override + public DataLoader idLoaderOddEvenExceptions(DataLoaderOptions options, List> loadCalls) { + return newDataLoader(keys -> { + loadCalls.add(new ArrayList<>(keys)); + + List errors = new ArrayList<>(); + for (Integer key : keys) { + if (key % 2 == 0) { + errors.add(key); + } else { + errors.add(new IllegalStateException("Error")); + } } - } - return CompletableFuture.completedFuture(errors); - }, options); + return CompletableFuture.completedFuture(errors); + }, options); + } } - private BatchLoader keysAsValues() { - return CompletableFuture::completedFuture; + private static class MappedDataLoaderFactory implements TestDataLoaderFactory { + + @Override + public DataLoader idLoader( + DataLoaderOptions options, List> loadCalls) { + return newMappedDataLoader((keys) -> { + loadCalls.add(new ArrayList<>(keys)); + Map map = new HashMap<>(); + //noinspection unchecked + keys.forEach(k -> map.put(k, (V) k)); + return CompletableFuture.completedFuture(map); + }, options); + } + + @Override + public DataLoader idLoaderBlowsUps(DataLoaderOptions options, List> loadCalls) { + return newMappedDataLoader((keys) -> { + loadCalls.add(new ArrayList<>(keys)); + return futureError(); + }, options); + } + + @Override + public DataLoader idLoaderAllExceptions( + DataLoaderOptions options, List> loadCalls) { + return newMappedDataLoader(keys -> { + loadCalls.add(new ArrayList<>(keys)); + Map errorByKey = new HashMap<>(); + keys.forEach(k -> errorByKey.put(k, new IllegalStateException("Error"))); + return CompletableFuture.completedFuture(errorByKey); + }, options); + } + + @Override + public DataLoader idLoaderOddEvenExceptions( + DataLoaderOptions options, List> loadCalls) { + return newMappedDataLoader(keys -> { + loadCalls.add(new ArrayList<>(keys)); + + Map errorByKey = new HashMap<>(); + for (Integer key : keys) { + if (key % 2 == 0) { + errorByKey.put(key, key); + } else { + errorByKey.put(key, new IllegalStateException("Error")); + } + } + return CompletableFuture.completedFuture(errorByKey); + }, options); + } } private static class ThrowingCacheMap extends CustomCacheMap { From a93112a79f7951ec39aae496739ee009ea9bbe9b Mon Sep 17 00:00:00 2001 From: bbaker Date: Mon, 20 May 2024 12:36:44 +1000 Subject: [PATCH 023/156] reactive streams support branch - getting it compiling --- src/main/java/org/dataloader/BatchPublisher.java | 2 +- .../java/org/dataloader/DataLoaderBatchPublisherTest.java | 6 +++--- .../org/dataloader/DataLoaderMappedBatchPublisherTest.java | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/dataloader/BatchPublisher.java b/src/main/java/org/dataloader/BatchPublisher.java index 5ab41e11..efc222a6 100644 --- a/src/main/java/org/dataloader/BatchPublisher.java +++ b/src/main/java/org/dataloader/BatchPublisher.java @@ -11,7 +11,7 @@ * the future returned by {@link DataLoader#load(Object)} to complete as soon as the individual value is available * (rather than when all values have been retrieved). *

- * NOTE: It is required that {@link Subscriber#onNext(V)} is invoked on each value in the same order as + * NOTE: It is required that {@link Subscriber#onNext(Object)} is invoked on each value in the same order as * the provided keys. * * @param type parameter indicating the type of keys to use for data load requests. diff --git a/src/test/java/org/dataloader/DataLoaderBatchPublisherTest.java b/src/test/java/org/dataloader/DataLoaderBatchPublisherTest.java index 84a8b181..e14d9f71 100644 --- a/src/test/java/org/dataloader/DataLoaderBatchPublisherTest.java +++ b/src/test/java/org/dataloader/DataLoaderBatchPublisherTest.java @@ -5,7 +5,7 @@ import org.dataloader.fixtures.User; import org.dataloader.fixtures.UserManager; import org.dataloader.impl.CompletableFutureKit; -import org.junit.Test; +import org.junit.jupiter.api.Test; import reactor.core.publisher.Flux; import java.util.ArrayList; @@ -30,12 +30,12 @@ import static org.dataloader.DataLoaderOptions.newOptions; import static org.dataloader.fixtures.TestKit.listFrom; import static org.dataloader.impl.CompletableFutureKit.cause; +import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertThat; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; public class DataLoaderBatchPublisherTest { diff --git a/src/test/java/org/dataloader/DataLoaderMappedBatchPublisherTest.java b/src/test/java/org/dataloader/DataLoaderMappedBatchPublisherTest.java index c16c58f7..5b9ca0b4 100644 --- a/src/test/java/org/dataloader/DataLoaderMappedBatchPublisherTest.java +++ b/src/test/java/org/dataloader/DataLoaderMappedBatchPublisherTest.java @@ -1,6 +1,6 @@ package org.dataloader; -import org.junit.Test; +import org.junit.jupiter.api.Test; import reactor.core.publisher.Flux; import java.util.ArrayList; @@ -19,10 +19,10 @@ import static org.dataloader.DataLoaderOptions.newOptions; import static org.dataloader.fixtures.TestKit.listFrom; import static org.dataloader.impl.CompletableFutureKit.cause; +import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; -import static org.junit.Assert.assertThat; public class DataLoaderMappedBatchPublisherTest { From 74567fe99051ea6db7329bbe49683424cebcd3ed Mon Sep 17 00:00:00 2001 From: bbaker Date: Tue, 21 May 2024 10:34:56 +1000 Subject: [PATCH 024/156] Making the Subscribers use a common base class --- .../java/org/dataloader/DataLoaderHelper.java | 167 +++++++++--------- 1 file changed, 88 insertions(+), 79 deletions(-) diff --git a/src/main/java/org/dataloader/DataLoaderHelper.java b/src/main/java/org/dataloader/DataLoaderHelper.java index ee8d78b7..3e4bf6e7 100644 --- a/src/main/java/org/dataloader/DataLoaderHelper.java +++ b/src/main/java/org/dataloader/DataLoaderHelper.java @@ -618,24 +618,23 @@ private static DispatchResult emptyDispatchResult() { return (DispatchResult) EMPTY_DISPATCH_RESULT; } - private class DataLoaderSubscriber implements Subscriber { + private abstract class DataLoaderSubscriberBase implements Subscriber { - private final CompletableFuture> valuesFuture; - private final List keys; - private final List callContexts; - private final List> queuedFutures; + final CompletableFuture> valuesFuture; + final List keys; + final List callContexts; + final List> queuedFutures; - private final List clearCacheKeys = new ArrayList<>(); - private final List completedValues = new ArrayList<>(); - private int idx = 0; - private boolean onErrorCalled = false; - private boolean onCompleteCalled = false; + List clearCacheKeys = new ArrayList<>(); + List completedValues = new ArrayList<>(); + boolean onErrorCalled = false; + boolean onCompleteCalled = false; - private DataLoaderSubscriber( - CompletableFuture> valuesFuture, - List keys, - List callContexts, - List> queuedFutures + DataLoaderSubscriberBase( + CompletableFuture> valuesFuture, + List keys, + List callContexts, + List> queuedFutures ) { this.valuesFuture = valuesFuture; this.keys = keys; @@ -648,40 +647,87 @@ public void onSubscribe(Subscription subscription) { subscription.request(keys.size()); } - // onNext may be called by multiple threads - for the time being, we pass 'synchronized' to guarantee - // correctness (at the cost of speed). @Override - public synchronized void onNext(V value) { + public void onNext(T v) { assertState(!onErrorCalled, () -> "onError has already been called; onNext may not be invoked."); assertState(!onCompleteCalled, () -> "onComplete has already been called; onNext may not be invoked."); + } - K key = keys.get(idx); - Object callContext = callContexts.get(idx); - CompletableFuture future = queuedFutures.get(idx); + @Override + public void onComplete() { + assertState(!onErrorCalled, () -> "onError has already been called; onComplete may not be invoked."); + onCompleteCalled = true; + } + + @Override + public void onError(Throwable throwable) { + assertState(!onCompleteCalled, () -> "onComplete has already been called; onError may not be invoked."); + onErrorCalled = true; + + stats.incrementBatchLoadExceptionCount(new IncrementBatchLoadExceptionCountStatisticsContext<>(keys, callContexts)); + } + + /* + * A value has arrived - how do we complete the future that's associated with it in a common way + */ + void onNextValue(K key, V value, Object callContext, CompletableFuture future) { if (value instanceof Try) { // we allow the batch loader to return a Try so we can better represent a computation // that might have worked or not. + //noinspection unchecked Try tryValue = (Try) value; if (tryValue.isSuccess()) { future.complete(tryValue.get()); } else { stats.incrementLoadErrorCount(new IncrementLoadErrorCountStatisticsContext<>(key, callContext)); future.completeExceptionally(tryValue.getThrowable()); - clearCacheKeys.add(keys.get(idx)); + clearCacheKeys.add(key); } } else { future.complete(value); } + } + + Throwable unwrapThrowable(Throwable ex) { + if (ex instanceof CompletionException) { + ex = ex.getCause(); + } + return ex; + } + } + + private class DataLoaderSubscriber extends DataLoaderSubscriberBase { + + private int idx = 0; + + private DataLoaderSubscriber( + CompletableFuture> valuesFuture, + List keys, + List callContexts, + List> queuedFutures + ) { + super(valuesFuture, keys, callContexts, queuedFutures); + } + + // onNext may be called by multiple threads - for the time being, we pass 'synchronized' to guarantee + // correctness (at the cost of speed). + @Override + public synchronized void onNext(V value) { + super.onNext(value); + + K key = keys.get(idx); + Object callContext = callContexts.get(idx); + CompletableFuture future = queuedFutures.get(idx); + onNextValue(key, value, callContext, future); completedValues.add(value); idx++; } + @Override public void onComplete() { - assertState(!onErrorCalled, () -> "onError has already been called; onComplete may not be invoked."); - onCompleteCalled = true; - + super.onComplete(); assertResultSize(keys, completedValues); possiblyClearCacheEntriesOnExceptions(clearCacheKeys); @@ -690,13 +736,8 @@ public void onComplete() { @Override public void onError(Throwable ex) { - assertState(!onCompleteCalled, () -> "onComplete has already been called; onError may not be invoked."); - onErrorCalled = true; - - stats.incrementBatchLoadExceptionCount(new IncrementBatchLoadExceptionCountStatisticsContext<>(keys, callContexts)); - if (ex instanceof CompletionException) { - ex = ex.getCause(); - } + super.onError(ex); + ex = unwrapThrowable(ex); // Set the remaining keys to the exception. for (int i = idx; i < queuedFutures.size(); i++) { K key = keys.get(i); @@ -706,32 +747,23 @@ public void onError(Throwable ex) { dataLoader.clear(key); } } + } - private class DataLoaderMapEntrySubscriber implements Subscriber> { - private final CompletableFuture> valuesFuture; - private final List keys; - private final List callContexts; - private final List> queuedFutures; + private class DataLoaderMapEntrySubscriber extends DataLoaderSubscriberBase> { + private final Map callContextByKey; private final Map> queuedFutureByKey; - - private final List clearCacheKeys = new ArrayList<>(); private final Map completedValuesByKey = new HashMap<>(); - private boolean onErrorCalled = false; - private boolean onCompleteCalled = false; + private DataLoaderMapEntrySubscriber( - CompletableFuture> valuesFuture, - List keys, - List callContexts, - List> queuedFutures + CompletableFuture> valuesFuture, + List keys, + List callContexts, + List> queuedFutures ) { - this.valuesFuture = valuesFuture; - this.keys = keys; - this.callContexts = callContexts; - this.queuedFutures = queuedFutures; - + super(valuesFuture,keys,callContexts,queuedFutures); this.callContextByKey = new HashMap<>(); this.queuedFutureByKey = new HashMap<>(); for (int idx = 0; idx < queuedFutures.size(); idx++) { @@ -743,42 +775,24 @@ private DataLoaderMapEntrySubscriber( } } - @Override - public void onSubscribe(Subscription subscription) { - subscription.request(keys.size()); - } @Override public void onNext(Map.Entry entry) { - assertState(!onErrorCalled, () -> "onError has already been called; onNext may not be invoked."); - assertState(!onCompleteCalled, () -> "onComplete has already been called; onNext may not be invoked."); + super.onNext(entry); K key = entry.getKey(); V value = entry.getValue(); Object callContext = callContextByKey.get(key); CompletableFuture future = queuedFutureByKey.get(key); - if (value instanceof Try) { - // we allow the batch loader to return a Try so we can better represent a computation - // that might have worked or not. - Try tryValue = (Try) value; - if (tryValue.isSuccess()) { - future.complete(tryValue.get()); - } else { - stats.incrementLoadErrorCount(new IncrementLoadErrorCountStatisticsContext<>(key, callContext)); - future.completeExceptionally(tryValue.getThrowable()); - clearCacheKeys.add(key); - } - } else { - future.complete(value); - } + + onNextValue(key, value, callContext, future); completedValuesByKey.put(key, value); } @Override public void onComplete() { - assertState(!onErrorCalled, () -> "onError has already been called; onComplete may not be invoked."); - onCompleteCalled = true; + super.onComplete(); possiblyClearCacheEntriesOnExceptions(clearCacheKeys); List values = new ArrayList<>(keys.size()); @@ -791,13 +805,8 @@ public void onComplete() { @Override public void onError(Throwable ex) { - assertState(!onCompleteCalled, () -> "onComplete has already been called; onError may not be invoked."); - onErrorCalled = true; - - stats.incrementBatchLoadExceptionCount(new IncrementBatchLoadExceptionCountStatisticsContext<>(keys, callContexts)); - if (ex instanceof CompletionException) { - ex = ex.getCause(); - } + super.onError(ex); + ex = unwrapThrowable(ex); // Complete the futures for the remaining keys with the exception. for (int idx = 0; idx < queuedFutures.size(); idx++) { K key = keys.get(idx); From 4396624894af67de8462458133c81d4827283bb4 Mon Sep 17 00:00:00 2001 From: bbaker Date: Tue, 21 May 2024 10:38:24 +1000 Subject: [PATCH 025/156] Making the Subscribers use a common base class- synchronized on each method --- src/main/java/org/dataloader/DataLoaderHelper.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/dataloader/DataLoaderHelper.java b/src/main/java/org/dataloader/DataLoaderHelper.java index 3e4bf6e7..f88890c3 100644 --- a/src/main/java/org/dataloader/DataLoaderHelper.java +++ b/src/main/java/org/dataloader/DataLoaderHelper.java @@ -726,7 +726,7 @@ public synchronized void onNext(V value) { @Override - public void onComplete() { + public synchronized void onComplete() { super.onComplete(); assertResultSize(keys, completedValues); @@ -735,7 +735,7 @@ public void onComplete() { } @Override - public void onError(Throwable ex) { + public synchronized void onError(Throwable ex) { super.onError(ex); ex = unwrapThrowable(ex); // Set the remaining keys to the exception. @@ -777,7 +777,7 @@ private DataLoaderMapEntrySubscriber( @Override - public void onNext(Map.Entry entry) { + public synchronized void onNext(Map.Entry entry) { super.onNext(entry); K key = entry.getKey(); V value = entry.getValue(); @@ -791,7 +791,7 @@ public void onNext(Map.Entry entry) { } @Override - public void onComplete() { + public synchronized void onComplete() { super.onComplete(); possiblyClearCacheEntriesOnExceptions(clearCacheKeys); @@ -804,7 +804,7 @@ public void onComplete() { } @Override - public void onError(Throwable ex) { + public synchronized void onError(Throwable ex) { super.onError(ex); ex = unwrapThrowable(ex); // Complete the futures for the remaining keys with the exception. From 8a6448363e67ee1262bce947775c34aeeff7735a Mon Sep 17 00:00:00 2001 From: bbaker Date: Tue, 21 May 2024 13:27:54 +1000 Subject: [PATCH 026/156] Making the Subscribers use a common base class- now with failing test case --- .../java/org/dataloader/DataLoaderBatchPublisherTest.java | 6 ++++-- .../org/dataloader/DataLoaderMappedBatchPublisherTest.java | 4 +++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/test/java/org/dataloader/DataLoaderBatchPublisherTest.java b/src/test/java/org/dataloader/DataLoaderBatchPublisherTest.java index e14d9f71..607c1e68 100644 --- a/src/test/java/org/dataloader/DataLoaderBatchPublisherTest.java +++ b/src/test/java/org/dataloader/DataLoaderBatchPublisherTest.java @@ -385,18 +385,20 @@ public void should_Resolve_to_error_to_indicate_failure() throws ExecutionExcept DataLoader evenLoader = idLoaderOddEvenExceptions(new DataLoaderOptions(), loadCalls); CompletableFuture future1 = evenLoader.load(1); - evenLoader.dispatch(); + CompletableFuture> dispatchCF = evenLoader.dispatch(); await().until(future1::isDone); assertThat(future1.isCompletedExceptionally(), is(true)); assertThat(cause(future1), instanceOf(IllegalStateException.class)); + assertThat(dispatchCF.isCompletedExceptionally(), is(true)); CompletableFuture future2 = evenLoader.load(2); - evenLoader.dispatch(); + dispatchCF = evenLoader.dispatch(); await().until(future2::isDone); assertThat(future2.get(), equalTo(2)); assertThat(loadCalls, equalTo(asList(singletonList(1), singletonList(2)))); + assertThat(dispatchCF.isCompletedExceptionally(), is(true)); } // Accept any kind of key. diff --git a/src/test/java/org/dataloader/DataLoaderMappedBatchPublisherTest.java b/src/test/java/org/dataloader/DataLoaderMappedBatchPublisherTest.java index 5b9ca0b4..aa9ee7be 100644 --- a/src/test/java/org/dataloader/DataLoaderMappedBatchPublisherTest.java +++ b/src/test/java/org/dataloader/DataLoaderMappedBatchPublisherTest.java @@ -116,7 +116,7 @@ public void should_Propagate_error_to_all_loads() { CompletableFuture future1 = errorLoader.load(1); CompletableFuture future2 = errorLoader.load(2); - errorLoader.dispatch(); + CompletableFuture> dispatchedCF = errorLoader.dispatch(); await().until(future1::isDone); @@ -132,6 +132,8 @@ public void should_Propagate_error_to_all_loads() { assertThat(cause.getMessage(), equalTo(cause.getMessage())); assertThat(loadCalls, equalTo(singletonList(asList(1, 2)))); + + assertThat(dispatchedCF.isCompletedExceptionally(),equalTo(true)); } @Test From 3e8ac9cf5a1123304329fb08dd57811f181669ca Mon Sep 17 00:00:00 2001 From: bbaker Date: Tue, 21 May 2024 16:02:27 +1000 Subject: [PATCH 027/156] Making the Subscribers use a common base class- fail the overall CF onError --- src/main/java/org/dataloader/DataLoaderHelper.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/dataloader/DataLoaderHelper.java b/src/main/java/org/dataloader/DataLoaderHelper.java index f88890c3..7b3a4239 100644 --- a/src/main/java/org/dataloader/DataLoaderHelper.java +++ b/src/main/java/org/dataloader/DataLoaderHelper.java @@ -746,6 +746,7 @@ public synchronized void onError(Throwable ex) { // clear any cached view of this key because they all failed dataLoader.clear(key); } + valuesFuture.completeExceptionally(ex); } } @@ -763,7 +764,7 @@ private DataLoaderMapEntrySubscriber( List callContexts, List> queuedFutures ) { - super(valuesFuture,keys,callContexts,queuedFutures); + super(valuesFuture, keys, callContexts, queuedFutures); this.callContextByKey = new HashMap<>(); this.queuedFutureByKey = new HashMap<>(); for (int idx = 0; idx < queuedFutures.size(); idx++) { @@ -817,6 +818,8 @@ public synchronized void onError(Throwable ex) { dataLoader.clear(key); } } + valuesFuture.completeExceptionally(ex); } + } } From eb2b40cc2c50300ef5c913c61737f553a8549a65 Mon Sep 17 00:00:00 2001 From: Alexandre Carlton Date: Mon, 20 May 2024 22:26:16 +1000 Subject: [PATCH 028/156] Inline BatchPublisher tests into DataLoaderTest We now have the same coverage but with less code. Note that: - this is currently failing on 'duplicate keys when caching disabled'. - we still need to add tests that only make sense for the Publisher variants (e.g. half-completed keys). --- .../DataLoaderBatchPublisherTest.java | 1096 ----------------- .../DataLoaderMappedBatchPublisherTest.java | 175 --- .../java/org/dataloader/DataLoaderTest.java | 132 +- 3 files changed, 116 insertions(+), 1287 deletions(-) delete mode 100644 src/test/java/org/dataloader/DataLoaderBatchPublisherTest.java delete mode 100644 src/test/java/org/dataloader/DataLoaderMappedBatchPublisherTest.java diff --git a/src/test/java/org/dataloader/DataLoaderBatchPublisherTest.java b/src/test/java/org/dataloader/DataLoaderBatchPublisherTest.java deleted file mode 100644 index e14d9f71..00000000 --- a/src/test/java/org/dataloader/DataLoaderBatchPublisherTest.java +++ /dev/null @@ -1,1096 +0,0 @@ -package org.dataloader; - -import org.dataloader.fixtures.CustomCacheMap; -import org.dataloader.fixtures.JsonObject; -import org.dataloader.fixtures.User; -import org.dataloader.fixtures.UserManager; -import org.dataloader.impl.CompletableFutureKit; -import org.junit.jupiter.api.Test; -import reactor.core.publisher.Flux; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionStage; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.Supplier; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static java.util.Arrays.asList; -import static java.util.Collections.emptyList; -import static java.util.Collections.singletonList; -import static org.awaitility.Awaitility.await; -import static org.dataloader.DataLoaderFactory.newDataLoader; -import static org.dataloader.DataLoaderFactory.newPublisherDataLoader; -import static org.dataloader.DataLoaderFactory.newPublisherDataLoaderWithTry; -import static org.dataloader.DataLoaderOptions.newOptions; -import static org.dataloader.fixtures.TestKit.listFrom; -import static org.dataloader.impl.CompletableFutureKit.cause; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.empty; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.instanceOf; -import static org.hamcrest.Matchers.is; -import static org.junit.jupiter.api.Assertions.assertArrayEquals; - -public class DataLoaderBatchPublisherTest { - - @Test - public void should_Build_a_really_really_simple_data_loader() { - AtomicBoolean success = new AtomicBoolean(); - DataLoader identityLoader = newPublisherDataLoader(keysAsValues()); - - CompletionStage future1 = identityLoader.load(1); - - future1.thenAccept(value -> { - assertThat(value, equalTo(1)); - success.set(true); - }); - identityLoader.dispatch(); - await().untilAtomic(success, is(true)); - } - - @Test - public void should_Support_loading_multiple_keys_in_one_call() { - AtomicBoolean success = new AtomicBoolean(); - DataLoader identityLoader = newPublisherDataLoader(keysAsValues()); - - CompletionStage> futureAll = identityLoader.loadMany(asList(1, 2)); - futureAll.thenAccept(promisedValues -> { - assertThat(promisedValues.size(), is(2)); - success.set(true); - }); - identityLoader.dispatch(); - await().untilAtomic(success, is(true)); - assertThat(futureAll.toCompletableFuture().join(), equalTo(asList(1, 2))); - } - - @Test - public void should_Resolve_to_empty_list_when_no_keys_supplied() { - AtomicBoolean success = new AtomicBoolean(); - DataLoader identityLoader = newPublisherDataLoader(keysAsValues()); - CompletableFuture> futureEmpty = identityLoader.loadMany(emptyList()); - futureEmpty.thenAccept(promisedValues -> { - assertThat(promisedValues.size(), is(0)); - success.set(true); - }); - identityLoader.dispatch(); - await().untilAtomic(success, is(true)); - assertThat(futureEmpty.join(), empty()); - } - - @Test - public void should_Return_zero_entries_dispatched_when_no_keys_supplied() { - AtomicBoolean success = new AtomicBoolean(); - DataLoader identityLoader = newPublisherDataLoader(keysAsValues()); - CompletableFuture> futureEmpty = identityLoader.loadMany(emptyList()); - futureEmpty.thenAccept(promisedValues -> { - assertThat(promisedValues.size(), is(0)); - success.set(true); - }); - DispatchResult dispatchResult = identityLoader.dispatchWithCounts(); - await().untilAtomic(success, is(true)); - assertThat(dispatchResult.getKeysCount(), equalTo(0)); - } - - @Test - public void should_Batch_multiple_requests() throws ExecutionException, InterruptedException { - List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); - - CompletableFuture future1 = identityLoader.load(1); - CompletableFuture future2 = identityLoader.load(2); - identityLoader.dispatch(); - - await().until(() -> future1.isDone() && future2.isDone()); - assertThat(future1.get(), equalTo(1)); - assertThat(future2.get(), equalTo(2)); - assertThat(loadCalls, equalTo(singletonList(asList(1, 2)))); - } - - @Test - public void should_Return_number_of_batched_entries() { - List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); - - CompletableFuture future1 = identityLoader.load(1); - CompletableFuture future2 = identityLoader.load(2); - DispatchResult dispatchResult = identityLoader.dispatchWithCounts(); - - await().until(() -> future1.isDone() && future2.isDone()); - assertThat(dispatchResult.getKeysCount(), equalTo(2)); // its two because it's the number dispatched (by key) not the load calls - assertThat(dispatchResult.getPromisedResults().isDone(), equalTo(true)); - } - - @Test - public void should_Coalesce_identical_requests() throws ExecutionException, InterruptedException { - List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); - - CompletableFuture future1a = identityLoader.load(1); - CompletableFuture future1b = identityLoader.load(1); - assertThat(future1a, equalTo(future1b)); - identityLoader.dispatch(); - - await().until(future1a::isDone); - assertThat(future1a.get(), equalTo(1)); - assertThat(future1b.get(), equalTo(1)); - assertThat(loadCalls, equalTo(singletonList(singletonList(1)))); - } - - @Test - public void should_Cache_repeated_requests() throws ExecutionException, InterruptedException { - List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); - - CompletableFuture future1 = identityLoader.load("A"); - CompletableFuture future2 = identityLoader.load("B"); - identityLoader.dispatch(); - - await().until(() -> future1.isDone() && future2.isDone()); - assertThat(future1.get(), equalTo("A")); - assertThat(future2.get(), equalTo("B")); - assertThat(loadCalls, equalTo(singletonList(asList("A", "B")))); - - CompletableFuture future1a = identityLoader.load("A"); - CompletableFuture future3 = identityLoader.load("C"); - identityLoader.dispatch(); - - await().until(() -> future1a.isDone() && future3.isDone()); - assertThat(future1a.get(), equalTo("A")); - assertThat(future3.get(), equalTo("C")); - assertThat(loadCalls, equalTo(asList(asList("A", "B"), singletonList("C")))); - - CompletableFuture future1b = identityLoader.load("A"); - CompletableFuture future2a = identityLoader.load("B"); - CompletableFuture future3a = identityLoader.load("C"); - identityLoader.dispatch(); - - await().until(() -> future1b.isDone() && future2a.isDone() && future3a.isDone()); - assertThat(future1b.get(), equalTo("A")); - assertThat(future2a.get(), equalTo("B")); - assertThat(future3a.get(), equalTo("C")); - assertThat(loadCalls, equalTo(asList(asList("A", "B"), singletonList("C")))); - } - - @Test - public void should_Not_redispatch_previous_load() throws ExecutionException, InterruptedException { - List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); - - CompletableFuture future1 = identityLoader.load("A"); - identityLoader.dispatch(); - - CompletableFuture future2 = identityLoader.load("B"); - identityLoader.dispatch(); - - await().until(() -> future1.isDone() && future2.isDone()); - assertThat(future1.get(), equalTo("A")); - assertThat(future2.get(), equalTo("B")); - assertThat(loadCalls, equalTo(asList(singletonList("A"), singletonList("B")))); - } - - @Test - public void should_Cache_on_redispatch() throws ExecutionException, InterruptedException { - List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); - - CompletableFuture future1 = identityLoader.load("A"); - identityLoader.dispatch(); - - CompletableFuture> future2 = identityLoader.loadMany(asList("A", "B")); - identityLoader.dispatch(); - - await().until(() -> future1.isDone() && future2.isDone()); - assertThat(future1.get(), equalTo("A")); - assertThat(future2.get(), equalTo(asList("A", "B"))); - assertThat(loadCalls, equalTo(asList(singletonList("A"), singletonList("B")))); - } - - @Test - public void should_Clear_single_value_in_loader() throws ExecutionException, InterruptedException { - List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); - - CompletableFuture future1 = identityLoader.load("A"); - CompletableFuture future2 = identityLoader.load("B"); - identityLoader.dispatch(); - - await().until(() -> future1.isDone() && future2.isDone()); - assertThat(future1.get(), equalTo("A")); - assertThat(future2.get(), equalTo("B")); - assertThat(loadCalls, equalTo(singletonList(asList("A", "B")))); - - // fluency - DataLoader dl = identityLoader.clear("A"); - assertThat(dl, equalTo(identityLoader)); - - CompletableFuture future1a = identityLoader.load("A"); - CompletableFuture future2a = identityLoader.load("B"); - identityLoader.dispatch(); - - await().until(() -> future1a.isDone() && future2a.isDone()); - assertThat(future1a.get(), equalTo("A")); - assertThat(future2a.get(), equalTo("B")); - assertThat(loadCalls, equalTo(asList(asList("A", "B"), singletonList("A")))); - } - - @Test - public void should_Clear_all_values_in_loader() throws ExecutionException, InterruptedException { - List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); - - CompletableFuture future1 = identityLoader.load("A"); - CompletableFuture future2 = identityLoader.load("B"); - identityLoader.dispatch(); - - await().until(() -> future1.isDone() && future2.isDone()); - assertThat(future1.get(), equalTo("A")); - assertThat(future2.get(), equalTo("B")); - assertThat(loadCalls, equalTo(singletonList(asList("A", "B")))); - - DataLoader dlFluent = identityLoader.clearAll(); - assertThat(dlFluent, equalTo(identityLoader)); // fluency - - CompletableFuture future1a = identityLoader.load("A"); - CompletableFuture future2a = identityLoader.load("B"); - identityLoader.dispatch(); - - await().until(() -> future1a.isDone() && future2a.isDone()); - assertThat(future1a.get(), equalTo("A")); - assertThat(future2a.get(), equalTo("B")); - assertThat(loadCalls, equalTo(asList(asList("A", "B"), asList("A", "B")))); - } - - @Test - public void should_Allow_priming_the_cache() throws ExecutionException, InterruptedException { - List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); - - DataLoader dlFluency = identityLoader.prime("A", "A"); - assertThat(dlFluency, equalTo(identityLoader)); - - CompletableFuture future1 = identityLoader.load("A"); - CompletableFuture future2 = identityLoader.load("B"); - identityLoader.dispatch(); - - await().until(() -> future1.isDone() && future2.isDone()); - assertThat(future1.get(), equalTo("A")); - assertThat(future2.get(), equalTo("B")); - assertThat(loadCalls, equalTo(singletonList(singletonList("B")))); - } - - @Test - public void should_Not_prime_keys_that_already_exist() throws ExecutionException, InterruptedException { - List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); - - identityLoader.prime("A", "X"); - - CompletableFuture future1 = identityLoader.load("A"); - CompletableFuture future2 = identityLoader.load("B"); - CompletableFuture> composite = identityLoader.dispatch(); - - await().until(composite::isDone); - assertThat(future1.get(), equalTo("X")); - assertThat(future2.get(), equalTo("B")); - - identityLoader.prime("A", "Y"); - identityLoader.prime("B", "Y"); - - CompletableFuture future1a = identityLoader.load("A"); - CompletableFuture future2a = identityLoader.load("B"); - CompletableFuture> composite2 = identityLoader.dispatch(); - - await().until(composite2::isDone); - assertThat(future1a.get(), equalTo("X")); - assertThat(future2a.get(), equalTo("B")); - assertThat(loadCalls, equalTo(singletonList(singletonList("B")))); - } - - @Test - public void should_Allow_to_forcefully_prime_the_cache() throws ExecutionException, InterruptedException { - List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); - - identityLoader.prime("A", "X"); - - CompletableFuture future1 = identityLoader.load("A"); - CompletableFuture future2 = identityLoader.load("B"); - CompletableFuture> composite = identityLoader.dispatch(); - - await().until(composite::isDone); - assertThat(future1.get(), equalTo("X")); - assertThat(future2.get(), equalTo("B")); - - identityLoader.clear("A").prime("A", "Y"); - identityLoader.clear("B").prime("B", "Y"); - - CompletableFuture future1a = identityLoader.load("A"); - CompletableFuture future2a = identityLoader.load("B"); - CompletableFuture> composite2 = identityLoader.dispatch(); - - await().until(composite2::isDone); - assertThat(future1a.get(), equalTo("Y")); - assertThat(future2a.get(), equalTo("Y")); - assertThat(loadCalls, equalTo(singletonList(singletonList("B")))); - } - - @Test - public void should_Allow_priming_the_cache_with_a_future() throws ExecutionException, InterruptedException { - List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); - - DataLoader dlFluency = identityLoader.prime("A", CompletableFuture.completedFuture("A")); - assertThat(dlFluency, equalTo(identityLoader)); - - CompletableFuture future1 = identityLoader.load("A"); - CompletableFuture future2 = identityLoader.load("B"); - identityLoader.dispatch(); - - await().until(() -> future1.isDone() && future2.isDone()); - assertThat(future1.get(), equalTo("A")); - assertThat(future2.get(), equalTo("B")); - assertThat(loadCalls, equalTo(singletonList(singletonList("B")))); - } - - @Test - public void should_not_Cache_failed_fetches_on_complete_failure() { - List> loadCalls = new ArrayList<>(); - DataLoader errorLoader = idLoaderBlowsUps(new DataLoaderOptions(), loadCalls); - - CompletableFuture future1 = errorLoader.load(1); - errorLoader.dispatch(); - - await().until(future1::isDone); - assertThat(future1.isCompletedExceptionally(), is(true)); - assertThat(cause(future1), instanceOf(IllegalStateException.class)); - - CompletableFuture future2 = errorLoader.load(1); - errorLoader.dispatch(); - - await().until(future2::isDone); - assertThat(future2.isCompletedExceptionally(), is(true)); - assertThat(cause(future2), instanceOf(IllegalStateException.class)); - assertThat(loadCalls, equalTo(asList(singletonList(1), singletonList(1)))); - } - - @Test - public void should_Resolve_to_error_to_indicate_failure() throws ExecutionException, InterruptedException { - List> loadCalls = new ArrayList<>(); - DataLoader evenLoader = idLoaderOddEvenExceptions(new DataLoaderOptions(), loadCalls); - - CompletableFuture future1 = evenLoader.load(1); - evenLoader.dispatch(); - - await().until(future1::isDone); - assertThat(future1.isCompletedExceptionally(), is(true)); - assertThat(cause(future1), instanceOf(IllegalStateException.class)); - - CompletableFuture future2 = evenLoader.load(2); - evenLoader.dispatch(); - - await().until(future2::isDone); - assertThat(future2.get(), equalTo(2)); - assertThat(loadCalls, equalTo(asList(singletonList(1), singletonList(2)))); - } - - // Accept any kind of key. - - @Test - public void should_Represent_failures_and_successes_simultaneously() throws ExecutionException, InterruptedException { - AtomicBoolean success = new AtomicBoolean(); - List> loadCalls = new ArrayList<>(); - DataLoader evenLoader = idLoaderOddEvenExceptions(new DataLoaderOptions(), loadCalls); - - CompletableFuture future1 = evenLoader.load(1); - CompletableFuture future2 = evenLoader.load(2); - CompletableFuture future3 = evenLoader.load(3); - CompletableFuture future4 = evenLoader.load(4); - CompletableFuture> result = evenLoader.dispatch(); - result.thenAccept(promisedValues -> success.set(true)); - - await().untilAtomic(success, is(true)); - - assertThat(future1.isCompletedExceptionally(), is(true)); - assertThat(cause(future1), instanceOf(IllegalStateException.class)); - assertThat(future2.get(), equalTo(2)); - assertThat(future3.isCompletedExceptionally(), is(true)); - assertThat(future4.get(), equalTo(4)); - - assertThat(loadCalls, equalTo(singletonList(asList(1, 2, 3, 4)))); - } - - // Accepts options - - @Test - public void should_Cache_failed_fetches() { - List> loadCalls = new ArrayList<>(); - DataLoader errorLoader = idLoaderAllExceptions(new DataLoaderOptions(), loadCalls); - - CompletableFuture future1 = errorLoader.load(1); - errorLoader.dispatch(); - - await().until(future1::isDone); - assertThat(future1.isCompletedExceptionally(), is(true)); - assertThat(cause(future1), instanceOf(IllegalStateException.class)); - - CompletableFuture future2 = errorLoader.load(1); - errorLoader.dispatch(); - - await().until(future2::isDone); - assertThat(future2.isCompletedExceptionally(), is(true)); - assertThat(cause(future2), instanceOf(IllegalStateException.class)); - - assertThat(loadCalls, equalTo(singletonList(singletonList(1)))); - } - - @Test - public void should_NOT_Cache_failed_fetches_if_told_not_too() { - DataLoaderOptions options = DataLoaderOptions.newOptions().setCachingExceptionsEnabled(false); - List> loadCalls = new ArrayList<>(); - DataLoader errorLoader = idLoaderAllExceptions(options, loadCalls); - - CompletableFuture future1 = errorLoader.load(1); - errorLoader.dispatch(); - - await().until(future1::isDone); - assertThat(future1.isCompletedExceptionally(), is(true)); - assertThat(cause(future1), instanceOf(IllegalStateException.class)); - - CompletableFuture future2 = errorLoader.load(1); - errorLoader.dispatch(); - - await().until(future2::isDone); - assertThat(future2.isCompletedExceptionally(), is(true)); - assertThat(cause(future2), instanceOf(IllegalStateException.class)); - - assertThat(loadCalls, equalTo(asList(singletonList(1), singletonList(1)))); - } - - - // Accepts object key in custom cacheKey function - - @Test - public void should_Handle_priming_the_cache_with_an_error() { - List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); - - identityLoader.prime(1, new IllegalStateException("Error")); - - CompletableFuture future1 = identityLoader.load(1); - identityLoader.dispatch(); - - await().until(future1::isDone); - assertThat(future1.isCompletedExceptionally(), is(true)); - assertThat(cause(future1), instanceOf(IllegalStateException.class)); - assertThat(loadCalls, equalTo(emptyList())); - } - - @Test - public void should_Clear_values_from_cache_after_errors() { - List> loadCalls = new ArrayList<>(); - DataLoader errorLoader = idLoaderBlowsUps(new DataLoaderOptions(), loadCalls); - - CompletableFuture future1 = errorLoader.load(1); - future1.handle((value, t) -> { - if (t != null) { - // Presumably determine if this error is transient, and only clear the cache in that case. - errorLoader.clear(1); - } - return null; - }); - errorLoader.dispatch(); - - await().until(future1::isDone); - assertThat(future1.isCompletedExceptionally(), is(true)); - assertThat(cause(future1), instanceOf(IllegalStateException.class)); - - CompletableFuture future2 = errorLoader.load(1); - future2.handle((value, t) -> { - if (t != null) { - // Again, only do this if you can determine the error is transient. - errorLoader.clear(1); - } - return null; - }); - errorLoader.dispatch(); - - await().until(future2::isDone); - assertThat(future2.isCompletedExceptionally(), is(true)); - assertThat(cause(future2), instanceOf(IllegalStateException.class)); - assertThat(loadCalls, equalTo(asList(singletonList(1), singletonList(1)))); - } - - @Test - public void should_Propagate_error_to_all_loads() { - List> loadCalls = new ArrayList<>(); - DataLoader errorLoader = idLoaderBlowsUps(new DataLoaderOptions(), loadCalls); - - CompletableFuture future1 = errorLoader.load(1); - CompletableFuture future2 = errorLoader.load(2); - errorLoader.dispatch(); - - await().until(future1::isDone); - assertThat(future1.isCompletedExceptionally(), is(true)); - Throwable cause = cause(future1); - assert cause != null; - assertThat(cause, instanceOf(IllegalStateException.class)); - assertThat(cause.getMessage(), equalTo("Error")); - - await().until(future2::isDone); - cause = cause(future2); - assert cause != null; - assertThat(cause.getMessage(), equalTo(cause.getMessage())); - assertThat(loadCalls, equalTo(singletonList(asList(1, 2)))); - } - - @Test - public void should_Accept_objects_as_keys() { - List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); - - Object keyA = new Object(); - Object keyB = new Object(); - - // Fetches as expected - - identityLoader.load(keyA); - identityLoader.load(keyB); - - identityLoader.dispatch().thenAccept(promisedValues -> { - assertThat(promisedValues.get(0), equalTo(keyA)); - assertThat(promisedValues.get(1), equalTo(keyB)); - }); - - assertThat(loadCalls.size(), equalTo(1)); - assertThat(loadCalls.get(0).size(), equalTo(2)); - assertThat(loadCalls.get(0).toArray()[0], equalTo(keyA)); - assertThat(loadCalls.get(0).toArray()[1], equalTo(keyB)); - - // Caching - identityLoader.clear(keyA); - //noinspection SuspiciousMethodCalls - loadCalls.remove(keyA); - - identityLoader.load(keyA); - identityLoader.load(keyB); - - identityLoader.dispatch().thenAccept(promisedValues -> { - assertThat(promisedValues.get(0), equalTo(keyA)); - assertThat(identityLoader.getCacheKey(keyB), equalTo(keyB)); - }); - - assertThat(loadCalls.size(), equalTo(2)); - assertThat(loadCalls.get(1).size(), equalTo(1)); - assertThat(loadCalls.get(1).toArray()[0], equalTo(keyA)); - } - - @Test - public void should_Disable_caching() throws ExecutionException, InterruptedException { - List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = - idLoader(newOptions().setCachingEnabled(false), loadCalls); - - CompletableFuture future1 = identityLoader.load("A"); - CompletableFuture future2 = identityLoader.load("B"); - identityLoader.dispatch(); - - await().until(() -> future1.isDone() && future2.isDone()); - assertThat(future1.get(), equalTo("A")); - assertThat(future2.get(), equalTo("B")); - assertThat(loadCalls, equalTo(singletonList(asList("A", "B")))); - - CompletableFuture future1a = identityLoader.load("A"); - CompletableFuture future3 = identityLoader.load("C"); - identityLoader.dispatch(); - - await().until(() -> future1a.isDone() && future3.isDone()); - assertThat(future1a.get(), equalTo("A")); - assertThat(future3.get(), equalTo("C")); - assertThat(loadCalls, equalTo(asList(asList("A", "B"), asList("A", "C")))); - - CompletableFuture future1b = identityLoader.load("A"); - CompletableFuture future2a = identityLoader.load("B"); - CompletableFuture future3a = identityLoader.load("C"); - identityLoader.dispatch(); - - await().until(() -> future1b.isDone() && future2a.isDone() && future3a.isDone()); - assertThat(future1b.get(), equalTo("A")); - assertThat(future2a.get(), equalTo("B")); - assertThat(future3a.get(), equalTo("C")); - assertThat(loadCalls, equalTo(asList(asList("A", "B"), - asList("A", "C"), asList("A", "B", "C")))); - } - - @Test - public void should_work_with_duplicate_keys_when_caching_disabled() throws ExecutionException, InterruptedException { - List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = - idLoader(newOptions().setCachingEnabled(false), loadCalls); - - CompletableFuture future1 = identityLoader.load("A"); - CompletableFuture future2 = identityLoader.load("B"); - CompletableFuture future3 = identityLoader.load("A"); - identityLoader.dispatch(); - - await().until(() -> future1.isDone() && future2.isDone() && future3.isDone()); - assertThat(future1.get(), equalTo("A")); - assertThat(future2.get(), equalTo("B")); - assertThat(future3.get(), equalTo("A")); - assertThat(loadCalls, equalTo(singletonList(asList("A", "B", "A")))); - } - - @Test - public void should_work_with_duplicate_keys_when_caching_enabled() throws ExecutionException, InterruptedException { - List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = - idLoader(newOptions().setCachingEnabled(true), loadCalls); - - CompletableFuture future1 = identityLoader.load("A"); - CompletableFuture future2 = identityLoader.load("B"); - CompletableFuture future3 = identityLoader.load("A"); - identityLoader.dispatch(); - - await().until(() -> future1.isDone() && future2.isDone() && future3.isDone()); - assertThat(future1.get(), equalTo("A")); - assertThat(future2.get(), equalTo("B")); - assertThat(future3.get(), equalTo("A")); - assertThat(loadCalls, equalTo(singletonList(asList("A", "B")))); - } - - // It is resilient to job queue ordering - - @Test - public void should_Accept_objects_with_a_complex_key() throws ExecutionException, InterruptedException { - List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().setCacheKeyFunction(getJsonObjectCacheMapFn()); - DataLoader identityLoader = idLoader(options, loadCalls); - - JsonObject key1 = new JsonObject().put("id", 123); - JsonObject key2 = new JsonObject().put("id", 123); - - CompletableFuture future1 = identityLoader.load(key1); - CompletableFuture future2 = identityLoader.load(key2); - identityLoader.dispatch(); - - await().until(() -> future1.isDone() && future2.isDone()); - assertThat(loadCalls, equalTo(singletonList(singletonList(key1)))); - assertThat(future1.get(), equalTo(key1)); - assertThat(future2.get(), equalTo(key1)); - } - - // Helper methods - - @Test - public void should_Clear_objects_with_complex_key() throws ExecutionException, InterruptedException { - List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().setCacheKeyFunction(getJsonObjectCacheMapFn()); - DataLoader identityLoader = idLoader(options, loadCalls); - - JsonObject key1 = new JsonObject().put("id", 123); - JsonObject key2 = new JsonObject().put("id", 123); - - CompletableFuture future1 = identityLoader.load(key1); - identityLoader.dispatch(); - - await().until(future1::isDone); - identityLoader.clear(key2); // clear equivalent object key - - CompletableFuture future2 = identityLoader.load(key1); - identityLoader.dispatch(); - - await().until(future2::isDone); - assertThat(loadCalls, equalTo(asList(singletonList(key1), singletonList(key1)))); - assertThat(future1.get(), equalTo(key1)); - assertThat(future2.get(), equalTo(key1)); - } - - @Test - public void should_Accept_objects_with_different_order_of_keys() throws ExecutionException, InterruptedException { - List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().setCacheKeyFunction(getJsonObjectCacheMapFn()); - DataLoader identityLoader = idLoader(options, loadCalls); - - JsonObject key1 = new JsonObject().put("a", 123).put("b", 321); - JsonObject key2 = new JsonObject().put("b", 321).put("a", 123); - - // Fetches as expected - - CompletableFuture future1 = identityLoader.load(key1); - CompletableFuture future2 = identityLoader.load(key2); - identityLoader.dispatch(); - - await().until(() -> future1.isDone() && future2.isDone()); - assertThat(loadCalls, equalTo(singletonList(singletonList(key1)))); - assertThat(loadCalls.size(), equalTo(1)); - assertThat(future1.get(), equalTo(key1)); - assertThat(future2.get(), equalTo(key2)); - } - - @Test - public void should_Allow_priming_the_cache_with_an_object_key() throws ExecutionException, InterruptedException { - List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().setCacheKeyFunction(getJsonObjectCacheMapFn()); - DataLoader identityLoader = idLoader(options, loadCalls); - - JsonObject key1 = new JsonObject().put("id", 123); - JsonObject key2 = new JsonObject().put("id", 123); - - identityLoader.prime(key1, key1); - - CompletableFuture future1 = identityLoader.load(key1); - CompletableFuture future2 = identityLoader.load(key2); - identityLoader.dispatch(); - - await().until(() -> future1.isDone() && future2.isDone()); - assertThat(loadCalls, equalTo(emptyList())); - assertThat(future1.get(), equalTo(key1)); - assertThat(future2.get(), equalTo(key1)); - } - - @Test - public void should_Accept_a_custom_cache_map_implementation() throws ExecutionException, InterruptedException { - CustomCacheMap customMap = new CustomCacheMap(); - List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().setCacheMap(customMap); - DataLoader identityLoader = idLoader(options, loadCalls); - - // Fetches as expected - - CompletableFuture future1 = identityLoader.load("a"); - CompletableFuture future2 = identityLoader.load("b"); - CompletableFuture> composite = identityLoader.dispatch(); - - await().until(composite::isDone); - assertThat(future1.get(), equalTo("a")); - assertThat(future2.get(), equalTo("b")); - - assertThat(loadCalls, equalTo(singletonList(asList("a", "b")))); - assertArrayEquals(customMap.stash.keySet().toArray(), asList("a", "b").toArray()); - - CompletableFuture future3 = identityLoader.load("c"); - CompletableFuture future2a = identityLoader.load("b"); - composite = identityLoader.dispatch(); - - await().until(composite::isDone); - assertThat(future3.get(), equalTo("c")); - assertThat(future2a.get(), equalTo("b")); - - assertThat(loadCalls, equalTo(asList(asList("a", "b"), singletonList("c")))); - assertArrayEquals(customMap.stash.keySet().toArray(), asList("a", "b", "c").toArray()); - - // Supports clear - - identityLoader.clear("b"); - assertArrayEquals(customMap.stash.keySet().toArray(), asList("a", "c").toArray()); - - CompletableFuture future2b = identityLoader.load("b"); - composite = identityLoader.dispatch(); - - await().until(composite::isDone); - assertThat(future2b.get(), equalTo("b")); - assertThat(loadCalls, equalTo(asList(asList("a", "b"), - singletonList("c"), singletonList("b")))); - assertArrayEquals(customMap.stash.keySet().toArray(), asList("a", "c", "b").toArray()); - - // Supports clear all - - identityLoader.clearAll(); - assertArrayEquals(customMap.stash.keySet().toArray(), emptyList().toArray()); - } - - @Test - public void should_degrade_gracefully_if_cache_get_throws() { - CacheMap cache = new ThrowingCacheMap(); - DataLoaderOptions options = newOptions().setCachingEnabled(true).setCacheMap(cache); - List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idLoader(options, loadCalls); - - assertThat(identityLoader.getIfPresent("a"), equalTo(Optional.empty())); - - CompletableFuture future = identityLoader.load("a"); - identityLoader.dispatch(); - assertThat(future.join(), equalTo("a")); - } - - @Test - public void batching_disabled_should_dispatch_immediately() { - List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().setBatchingEnabled(false); - DataLoader identityLoader = idLoader(options, loadCalls); - - CompletableFuture fa = identityLoader.load("A"); - CompletableFuture fb = identityLoader.load("B"); - - // caching is on still - CompletableFuture fa1 = identityLoader.load("A"); - CompletableFuture fb1 = identityLoader.load("B"); - - List values = CompletableFutureKit.allOf(asList(fa, fb, fa1, fb1)).join(); - - assertThat(fa.join(), equalTo("A")); - assertThat(fb.join(), equalTo("B")); - assertThat(fa1.join(), equalTo("A")); - assertThat(fb1.join(), equalTo("B")); - - assertThat(values, equalTo(asList("A", "B", "A", "B"))); - - assertThat(loadCalls, equalTo(asList( - singletonList("A"), - singletonList("B")))); - - } - - @Test - public void batching_disabled_and_caching_disabled_should_dispatch_immediately_and_forget() { - List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().setBatchingEnabled(false).setCachingEnabled(false); - DataLoader identityLoader = idLoader(options, loadCalls); - - CompletableFuture fa = identityLoader.load("A"); - CompletableFuture fb = identityLoader.load("B"); - - // caching is off - CompletableFuture fa1 = identityLoader.load("A"); - CompletableFuture fb1 = identityLoader.load("B"); - - List values = CompletableFutureKit.allOf(asList(fa, fb, fa1, fb1)).join(); - - assertThat(fa.join(), equalTo("A")); - assertThat(fb.join(), equalTo("B")); - assertThat(fa1.join(), equalTo("A")); - assertThat(fb1.join(), equalTo("B")); - - assertThat(values, equalTo(asList("A", "B", "A", "B"))); - - assertThat(loadCalls, equalTo(asList( - singletonList("A"), - singletonList("B"), - singletonList("A"), - singletonList("B") - ))); - - } - - @Test - public void batches_multiple_requests_with_max_batch_size() { - List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idLoader(newOptions().setMaxBatchSize(2), loadCalls); - - CompletableFuture f1 = identityLoader.load(1); - CompletableFuture f2 = identityLoader.load(2); - CompletableFuture f3 = identityLoader.load(3); - - identityLoader.dispatch(); - - CompletableFuture.allOf(f1, f2, f3).join(); - - assertThat(f1.join(), equalTo(1)); - assertThat(f2.join(), equalTo(2)); - assertThat(f3.join(), equalTo(3)); - - assertThat(loadCalls, equalTo(asList(asList(1, 2), singletonList(3)))); - - } - - @Test - public void can_split_max_batch_sizes_correctly() { - List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idLoader(newOptions().setMaxBatchSize(5), loadCalls); - - for (int i = 0; i < 21; i++) { - identityLoader.load(i); - } - List> expectedCalls = new ArrayList<>(); - expectedCalls.add(listFrom(0, 5)); - expectedCalls.add(listFrom(5, 10)); - expectedCalls.add(listFrom(10, 15)); - expectedCalls.add(listFrom(15, 20)); - expectedCalls.add(listFrom(20, 21)); - - List result = identityLoader.dispatch().join(); - - assertThat(result, equalTo(listFrom(0, 21))); - assertThat(loadCalls, equalTo(expectedCalls)); - - } - - @Test - public void should_Batch_loads_occurring_within_futures() { - List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idLoader(newOptions(), loadCalls); - - Supplier nullValue = () -> null; - - AtomicBoolean v4Called = new AtomicBoolean(); - - CompletableFuture.supplyAsync(nullValue).thenAccept(v1 -> { - identityLoader.load("a"); - CompletableFuture.supplyAsync(nullValue).thenAccept(v2 -> { - identityLoader.load("b"); - CompletableFuture.supplyAsync(nullValue).thenAccept(v3 -> { - identityLoader.load("c"); - CompletableFuture.supplyAsync(nullValue).thenAccept( - v4 -> { - identityLoader.load("d"); - v4Called.set(true); - }); - }); - }); - }); - - await().untilTrue(v4Called); - - identityLoader.dispatchAndJoin(); - - assertThat(loadCalls, equalTo( - singletonList(asList("a", "b", "c", "d")))); - } - - @Test - public void can_call_a_loader_from_a_loader() throws Exception { - List> deepLoadCalls = new ArrayList<>(); - DataLoader deepLoader = newDataLoader(keys -> { - deepLoadCalls.add(keys); - return CompletableFuture.completedFuture(keys); - }); - - List> aLoadCalls = new ArrayList<>(); - DataLoader aLoader = newDataLoader(keys -> { - aLoadCalls.add(keys); - return deepLoader.loadMany(keys); - }); - - List> bLoadCalls = new ArrayList<>(); - DataLoader bLoader = newDataLoader(keys -> { - bLoadCalls.add(keys); - return deepLoader.loadMany(keys); - }); - - CompletableFuture a1 = aLoader.load("A1"); - CompletableFuture a2 = aLoader.load("A2"); - CompletableFuture b1 = bLoader.load("B1"); - CompletableFuture b2 = bLoader.load("B2"); - - CompletableFuture.allOf( - aLoader.dispatch(), - deepLoader.dispatch(), - bLoader.dispatch(), - deepLoader.dispatch() - ).join(); - - assertThat(a1.get(), equalTo("A1")); - assertThat(a2.get(), equalTo("A2")); - assertThat(b1.get(), equalTo("B1")); - assertThat(b2.get(), equalTo("B2")); - - assertThat(aLoadCalls, equalTo( - singletonList(asList("A1", "A2")))); - - assertThat(bLoadCalls, equalTo( - singletonList(asList("B1", "B2")))); - - assertThat(deepLoadCalls, equalTo( - asList(asList("A1", "A2"), asList("B1", "B2")))); - } - - @Test - public void should_allow_composition_of_data_loader_calls() { - UserManager userManager = new UserManager(); - - BatchLoader userBatchLoader = userIds -> CompletableFuture - .supplyAsync(() -> userIds - .stream() - .map(userManager::loadUserById) - .collect(Collectors.toList())); - DataLoader userLoader = newDataLoader(userBatchLoader); - - AtomicBoolean gandalfCalled = new AtomicBoolean(false); - AtomicBoolean sarumanCalled = new AtomicBoolean(false); - - userLoader.load(1L) - .thenAccept(user -> userLoader.load(user.getInvitedByID()) - .thenAccept(invitedBy -> { - gandalfCalled.set(true); - assertThat(invitedBy.getName(), equalTo("Manwë")); - })); - - userLoader.load(2L) - .thenAccept(user -> userLoader.load(user.getInvitedByID()) - .thenAccept(invitedBy -> { - sarumanCalled.set(true); - assertThat(invitedBy.getName(), equalTo("Aulë")); - })); - - List allResults = userLoader.dispatchAndJoin(); - - await().untilTrue(gandalfCalled); - await().untilTrue(sarumanCalled); - - assertThat(allResults.size(), equalTo(4)); - } - - private static CacheKey getJsonObjectCacheMapFn() { - return key -> key.stream() - .map(entry -> entry.getKey() + ":" + entry.getValue()) - .sorted() - .collect(Collectors.joining()); - } - - private static DataLoader idLoader(DataLoaderOptions options, List> loadCalls) { - return newPublisherDataLoader((BatchPublisher) (keys, subscriber) -> { - loadCalls.add(new ArrayList<>(keys)); - Flux.fromIterable(keys).subscribe(subscriber); - }, options); - } - - private static DataLoader idLoaderBlowsUps( - DataLoaderOptions options, List> loadCalls) { - return newPublisherDataLoader((BatchPublisher) (keys, subscriber) -> { - loadCalls.add(new ArrayList<>(keys)); - Flux.error(new IllegalStateException("Error")).subscribe(subscriber); - }, options); - } - - private static DataLoader idLoaderAllExceptions( - DataLoaderOptions options, List> loadCalls) { - return newPublisherDataLoaderWithTry((BatchPublisher>) (keys, subscriber) -> { - loadCalls.add(new ArrayList<>(keys)); - Stream> failures = keys.stream().map(k -> Try.failed(new IllegalStateException("Error"))); - Flux.fromStream(failures).subscribe(subscriber); - }, options); - } - - private static DataLoader idLoaderOddEvenExceptions( - DataLoaderOptions options, List> loadCalls) { - return newPublisherDataLoaderWithTry((BatchPublisher>) (keys, subscriber) -> { - loadCalls.add(new ArrayList<>(keys)); - - List> errors = new ArrayList<>(); - for (Integer key : keys) { - if (key % 2 == 0) { - errors.add(Try.succeeded(key)); - } else { - errors.add(Try.failed(new IllegalStateException("Error"))); - } - } - Flux.fromIterable(errors).subscribe(subscriber); - }, options); - } - - private static BatchPublisher keysAsValues() { - return (keys, subscriber) -> Flux.fromIterable(keys).subscribe(subscriber); - } - - private static class ThrowingCacheMap extends CustomCacheMap { - @Override - public CompletableFuture get(String key) { - throw new RuntimeException("Cache implementation failed."); - } - } -} diff --git a/src/test/java/org/dataloader/DataLoaderMappedBatchPublisherTest.java b/src/test/java/org/dataloader/DataLoaderMappedBatchPublisherTest.java deleted file mode 100644 index 5b9ca0b4..00000000 --- a/src/test/java/org/dataloader/DataLoaderMappedBatchPublisherTest.java +++ /dev/null @@ -1,175 +0,0 @@ -package org.dataloader; - -import org.junit.jupiter.api.Test; -import reactor.core.publisher.Flux; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.atomic.AtomicInteger; - -import static java.util.Arrays.asList; -import static java.util.Collections.singletonList; -import static org.awaitility.Awaitility.await; -import static org.dataloader.DataLoaderFactory.newMappedPublisherDataLoader; -import static org.dataloader.DataLoaderOptions.newOptions; -import static org.dataloader.fixtures.TestKit.listFrom; -import static org.dataloader.impl.CompletableFutureKit.cause; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.instanceOf; -import static org.hamcrest.Matchers.is; - -public class DataLoaderMappedBatchPublisherTest { - - MappedBatchPublisher evensOnlyMappedBatchLoader = (keys, subscriber) -> { - Map mapOfResults = new HashMap<>(); - - AtomicInteger index = new AtomicInteger(); - keys.forEach(k -> { - int i = index.getAndIncrement(); - if (i % 2 == 0) { - mapOfResults.put(k, k); - } - }); - Flux.fromIterable(mapOfResults.entrySet()).subscribe(subscriber); - }; - - private static DataLoader idMapLoader(DataLoaderOptions options, List> loadCalls) { - MappedBatchPublisher kvBatchLoader = (keys, subscriber) -> { - loadCalls.add(new ArrayList<>(keys)); - Map map = new HashMap<>(); - //noinspection unchecked - keys.forEach(k -> map.put(k, (V) k)); - Flux.fromIterable(map.entrySet()).subscribe(subscriber); - }; - return DataLoaderFactory.newMappedPublisherDataLoader(kvBatchLoader, options); - } - - private static DataLoader idMapLoaderBlowsUps( - DataLoaderOptions options, List> loadCalls) { - return newMappedPublisherDataLoader((MappedBatchPublisher) (keys, subscriber) -> { - loadCalls.add(new ArrayList<>(keys)); - Flux.>error(new IllegalStateException("Error")).subscribe(subscriber); - }, options); - } - - - @Test - public void basic_map_batch_loading() { - DataLoader loader = DataLoaderFactory.newMappedPublisherDataLoader(evensOnlyMappedBatchLoader); - - loader.load("A"); - loader.load("B"); - loader.loadMany(asList("C", "D")); - - List results = loader.dispatchAndJoin(); - - assertThat(results.size(), equalTo(4)); - assertThat(results, equalTo(asList("A", null, "C", null))); - } - - @Test - public void should_map_Batch_multiple_requests() throws ExecutionException, InterruptedException { - List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idMapLoader(new DataLoaderOptions(), loadCalls); - - CompletableFuture future1 = identityLoader.load(1); - CompletableFuture future2 = identityLoader.load(2); - identityLoader.dispatch(); - - await().until(() -> future1.isDone() && future2.isDone()); - assertThat(future1.get(), equalTo(1)); - assertThat(future2.get(), equalTo(2)); - assertThat(loadCalls, equalTo(singletonList(asList(1, 2)))); - } - - @Test - public void can_split_max_batch_sizes_correctly() { - List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = idMapLoader(newOptions().setMaxBatchSize(5), loadCalls); - - for (int i = 0; i < 21; i++) { - identityLoader.load(i); - } - List> expectedCalls = new ArrayList<>(); - expectedCalls.add(listFrom(0, 5)); - expectedCalls.add(listFrom(5, 10)); - expectedCalls.add(listFrom(10, 15)); - expectedCalls.add(listFrom(15, 20)); - expectedCalls.add(listFrom(20, 21)); - - List result = identityLoader.dispatch().join(); - - assertThat(result, equalTo(listFrom(0, 21))); - assertThat(loadCalls, equalTo(expectedCalls)); - } - - @Test - public void should_Propagate_error_to_all_loads() { - List> loadCalls = new ArrayList<>(); - DataLoader errorLoader = idMapLoaderBlowsUps(new DataLoaderOptions(), loadCalls); - - CompletableFuture future1 = errorLoader.load(1); - CompletableFuture future2 = errorLoader.load(2); - errorLoader.dispatch(); - - await().until(future1::isDone); - - assertThat(future1.isCompletedExceptionally(), is(true)); - Throwable cause = cause(future1); - assert cause != null; - assertThat(cause, instanceOf(IllegalStateException.class)); - assertThat(cause.getMessage(), equalTo("Error")); - - await().until(future2::isDone); - cause = cause(future2); - assert cause != null; - assertThat(cause.getMessage(), equalTo(cause.getMessage())); - - assertThat(loadCalls, equalTo(singletonList(asList(1, 2)))); - } - - @Test - public void should_work_with_duplicate_keys_when_caching_disabled() throws ExecutionException, InterruptedException { - List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = - idMapLoader(newOptions().setCachingEnabled(false), loadCalls); - - CompletableFuture future1 = identityLoader.load("A"); - CompletableFuture future2 = identityLoader.load("B"); - CompletableFuture future3 = identityLoader.load("A"); - identityLoader.dispatch(); - - await().until(() -> future1.isDone() && future2.isDone() && future3.isDone()); - assertThat(future1.get(), equalTo("A")); - assertThat(future2.get(), equalTo("B")); - assertThat(future3.get(), equalTo("A")); - - // the map batch functions use a set of keys as input and hence remove duplicates unlike list variant - assertThat(loadCalls, equalTo(singletonList(asList("A", "B")))); - } - - @Test - public void should_work_with_duplicate_keys_when_caching_enabled() throws ExecutionException, InterruptedException { - List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = - idMapLoader(newOptions().setCachingEnabled(true), loadCalls); - - CompletableFuture future1 = identityLoader.load("A"); - CompletableFuture future2 = identityLoader.load("B"); - CompletableFuture future3 = identityLoader.load("A"); - identityLoader.dispatch(); - - await().until(() -> future1.isDone() && future2.isDone() && future3.isDone()); - assertThat(future1.get(), equalTo("A")); - assertThat(future2.get(), equalTo("B")); - assertThat(future3.get(), equalTo("A")); - assertThat(loadCalls, equalTo(singletonList(asList("A", "B")))); - } - -} diff --git a/src/test/java/org/dataloader/DataLoaderTest.java b/src/test/java/org/dataloader/DataLoaderTest.java index db71c1e9..ea4b2b91 100644 --- a/src/test/java/org/dataloader/DataLoaderTest.java +++ b/src/test/java/org/dataloader/DataLoaderTest.java @@ -27,6 +27,7 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; +import reactor.core.publisher.Flux; import java.util.ArrayList; import java.util.Collection; @@ -49,6 +50,10 @@ import static org.awaitility.Awaitility.await; import static org.dataloader.DataLoaderFactory.newDataLoader; import static org.dataloader.DataLoaderFactory.newMappedDataLoader; +import static org.dataloader.DataLoaderFactory.newMappedPublisherDataLoader; +import static org.dataloader.DataLoaderFactory.newMappedPublisherDataLoaderWithTry; +import static org.dataloader.DataLoaderFactory.newPublisherDataLoader; +import static org.dataloader.DataLoaderFactory.newPublisherDataLoaderWithTry; import static org.dataloader.DataLoaderOptions.newOptions; import static org.dataloader.fixtures.TestKit.futureError; import static org.dataloader.fixtures.TestKit.listFrom; @@ -727,7 +732,7 @@ public void should_work_with_duplicate_keys_when_caching_disabled(TestDataLoader assertThat(future1.get(), equalTo("A")); assertThat(future2.get(), equalTo("B")); assertThat(future3.get(), equalTo("A")); - if (factory instanceof ListDataLoaderFactory) { + if (factory instanceof ListDataLoaderFactory || factory instanceof PublisherDataLoaderFactory) { assertThat(loadCalls, equalTo(singletonList(asList("A", "B", "A")))); } else { assertThat(loadCalls, equalTo(singletonList(asList("A", "B")))); @@ -1147,32 +1152,30 @@ private static CacheKey getJsonObjectCacheMapFn() { private static Stream dataLoaderFactories() { return Stream.of( Arguments.of(Named.of("List DataLoader", new ListDataLoaderFactory())), - Arguments.of(Named.of("Mapped DataLoader", new MappedDataLoaderFactory())) + Arguments.of(Named.of("Mapped DataLoader", new MappedDataLoaderFactory())), + Arguments.of(Named.of("Publisher DataLoader", new PublisherDataLoaderFactory())), + Arguments.of(Named.of("Mapped Publisher DataLoader", new MappedPublisherDataLoaderFactory())) ); } public interface TestDataLoaderFactory { - DataLoader idLoader(DataLoaderOptions options, List> loadCalls); - DataLoader idLoaderBlowsUps(DataLoaderOptions options, List> loadCalls); + DataLoader idLoader(DataLoaderOptions options, List> loadCalls); + DataLoader idLoaderBlowsUps(DataLoaderOptions options, List> loadCalls); DataLoader idLoaderAllExceptions(DataLoaderOptions options, List> loadCalls); DataLoader idLoaderOddEvenExceptions(DataLoaderOptions options, List> loadCalls); } private static class ListDataLoaderFactory implements TestDataLoaderFactory { @Override - public DataLoader idLoader(DataLoaderOptions options, List> loadCalls) { + public DataLoader idLoader(DataLoaderOptions options, List> loadCalls) { return newDataLoader(keys -> { loadCalls.add(new ArrayList<>(keys)); - @SuppressWarnings("unchecked") - List values = keys.stream() - .map(k -> (V) k) - .collect(Collectors.toList()); - return CompletableFuture.completedFuture(values); + return CompletableFuture.completedFuture(keys); }, options); } @Override - public DataLoader idLoaderBlowsUps( + public DataLoader idLoaderBlowsUps( DataLoaderOptions options, List> loadCalls) { return newDataLoader(keys -> { loadCalls.add(new ArrayList<>(keys)); @@ -1211,19 +1214,18 @@ public DataLoader idLoaderOddEvenExceptions(DataLoaderOptions o private static class MappedDataLoaderFactory implements TestDataLoaderFactory { @Override - public DataLoader idLoader( + public DataLoader idLoader( DataLoaderOptions options, List> loadCalls) { return newMappedDataLoader((keys) -> { loadCalls.add(new ArrayList<>(keys)); - Map map = new HashMap<>(); - //noinspection unchecked - keys.forEach(k -> map.put(k, (V) k)); + Map map = new HashMap<>(); + keys.forEach(k -> map.put(k, k)); return CompletableFuture.completedFuture(map); }, options); } @Override - public DataLoader idLoaderBlowsUps(DataLoaderOptions options, List> loadCalls) { + public DataLoader idLoaderBlowsUps(DataLoaderOptions options, List> loadCalls) { return newMappedDataLoader((keys) -> { loadCalls.add(new ArrayList<>(keys)); return futureError(); @@ -1260,6 +1262,104 @@ public DataLoader idLoaderOddEvenExceptions( } } + private static class PublisherDataLoaderFactory implements TestDataLoaderFactory { + + @Override + public DataLoader idLoader( + DataLoaderOptions options, List> loadCalls) { + return newPublisherDataLoader((keys, subscriber) -> { + loadCalls.add(new ArrayList<>(keys)); + Flux.fromIterable(keys).subscribe(subscriber); + }, options); + } + + @Override + public DataLoader idLoaderBlowsUps(DataLoaderOptions options, List> loadCalls) { + return newPublisherDataLoader((keys, subscriber) -> { + loadCalls.add(new ArrayList<>(keys)); + Flux.error(new IllegalStateException("Error")).subscribe(subscriber); + }, options); + } + + @Override + public DataLoader idLoaderAllExceptions( + DataLoaderOptions options, List> loadCalls) { + return newPublisherDataLoaderWithTry((keys, subscriber) -> { + loadCalls.add(new ArrayList<>(keys)); + Stream> failures = keys.stream().map(k -> Try.failed(new IllegalStateException("Error"))); + Flux.fromStream(failures).subscribe(subscriber); + }, options); + } + + @Override + public DataLoader idLoaderOddEvenExceptions( + DataLoaderOptions options, List> loadCalls) { + return newPublisherDataLoaderWithTry((keys, subscriber) -> { + loadCalls.add(new ArrayList<>(keys)); + + List> errors = new ArrayList<>(); + for (Integer key : keys) { + if (key % 2 == 0) { + errors.add(Try.succeeded(key)); + } else { + errors.add(Try.failed(new IllegalStateException("Error"))); + } + } + Flux.fromIterable(errors).subscribe(subscriber); + }, options); + } + } + + private static class MappedPublisherDataLoaderFactory implements TestDataLoaderFactory { + + @Override + public DataLoader idLoader( + DataLoaderOptions options, List> loadCalls) { + return newMappedPublisherDataLoader((keys, subscriber) -> { + loadCalls.add(new ArrayList<>(keys)); + Map map = new HashMap<>(); + keys.forEach(k -> map.put(k, k)); + Flux.fromIterable(map.entrySet()).subscribe(subscriber); + }, options); + } + + @Override + public DataLoader idLoaderBlowsUps(DataLoaderOptions options, List> loadCalls) { + return newMappedPublisherDataLoader((keys, subscriber) -> { + loadCalls.add(new ArrayList<>(keys)); + Flux.>error(new IllegalStateException("Error")).subscribe(subscriber); + }, options); + } + + @Override + public DataLoader idLoaderAllExceptions( + DataLoaderOptions options, List> loadCalls) { + return newMappedPublisherDataLoaderWithTry((keys, subscriber) -> { + loadCalls.add(new ArrayList<>(keys)); + Stream>> failures = keys.stream().map(k -> Map.entry(k, Try.failed(new IllegalStateException("Error")))); + Flux.fromStream(failures).subscribe(subscriber); + }, options); + } + + @Override + public DataLoader idLoaderOddEvenExceptions( + DataLoaderOptions options, List> loadCalls) { + return newMappedPublisherDataLoaderWithTry((keys, subscriber) -> { + loadCalls.add(new ArrayList<>(keys)); + + Map> errorByKey = new HashMap<>(); + for (Integer key : keys) { + if (key % 2 == 0) { + errorByKey.put(key, Try.succeeded(key)); + } else { + errorByKey.put(key, Try.failed(new IllegalStateException("Error"))); + } + } + Flux.fromIterable(errorByKey.entrySet()).subscribe(subscriber); + }, options); + } + } + private static class ThrowingCacheMap extends CustomCacheMap { @Override public CompletableFuture get(String key) { From 651e5611f3beecf6a74f5388431033fd260704dd Mon Sep 17 00:00:00 2001 From: Alexandre Carlton Date: Mon, 20 May 2024 22:49:59 +1000 Subject: [PATCH 029/156] Fix MappedBatchPublisher loaders to work without cache If we did not cache the futures, then the MappedBatchPublisher DataLoader would not work as we were only completing the last future for a given key. --- .../java/org/dataloader/DataLoaderHelper.java | 20 ++++++++++--------- .../java/org/dataloader/DataLoaderTest.java | 6 +++--- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/dataloader/DataLoaderHelper.java b/src/main/java/org/dataloader/DataLoaderHelper.java index ee8d78b7..f4e3915d 100644 --- a/src/main/java/org/dataloader/DataLoaderHelper.java +++ b/src/main/java/org/dataloader/DataLoaderHelper.java @@ -714,7 +714,7 @@ private class DataLoaderMapEntrySubscriber implements Subscriber private final List callContexts; private final List> queuedFutures; private final Map callContextByKey; - private final Map> queuedFutureByKey; + private final Map>> queuedFuturesByKey; private final List clearCacheKeys = new ArrayList<>(); private final Map completedValuesByKey = new HashMap<>(); @@ -733,13 +733,13 @@ private DataLoaderMapEntrySubscriber( this.queuedFutures = queuedFutures; this.callContextByKey = new HashMap<>(); - this.queuedFutureByKey = new HashMap<>(); + this.queuedFuturesByKey = new HashMap<>(); for (int idx = 0; idx < queuedFutures.size(); idx++) { K key = keys.get(idx); Object callContext = callContexts.get(idx); CompletableFuture queuedFuture = queuedFutures.get(idx); callContextByKey.put(key, callContext); - queuedFutureByKey.put(key, queuedFuture); + queuedFuturesByKey.computeIfAbsent(key, k -> new ArrayList<>()).add(queuedFuture); } } @@ -756,20 +756,20 @@ public void onNext(Map.Entry entry) { V value = entry.getValue(); Object callContext = callContextByKey.get(key); - CompletableFuture future = queuedFutureByKey.get(key); + List> futures = queuedFuturesByKey.get(key); if (value instanceof Try) { // we allow the batch loader to return a Try so we can better represent a computation // that might have worked or not. Try tryValue = (Try) value; if (tryValue.isSuccess()) { - future.complete(tryValue.get()); + futures.forEach(f -> f.complete(tryValue.get())); } else { stats.incrementLoadErrorCount(new IncrementLoadErrorCountStatisticsContext<>(key, callContext)); - future.completeExceptionally(tryValue.getThrowable()); + futures.forEach(f -> f.completeExceptionally(tryValue.getThrowable())); clearCacheKeys.add(key); } } else { - future.complete(value); + futures.forEach(f -> f.complete(value)); } completedValuesByKey.put(key, value); @@ -801,9 +801,11 @@ public void onError(Throwable ex) { // Complete the futures for the remaining keys with the exception. for (int idx = 0; idx < queuedFutures.size(); idx++) { K key = keys.get(idx); - CompletableFuture future = queuedFutureByKey.get(key); + List> futures = queuedFuturesByKey.get(key); if (!completedValuesByKey.containsKey(key)) { - future.completeExceptionally(ex); + for (CompletableFuture future : futures) { + future.completeExceptionally(ex); + } // clear any cached view of this key because they all failed dataLoader.clear(key); } diff --git a/src/test/java/org/dataloader/DataLoaderTest.java b/src/test/java/org/dataloader/DataLoaderTest.java index ea4b2b91..a87c77d5 100644 --- a/src/test/java/org/dataloader/DataLoaderTest.java +++ b/src/test/java/org/dataloader/DataLoaderTest.java @@ -732,10 +732,10 @@ public void should_work_with_duplicate_keys_when_caching_disabled(TestDataLoader assertThat(future1.get(), equalTo("A")); assertThat(future2.get(), equalTo("B")); assertThat(future3.get(), equalTo("A")); - if (factory instanceof ListDataLoaderFactory || factory instanceof PublisherDataLoaderFactory) { - assertThat(loadCalls, equalTo(singletonList(asList("A", "B", "A")))); - } else { + if (factory instanceof MappedDataLoaderFactory) { assertThat(loadCalls, equalTo(singletonList(asList("A", "B")))); + } else { + assertThat(loadCalls, equalTo(singletonList(asList("A", "B", "A")))); } } From 6d3c4eb817a3d1f6b35047a9f9ff48c6f6381168 Mon Sep 17 00:00:00 2001 From: bbaker Date: Wed, 22 May 2024 10:55:21 +1000 Subject: [PATCH 030/156] Making the Subscribers use a common base class - merged in main branch --- .../java/org/dataloader/DataLoaderHelper.java | 33 +++++++++++-------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/dataloader/DataLoaderHelper.java b/src/main/java/org/dataloader/DataLoaderHelper.java index 7b3a4239..d01b9307 100644 --- a/src/main/java/org/dataloader/DataLoaderHelper.java +++ b/src/main/java/org/dataloader/DataLoaderHelper.java @@ -155,11 +155,13 @@ CompletableFuture load(K key, Object loadContext) { } } + @SuppressWarnings("unchecked") Object getCacheKey(K key) { return loaderOptions.cacheKeyFunction().isPresent() ? loaderOptions.cacheKeyFunction().get().getKey(key) : key; } + @SuppressWarnings("unchecked") Object getCacheKeyWithContext(K key, Object context) { return loaderOptions.cacheKeyFunction().isPresent() ? loaderOptions.cacheKeyFunction().get().getKeyWithContext(key, context) : key; @@ -511,6 +513,7 @@ private CompletableFuture> invokeBatchPublisher(List keys, List loadFunction = (BatchPublisherWithContext) batchLoadFunction; if (batchLoaderScheduler != null) { BatchLoaderScheduler.ScheduledBatchPublisherCall loadCall = () -> loadFunction.load(keys, subscriber, environment); @@ -519,6 +522,7 @@ private CompletableFuture> invokeBatchPublisher(List keys, List loadFunction = (BatchPublisher) batchLoadFunction; if (batchLoaderScheduler != null) { BatchLoaderScheduler.ScheduledBatchPublisherCall loadCall = () -> loadFunction.load(keys, subscriber); @@ -536,6 +540,7 @@ private CompletableFuture> invokeMappedBatchPublisher(List keys, List BatchLoaderScheduler batchLoaderScheduler = loaderOptions.getBatchLoaderScheduler(); if (batchLoadFunction instanceof MappedBatchPublisherWithContext) { + //noinspection unchecked MappedBatchPublisherWithContext loadFunction = (MappedBatchPublisherWithContext) batchLoadFunction; if (batchLoaderScheduler != null) { BatchLoaderScheduler.ScheduledBatchPublisherCall loadCall = () -> loadFunction.load(keys, subscriber, environment); @@ -544,6 +549,7 @@ private CompletableFuture> invokeMappedBatchPublisher(List keys, List loadFunction.load(keys, subscriber, environment); } } else { + //noinspection unchecked MappedBatchPublisher loadFunction = (MappedBatchPublisher) batchLoadFunction; if (batchLoaderScheduler != null) { BatchLoaderScheduler.ScheduledBatchPublisherCall loadCall = () -> loadFunction.load(keys, subscriber); @@ -670,21 +676,21 @@ public void onError(Throwable throwable) { /* * A value has arrived - how do we complete the future that's associated with it in a common way */ - void onNextValue(K key, V value, Object callContext, CompletableFuture future) { + void onNextValue(K key, V value, Object callContext, List> futures) { if (value instanceof Try) { // we allow the batch loader to return a Try so we can better represent a computation // that might have worked or not. //noinspection unchecked Try tryValue = (Try) value; if (tryValue.isSuccess()) { - future.complete(tryValue.get()); + futures.forEach(f -> f.complete(tryValue.get())); } else { stats.incrementLoadErrorCount(new IncrementLoadErrorCountStatisticsContext<>(key, callContext)); - future.completeExceptionally(tryValue.getThrowable()); + futures.forEach(f -> f.completeExceptionally(tryValue.getThrowable())); clearCacheKeys.add(key); } } else { - future.complete(value); + futures.forEach(f -> f.complete(value)); } } @@ -718,7 +724,7 @@ public synchronized void onNext(V value) { K key = keys.get(idx); Object callContext = callContexts.get(idx); CompletableFuture future = queuedFutures.get(idx); - onNextValue(key, value, callContext, future); + onNextValue(key, value, callContext, List.of(future)); completedValues.add(value); idx++; @@ -754,7 +760,7 @@ public synchronized void onError(Throwable ex) { private class DataLoaderMapEntrySubscriber extends DataLoaderSubscriberBase> { private final Map callContextByKey; - private final Map> queuedFutureByKey; + private final Map>> queuedFuturesByKey; private final Map completedValuesByKey = new HashMap<>(); @@ -766,13 +772,13 @@ private DataLoaderMapEntrySubscriber( ) { super(valuesFuture, keys, callContexts, queuedFutures); this.callContextByKey = new HashMap<>(); - this.queuedFutureByKey = new HashMap<>(); + this.queuedFuturesByKey = new HashMap<>(); for (int idx = 0; idx < queuedFutures.size(); idx++) { K key = keys.get(idx); Object callContext = callContexts.get(idx); CompletableFuture queuedFuture = queuedFutures.get(idx); callContextByKey.put(key, callContext); - queuedFutureByKey.put(key, queuedFuture); + queuedFuturesByKey.computeIfAbsent(key, k -> new ArrayList<>()).add(queuedFuture); } } @@ -784,9 +790,9 @@ public synchronized void onNext(Map.Entry entry) { V value = entry.getValue(); Object callContext = callContextByKey.get(key); - CompletableFuture future = queuedFutureByKey.get(key); + List> futures = queuedFuturesByKey.get(key); - onNextValue(key, value, callContext, future); + onNextValue(key, value, callContext, futures); completedValuesByKey.put(key, value); } @@ -811,15 +817,16 @@ public synchronized void onError(Throwable ex) { // Complete the futures for the remaining keys with the exception. for (int idx = 0; idx < queuedFutures.size(); idx++) { K key = keys.get(idx); - CompletableFuture future = queuedFutureByKey.get(key); + List> futures = queuedFuturesByKey.get(key); if (!completedValuesByKey.containsKey(key)) { - future.completeExceptionally(ex); + for (CompletableFuture future : futures) { + future.completeExceptionally(ex); + } // clear any cached view of this key because they all failed dataLoader.clear(key); } } valuesFuture.completeExceptionally(ex); } - } } From 034c68f4ca4faa17cf762230891324b04b3df7be Mon Sep 17 00:00:00 2001 From: bbaker Date: Wed, 22 May 2024 20:25:48 +1000 Subject: [PATCH 031/156] More tests for Publishers --- .../java/org/dataloader/DataLoaderHelper.java | 2 +- .../java/org/dataloader/DataLoaderTest.java | 190 +++++++++++++++--- 2 files changed, 163 insertions(+), 29 deletions(-) diff --git a/src/main/java/org/dataloader/DataLoaderHelper.java b/src/main/java/org/dataloader/DataLoaderHelper.java index d01b9307..cf3fc6e6 100644 --- a/src/main/java/org/dataloader/DataLoaderHelper.java +++ b/src/main/java/org/dataloader/DataLoaderHelper.java @@ -790,7 +790,7 @@ public synchronized void onNext(Map.Entry entry) { V value = entry.getValue(); Object callContext = callContextByKey.get(key); - List> futures = queuedFuturesByKey.get(key); + List> futures = queuedFuturesByKey.getOrDefault(key, List.of()); onNextValue(key, value, callContext, futures); diff --git a/src/test/java/org/dataloader/DataLoaderTest.java b/src/test/java/org/dataloader/DataLoaderTest.java index a87c77d5..c8d45627 100644 --- a/src/test/java/org/dataloader/DataLoaderTest.java +++ b/src/test/java/org/dataloader/DataLoaderTest.java @@ -16,12 +16,14 @@ package org.dataloader; +import org.awaitility.Duration; import org.dataloader.fixtures.CustomCacheMap; import org.dataloader.fixtures.JsonObject; import org.dataloader.fixtures.TestKit; import org.dataloader.fixtures.User; import org.dataloader.fixtures.UserManager; import org.dataloader.impl.CompletableFutureKit; +import org.dataloader.impl.DataLoaderAssertionException; import org.junit.jupiter.api.Named; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -35,6 +37,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; import java.util.concurrent.ExecutionException; @@ -47,6 +50,7 @@ import static java.util.Arrays.asList; import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; +import static java.util.concurrent.CompletableFuture.*; import static org.awaitility.Awaitility.await; import static org.dataloader.DataLoaderFactory.newDataLoader; import static org.dataloader.DataLoaderFactory.newMappedDataLoader; @@ -104,7 +108,7 @@ public void basic_map_batch_loading() { mapOfResults.put(k, k); } }); - return CompletableFuture.completedFuture(mapOfResults); + return completedFuture(mapOfResults); }; DataLoader loader = DataLoaderFactory.newMappedDataLoader(evensOnlyMappedBatchLoader); @@ -424,7 +428,7 @@ public void should_Allow_priming_the_cache_with_a_future(TestDataLoaderFactory f List> loadCalls = new ArrayList<>(); DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), loadCalls); - DataLoader dlFluency = identityLoader.prime("A", CompletableFuture.completedFuture("A")); + DataLoader dlFluency = identityLoader.prime("A", completedFuture("A")); assertThat(dlFluency, equalTo(identityLoader)); CompletableFuture future1 = identityLoader.load("A"); @@ -992,7 +996,7 @@ public void batches_multiple_requests_with_max_batch_size(TestDataLoaderFactory identityLoader.dispatch(); - CompletableFuture.allOf(f1, f2, f3).join(); + allOf(f1, f2, f3).join(); assertThat(f1.join(), equalTo(1)); assertThat(f2.join(), equalTo(2)); @@ -1035,13 +1039,13 @@ public void should_Batch_loads_occurring_within_futures(TestDataLoaderFactory fa AtomicBoolean v4Called = new AtomicBoolean(); - CompletableFuture.supplyAsync(nullValue).thenAccept(v1 -> { + supplyAsync(nullValue).thenAccept(v1 -> { identityLoader.load("a"); - CompletableFuture.supplyAsync(nullValue).thenAccept(v2 -> { + supplyAsync(nullValue).thenAccept(v2 -> { identityLoader.load("b"); - CompletableFuture.supplyAsync(nullValue).thenAccept(v3 -> { + supplyAsync(nullValue).thenAccept(v3 -> { identityLoader.load("c"); - CompletableFuture.supplyAsync(nullValue).thenAccept( + supplyAsync(nullValue).thenAccept( v4 -> { identityLoader.load("d"); v4Called.set(true); @@ -1058,12 +1062,68 @@ public void should_Batch_loads_occurring_within_futures(TestDataLoaderFactory fa singletonList(asList("a", "b", "c", "d")))); } + @ParameterizedTest + @MethodSource("dataLoaderFactories") + public void should_blowup_after_N_keys(TestDataLoaderFactory factory) { + if (!(factory instanceof TestReactiveDataLoaderFactory)) { + return; + } + // + // if we blow up after emitting N keys, the N keys should work but the rest of the keys + // should be exceptional + DataLoader identityLoader = ((TestReactiveDataLoaderFactory) factory).idLoaderBlowsUpsAfterN(3, new DataLoaderOptions(), new ArrayList<>()); + CompletableFuture cf1 = identityLoader.load(1); + CompletableFuture cf2 = identityLoader.load(2); + CompletableFuture cf3 = identityLoader.load(3); + CompletableFuture cf4 = identityLoader.load(4); + CompletableFuture cf5 = identityLoader.load(5); + identityLoader.dispatch(); + await().until(cf5::isDone); + + assertThat(cf1.join(), equalTo(1)); + assertThat(cf2.join(), equalTo(2)); + assertThat(cf3.join(), equalTo(3)); + assertThat(cf4.isCompletedExceptionally(), is(true)); + assertThat(cf5.isCompletedExceptionally(), is(true)); + + } + + @ParameterizedTest + @MethodSource("dataLoaderFactories") + public void should_assert_values_size_equals_key_size(TestDataLoaderFactory factory) { + // + // what happens if we want 4 values but are only given 2 back say + // + DataLoader identityLoader = factory.onlyReturnsNValues(2, new DataLoaderOptions(), new ArrayList<>()); + CompletableFuture cf1 = identityLoader.load("A"); + CompletableFuture cf2 = identityLoader.load("B"); + CompletableFuture cf3 = identityLoader.load("C"); + CompletableFuture cf4 = identityLoader.load("D"); + identityLoader.dispatch(); + + await().atMost(Duration.FIVE_HUNDRED_MILLISECONDS).until(() -> cf1.isDone() && cf2.isDone() && cf3.isDone() && cf4.isDone()); + + if (factory instanceof ListDataLoaderFactory | factory instanceof PublisherDataLoaderFactory) { + assertThat(cause(cf1), instanceOf(DataLoaderAssertionException.class)); + assertThat(cause(cf2), instanceOf(DataLoaderAssertionException.class)); + assertThat(cause(cf3), instanceOf(DataLoaderAssertionException.class)); + assertThat(cause(cf4), instanceOf(DataLoaderAssertionException.class)); + } else { + // with the maps it's ok to have fewer results + assertThat(cf1.join(), equalTo("A")); + assertThat(cf2.join(), equalTo("B")); + assertThat(cf3.join(), equalTo(null)); + assertThat(cf4.join(), equalTo(null)); + } + + } + @Test public void can_call_a_loader_from_a_loader() throws Exception { List> deepLoadCalls = new ArrayList<>(); DataLoader deepLoader = newDataLoader(keys -> { deepLoadCalls.add(keys); - return CompletableFuture.completedFuture(keys); + return completedFuture(keys); }); List> aLoadCalls = new ArrayList<>(); @@ -1083,7 +1143,7 @@ public void can_call_a_loader_from_a_loader() throws Exception { CompletableFuture b1 = bLoader.load("B1"); CompletableFuture b2 = bLoader.load("B2"); - CompletableFuture.allOf( + allOf( aLoader.dispatch(), deepLoader.dispatch(), bLoader.dispatch(), @@ -1109,11 +1169,10 @@ public void can_call_a_loader_from_a_loader() throws Exception { public void should_allow_composition_of_data_loader_calls() { UserManager userManager = new UserManager(); - BatchLoader userBatchLoader = userIds -> CompletableFuture - .supplyAsync(() -> userIds - .stream() - .map(userManager::loadUserById) - .collect(Collectors.toList())); + BatchLoader userBatchLoader = userIds -> supplyAsync(() -> userIds + .stream() + .map(userManager::loadUserById) + .collect(Collectors.toList())); DataLoader userLoader = newDataLoader(userBatchLoader); AtomicBoolean gandalfCalled = new AtomicBoolean(false); @@ -1160,9 +1219,18 @@ private static Stream dataLoaderFactories() { public interface TestDataLoaderFactory { DataLoader idLoader(DataLoaderOptions options, List> loadCalls); + DataLoader idLoaderBlowsUps(DataLoaderOptions options, List> loadCalls); + DataLoader idLoaderAllExceptions(DataLoaderOptions options, List> loadCalls); + DataLoader idLoaderOddEvenExceptions(DataLoaderOptions options, List> loadCalls); + + DataLoader onlyReturnsNValues(int N, DataLoaderOptions options, ArrayList loadCalls); + } + + public interface TestReactiveDataLoaderFactory { + DataLoader idLoaderBlowsUpsAfterN(int N, DataLoaderOptions options, List> loadCalls); } private static class ListDataLoaderFactory implements TestDataLoaderFactory { @@ -1170,7 +1238,7 @@ private static class ListDataLoaderFactory implements TestDataLoaderFactory { public DataLoader idLoader(DataLoaderOptions options, List> loadCalls) { return newDataLoader(keys -> { loadCalls.add(new ArrayList<>(keys)); - return CompletableFuture.completedFuture(keys); + return completedFuture(keys); }, options); } @@ -1189,7 +1257,7 @@ public DataLoader idLoaderAllExceptions(DataLoaderOptions options loadCalls.add(new ArrayList<>(keys)); List errors = keys.stream().map(k -> new IllegalStateException("Error")).collect(Collectors.toList()); - return CompletableFuture.completedFuture(errors); + return completedFuture(errors); }, options); } @@ -1206,7 +1274,15 @@ public DataLoader idLoaderOddEvenExceptions(DataLoaderOptions o errors.add(new IllegalStateException("Error")); } } - return CompletableFuture.completedFuture(errors); + return completedFuture(errors); + }, options); + } + + @Override + public DataLoader onlyReturnsNValues(int N, DataLoaderOptions options, ArrayList loadCalls) { + return newDataLoader(keys -> { + loadCalls.add(new ArrayList<>(keys)); + return completedFuture(keys.subList(0, N)); }, options); } } @@ -1220,7 +1296,7 @@ public DataLoader idLoader( loadCalls.add(new ArrayList<>(keys)); Map map = new HashMap<>(); keys.forEach(k -> map.put(k, k)); - return CompletableFuture.completedFuture(map); + return completedFuture(map); }, options); } @@ -1239,7 +1315,7 @@ public DataLoader idLoaderAllExceptions( loadCalls.add(new ArrayList<>(keys)); Map errorByKey = new HashMap<>(); keys.forEach(k -> errorByKey.put(k, new IllegalStateException("Error"))); - return CompletableFuture.completedFuture(errorByKey); + return completedFuture(errorByKey); }, options); } @@ -1257,16 +1333,28 @@ public DataLoader idLoaderOddEvenExceptions( errorByKey.put(key, new IllegalStateException("Error")); } } - return CompletableFuture.completedFuture(errorByKey); + return completedFuture(errorByKey); + }, options); + } + + @Override + public DataLoader onlyReturnsNValues(int N, DataLoaderOptions options, ArrayList loadCalls) { + return newMappedDataLoader(keys -> { + loadCalls.add(new ArrayList<>(keys)); + + Map collect = List.copyOf(keys).subList(0, N).stream().collect(Collectors.toMap( + k -> k, v -> v + )); + return completedFuture(collect); }, options); } } - private static class PublisherDataLoaderFactory implements TestDataLoaderFactory { + private static class PublisherDataLoaderFactory implements TestDataLoaderFactory, TestReactiveDataLoaderFactory { @Override public DataLoader idLoader( - DataLoaderOptions options, List> loadCalls) { + DataLoaderOptions options, List> loadCalls) { return newPublisherDataLoader((keys, subscriber) -> { loadCalls.add(new ArrayList<>(keys)); Flux.fromIterable(keys).subscribe(subscriber); @@ -1283,7 +1371,7 @@ public DataLoader idLoaderBlowsUps(DataLoaderOptions options, List DataLoader idLoaderAllExceptions( - DataLoaderOptions options, List> loadCalls) { + DataLoaderOptions options, List> loadCalls) { return newPublisherDataLoaderWithTry((keys, subscriber) -> { loadCalls.add(new ArrayList<>(keys)); Stream> failures = keys.stream().map(k -> Try.failed(new IllegalStateException("Error"))); @@ -1293,7 +1381,7 @@ public DataLoader idLoaderAllExceptions( @Override public DataLoader idLoaderOddEvenExceptions( - DataLoaderOptions options, List> loadCalls) { + DataLoaderOptions options, List> loadCalls) { return newPublisherDataLoaderWithTry((keys, subscriber) -> { loadCalls.add(new ArrayList<>(keys)); @@ -1308,13 +1396,36 @@ public DataLoader idLoaderOddEvenExceptions( Flux.fromIterable(errors).subscribe(subscriber); }, options); } + + @Override + public DataLoader idLoaderBlowsUpsAfterN(int N, DataLoaderOptions options, List> loadCalls) { + return newPublisherDataLoader((keys, subscriber) -> { + loadCalls.add(new ArrayList<>(keys)); + + List nKeys = keys.subList(0, N); + Flux subFlux = Flux.fromIterable(nKeys); + subFlux.concatWith(Flux.error(new IllegalStateException("Error"))) + .subscribe(subscriber); + }, options); + } + + @Override + public DataLoader onlyReturnsNValues(int N, DataLoaderOptions options, ArrayList loadCalls) { + return newPublisherDataLoader((keys, subscriber) -> { + loadCalls.add(new ArrayList<>(keys)); + + List nKeys = keys.subList(0, N); + Flux.fromIterable(nKeys) + .subscribe(subscriber); + }, options); + } } - private static class MappedPublisherDataLoaderFactory implements TestDataLoaderFactory { + private static class MappedPublisherDataLoaderFactory implements TestDataLoaderFactory, TestReactiveDataLoaderFactory { @Override public DataLoader idLoader( - DataLoaderOptions options, List> loadCalls) { + DataLoaderOptions options, List> loadCalls) { return newMappedPublisherDataLoader((keys, subscriber) -> { loadCalls.add(new ArrayList<>(keys)); Map map = new HashMap<>(); @@ -1333,7 +1444,7 @@ public DataLoader idLoaderBlowsUps(DataLoaderOptions options, List DataLoader idLoaderAllExceptions( - DataLoaderOptions options, List> loadCalls) { + DataLoaderOptions options, List> loadCalls) { return newMappedPublisherDataLoaderWithTry((keys, subscriber) -> { loadCalls.add(new ArrayList<>(keys)); Stream>> failures = keys.stream().map(k -> Map.entry(k, Try.failed(new IllegalStateException("Error")))); @@ -1343,7 +1454,7 @@ public DataLoader idLoaderAllExceptions( @Override public DataLoader idLoaderOddEvenExceptions( - DataLoaderOptions options, List> loadCalls) { + DataLoaderOptions options, List> loadCalls) { return newMappedPublisherDataLoaderWithTry((keys, subscriber) -> { loadCalls.add(new ArrayList<>(keys)); @@ -1358,6 +1469,29 @@ public DataLoader idLoaderOddEvenExceptions( Flux.fromIterable(errorByKey.entrySet()).subscribe(subscriber); }, options); } + + @Override + public DataLoader idLoaderBlowsUpsAfterN(int N, DataLoaderOptions options, List> loadCalls) { + return newMappedPublisherDataLoader((keys, subscriber) -> { + loadCalls.add(new ArrayList<>(keys)); + + List nKeys = keys.subList(0, N); + Flux> subFlux = Flux.fromIterable(nKeys).map(k -> Map.entry(k, k)); + subFlux.concatWith(Flux.error(new IllegalStateException("Error"))) + .subscribe(subscriber); + }, options); + } + + @Override + public DataLoader onlyReturnsNValues(int N, DataLoaderOptions options, ArrayList loadCalls) { + return newMappedPublisherDataLoader((keys, subscriber) -> { + loadCalls.add(new ArrayList<>(keys)); + + List nKeys = keys.subList(0, N); + Flux.fromIterable(nKeys).map(k -> Map.entry(k, k)) + .subscribe(subscriber); + }, options); + } } private static class ThrowingCacheMap extends CustomCacheMap { From d62687f9daa16277dff4ccbbb62074f61064ad7d Mon Sep 17 00:00:00 2001 From: bbaker Date: Thu, 23 May 2024 10:28:07 +1000 Subject: [PATCH 032/156] Make builds run --- .github/workflows/pull_request.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 13a366a9..02585672 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -7,6 +7,8 @@ on: pull_request: branches: - master + - reactive-streams-branch + - '**' jobs: buildAndTest: runs-on: ubuntu-latest From 8b344dbdba430c2ca56896112c0b9f85a2d01cb4 Mon Sep 17 00:00:00 2001 From: bbaker Date: Thu, 23 May 2024 15:46:21 +1000 Subject: [PATCH 033/156] Now the builds pass - broken out the fixtures --- .../java/org/dataloader/DataLoaderHelper.java | 50 ++- .../java/org/dataloader/DataLoaderTest.java | 343 +++--------------- .../java/org/dataloader/fixtures/TestKit.java | 9 + .../parameterized/ListDataLoaderFactory.java | 79 ++++ .../MappedDataLoaderFactory.java | 95 +++++ .../MappedPublisherDataLoaderFactory.java | 104 ++++++ .../PublisherDataLoaderFactory.java | 100 +++++ .../parameterized/TestDataLoaderFactory.java | 22 ++ .../TestReactiveDataLoaderFactory.java | 11 + 9 files changed, 516 insertions(+), 297 deletions(-) create mode 100644 src/test/java/org/dataloader/fixtures/parameterized/ListDataLoaderFactory.java create mode 100644 src/test/java/org/dataloader/fixtures/parameterized/MappedDataLoaderFactory.java create mode 100644 src/test/java/org/dataloader/fixtures/parameterized/MappedPublisherDataLoaderFactory.java create mode 100644 src/test/java/org/dataloader/fixtures/parameterized/PublisherDataLoaderFactory.java create mode 100644 src/test/java/org/dataloader/fixtures/parameterized/TestDataLoaderFactory.java create mode 100644 src/test/java/org/dataloader/fixtures/parameterized/TestReactiveDataLoaderFactory.java diff --git a/src/main/java/org/dataloader/DataLoaderHelper.java b/src/main/java/org/dataloader/DataLoaderHelper.java index cf3fc6e6..edbf3484 100644 --- a/src/main/java/org/dataloader/DataLoaderHelper.java +++ b/src/main/java/org/dataloader/DataLoaderHelper.java @@ -3,6 +3,7 @@ import org.dataloader.annotations.GuardedBy; import org.dataloader.annotations.Internal; import org.dataloader.impl.CompletableFutureKit; +import org.dataloader.impl.DataLoaderAssertionException; import org.dataloader.scheduler.BatchLoaderScheduler; import org.dataloader.stats.StatisticsCollector; import org.dataloader.stats.context.IncrementBatchLoadCountByStatisticsContext; @@ -624,6 +625,15 @@ private static DispatchResult emptyDispatchResult() { return (DispatchResult) EMPTY_DISPATCH_RESULT; } + /********************************************************************************************** + * ******************************************************************************************** + *

+ * The reactive support classes start here + * + * @param for two + ********************************************************************************************** + ********************************************************************************************** + */ private abstract class DataLoaderSubscriberBase implements Subscriber { final CompletableFuture> valuesFuture; @@ -721,6 +731,11 @@ private DataLoaderSubscriber( public synchronized void onNext(V value) { super.onNext(value); + if (idx >= keys.size()) { + // hang on they have given us more values than we asked for in keys + // we cant handle this + return; + } K key = keys.get(idx); Object callContext = callContexts.get(idx); CompletableFuture future = queuedFutures.get(idx); @@ -734,8 +749,16 @@ public synchronized void onNext(V value) { @Override public synchronized void onComplete() { super.onComplete(); - assertResultSize(keys, completedValues); - + if (keys.size() != completedValues.size()) { + // we have more or less values than promised + // we will go through all the outstanding promises and mark those that + // have not finished as failed + for (CompletableFuture queuedFuture : queuedFutures) { + if (!queuedFuture.isDone()) { + queuedFuture.completeExceptionally(new DataLoaderAssertionException("The size of the promised values MUST be the same size as the key list")); + } + } + } possiblyClearCacheEntriesOnExceptions(clearCacheKeys); valuesFuture.complete(completedValues); } @@ -748,9 +771,11 @@ public synchronized void onError(Throwable ex) { for (int i = idx; i < queuedFutures.size(); i++) { K key = keys.get(i); CompletableFuture future = queuedFutures.get(i); - future.completeExceptionally(ex); - // clear any cached view of this key because they all failed - dataLoader.clear(key); + if (! future.isDone()) { + future.completeExceptionally(ex); + // clear any cached view of this key because it failed + dataLoader.clear(key); + } } valuesFuture.completeExceptionally(ex); } @@ -794,7 +819,10 @@ public synchronized void onNext(Map.Entry entry) { onNextValue(key, value, callContext, futures); - completedValuesByKey.put(key, value); + // did we have an actual key for this value - ignore it if they send us one outside the key set + if (!futures.isEmpty()) { + completedValuesByKey.put(key, value); + } } @Override @@ -806,6 +834,16 @@ public synchronized void onComplete() { for (K key : keys) { V value = completedValuesByKey.get(key); values.add(value); + + List> futures = queuedFuturesByKey.getOrDefault(key, List.of()); + for (CompletableFuture future : futures) { + if (! future.isDone()) { + // we have a future that never came back for that key + // but the publisher is done sending in data - it must be null + // e.g. for key X when found no value + future.complete(null); + } + } } valuesFuture.complete(values); } diff --git a/src/test/java/org/dataloader/DataLoaderTest.java b/src/test/java/org/dataloader/DataLoaderTest.java index c8d45627..1f748fb1 100644 --- a/src/test/java/org/dataloader/DataLoaderTest.java +++ b/src/test/java/org/dataloader/DataLoaderTest.java @@ -19,9 +19,14 @@ import org.awaitility.Duration; import org.dataloader.fixtures.CustomCacheMap; import org.dataloader.fixtures.JsonObject; -import org.dataloader.fixtures.TestKit; import org.dataloader.fixtures.User; import org.dataloader.fixtures.UserManager; +import org.dataloader.fixtures.parameterized.ListDataLoaderFactory; +import org.dataloader.fixtures.parameterized.MappedDataLoaderFactory; +import org.dataloader.fixtures.parameterized.MappedPublisherDataLoaderFactory; +import org.dataloader.fixtures.parameterized.PublisherDataLoaderFactory; +import org.dataloader.fixtures.parameterized.TestDataLoaderFactory; +import org.dataloader.fixtures.parameterized.TestReactiveDataLoaderFactory; import org.dataloader.impl.CompletableFutureKit; import org.dataloader.impl.DataLoaderAssertionException; import org.junit.jupiter.api.Named; @@ -29,7 +34,6 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; -import reactor.core.publisher.Flux; import java.util.ArrayList; import java.util.Collection; @@ -37,7 +41,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; import java.util.concurrent.ExecutionException; @@ -59,7 +62,7 @@ import static org.dataloader.DataLoaderFactory.newPublisherDataLoader; import static org.dataloader.DataLoaderFactory.newPublisherDataLoaderWithTry; import static org.dataloader.DataLoaderOptions.newOptions; -import static org.dataloader.fixtures.TestKit.futureError; +import static org.dataloader.fixtures.TestKit.areAllDone; import static org.dataloader.fixtures.TestKit.listFrom; import static org.dataloader.impl.CompletableFutureKit.cause; import static org.hamcrest.MatcherAssert.assertThat; @@ -68,6 +71,7 @@ import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.fail; /** * Tests for {@link DataLoader}. @@ -1090,7 +1094,7 @@ public void should_blowup_after_N_keys(TestDataLoaderFactory factory) { @ParameterizedTest @MethodSource("dataLoaderFactories") - public void should_assert_values_size_equals_key_size(TestDataLoaderFactory factory) { + public void when_values_size_are_less_then_key_size(TestDataLoaderFactory factory) { // // what happens if we want 4 values but are only given 2 back say // @@ -1101,13 +1105,19 @@ public void should_assert_values_size_equals_key_size(TestDataLoaderFactory fact CompletableFuture cf4 = identityLoader.load("D"); identityLoader.dispatch(); - await().atMost(Duration.FIVE_HUNDRED_MILLISECONDS).until(() -> cf1.isDone() && cf2.isDone() && cf3.isDone() && cf4.isDone()); + await().atMost(Duration.FIVE_SECONDS).until(() -> areAllDone(cf1, cf2, cf3, cf4)); - if (factory instanceof ListDataLoaderFactory | factory instanceof PublisherDataLoaderFactory) { + if (factory instanceof ListDataLoaderFactory) { assertThat(cause(cf1), instanceOf(DataLoaderAssertionException.class)); assertThat(cause(cf2), instanceOf(DataLoaderAssertionException.class)); assertThat(cause(cf3), instanceOf(DataLoaderAssertionException.class)); assertThat(cause(cf4), instanceOf(DataLoaderAssertionException.class)); + } else if (factory instanceof PublisherDataLoaderFactory) { + // some have completed progressively but the other never did + assertThat(cf1.join(), equalTo("A")); + assertThat(cf2.join(), equalTo("B")); + assertThat(cause(cf3), instanceOf(DataLoaderAssertionException.class)); + assertThat(cause(cf4), instanceOf(DataLoaderAssertionException.class)); } else { // with the maps it's ok to have fewer results assertThat(cf1.join(), equalTo("A")); @@ -1115,7 +1125,34 @@ public void should_assert_values_size_equals_key_size(TestDataLoaderFactory fact assertThat(cf3.join(), equalTo(null)); assertThat(cf4.join(), equalTo(null)); } + } + + @ParameterizedTest + @MethodSource("dataLoaderFactories") + public void when_values_size_are_more_then_key_size(TestDataLoaderFactory factory) { + // + // what happens if we want 4 values but only given 6 back say + // + DataLoader identityLoader = factory.idLoaderReturnsTooMany(2, new DataLoaderOptions(), new ArrayList<>()); + CompletableFuture cf1 = identityLoader.load("A"); + CompletableFuture cf2 = identityLoader.load("B"); + CompletableFuture cf3 = identityLoader.load("C"); + CompletableFuture cf4 = identityLoader.load("D"); + identityLoader.dispatch(); + await().atMost(Duration.FIVE_SECONDS).until(() -> areAllDone(cf1, cf2, cf3, cf4)); + + if (factory instanceof ListDataLoaderFactory) { + assertThat(cause(cf1), instanceOf(DataLoaderAssertionException.class)); + assertThat(cause(cf2), instanceOf(DataLoaderAssertionException.class)); + assertThat(cause(cf3), instanceOf(DataLoaderAssertionException.class)); + assertThat(cause(cf4), instanceOf(DataLoaderAssertionException.class)); + } else { + assertThat(cf1.join(), equalTo("A")); + assertThat(cf2.join(), equalTo("B")); + assertThat(cf3.join(), equalTo("C")); + assertThat(cf4.join(), equalTo("D")); + } } @Test @@ -1208,6 +1245,14 @@ private static CacheKey getJsonObjectCacheMapFn() { .collect(Collectors.joining()); } + private static class ThrowingCacheMap extends CustomCacheMap { + + @Override + public CompletableFuture get(String key) { + throw new RuntimeException("Cache implementation failed."); + } + } + private static Stream dataLoaderFactories() { return Stream.of( Arguments.of(Named.of("List DataLoader", new ListDataLoaderFactory())), @@ -1216,289 +1261,5 @@ private static Stream dataLoaderFactories() { Arguments.of(Named.of("Mapped Publisher DataLoader", new MappedPublisherDataLoaderFactory())) ); } - - public interface TestDataLoaderFactory { - DataLoader idLoader(DataLoaderOptions options, List> loadCalls); - - DataLoader idLoaderBlowsUps(DataLoaderOptions options, List> loadCalls); - - DataLoader idLoaderAllExceptions(DataLoaderOptions options, List> loadCalls); - - DataLoader idLoaderOddEvenExceptions(DataLoaderOptions options, List> loadCalls); - - DataLoader onlyReturnsNValues(int N, DataLoaderOptions options, ArrayList loadCalls); - } - - public interface TestReactiveDataLoaderFactory { - DataLoader idLoaderBlowsUpsAfterN(int N, DataLoaderOptions options, List> loadCalls); - } - - private static class ListDataLoaderFactory implements TestDataLoaderFactory { - @Override - public DataLoader idLoader(DataLoaderOptions options, List> loadCalls) { - return newDataLoader(keys -> { - loadCalls.add(new ArrayList<>(keys)); - return completedFuture(keys); - }, options); - } - - @Override - public DataLoader idLoaderBlowsUps( - DataLoaderOptions options, List> loadCalls) { - return newDataLoader(keys -> { - loadCalls.add(new ArrayList<>(keys)); - return TestKit.futureError(); - }, options); - } - - @Override - public DataLoader idLoaderAllExceptions(DataLoaderOptions options, List> loadCalls) { - return newDataLoader(keys -> { - loadCalls.add(new ArrayList<>(keys)); - - List errors = keys.stream().map(k -> new IllegalStateException("Error")).collect(Collectors.toList()); - return completedFuture(errors); - }, options); - } - - @Override - public DataLoader idLoaderOddEvenExceptions(DataLoaderOptions options, List> loadCalls) { - return newDataLoader(keys -> { - loadCalls.add(new ArrayList<>(keys)); - - List errors = new ArrayList<>(); - for (Integer key : keys) { - if (key % 2 == 0) { - errors.add(key); - } else { - errors.add(new IllegalStateException("Error")); - } - } - return completedFuture(errors); - }, options); - } - - @Override - public DataLoader onlyReturnsNValues(int N, DataLoaderOptions options, ArrayList loadCalls) { - return newDataLoader(keys -> { - loadCalls.add(new ArrayList<>(keys)); - return completedFuture(keys.subList(0, N)); - }, options); - } - } - - private static class MappedDataLoaderFactory implements TestDataLoaderFactory { - - @Override - public DataLoader idLoader( - DataLoaderOptions options, List> loadCalls) { - return newMappedDataLoader((keys) -> { - loadCalls.add(new ArrayList<>(keys)); - Map map = new HashMap<>(); - keys.forEach(k -> map.put(k, k)); - return completedFuture(map); - }, options); - } - - @Override - public DataLoader idLoaderBlowsUps(DataLoaderOptions options, List> loadCalls) { - return newMappedDataLoader((keys) -> { - loadCalls.add(new ArrayList<>(keys)); - return futureError(); - }, options); - } - - @Override - public DataLoader idLoaderAllExceptions( - DataLoaderOptions options, List> loadCalls) { - return newMappedDataLoader(keys -> { - loadCalls.add(new ArrayList<>(keys)); - Map errorByKey = new HashMap<>(); - keys.forEach(k -> errorByKey.put(k, new IllegalStateException("Error"))); - return completedFuture(errorByKey); - }, options); - } - - @Override - public DataLoader idLoaderOddEvenExceptions( - DataLoaderOptions options, List> loadCalls) { - return newMappedDataLoader(keys -> { - loadCalls.add(new ArrayList<>(keys)); - - Map errorByKey = new HashMap<>(); - for (Integer key : keys) { - if (key % 2 == 0) { - errorByKey.put(key, key); - } else { - errorByKey.put(key, new IllegalStateException("Error")); - } - } - return completedFuture(errorByKey); - }, options); - } - - @Override - public DataLoader onlyReturnsNValues(int N, DataLoaderOptions options, ArrayList loadCalls) { - return newMappedDataLoader(keys -> { - loadCalls.add(new ArrayList<>(keys)); - - Map collect = List.copyOf(keys).subList(0, N).stream().collect(Collectors.toMap( - k -> k, v -> v - )); - return completedFuture(collect); - }, options); - } - } - - private static class PublisherDataLoaderFactory implements TestDataLoaderFactory, TestReactiveDataLoaderFactory { - - @Override - public DataLoader idLoader( - DataLoaderOptions options, List> loadCalls) { - return newPublisherDataLoader((keys, subscriber) -> { - loadCalls.add(new ArrayList<>(keys)); - Flux.fromIterable(keys).subscribe(subscriber); - }, options); - } - - @Override - public DataLoader idLoaderBlowsUps(DataLoaderOptions options, List> loadCalls) { - return newPublisherDataLoader((keys, subscriber) -> { - loadCalls.add(new ArrayList<>(keys)); - Flux.error(new IllegalStateException("Error")).subscribe(subscriber); - }, options); - } - - @Override - public DataLoader idLoaderAllExceptions( - DataLoaderOptions options, List> loadCalls) { - return newPublisherDataLoaderWithTry((keys, subscriber) -> { - loadCalls.add(new ArrayList<>(keys)); - Stream> failures = keys.stream().map(k -> Try.failed(new IllegalStateException("Error"))); - Flux.fromStream(failures).subscribe(subscriber); - }, options); - } - - @Override - public DataLoader idLoaderOddEvenExceptions( - DataLoaderOptions options, List> loadCalls) { - return newPublisherDataLoaderWithTry((keys, subscriber) -> { - loadCalls.add(new ArrayList<>(keys)); - - List> errors = new ArrayList<>(); - for (Integer key : keys) { - if (key % 2 == 0) { - errors.add(Try.succeeded(key)); - } else { - errors.add(Try.failed(new IllegalStateException("Error"))); - } - } - Flux.fromIterable(errors).subscribe(subscriber); - }, options); - } - - @Override - public DataLoader idLoaderBlowsUpsAfterN(int N, DataLoaderOptions options, List> loadCalls) { - return newPublisherDataLoader((keys, subscriber) -> { - loadCalls.add(new ArrayList<>(keys)); - - List nKeys = keys.subList(0, N); - Flux subFlux = Flux.fromIterable(nKeys); - subFlux.concatWith(Flux.error(new IllegalStateException("Error"))) - .subscribe(subscriber); - }, options); - } - - @Override - public DataLoader onlyReturnsNValues(int N, DataLoaderOptions options, ArrayList loadCalls) { - return newPublisherDataLoader((keys, subscriber) -> { - loadCalls.add(new ArrayList<>(keys)); - - List nKeys = keys.subList(0, N); - Flux.fromIterable(nKeys) - .subscribe(subscriber); - }, options); - } - } - - private static class MappedPublisherDataLoaderFactory implements TestDataLoaderFactory, TestReactiveDataLoaderFactory { - - @Override - public DataLoader idLoader( - DataLoaderOptions options, List> loadCalls) { - return newMappedPublisherDataLoader((keys, subscriber) -> { - loadCalls.add(new ArrayList<>(keys)); - Map map = new HashMap<>(); - keys.forEach(k -> map.put(k, k)); - Flux.fromIterable(map.entrySet()).subscribe(subscriber); - }, options); - } - - @Override - public DataLoader idLoaderBlowsUps(DataLoaderOptions options, List> loadCalls) { - return newMappedPublisherDataLoader((keys, subscriber) -> { - loadCalls.add(new ArrayList<>(keys)); - Flux.>error(new IllegalStateException("Error")).subscribe(subscriber); - }, options); - } - - @Override - public DataLoader idLoaderAllExceptions( - DataLoaderOptions options, List> loadCalls) { - return newMappedPublisherDataLoaderWithTry((keys, subscriber) -> { - loadCalls.add(new ArrayList<>(keys)); - Stream>> failures = keys.stream().map(k -> Map.entry(k, Try.failed(new IllegalStateException("Error")))); - Flux.fromStream(failures).subscribe(subscriber); - }, options); - } - - @Override - public DataLoader idLoaderOddEvenExceptions( - DataLoaderOptions options, List> loadCalls) { - return newMappedPublisherDataLoaderWithTry((keys, subscriber) -> { - loadCalls.add(new ArrayList<>(keys)); - - Map> errorByKey = new HashMap<>(); - for (Integer key : keys) { - if (key % 2 == 0) { - errorByKey.put(key, Try.succeeded(key)); - } else { - errorByKey.put(key, Try.failed(new IllegalStateException("Error"))); - } - } - Flux.fromIterable(errorByKey.entrySet()).subscribe(subscriber); - }, options); - } - - @Override - public DataLoader idLoaderBlowsUpsAfterN(int N, DataLoaderOptions options, List> loadCalls) { - return newMappedPublisherDataLoader((keys, subscriber) -> { - loadCalls.add(new ArrayList<>(keys)); - - List nKeys = keys.subList(0, N); - Flux> subFlux = Flux.fromIterable(nKeys).map(k -> Map.entry(k, k)); - subFlux.concatWith(Flux.error(new IllegalStateException("Error"))) - .subscribe(subscriber); - }, options); - } - - @Override - public DataLoader onlyReturnsNValues(int N, DataLoaderOptions options, ArrayList loadCalls) { - return newMappedPublisherDataLoader((keys, subscriber) -> { - loadCalls.add(new ArrayList<>(keys)); - - List nKeys = keys.subList(0, N); - Flux.fromIterable(nKeys).map(k -> Map.entry(k, k)) - .subscribe(subscriber); - }, options); - } - } - - private static class ThrowingCacheMap extends CustomCacheMap { - @Override - public CompletableFuture get(String key) { - throw new RuntimeException("Cache implementation failed."); - } - } } diff --git a/src/test/java/org/dataloader/fixtures/TestKit.java b/src/test/java/org/dataloader/fixtures/TestKit.java index adffb069..c22988d0 100644 --- a/src/test/java/org/dataloader/fixtures/TestKit.java +++ b/src/test/java/org/dataloader/fixtures/TestKit.java @@ -131,4 +131,13 @@ public static Set asSet(T... elements) { public static Set asSet(Collection elements) { return new LinkedHashSet<>(elements); } + + public static boolean areAllDone(CompletableFuture... cfs) { + for (CompletableFuture cf : cfs) { + if (! cf.isDone()) { + return false; + } + } + return true; + } } diff --git a/src/test/java/org/dataloader/fixtures/parameterized/ListDataLoaderFactory.java b/src/test/java/org/dataloader/fixtures/parameterized/ListDataLoaderFactory.java new file mode 100644 index 00000000..ee1f1d7d --- /dev/null +++ b/src/test/java/org/dataloader/fixtures/parameterized/ListDataLoaderFactory.java @@ -0,0 +1,79 @@ +package org.dataloader.fixtures.parameterized; + +import org.dataloader.DataLoader; +import org.dataloader.DataLoaderOptions; +import org.dataloader.fixtures.TestKit; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; + +import static java.util.concurrent.CompletableFuture.completedFuture; +import static org.dataloader.DataLoaderFactory.newDataLoader; + +public class ListDataLoaderFactory implements TestDataLoaderFactory { + @Override + public DataLoader idLoader(DataLoaderOptions options, List> loadCalls) { + return newDataLoader(keys -> { + loadCalls.add(new ArrayList<>(keys)); + return completedFuture(keys); + }, options); + } + + @Override + public DataLoader idLoaderBlowsUps( + DataLoaderOptions options, List> loadCalls) { + return newDataLoader(keys -> { + loadCalls.add(new ArrayList<>(keys)); + return TestKit.futureError(); + }, options); + } + + @Override + public DataLoader idLoaderAllExceptions(DataLoaderOptions options, List> loadCalls) { + return newDataLoader(keys -> { + loadCalls.add(new ArrayList<>(keys)); + + List errors = keys.stream().map(k -> new IllegalStateException("Error")).collect(Collectors.toList()); + return completedFuture(errors); + }, options); + } + + @Override + public DataLoader idLoaderOddEvenExceptions(DataLoaderOptions options, List> loadCalls) { + return newDataLoader(keys -> { + loadCalls.add(new ArrayList<>(keys)); + + List errors = new ArrayList<>(); + for (Integer key : keys) { + if (key % 2 == 0) { + errors.add(key); + } else { + errors.add(new IllegalStateException("Error")); + } + } + return completedFuture(errors); + }, options); + } + + @Override + public DataLoader onlyReturnsNValues(int N, DataLoaderOptions options, ArrayList loadCalls) { + return newDataLoader(keys -> { + loadCalls.add(new ArrayList<>(keys)); + return completedFuture(keys.subList(0, N)); + }, options); + } + + @Override + public DataLoader idLoaderReturnsTooMany(int howManyMore, DataLoaderOptions options, ArrayList loadCalls) { + return newDataLoader(keys -> { + loadCalls.add(new ArrayList<>(keys)); + List l = new ArrayList<>(keys); + for (int i = 0; i < howManyMore; i++) { + l.add("extra-" + i); + } + return completedFuture(l); + }, options); + } +} diff --git a/src/test/java/org/dataloader/fixtures/parameterized/MappedDataLoaderFactory.java b/src/test/java/org/dataloader/fixtures/parameterized/MappedDataLoaderFactory.java new file mode 100644 index 00000000..8f41441a --- /dev/null +++ b/src/test/java/org/dataloader/fixtures/parameterized/MappedDataLoaderFactory.java @@ -0,0 +1,95 @@ +package org.dataloader.fixtures.parameterized; + +import org.dataloader.DataLoader; +import org.dataloader.DataLoaderOptions; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static java.util.concurrent.CompletableFuture.completedFuture; +import static org.dataloader.DataLoaderFactory.newMappedDataLoader; +import static org.dataloader.fixtures.TestKit.futureError; + +public class MappedDataLoaderFactory implements TestDataLoaderFactory { + + @Override + public DataLoader idLoader( + DataLoaderOptions options, List> loadCalls) { + return newMappedDataLoader((keys) -> { + loadCalls.add(new ArrayList<>(keys)); + Map map = new HashMap<>(); + keys.forEach(k -> map.put(k, k)); + return completedFuture(map); + }, options); + } + + @Override + public DataLoader idLoaderBlowsUps(DataLoaderOptions options, List> loadCalls) { + return newMappedDataLoader((keys) -> { + loadCalls.add(new ArrayList<>(keys)); + return futureError(); + }, options); + } + + @Override + public DataLoader idLoaderAllExceptions( + DataLoaderOptions options, List> loadCalls) { + return newMappedDataLoader(keys -> { + loadCalls.add(new ArrayList<>(keys)); + Map errorByKey = new HashMap<>(); + keys.forEach(k -> errorByKey.put(k, new IllegalStateException("Error"))); + return completedFuture(errorByKey); + }, options); + } + + @Override + public DataLoader idLoaderOddEvenExceptions( + DataLoaderOptions options, List> loadCalls) { + return newMappedDataLoader(keys -> { + loadCalls.add(new ArrayList<>(keys)); + + Map errorByKey = new HashMap<>(); + for (Integer key : keys) { + if (key % 2 == 0) { + errorByKey.put(key, key); + } else { + errorByKey.put(key, new IllegalStateException("Error")); + } + } + return completedFuture(errorByKey); + }, options); + } + + @Override + public DataLoader onlyReturnsNValues(int N, DataLoaderOptions options, ArrayList loadCalls) { + return newMappedDataLoader(keys -> { + loadCalls.add(new ArrayList<>(keys)); + + Map collect = List.copyOf(keys).subList(0, N).stream().collect(Collectors.toMap( + k -> k, v -> v + )); + return completedFuture(collect); + }, options); + } + + @Override + public DataLoader idLoaderReturnsTooMany(int howManyMore, DataLoaderOptions options, ArrayList loadCalls) { + return newMappedDataLoader(keys -> { + loadCalls.add(new ArrayList<>(keys)); + + List l = new ArrayList<>(keys); + for (int i = 0; i < howManyMore; i++) { + l.add("extra-" + i); + } + + Map collect = l.stream().collect(Collectors.toMap( + k -> k, v -> v + )); + return completedFuture(collect); + }, options); + } +} diff --git a/src/test/java/org/dataloader/fixtures/parameterized/MappedPublisherDataLoaderFactory.java b/src/test/java/org/dataloader/fixtures/parameterized/MappedPublisherDataLoaderFactory.java new file mode 100644 index 00000000..f5c1ad5e --- /dev/null +++ b/src/test/java/org/dataloader/fixtures/parameterized/MappedPublisherDataLoaderFactory.java @@ -0,0 +1,104 @@ +package org.dataloader.fixtures.parameterized; + +import org.dataloader.DataLoader; +import org.dataloader.DataLoaderOptions; +import org.dataloader.Try; +import reactor.core.publisher.Flux; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; + +import static org.dataloader.DataLoaderFactory.newMappedPublisherDataLoader; +import static org.dataloader.DataLoaderFactory.newMappedPublisherDataLoaderWithTry; + +public class MappedPublisherDataLoaderFactory implements TestDataLoaderFactory, TestReactiveDataLoaderFactory { + + @Override + public DataLoader idLoader( + DataLoaderOptions options, List> loadCalls) { + return newMappedPublisherDataLoader((keys, subscriber) -> { + loadCalls.add(new ArrayList<>(keys)); + Map map = new HashMap<>(); + keys.forEach(k -> map.put(k, k)); + Flux.fromIterable(map.entrySet()).subscribe(subscriber); + }, options); + } + + @Override + public DataLoader idLoaderBlowsUps(DataLoaderOptions options, List> loadCalls) { + return newMappedPublisherDataLoader((keys, subscriber) -> { + loadCalls.add(new ArrayList<>(keys)); + Flux.>error(new IllegalStateException("Error")).subscribe(subscriber); + }, options); + } + + @Override + public DataLoader idLoaderAllExceptions( + DataLoaderOptions options, List> loadCalls) { + return newMappedPublisherDataLoaderWithTry((keys, subscriber) -> { + loadCalls.add(new ArrayList<>(keys)); + Stream>> failures = keys.stream().map(k -> Map.entry(k, Try.failed(new IllegalStateException("Error")))); + Flux.fromStream(failures).subscribe(subscriber); + }, options); + } + + @Override + public DataLoader idLoaderOddEvenExceptions( + DataLoaderOptions options, List> loadCalls) { + return newMappedPublisherDataLoaderWithTry((keys, subscriber) -> { + loadCalls.add(new ArrayList<>(keys)); + + Map> errorByKey = new HashMap<>(); + for (Integer key : keys) { + if (key % 2 == 0) { + errorByKey.put(key, Try.succeeded(key)); + } else { + errorByKey.put(key, Try.failed(new IllegalStateException("Error"))); + } + } + Flux.fromIterable(errorByKey.entrySet()).subscribe(subscriber); + }, options); + } + + @Override + public DataLoader idLoaderBlowsUpsAfterN(int N, DataLoaderOptions options, List> loadCalls) { + return newMappedPublisherDataLoader((keys, subscriber) -> { + loadCalls.add(new ArrayList<>(keys)); + + List nKeys = keys.subList(0, N); + Flux> subFlux = Flux.fromIterable(nKeys).map(k -> Map.entry(k, k)); + subFlux.concatWith(Flux.error(new IllegalStateException("Error"))) + .subscribe(subscriber); + }, options); + } + + @Override + public DataLoader onlyReturnsNValues(int N, DataLoaderOptions options, ArrayList loadCalls) { + return newMappedPublisherDataLoader((keys, subscriber) -> { + loadCalls.add(new ArrayList<>(keys)); + + List nKeys = keys.subList(0, N); + Flux.fromIterable(nKeys).map(k -> Map.entry(k, k)) + .subscribe(subscriber); + }, options); + } + + @Override + public DataLoader idLoaderReturnsTooMany(int howManyMore, DataLoaderOptions options, ArrayList loadCalls) { + return newMappedPublisherDataLoader((keys, subscriber) -> { + loadCalls.add(new ArrayList<>(keys)); + + List l = new ArrayList<>(keys); + for (int i = 0; i < howManyMore; i++) { + l.add("extra-" + i); + } + + Flux.fromIterable(l).map(k -> Map.entry(k, k)) + .subscribe(subscriber); + }, options); + } +} diff --git a/src/test/java/org/dataloader/fixtures/parameterized/PublisherDataLoaderFactory.java b/src/test/java/org/dataloader/fixtures/parameterized/PublisherDataLoaderFactory.java new file mode 100644 index 00000000..d75ff385 --- /dev/null +++ b/src/test/java/org/dataloader/fixtures/parameterized/PublisherDataLoaderFactory.java @@ -0,0 +1,100 @@ +package org.dataloader.fixtures.parameterized; + +import org.dataloader.DataLoader; +import org.dataloader.DataLoaderOptions; +import org.dataloader.Try; +import reactor.core.publisher.Flux; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.stream.Stream; + +import static org.dataloader.DataLoaderFactory.newPublisherDataLoader; +import static org.dataloader.DataLoaderFactory.newPublisherDataLoaderWithTry; + +public class PublisherDataLoaderFactory implements TestDataLoaderFactory, TestReactiveDataLoaderFactory { + + @Override + public DataLoader idLoader( + DataLoaderOptions options, List> loadCalls) { + return newPublisherDataLoader((keys, subscriber) -> { + loadCalls.add(new ArrayList<>(keys)); + Flux.fromIterable(keys).subscribe(subscriber); + }, options); + } + + @Override + public DataLoader idLoaderBlowsUps(DataLoaderOptions options, List> loadCalls) { + return newPublisherDataLoader((keys, subscriber) -> { + loadCalls.add(new ArrayList<>(keys)); + Flux.error(new IllegalStateException("Error")).subscribe(subscriber); + }, options); + } + + @Override + public DataLoader idLoaderAllExceptions( + DataLoaderOptions options, List> loadCalls) { + return newPublisherDataLoaderWithTry((keys, subscriber) -> { + loadCalls.add(new ArrayList<>(keys)); + Stream> failures = keys.stream().map(k -> Try.failed(new IllegalStateException("Error"))); + Flux.fromStream(failures).subscribe(subscriber); + }, options); + } + + @Override + public DataLoader idLoaderOddEvenExceptions( + DataLoaderOptions options, List> loadCalls) { + return newPublisherDataLoaderWithTry((keys, subscriber) -> { + loadCalls.add(new ArrayList<>(keys)); + + List> errors = new ArrayList<>(); + for (Integer key : keys) { + if (key % 2 == 0) { + errors.add(Try.succeeded(key)); + } else { + errors.add(Try.failed(new IllegalStateException("Error"))); + } + } + Flux.fromIterable(errors).subscribe(subscriber); + }, options); + } + + @Override + public DataLoader idLoaderBlowsUpsAfterN(int N, DataLoaderOptions options, List> loadCalls) { + return newPublisherDataLoader((keys, subscriber) -> { + loadCalls.add(new ArrayList<>(keys)); + + List nKeys = keys.subList(0, N); + Flux subFlux = Flux.fromIterable(nKeys); + subFlux.concatWith(Flux.error(new IllegalStateException("Error"))) + .subscribe(subscriber); + }, options); + } + + @Override + public DataLoader onlyReturnsNValues(int N, DataLoaderOptions options, ArrayList loadCalls) { + return newPublisherDataLoader((keys, subscriber) -> { + loadCalls.add(new ArrayList<>(keys)); + + List nKeys = keys.subList(0, N); + Flux.fromIterable(nKeys) + .subscribe(subscriber); + }, options); + } + + @Override + public DataLoader idLoaderReturnsTooMany(int howManyMore, DataLoaderOptions options, ArrayList loadCalls) { + return newPublisherDataLoader((keys, subscriber) -> { + loadCalls.add(new ArrayList<>(keys)); + + List l = new ArrayList<>(keys); + for (int i = 0; i < howManyMore; i++) { + l.add("extra-" + i); + } + + Flux.fromIterable(l) + .subscribe(subscriber); + }, options); + } +} diff --git a/src/test/java/org/dataloader/fixtures/parameterized/TestDataLoaderFactory.java b/src/test/java/org/dataloader/fixtures/parameterized/TestDataLoaderFactory.java new file mode 100644 index 00000000..8c1bc22d --- /dev/null +++ b/src/test/java/org/dataloader/fixtures/parameterized/TestDataLoaderFactory.java @@ -0,0 +1,22 @@ +package org.dataloader.fixtures.parameterized; + +import org.dataloader.DataLoader; +import org.dataloader.DataLoaderOptions; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +public interface TestDataLoaderFactory { + DataLoader idLoader(DataLoaderOptions options, List> loadCalls); + + DataLoader idLoaderBlowsUps(DataLoaderOptions options, List> loadCalls); + + DataLoader idLoaderAllExceptions(DataLoaderOptions options, List> loadCalls); + + DataLoader idLoaderOddEvenExceptions(DataLoaderOptions options, List> loadCalls); + + DataLoader onlyReturnsNValues(int N, DataLoaderOptions options, ArrayList loadCalls); + + DataLoader idLoaderReturnsTooMany(int howManyMore, DataLoaderOptions options, ArrayList loadCalls); +} diff --git a/src/test/java/org/dataloader/fixtures/parameterized/TestReactiveDataLoaderFactory.java b/src/test/java/org/dataloader/fixtures/parameterized/TestReactiveDataLoaderFactory.java new file mode 100644 index 00000000..d45932c1 --- /dev/null +++ b/src/test/java/org/dataloader/fixtures/parameterized/TestReactiveDataLoaderFactory.java @@ -0,0 +1,11 @@ +package org.dataloader.fixtures.parameterized; + +import org.dataloader.DataLoader; +import org.dataloader.DataLoaderOptions; + +import java.util.Collection; +import java.util.List; + +public interface TestReactiveDataLoaderFactory { + DataLoader idLoaderBlowsUpsAfterN(int N, DataLoaderOptions options, List> loadCalls); +} From 91d3036c0a7e3f400c8c5ac6c794020e69b8e311 Mon Sep 17 00:00:00 2001 From: bbaker Date: Fri, 24 May 2024 11:21:12 +1000 Subject: [PATCH 034/156] This moves the reactive code pout into its own package because DataLoaderHelper is way too big --- .../java/org/dataloader/DataLoaderHelper.java | 271 ++---------------- .../DataLoaderMapEntrySubscriber.java | 104 +++++++ .../reactive/DataLoaderSubscriber.java | 86 ++++++ .../reactive/DataLoaderSubscriberBase.java | 104 +++++++ .../reactive/HelperIntegration.java | 19 ++ 5 files changed, 337 insertions(+), 247 deletions(-) create mode 100644 src/main/java/org/dataloader/reactive/DataLoaderMapEntrySubscriber.java create mode 100644 src/main/java/org/dataloader/reactive/DataLoaderSubscriber.java create mode 100644 src/main/java/org/dataloader/reactive/DataLoaderSubscriberBase.java create mode 100644 src/main/java/org/dataloader/reactive/HelperIntegration.java diff --git a/src/main/java/org/dataloader/DataLoaderHelper.java b/src/main/java/org/dataloader/DataLoaderHelper.java index edbf3484..88bc73ba 100644 --- a/src/main/java/org/dataloader/DataLoaderHelper.java +++ b/src/main/java/org/dataloader/DataLoaderHelper.java @@ -3,7 +3,9 @@ import org.dataloader.annotations.GuardedBy; import org.dataloader.annotations.Internal; import org.dataloader.impl.CompletableFutureKit; -import org.dataloader.impl.DataLoaderAssertionException; +import org.dataloader.reactive.HelperIntegration; +import org.dataloader.reactive.DataLoaderMapEntrySubscriber; +import org.dataloader.reactive.DataLoaderSubscriber; import org.dataloader.scheduler.BatchLoaderScheduler; import org.dataloader.stats.StatisticsCollector; import org.dataloader.stats.context.IncrementBatchLoadCountByStatisticsContext; @@ -12,13 +14,11 @@ import org.dataloader.stats.context.IncrementLoadCountStatisticsContext; import org.dataloader.stats.context.IncrementLoadErrorCountStatisticsContext; import org.reactivestreams.Subscriber; -import org.reactivestreams.Subscription; import java.time.Clock; import java.time.Instant; import java.util.ArrayList; import java.util.Collection; -import java.util.HashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; @@ -510,7 +510,7 @@ private CompletableFuture> invokeMapBatchLoader(List keys, BatchLoade private CompletableFuture> invokeBatchPublisher(List keys, List keyContexts, List> queuedFutures, BatchLoaderEnvironment environment) { CompletableFuture> loadResult = new CompletableFuture<>(); - Subscriber subscriber = new DataLoaderSubscriber(loadResult, keys, keyContexts, queuedFutures); + Subscriber subscriber = new DataLoaderSubscriber<>(loadResult, keys, keyContexts, queuedFutures, helperIntegration()); BatchLoaderScheduler batchLoaderScheduler = loaderOptions.getBatchLoaderScheduler(); if (batchLoadFunction instanceof BatchPublisherWithContext) { @@ -535,9 +535,28 @@ private CompletableFuture> invokeBatchPublisher(List keys, List helperIntegration() { + return new HelperIntegration<>() { + @Override + public StatisticsCollector getStats() { + return stats; + } + + @Override + public void clearCacheView(K key) { + dataLoader.clear(key); + } + + @Override + public void clearCacheEntriesOnExceptions(List keys) { + possiblyClearCacheEntriesOnExceptions(keys); + } + }; + } + private CompletableFuture> invokeMappedBatchPublisher(List keys, List keyContexts, List> queuedFutures, BatchLoaderEnvironment environment) { CompletableFuture> loadResult = new CompletableFuture<>(); - Subscriber> subscriber = new DataLoaderMapEntrySubscriber(loadResult, keys, keyContexts, queuedFutures); + Subscriber> subscriber = new DataLoaderMapEntrySubscriber<>(loadResult, keys, keyContexts, queuedFutures, helperIntegration()); BatchLoaderScheduler batchLoaderScheduler = loaderOptions.getBatchLoaderScheduler(); if (batchLoadFunction instanceof MappedBatchPublisherWithContext) { @@ -625,246 +644,4 @@ private static DispatchResult emptyDispatchResult() { return (DispatchResult) EMPTY_DISPATCH_RESULT; } - /********************************************************************************************** - * ******************************************************************************************** - *

- * The reactive support classes start here - * - * @param for two - ********************************************************************************************** - ********************************************************************************************** - */ - private abstract class DataLoaderSubscriberBase implements Subscriber { - - final CompletableFuture> valuesFuture; - final List keys; - final List callContexts; - final List> queuedFutures; - - List clearCacheKeys = new ArrayList<>(); - List completedValues = new ArrayList<>(); - boolean onErrorCalled = false; - boolean onCompleteCalled = false; - - DataLoaderSubscriberBase( - CompletableFuture> valuesFuture, - List keys, - List callContexts, - List> queuedFutures - ) { - this.valuesFuture = valuesFuture; - this.keys = keys; - this.callContexts = callContexts; - this.queuedFutures = queuedFutures; - } - - @Override - public void onSubscribe(Subscription subscription) { - subscription.request(keys.size()); - } - - @Override - public void onNext(T v) { - assertState(!onErrorCalled, () -> "onError has already been called; onNext may not be invoked."); - assertState(!onCompleteCalled, () -> "onComplete has already been called; onNext may not be invoked."); - } - - @Override - public void onComplete() { - assertState(!onErrorCalled, () -> "onError has already been called; onComplete may not be invoked."); - onCompleteCalled = true; - } - - @Override - public void onError(Throwable throwable) { - assertState(!onCompleteCalled, () -> "onComplete has already been called; onError may not be invoked."); - onErrorCalled = true; - - stats.incrementBatchLoadExceptionCount(new IncrementBatchLoadExceptionCountStatisticsContext<>(keys, callContexts)); - } - - /* - * A value has arrived - how do we complete the future that's associated with it in a common way - */ - void onNextValue(K key, V value, Object callContext, List> futures) { - if (value instanceof Try) { - // we allow the batch loader to return a Try so we can better represent a computation - // that might have worked or not. - //noinspection unchecked - Try tryValue = (Try) value; - if (tryValue.isSuccess()) { - futures.forEach(f -> f.complete(tryValue.get())); - } else { - stats.incrementLoadErrorCount(new IncrementLoadErrorCountStatisticsContext<>(key, callContext)); - futures.forEach(f -> f.completeExceptionally(tryValue.getThrowable())); - clearCacheKeys.add(key); - } - } else { - futures.forEach(f -> f.complete(value)); - } - } - - Throwable unwrapThrowable(Throwable ex) { - if (ex instanceof CompletionException) { - ex = ex.getCause(); - } - return ex; - } - } - - private class DataLoaderSubscriber extends DataLoaderSubscriberBase { - - private int idx = 0; - - private DataLoaderSubscriber( - CompletableFuture> valuesFuture, - List keys, - List callContexts, - List> queuedFutures - ) { - super(valuesFuture, keys, callContexts, queuedFutures); - } - - // onNext may be called by multiple threads - for the time being, we pass 'synchronized' to guarantee - // correctness (at the cost of speed). - @Override - public synchronized void onNext(V value) { - super.onNext(value); - - if (idx >= keys.size()) { - // hang on they have given us more values than we asked for in keys - // we cant handle this - return; - } - K key = keys.get(idx); - Object callContext = callContexts.get(idx); - CompletableFuture future = queuedFutures.get(idx); - onNextValue(key, value, callContext, List.of(future)); - - completedValues.add(value); - idx++; - } - - - @Override - public synchronized void onComplete() { - super.onComplete(); - if (keys.size() != completedValues.size()) { - // we have more or less values than promised - // we will go through all the outstanding promises and mark those that - // have not finished as failed - for (CompletableFuture queuedFuture : queuedFutures) { - if (!queuedFuture.isDone()) { - queuedFuture.completeExceptionally(new DataLoaderAssertionException("The size of the promised values MUST be the same size as the key list")); - } - } - } - possiblyClearCacheEntriesOnExceptions(clearCacheKeys); - valuesFuture.complete(completedValues); - } - - @Override - public synchronized void onError(Throwable ex) { - super.onError(ex); - ex = unwrapThrowable(ex); - // Set the remaining keys to the exception. - for (int i = idx; i < queuedFutures.size(); i++) { - K key = keys.get(i); - CompletableFuture future = queuedFutures.get(i); - if (! future.isDone()) { - future.completeExceptionally(ex); - // clear any cached view of this key because it failed - dataLoader.clear(key); - } - } - valuesFuture.completeExceptionally(ex); - } - - } - - private class DataLoaderMapEntrySubscriber extends DataLoaderSubscriberBase> { - - private final Map callContextByKey; - private final Map>> queuedFuturesByKey; - private final Map completedValuesByKey = new HashMap<>(); - - - private DataLoaderMapEntrySubscriber( - CompletableFuture> valuesFuture, - List keys, - List callContexts, - List> queuedFutures - ) { - super(valuesFuture, keys, callContexts, queuedFutures); - this.callContextByKey = new HashMap<>(); - this.queuedFuturesByKey = new HashMap<>(); - for (int idx = 0; idx < queuedFutures.size(); idx++) { - K key = keys.get(idx); - Object callContext = callContexts.get(idx); - CompletableFuture queuedFuture = queuedFutures.get(idx); - callContextByKey.put(key, callContext); - queuedFuturesByKey.computeIfAbsent(key, k -> new ArrayList<>()).add(queuedFuture); - } - } - - - @Override - public synchronized void onNext(Map.Entry entry) { - super.onNext(entry); - K key = entry.getKey(); - V value = entry.getValue(); - - Object callContext = callContextByKey.get(key); - List> futures = queuedFuturesByKey.getOrDefault(key, List.of()); - - onNextValue(key, value, callContext, futures); - - // did we have an actual key for this value - ignore it if they send us one outside the key set - if (!futures.isEmpty()) { - completedValuesByKey.put(key, value); - } - } - - @Override - public synchronized void onComplete() { - super.onComplete(); - - possiblyClearCacheEntriesOnExceptions(clearCacheKeys); - List values = new ArrayList<>(keys.size()); - for (K key : keys) { - V value = completedValuesByKey.get(key); - values.add(value); - - List> futures = queuedFuturesByKey.getOrDefault(key, List.of()); - for (CompletableFuture future : futures) { - if (! future.isDone()) { - // we have a future that never came back for that key - // but the publisher is done sending in data - it must be null - // e.g. for key X when found no value - future.complete(null); - } - } - } - valuesFuture.complete(values); - } - - @Override - public synchronized void onError(Throwable ex) { - super.onError(ex); - ex = unwrapThrowable(ex); - // Complete the futures for the remaining keys with the exception. - for (int idx = 0; idx < queuedFutures.size(); idx++) { - K key = keys.get(idx); - List> futures = queuedFuturesByKey.get(key); - if (!completedValuesByKey.containsKey(key)) { - for (CompletableFuture future : futures) { - future.completeExceptionally(ex); - } - // clear any cached view of this key because they all failed - dataLoader.clear(key); - } - } - valuesFuture.completeExceptionally(ex); - } - } } diff --git a/src/main/java/org/dataloader/reactive/DataLoaderMapEntrySubscriber.java b/src/main/java/org/dataloader/reactive/DataLoaderMapEntrySubscriber.java new file mode 100644 index 00000000..839a8c63 --- /dev/null +++ b/src/main/java/org/dataloader/reactive/DataLoaderMapEntrySubscriber.java @@ -0,0 +1,104 @@ +package org.dataloader.reactive; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; + +/** + * This class can be used to subscribe to a {@link org.reactivestreams.Publisher} and then + * have the values it receives complete the data loader keys in a map lookup fashion. + *

+ * This is a reactive version of {@link org.dataloader.MappedBatchLoader} + * + * @param the type of keys + * @param the type of values + */ +public class DataLoaderMapEntrySubscriber extends DataLoaderSubscriberBase> { + + private final Map callContextByKey; + private final Map>> queuedFuturesByKey; + private final Map completedValuesByKey = new HashMap<>(); + + + public DataLoaderMapEntrySubscriber( + CompletableFuture> valuesFuture, + List keys, + List callContexts, + List> queuedFutures, + HelperIntegration helperIntegration + + ) { + super(valuesFuture, keys, callContexts, queuedFutures, helperIntegration); + this.callContextByKey = new HashMap<>(); + this.queuedFuturesByKey = new HashMap<>(); + for (int idx = 0; idx < queuedFutures.size(); idx++) { + K key = keys.get(idx); + Object callContext = callContexts.get(idx); + CompletableFuture queuedFuture = queuedFutures.get(idx); + callContextByKey.put(key, callContext); + queuedFuturesByKey.computeIfAbsent(key, k -> new ArrayList<>()).add(queuedFuture); + } + } + + + @Override + public synchronized void onNext(Map.Entry entry) { + super.onNext(entry); + K key = entry.getKey(); + V value = entry.getValue(); + + Object callContext = callContextByKey.get(key); + List> futures = queuedFuturesByKey.getOrDefault(key, List.of()); + + onNextValue(key, value, callContext, futures); + + // did we have an actual key for this value - ignore it if they send us one outside the key set + if (!futures.isEmpty()) { + completedValuesByKey.put(key, value); + } + } + + @Override + public synchronized void onComplete() { + super.onComplete(); + + possiblyClearCacheEntriesOnExceptions(); + List values = new ArrayList<>(keys.size()); + for (K key : keys) { + V value = completedValuesByKey.get(key); + values.add(value); + + List> futures = queuedFuturesByKey.getOrDefault(key, List.of()); + for (CompletableFuture future : futures) { + if (!future.isDone()) { + // we have a future that never came back for that key + // but the publisher is done sending in data - it must be null + // e.g. for key X when found no value + future.complete(null); + } + } + } + valuesFuture.complete(values); + } + + @Override + public synchronized void onError(Throwable ex) { + super.onError(ex); + ex = unwrapThrowable(ex); + // Complete the futures for the remaining keys with the exception. + for (int idx = 0; idx < queuedFutures.size(); idx++) { + K key = keys.get(idx); + List> futures = queuedFuturesByKey.get(key); + if (!completedValuesByKey.containsKey(key)) { + for (CompletableFuture future : futures) { + future.completeExceptionally(ex); + } + // clear any cached view of this key because they all failed + helperIntegration.clearCacheView(key); + } + } + valuesFuture.completeExceptionally(ex); + } +} diff --git a/src/main/java/org/dataloader/reactive/DataLoaderSubscriber.java b/src/main/java/org/dataloader/reactive/DataLoaderSubscriber.java new file mode 100644 index 00000000..a0d3ee60 --- /dev/null +++ b/src/main/java/org/dataloader/reactive/DataLoaderSubscriber.java @@ -0,0 +1,86 @@ +package org.dataloader.reactive; + +import org.dataloader.impl.DataLoaderAssertionException; + +import java.util.List; +import java.util.concurrent.CompletableFuture; + +/** + * This class can be used to subscribe to a {@link org.reactivestreams.Publisher} and then + * have the values it receives complete the data loader keys. The keys and values must be + * in index order. + *

+ * This is a reactive version of {@link org.dataloader.BatchLoader} + * + * @param the type of keys + * @param the type of values + */ +public class DataLoaderSubscriber extends DataLoaderSubscriberBase { + + private int idx = 0; + + public DataLoaderSubscriber( + CompletableFuture> valuesFuture, + List keys, + List callContexts, + List> queuedFutures, + HelperIntegration helperIntegration + ) { + super(valuesFuture, keys, callContexts, queuedFutures, helperIntegration); + } + + // onNext may be called by multiple threads - for the time being, we pass 'synchronized' to guarantee + // correctness (at the cost of speed). + @Override + public synchronized void onNext(V value) { + super.onNext(value); + + if (idx >= keys.size()) { + // hang on they have given us more values than we asked for in keys + // we cant handle this + return; + } + K key = keys.get(idx); + Object callContext = callContexts.get(idx); + CompletableFuture future = queuedFutures.get(idx); + onNextValue(key, value, callContext, List.of(future)); + + completedValues.add(value); + idx++; + } + + + @Override + public synchronized void onComplete() { + super.onComplete(); + if (keys.size() != completedValues.size()) { + // we have more or less values than promised + // we will go through all the outstanding promises and mark those that + // have not finished as failed + for (CompletableFuture queuedFuture : queuedFutures) { + if (!queuedFuture.isDone()) { + queuedFuture.completeExceptionally(new DataLoaderAssertionException("The size of the promised values MUST be the same size as the key list")); + } + } + } + possiblyClearCacheEntriesOnExceptions(); + valuesFuture.complete(completedValues); + } + + @Override + public synchronized void onError(Throwable ex) { + super.onError(ex); + ex = unwrapThrowable(ex); + // Set the remaining keys to the exception. + for (int i = idx; i < queuedFutures.size(); i++) { + K key = keys.get(i); + CompletableFuture future = queuedFutures.get(i); + if (!future.isDone()) { + future.completeExceptionally(ex); + // clear any cached view of this key because it failed + helperIntegration.clearCacheView(key); + } + } + valuesFuture.completeExceptionally(ex); + } +} diff --git a/src/main/java/org/dataloader/reactive/DataLoaderSubscriberBase.java b/src/main/java/org/dataloader/reactive/DataLoaderSubscriberBase.java new file mode 100644 index 00000000..e2cb01d5 --- /dev/null +++ b/src/main/java/org/dataloader/reactive/DataLoaderSubscriberBase.java @@ -0,0 +1,104 @@ +package org.dataloader.reactive; + +import org.dataloader.Try; +import org.dataloader.stats.context.IncrementBatchLoadExceptionCountStatisticsContext; +import org.dataloader.stats.context.IncrementLoadErrorCountStatisticsContext; +import org.reactivestreams.Subscriber; +import org.reactivestreams.Subscription; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; + +import static org.dataloader.impl.Assertions.assertState; + +/** + * The base class for our reactive subscriber support + * + * @param for two + */ +abstract class DataLoaderSubscriberBase implements Subscriber { + + final CompletableFuture> valuesFuture; + final List keys; + final List callContexts; + final List> queuedFutures; + final HelperIntegration helperIntegration; + + List clearCacheKeys = new ArrayList<>(); + List completedValues = new ArrayList<>(); + boolean onErrorCalled = false; + boolean onCompleteCalled = false; + + DataLoaderSubscriberBase( + CompletableFuture> valuesFuture, + List keys, + List callContexts, + List> queuedFutures, + HelperIntegration helperIntegration + ) { + this.valuesFuture = valuesFuture; + this.keys = keys; + this.callContexts = callContexts; + this.queuedFutures = queuedFutures; + this.helperIntegration = helperIntegration; + } + + @Override + public void onSubscribe(Subscription subscription) { + subscription.request(keys.size()); + } + + @Override + public void onNext(T v) { + assertState(!onErrorCalled, () -> "onError has already been called; onNext may not be invoked."); + assertState(!onCompleteCalled, () -> "onComplete has already been called; onNext may not be invoked."); + } + + @Override + public void onComplete() { + assertState(!onErrorCalled, () -> "onError has already been called; onComplete may not be invoked."); + onCompleteCalled = true; + } + + @Override + public void onError(Throwable throwable) { + assertState(!onCompleteCalled, () -> "onComplete has already been called; onError may not be invoked."); + onErrorCalled = true; + + helperIntegration.getStats().incrementBatchLoadExceptionCount(new IncrementBatchLoadExceptionCountStatisticsContext<>(keys, callContexts)); + } + + /* + * A value has arrived - how do we complete the future that's associated with it in a common way + */ + void onNextValue(K key, V value, Object callContext, List> futures) { + if (value instanceof Try) { + // we allow the batch loader to return a Try so we can better represent a computation + // that might have worked or not. + //noinspection unchecked + Try tryValue = (Try) value; + if (tryValue.isSuccess()) { + futures.forEach(f -> f.complete(tryValue.get())); + } else { + helperIntegration.getStats().incrementLoadErrorCount(new IncrementLoadErrorCountStatisticsContext<>(key, callContext)); + futures.forEach(f -> f.completeExceptionally(tryValue.getThrowable())); + clearCacheKeys.add(key); + } + } else { + futures.forEach(f -> f.complete(value)); + } + } + + Throwable unwrapThrowable(Throwable ex) { + if (ex instanceof CompletionException) { + ex = ex.getCause(); + } + return ex; + } + + void possiblyClearCacheEntriesOnExceptions() { + helperIntegration.clearCacheEntriesOnExceptions(clearCacheKeys); + } +} diff --git a/src/main/java/org/dataloader/reactive/HelperIntegration.java b/src/main/java/org/dataloader/reactive/HelperIntegration.java new file mode 100644 index 00000000..49724cb5 --- /dev/null +++ b/src/main/java/org/dataloader/reactive/HelperIntegration.java @@ -0,0 +1,19 @@ +package org.dataloader.reactive; + +import org.dataloader.stats.StatisticsCollector; + +import java.util.List; + +/** + * Just some callbacks to the data loader code to do common tasks + * + * @param for keys + */ +public interface HelperIntegration { + + StatisticsCollector getStats(); + + void clearCacheView(K key); + + void clearCacheEntriesOnExceptions(List keys); +} From e98621bb24e25bbfc859b9d7a84b30c407d55794 Mon Sep 17 00:00:00 2001 From: bbaker Date: Fri, 24 May 2024 11:30:42 +1000 Subject: [PATCH 035/156] renamed classes inline with their counterparts --- src/main/java/org/dataloader/DataLoaderHelper.java | 8 ++++---- ...erSubscriberBase.java => AbstractBatchSubscriber.java} | 4 ++-- .../{DataLoaderSubscriber.java => BatchSubscriber.java} | 4 ++-- ...MapEntrySubscriber.java => MappedBatchSubscriber.java} | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) rename src/main/java/org/dataloader/reactive/{DataLoaderSubscriberBase.java => AbstractBatchSubscriber.java} (97%) rename src/main/java/org/dataloader/reactive/{DataLoaderSubscriber.java => BatchSubscriber.java} (96%) rename src/main/java/org/dataloader/reactive/{DataLoaderMapEntrySubscriber.java => MappedBatchSubscriber.java} (96%) diff --git a/src/main/java/org/dataloader/DataLoaderHelper.java b/src/main/java/org/dataloader/DataLoaderHelper.java index 88bc73ba..33833bd5 100644 --- a/src/main/java/org/dataloader/DataLoaderHelper.java +++ b/src/main/java/org/dataloader/DataLoaderHelper.java @@ -4,8 +4,8 @@ import org.dataloader.annotations.Internal; import org.dataloader.impl.CompletableFutureKit; import org.dataloader.reactive.HelperIntegration; -import org.dataloader.reactive.DataLoaderMapEntrySubscriber; -import org.dataloader.reactive.DataLoaderSubscriber; +import org.dataloader.reactive.MappedBatchSubscriber; +import org.dataloader.reactive.BatchSubscriber; import org.dataloader.scheduler.BatchLoaderScheduler; import org.dataloader.stats.StatisticsCollector; import org.dataloader.stats.context.IncrementBatchLoadCountByStatisticsContext; @@ -510,7 +510,7 @@ private CompletableFuture> invokeMapBatchLoader(List keys, BatchLoade private CompletableFuture> invokeBatchPublisher(List keys, List keyContexts, List> queuedFutures, BatchLoaderEnvironment environment) { CompletableFuture> loadResult = new CompletableFuture<>(); - Subscriber subscriber = new DataLoaderSubscriber<>(loadResult, keys, keyContexts, queuedFutures, helperIntegration()); + Subscriber subscriber = new BatchSubscriber<>(loadResult, keys, keyContexts, queuedFutures, helperIntegration()); BatchLoaderScheduler batchLoaderScheduler = loaderOptions.getBatchLoaderScheduler(); if (batchLoadFunction instanceof BatchPublisherWithContext) { @@ -556,7 +556,7 @@ public void clearCacheEntriesOnExceptions(List keys) { private CompletableFuture> invokeMappedBatchPublisher(List keys, List keyContexts, List> queuedFutures, BatchLoaderEnvironment environment) { CompletableFuture> loadResult = new CompletableFuture<>(); - Subscriber> subscriber = new DataLoaderMapEntrySubscriber<>(loadResult, keys, keyContexts, queuedFutures, helperIntegration()); + Subscriber> subscriber = new MappedBatchSubscriber<>(loadResult, keys, keyContexts, queuedFutures, helperIntegration()); BatchLoaderScheduler batchLoaderScheduler = loaderOptions.getBatchLoaderScheduler(); if (batchLoadFunction instanceof MappedBatchPublisherWithContext) { diff --git a/src/main/java/org/dataloader/reactive/DataLoaderSubscriberBase.java b/src/main/java/org/dataloader/reactive/AbstractBatchSubscriber.java similarity index 97% rename from src/main/java/org/dataloader/reactive/DataLoaderSubscriberBase.java rename to src/main/java/org/dataloader/reactive/AbstractBatchSubscriber.java index e2cb01d5..578a33aa 100644 --- a/src/main/java/org/dataloader/reactive/DataLoaderSubscriberBase.java +++ b/src/main/java/org/dataloader/reactive/AbstractBatchSubscriber.java @@ -18,7 +18,7 @@ * * @param for two */ -abstract class DataLoaderSubscriberBase implements Subscriber { +abstract class AbstractBatchSubscriber implements Subscriber { final CompletableFuture> valuesFuture; final List keys; @@ -31,7 +31,7 @@ abstract class DataLoaderSubscriberBase implements Subscriber { boolean onErrorCalled = false; boolean onCompleteCalled = false; - DataLoaderSubscriberBase( + AbstractBatchSubscriber( CompletableFuture> valuesFuture, List keys, List callContexts, diff --git a/src/main/java/org/dataloader/reactive/DataLoaderSubscriber.java b/src/main/java/org/dataloader/reactive/BatchSubscriber.java similarity index 96% rename from src/main/java/org/dataloader/reactive/DataLoaderSubscriber.java rename to src/main/java/org/dataloader/reactive/BatchSubscriber.java index a0d3ee60..41b97ddb 100644 --- a/src/main/java/org/dataloader/reactive/DataLoaderSubscriber.java +++ b/src/main/java/org/dataloader/reactive/BatchSubscriber.java @@ -15,11 +15,11 @@ * @param the type of keys * @param the type of values */ -public class DataLoaderSubscriber extends DataLoaderSubscriberBase { +public class BatchSubscriber extends AbstractBatchSubscriber { private int idx = 0; - public DataLoaderSubscriber( + public BatchSubscriber( CompletableFuture> valuesFuture, List keys, List callContexts, diff --git a/src/main/java/org/dataloader/reactive/DataLoaderMapEntrySubscriber.java b/src/main/java/org/dataloader/reactive/MappedBatchSubscriber.java similarity index 96% rename from src/main/java/org/dataloader/reactive/DataLoaderMapEntrySubscriber.java rename to src/main/java/org/dataloader/reactive/MappedBatchSubscriber.java index 839a8c63..127061cf 100644 --- a/src/main/java/org/dataloader/reactive/DataLoaderMapEntrySubscriber.java +++ b/src/main/java/org/dataloader/reactive/MappedBatchSubscriber.java @@ -15,14 +15,14 @@ * @param the type of keys * @param the type of values */ -public class DataLoaderMapEntrySubscriber extends DataLoaderSubscriberBase> { +public class MappedBatchSubscriber extends AbstractBatchSubscriber> { private final Map callContextByKey; private final Map>> queuedFuturesByKey; private final Map completedValuesByKey = new HashMap<>(); - public DataLoaderMapEntrySubscriber( + public MappedBatchSubscriber( CompletableFuture> valuesFuture, List keys, List callContexts, From 6523015d12cd73a11e0622cb5e67d0e7e5e7c2e2 Mon Sep 17 00:00:00 2001 From: bbaker Date: Fri, 24 May 2024 11:40:52 +1000 Subject: [PATCH 036/156] made them non public and created a static factory support class --- .../java/org/dataloader/DataLoaderHelper.java | 12 +++-- .../reactive/AbstractBatchSubscriber.java | 4 +- ...bscriber.java => BatchSubscriberImpl.java} | 6 +-- .../reactive/HelperIntegration.java | 19 -------- ...er.java => MappedBatchSubscriberImpl.java} | 7 ++- .../dataloader/reactive/ReactiveSupport.java | 45 +++++++++++++++++++ 6 files changed, 58 insertions(+), 35 deletions(-) rename src/main/java/org/dataloader/reactive/{BatchSubscriber.java => BatchSubscriberImpl.java} (94%) delete mode 100644 src/main/java/org/dataloader/reactive/HelperIntegration.java rename src/main/java/org/dataloader/reactive/{MappedBatchSubscriber.java => MappedBatchSubscriberImpl.java} (95%) create mode 100644 src/main/java/org/dataloader/reactive/ReactiveSupport.java diff --git a/src/main/java/org/dataloader/DataLoaderHelper.java b/src/main/java/org/dataloader/DataLoaderHelper.java index 33833bd5..14ed9bf9 100644 --- a/src/main/java/org/dataloader/DataLoaderHelper.java +++ b/src/main/java/org/dataloader/DataLoaderHelper.java @@ -3,9 +3,7 @@ import org.dataloader.annotations.GuardedBy; import org.dataloader.annotations.Internal; import org.dataloader.impl.CompletableFutureKit; -import org.dataloader.reactive.HelperIntegration; -import org.dataloader.reactive.MappedBatchSubscriber; -import org.dataloader.reactive.BatchSubscriber; +import org.dataloader.reactive.ReactiveSupport; import org.dataloader.scheduler.BatchLoaderScheduler; import org.dataloader.stats.StatisticsCollector; import org.dataloader.stats.context.IncrementBatchLoadCountByStatisticsContext; @@ -510,7 +508,7 @@ private CompletableFuture> invokeMapBatchLoader(List keys, BatchLoade private CompletableFuture> invokeBatchPublisher(List keys, List keyContexts, List> queuedFutures, BatchLoaderEnvironment environment) { CompletableFuture> loadResult = new CompletableFuture<>(); - Subscriber subscriber = new BatchSubscriber<>(loadResult, keys, keyContexts, queuedFutures, helperIntegration()); + Subscriber subscriber = ReactiveSupport.batchSubscriber(loadResult, keys, keyContexts, queuedFutures, helperIntegration()); BatchLoaderScheduler batchLoaderScheduler = loaderOptions.getBatchLoaderScheduler(); if (batchLoadFunction instanceof BatchPublisherWithContext) { @@ -535,8 +533,8 @@ private CompletableFuture> invokeBatchPublisher(List keys, List helperIntegration() { - return new HelperIntegration<>() { + private ReactiveSupport.HelperIntegration helperIntegration() { + return new ReactiveSupport.HelperIntegration<>() { @Override public StatisticsCollector getStats() { return stats; @@ -556,7 +554,7 @@ public void clearCacheEntriesOnExceptions(List keys) { private CompletableFuture> invokeMappedBatchPublisher(List keys, List keyContexts, List> queuedFutures, BatchLoaderEnvironment environment) { CompletableFuture> loadResult = new CompletableFuture<>(); - Subscriber> subscriber = new MappedBatchSubscriber<>(loadResult, keys, keyContexts, queuedFutures, helperIntegration()); + Subscriber> subscriber = ReactiveSupport.mappedBatchSubscriber(loadResult, keys, keyContexts, queuedFutures, helperIntegration()); BatchLoaderScheduler batchLoaderScheduler = loaderOptions.getBatchLoaderScheduler(); if (batchLoadFunction instanceof MappedBatchPublisherWithContext) { diff --git a/src/main/java/org/dataloader/reactive/AbstractBatchSubscriber.java b/src/main/java/org/dataloader/reactive/AbstractBatchSubscriber.java index 578a33aa..c2f54380 100644 --- a/src/main/java/org/dataloader/reactive/AbstractBatchSubscriber.java +++ b/src/main/java/org/dataloader/reactive/AbstractBatchSubscriber.java @@ -24,7 +24,7 @@ abstract class AbstractBatchSubscriber implements Subscriber { final List keys; final List callContexts; final List> queuedFutures; - final HelperIntegration helperIntegration; + final ReactiveSupport.HelperIntegration helperIntegration; List clearCacheKeys = new ArrayList<>(); List completedValues = new ArrayList<>(); @@ -36,7 +36,7 @@ abstract class AbstractBatchSubscriber implements Subscriber { List keys, List callContexts, List> queuedFutures, - HelperIntegration helperIntegration + ReactiveSupport.HelperIntegration helperIntegration ) { this.valuesFuture = valuesFuture; this.keys = keys; diff --git a/src/main/java/org/dataloader/reactive/BatchSubscriber.java b/src/main/java/org/dataloader/reactive/BatchSubscriberImpl.java similarity index 94% rename from src/main/java/org/dataloader/reactive/BatchSubscriber.java rename to src/main/java/org/dataloader/reactive/BatchSubscriberImpl.java index 41b97ddb..d0b81100 100644 --- a/src/main/java/org/dataloader/reactive/BatchSubscriber.java +++ b/src/main/java/org/dataloader/reactive/BatchSubscriberImpl.java @@ -15,16 +15,16 @@ * @param the type of keys * @param the type of values */ -public class BatchSubscriber extends AbstractBatchSubscriber { +class BatchSubscriberImpl extends AbstractBatchSubscriber { private int idx = 0; - public BatchSubscriber( + BatchSubscriberImpl( CompletableFuture> valuesFuture, List keys, List callContexts, List> queuedFutures, - HelperIntegration helperIntegration + ReactiveSupport.HelperIntegration helperIntegration ) { super(valuesFuture, keys, callContexts, queuedFutures, helperIntegration); } diff --git a/src/main/java/org/dataloader/reactive/HelperIntegration.java b/src/main/java/org/dataloader/reactive/HelperIntegration.java deleted file mode 100644 index 49724cb5..00000000 --- a/src/main/java/org/dataloader/reactive/HelperIntegration.java +++ /dev/null @@ -1,19 +0,0 @@ -package org.dataloader.reactive; - -import org.dataloader.stats.StatisticsCollector; - -import java.util.List; - -/** - * Just some callbacks to the data loader code to do common tasks - * - * @param for keys - */ -public interface HelperIntegration { - - StatisticsCollector getStats(); - - void clearCacheView(K key); - - void clearCacheEntriesOnExceptions(List keys); -} diff --git a/src/main/java/org/dataloader/reactive/MappedBatchSubscriber.java b/src/main/java/org/dataloader/reactive/MappedBatchSubscriberImpl.java similarity index 95% rename from src/main/java/org/dataloader/reactive/MappedBatchSubscriber.java rename to src/main/java/org/dataloader/reactive/MappedBatchSubscriberImpl.java index 127061cf..d56efa0e 100644 --- a/src/main/java/org/dataloader/reactive/MappedBatchSubscriber.java +++ b/src/main/java/org/dataloader/reactive/MappedBatchSubscriberImpl.java @@ -15,20 +15,19 @@ * @param the type of keys * @param the type of values */ -public class MappedBatchSubscriber extends AbstractBatchSubscriber> { +class MappedBatchSubscriberImpl extends AbstractBatchSubscriber> { private final Map callContextByKey; private final Map>> queuedFuturesByKey; private final Map completedValuesByKey = new HashMap<>(); - public MappedBatchSubscriber( + MappedBatchSubscriberImpl( CompletableFuture> valuesFuture, List keys, List callContexts, List> queuedFutures, - HelperIntegration helperIntegration - + ReactiveSupport.HelperIntegration helperIntegration ) { super(valuesFuture, keys, callContexts, queuedFutures, helperIntegration); this.callContextByKey = new HashMap<>(); diff --git a/src/main/java/org/dataloader/reactive/ReactiveSupport.java b/src/main/java/org/dataloader/reactive/ReactiveSupport.java new file mode 100644 index 00000000..fc03bb0b --- /dev/null +++ b/src/main/java/org/dataloader/reactive/ReactiveSupport.java @@ -0,0 +1,45 @@ +package org.dataloader.reactive; + +import org.dataloader.stats.StatisticsCollector; +import org.reactivestreams.Subscriber; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; + +public class ReactiveSupport { + + public static Subscriber batchSubscriber( + CompletableFuture> valuesFuture, + List keys, + List callContexts, + List> queuedFutures, + ReactiveSupport.HelperIntegration helperIntegration + ) { + return new BatchSubscriberImpl<>(valuesFuture, keys, callContexts, queuedFutures, helperIntegration); + } + + public static Subscriber> mappedBatchSubscriber( + CompletableFuture> valuesFuture, + List keys, + List callContexts, + List> queuedFutures, + ReactiveSupport.HelperIntegration helperIntegration + ) { + return new MappedBatchSubscriberImpl<>(valuesFuture, keys, callContexts, queuedFutures, helperIntegration); + } + + /** + * Just some callbacks to the data loader code to do common tasks + * + * @param for keys + */ + public interface HelperIntegration { + + StatisticsCollector getStats(); + + void clearCacheView(K key); + + void clearCacheEntriesOnExceptions(List keys); + } +} From 170ccf8308cb4e86ed96b7bcd8a77cd6342999b5 Mon Sep 17 00:00:00 2001 From: bbaker Date: Fri, 24 May 2024 11:43:11 +1000 Subject: [PATCH 037/156] reorged method placement --- .../java/org/dataloader/DataLoaderHelper.java | 37 +++++++++---------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/src/main/java/org/dataloader/DataLoaderHelper.java b/src/main/java/org/dataloader/DataLoaderHelper.java index 14ed9bf9..62a7cb63 100644 --- a/src/main/java/org/dataloader/DataLoaderHelper.java +++ b/src/main/java/org/dataloader/DataLoaderHelper.java @@ -533,25 +533,6 @@ private CompletableFuture> invokeBatchPublisher(List keys, List helperIntegration() { - return new ReactiveSupport.HelperIntegration<>() { - @Override - public StatisticsCollector getStats() { - return stats; - } - - @Override - public void clearCacheView(K key) { - dataLoader.clear(key); - } - - @Override - public void clearCacheEntriesOnExceptions(List keys) { - possiblyClearCacheEntriesOnExceptions(keys); - } - }; - } - private CompletableFuture> invokeMappedBatchPublisher(List keys, List keyContexts, List> queuedFutures, BatchLoaderEnvironment environment) { CompletableFuture> loadResult = new CompletableFuture<>(); Subscriber> subscriber = ReactiveSupport.mappedBatchSubscriber(loadResult, keys, keyContexts, queuedFutures, helperIntegration()); @@ -642,4 +623,22 @@ private static DispatchResult emptyDispatchResult() { return (DispatchResult) EMPTY_DISPATCH_RESULT; } + private ReactiveSupport.HelperIntegration helperIntegration() { + return new ReactiveSupport.HelperIntegration<>() { + @Override + public StatisticsCollector getStats() { + return stats; + } + + @Override + public void clearCacheView(K key) { + dataLoader.clear(key); + } + + @Override + public void clearCacheEntriesOnExceptions(List keys) { + possiblyClearCacheEntriesOnExceptions(keys); + } + }; + } } From 4b9356e24d4966160be08eb7baf65ef357644da5 Mon Sep 17 00:00:00 2001 From: bbaker Date: Fri, 24 May 2024 13:15:47 +1000 Subject: [PATCH 038/156] Added javadoc to publisher interfaces --- .../java/org/dataloader/BatchPublisher.java | 20 +++++++++++++--- .../dataloader/BatchPublisherWithContext.java | 24 ++++++++++++++++++- .../org/dataloader/MappedBatchPublisher.java | 12 +++++++++- .../MappedBatchPublisherWithContext.java | 21 +++++++++++++++- 4 files changed, 71 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/dataloader/BatchPublisher.java b/src/main/java/org/dataloader/BatchPublisher.java index efc222a6..c4992266 100644 --- a/src/main/java/org/dataloader/BatchPublisher.java +++ b/src/main/java/org/dataloader/BatchPublisher.java @@ -7,16 +7,30 @@ /** * A function that is invoked for batch loading a stream of data values indicated by the provided list of keys. *

- * The function will call the provided {@link Subscriber} to process the values it has retrieved to allow + * The function must call the provided {@link Subscriber} to process the values it has retrieved to allow * the future returned by {@link DataLoader#load(Object)} to complete as soon as the individual value is available * (rather than when all values have been retrieved). *

- * NOTE: It is required that {@link Subscriber#onNext(Object)} is invoked on each value in the same order as - * the provided keys. + * NOTE: It is required that {@link Subscriber#onNext(Object)} is invoked on each value in the same order as + * the provided keys and that you provide a value for every key provided. * * @param type parameter indicating the type of keys to use for data load requests. * @param type parameter indicating the type of values returned + * @see BatchLoader for the non-reactive version */ public interface BatchPublisher { + /** + * Called to batch the provided keys into a stream of values. You must provide + * the same number of values as there as keys, and they must be in the order of the keys. + *

+ * The idiomatic approach would be to create a reactive {@link org.reactivestreams.Publisher} that provides + * the values given the keys and then subscribe to it with the provided {@link Subscriber}. + *

+ * NOTE: It is required that {@link Subscriber#onNext(Object)} is invoked on each value in the same order as + * the provided keys and that you provide a value for every key provided. + * + * @param keys the collection of keys to load + * @param subscriber as values arrive you must call the subscriber for each value + */ void load(List keys, Subscriber subscriber); } diff --git a/src/main/java/org/dataloader/BatchPublisherWithContext.java b/src/main/java/org/dataloader/BatchPublisherWithContext.java index effda90e..4eadfe95 100644 --- a/src/main/java/org/dataloader/BatchPublisherWithContext.java +++ b/src/main/java/org/dataloader/BatchPublisherWithContext.java @@ -5,8 +5,30 @@ import java.util.List; /** - * An {@link BatchPublisher} with a {@link BatchLoaderEnvironment} provided as an extra parameter to {@link #load}. + * This form of {@link BatchPublisher} is given a {@link org.dataloader.BatchLoaderEnvironment} object + * that encapsulates the calling context. A typical use case is passing in security credentials or database details + * for example. + *

+ * See {@link BatchPublisher} for more details on the design invariants that you must implement in order to + * use this interface. */ public interface BatchPublisherWithContext { + /** + * Called to batch the provided keys into a stream of values. You must provide + * the same number of values as there as keys, and they must be in the order of the keys. + *

+ * The idiomatic approach would be to create a reactive {@link org.reactivestreams.Publisher} that provides + * the values given the keys and then subscribe to it with the provided {@link Subscriber}. + *

+ * NOTE: It is required that {@link Subscriber#onNext(Object)} is invoked on each value in the same order as + * the provided keys and that you provide a value for every key provided. + *

+ * This is given an environment object to that maybe be useful during the call. A typical use case + * is passing in security credentials or database details for example. + * + * @param keys the collection of keys to load + * @param subscriber as values arrive you must call the subscriber for each value + * @param environment an environment object that can help with the call + */ void load(List keys, Subscriber subscriber, BatchLoaderEnvironment environment); } diff --git a/src/main/java/org/dataloader/MappedBatchPublisher.java b/src/main/java/org/dataloader/MappedBatchPublisher.java index 9b3fcb95..398e8800 100644 --- a/src/main/java/org/dataloader/MappedBatchPublisher.java +++ b/src/main/java/org/dataloader/MappedBatchPublisher.java @@ -8,13 +8,23 @@ /** * A function that is invoked for batch loading a stream of data values indicated by the provided list of keys. *

- * The function will call the provided {@link Subscriber} to process the key/value pairs it has retrieved to allow + * The function must call the provided {@link Subscriber} to process the key/value pairs it has retrieved to allow * the future returned by {@link DataLoader#load(Object)} to complete as soon as the individual value is available * (rather than when all values have been retrieved). * * @param type parameter indicating the type of keys to use for data load requests. * @param type parameter indicating the type of values returned + * @see MappedBatchLoader for the non-reactive version */ public interface MappedBatchPublisher { + /** + * Called to batch the provided keys into a stream of map entries of keys and values. + *

+ * The idiomatic approach would be to create a reactive {@link org.reactivestreams.Publisher} that provides + * the values given the keys and then subscribe to it with the provided {@link Subscriber}. + * + * @param keys the collection of keys to load + * @param subscriber as values arrive you must call the subscriber for each value + */ void load(List keys, Subscriber> subscriber); } diff --git a/src/main/java/org/dataloader/MappedBatchPublisherWithContext.java b/src/main/java/org/dataloader/MappedBatchPublisherWithContext.java index 4810111a..2e94152f 100644 --- a/src/main/java/org/dataloader/MappedBatchPublisherWithContext.java +++ b/src/main/java/org/dataloader/MappedBatchPublisherWithContext.java @@ -6,8 +6,27 @@ import java.util.Map; /** - * A {@link MappedBatchPublisher} with a {@link BatchLoaderEnvironment} provided as an extra parameter to {@link #load}. + * This form of {@link MappedBatchPublisher} is given a {@link org.dataloader.BatchLoaderEnvironment} object + * that encapsulates the calling context. A typical use case is passing in security credentials or database details + * for example. + *

+ * See {@link MappedBatchPublisher} for more details on the design invariants that you must implement in order to + * use this interface. */ public interface MappedBatchPublisherWithContext { + + /** + * Called to batch the provided keys into a stream of map entries of keys and values. + *

+ * The idiomatic approach would be to create a reactive {@link org.reactivestreams.Publisher} that provides + * the values given the keys and then subscribe to it with the provided {@link Subscriber}. + *

+ * This is given an environment object to that maybe be useful during the call. A typical use case + * is passing in security credentials or database details for example. + * + * @param keys the collection of keys to load + * @param subscriber as values arrive you must call the subscriber for each value + * @param environment an environment object that can help with the call + */ void load(List keys, Subscriber> subscriber, BatchLoaderEnvironment environment); } From 3c3cc99e8e5346eea25080319b19173534b278df Mon Sep 17 00:00:00 2001 From: Alexandre Carlton Date: Sun, 26 May 2024 22:38:56 +1000 Subject: [PATCH 039/156] Have MappedBatchPublisher take in a Set keys This is more symmetric with `MappedbatchLoader` and preserves efficiency; we do not need to emit a `Map.Entry` for duplicate keys (given the strong intention that this will be used to create a `Map`). --- src/main/java/org/dataloader/DataLoaderHelper.java | 6 +++--- .../java/org/dataloader/MappedBatchPublisher.java | 4 ++-- src/test/java/org/dataloader/DataLoaderTest.java | 2 +- .../java/org/dataloader/fixtures/UserManager.java | 11 +++++++++++ .../MappedPublisherDataLoaderFactory.java | 8 ++++++-- 5 files changed, 23 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/dataloader/DataLoaderHelper.java b/src/main/java/org/dataloader/DataLoaderHelper.java index 62a7cb63..9cd38d6c 100644 --- a/src/main/java/org/dataloader/DataLoaderHelper.java +++ b/src/main/java/org/dataloader/DataLoaderHelper.java @@ -536,7 +536,7 @@ private CompletableFuture> invokeBatchPublisher(List keys, List> invokeMappedBatchPublisher(List keys, List keyContexts, List> queuedFutures, BatchLoaderEnvironment environment) { CompletableFuture> loadResult = new CompletableFuture<>(); Subscriber> subscriber = ReactiveSupport.mappedBatchSubscriber(loadResult, keys, keyContexts, queuedFutures, helperIntegration()); - + Set setOfKeys = new LinkedHashSet<>(keys); BatchLoaderScheduler batchLoaderScheduler = loaderOptions.getBatchLoaderScheduler(); if (batchLoadFunction instanceof MappedBatchPublisherWithContext) { //noinspection unchecked @@ -551,10 +551,10 @@ private CompletableFuture> invokeMappedBatchPublisher(List keys, List //noinspection unchecked MappedBatchPublisher loadFunction = (MappedBatchPublisher) batchLoadFunction; if (batchLoaderScheduler != null) { - BatchLoaderScheduler.ScheduledBatchPublisherCall loadCall = () -> loadFunction.load(keys, subscriber); + BatchLoaderScheduler.ScheduledBatchPublisherCall loadCall = () -> loadFunction.load(setOfKeys, subscriber); batchLoaderScheduler.scheduleBatchPublisher(loadCall, keys, null); } else { - loadFunction.load(keys, subscriber); + loadFunction.load(setOfKeys, subscriber); } } return loadResult; diff --git a/src/main/java/org/dataloader/MappedBatchPublisher.java b/src/main/java/org/dataloader/MappedBatchPublisher.java index 398e8800..754ee52c 100644 --- a/src/main/java/org/dataloader/MappedBatchPublisher.java +++ b/src/main/java/org/dataloader/MappedBatchPublisher.java @@ -2,8 +2,8 @@ import org.reactivestreams.Subscriber; -import java.util.List; import java.util.Map; +import java.util.Set; /** * A function that is invoked for batch loading a stream of data values indicated by the provided list of keys. @@ -26,5 +26,5 @@ public interface MappedBatchPublisher { * @param keys the collection of keys to load * @param subscriber as values arrive you must call the subscriber for each value */ - void load(List keys, Subscriber> subscriber); + void load(Set keys, Subscriber> subscriber); } diff --git a/src/test/java/org/dataloader/DataLoaderTest.java b/src/test/java/org/dataloader/DataLoaderTest.java index 1f748fb1..1ce34ea9 100644 --- a/src/test/java/org/dataloader/DataLoaderTest.java +++ b/src/test/java/org/dataloader/DataLoaderTest.java @@ -740,7 +740,7 @@ public void should_work_with_duplicate_keys_when_caching_disabled(TestDataLoader assertThat(future1.get(), equalTo("A")); assertThat(future2.get(), equalTo("B")); assertThat(future3.get(), equalTo("A")); - if (factory instanceof MappedDataLoaderFactory) { + if (factory instanceof MappedDataLoaderFactory || factory instanceof MappedPublisherDataLoaderFactory) { assertThat(loadCalls, equalTo(singletonList(asList("A", "B")))); } else { assertThat(loadCalls, equalTo(singletonList(asList("A", "B", "A")))); diff --git a/src/test/java/org/dataloader/fixtures/UserManager.java b/src/test/java/org/dataloader/fixtures/UserManager.java index 24fee0d2..4fed3f71 100644 --- a/src/test/java/org/dataloader/fixtures/UserManager.java +++ b/src/test/java/org/dataloader/fixtures/UserManager.java @@ -1,5 +1,8 @@ package org.dataloader.fixtures; +import org.reactivestreams.Subscriber; +import reactor.core.publisher.Flux; + import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; @@ -52,6 +55,14 @@ public List loadUsersById(List userIds) { return userIds.stream().map(this::loadUserById).collect(Collectors.toList()); } + public void publishUsersById(List userIds, Subscriber userSubscriber) { + Flux.fromIterable(loadUsersById(userIds)).subscribe(userSubscriber); + } + + public void publishUsersById(Set userIds, Subscriber> userEntrySubscriber) { + Flux.fromIterable(loadMapOfUsersByIds(null, userIds).entrySet()).subscribe(userEntrySubscriber); + } + public Map loadMapOfUsersByIds(SecurityCtx callCtx, Set userIds) { Map map = new HashMap<>(); userIds.forEach(userId -> { diff --git a/src/test/java/org/dataloader/fixtures/parameterized/MappedPublisherDataLoaderFactory.java b/src/test/java/org/dataloader/fixtures/parameterized/MappedPublisherDataLoaderFactory.java index f5c1ad5e..9c923304 100644 --- a/src/test/java/org/dataloader/fixtures/parameterized/MappedPublisherDataLoaderFactory.java +++ b/src/test/java/org/dataloader/fixtures/parameterized/MappedPublisherDataLoaderFactory.java @@ -10,8 +10,12 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; import java.util.stream.Stream; +import static java.util.stream.Collectors.toList; +import static java.util.stream.Collectors.toSet; import static org.dataloader.DataLoaderFactory.newMappedPublisherDataLoader; import static org.dataloader.DataLoaderFactory.newMappedPublisherDataLoaderWithTry; @@ -69,7 +73,7 @@ public DataLoader idLoaderBlowsUpsAfterN(int N, DataLoaderOptions opti return newMappedPublisherDataLoader((keys, subscriber) -> { loadCalls.add(new ArrayList<>(keys)); - List nKeys = keys.subList(0, N); + List nKeys = keys.stream().limit(N).collect(toList()); Flux> subFlux = Flux.fromIterable(nKeys).map(k -> Map.entry(k, k)); subFlux.concatWith(Flux.error(new IllegalStateException("Error"))) .subscribe(subscriber); @@ -81,7 +85,7 @@ public DataLoader onlyReturnsNValues(int N, DataLoaderOptions op return newMappedPublisherDataLoader((keys, subscriber) -> { loadCalls.add(new ArrayList<>(keys)); - List nKeys = keys.subList(0, N); + List nKeys = keys.stream().limit(N).collect(toList()); Flux.fromIterable(nKeys).map(k -> Map.entry(k, k)) .subscribe(subscriber); }, options); From 2e828581ab89a7fc75ad09433186229f8644cf71 Mon Sep 17 00:00:00 2001 From: Alexandre Carlton Date: Sun, 26 May 2024 23:02:03 +1000 Subject: [PATCH 040/156] Add README sections for `*BatchPublisher` --- README.md | 66 +++++++++++++++++++++++++++++++ src/test/java/ReadmeExamples.java | 25 +++++++++++- 2 files changed, 90 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 24a65f68..9ffe265c 100644 --- a/README.md +++ b/README.md @@ -286,6 +286,66 @@ For example, let's assume you want to load users from a database, you could prob // ... ``` +### Returning a stream of results from your batch publisher + +It may be that your batch loader function is a [Reactive Streams](https://www.reactive-streams.org/) [Publisher](https://www.reactive-streams.org/reactive-streams-1.0.3-javadoc/org/reactivestreams/Publisher.html), where values are emitted as an asynchronous stream. + +For example, let's say you wanted to load many users from a service without forcing the service to load all +users into its memory (which may exert considerable pressure on it). + +A `org.dataloader.BatchPublisher` may be used to load this data: + +```java + BatchPublisher batchPublisher = new BatchPublisher() { + @Override + public void load(List userIds, Subscriber userSubscriber) { + userManager.publishUsersById(userIds, userSubscriber); + } + }; + DataLoader userLoader = DataLoaderFactory.newPublisherDataLoader(batchPublisher); + + // ... +``` + +Rather than waiting for all values to be returned, this `DataLoader` will complete +the `CompletableFuture` returned by `Dataloader#load(Long)` as each value is +processed. + +If an exception is thrown, the remaining futures yet to be completed are completed +exceptionally. + +You *MUST* ensure that the values are streamed in the same order as the keys provided, +with the same cardinality (i.e. the number of values must match the number of keys). +Failing to do so will result in incorrect data being returned from `DataLoader#load`. + + +### Returning a mapped stream of results from your batch publisher + +Your publisher may not necessarily return values in the same order in which it processes keys. + +For example, let's say your batch publisher function loads user data which is spread across shards, +with some shards responding more quickly than others. + +In instances like these, `org.dataloader.MappedBatchPublisher` can be used. + +```java + MappedBatchPublisher mappedBatchPublisher = new MappedBatchPublisher() { + @Override + public void load(Set userIds, Subscriber> userEntrySubscriber) { + userManager.publishUsersById(userIds, userEntrySubscriber); + } + }; + DataLoader userLoader = DataLoaderFactory.newMappedPublisherDataLoader(mappedBatchPublisher); + + // ... +``` + +Like the `BatchPublisher`, if an exception is thrown, the remaining futures yet to be completed are completed +exceptionally. + +Unlike the `BatchPublisher`, however, it is not necessary to return values in the same order as the provided keys, +or even the same number of values. + ### Error object is not a thing in a type safe Java world In the reference JS implementation if the batch loader returns an `Error` object back from the `load()` promise is rejected @@ -541,6 +601,12 @@ The following is a `BatchLoaderScheduler` that waits 10 milliseconds before invo return scheduledCall.invoke(); }).thenCompose(Function.identity()); } + + @Override + public void scheduleBatchPublisher(ScheduledBatchPublisherCall scheduledCall, List keys, BatchLoaderEnvironment environment) { + snooze(10); + scheduledCall.invoke(); + } }; ``` diff --git a/src/test/java/ReadmeExamples.java b/src/test/java/ReadmeExamples.java index 31354eaa..a20c0ea9 100644 --- a/src/test/java/ReadmeExamples.java +++ b/src/test/java/ReadmeExamples.java @@ -1,11 +1,13 @@ import org.dataloader.BatchLoader; import org.dataloader.BatchLoaderEnvironment; import org.dataloader.BatchLoaderWithContext; +import org.dataloader.BatchPublisher; import org.dataloader.CacheMap; import org.dataloader.DataLoader; import org.dataloader.DataLoaderFactory; import org.dataloader.DataLoaderOptions; import org.dataloader.MappedBatchLoaderWithContext; +import org.dataloader.MappedBatchPublisher; import org.dataloader.Try; import org.dataloader.fixtures.SecurityCtx; import org.dataloader.fixtures.User; @@ -15,6 +17,7 @@ import org.dataloader.scheduler.BatchLoaderScheduler; import org.dataloader.stats.Statistics; import org.dataloader.stats.ThreadLocalStatisticsCollector; +import org.reactivestreams.Subscriber; import java.time.Duration; import java.util.ArrayList; @@ -171,7 +174,7 @@ private void tryExample() { } } - private void tryBatcLoader() { + private void tryBatchLoader() { DataLoader dataLoader = DataLoaderFactory.newDataLoaderWithTry(new BatchLoader>() { @Override public CompletionStage>> load(List keys) { @@ -187,6 +190,26 @@ public CompletionStage>> load(List keys) { }); } + private void batchPublisher() { + BatchPublisher batchPublisher = new BatchPublisher() { + @Override + public void load(List userIds, Subscriber userSubscriber) { + userManager.publishUsersById(userIds, userSubscriber); + } + }; + DataLoader userLoader = DataLoaderFactory.newPublisherDataLoader(batchPublisher); + } + + private void mappedBatchPublisher() { + MappedBatchPublisher mappedBatchPublisher = new MappedBatchPublisher() { + @Override + public void load(Set userIds, Subscriber> userEntrySubscriber) { + userManager.publishUsersById(userIds, userEntrySubscriber); + } + }; + DataLoader userLoader = DataLoaderFactory.newMappedPublisherDataLoader(mappedBatchPublisher); + } + DataLoader userDataLoader; private void clearCacheOnError() { From d0820fc8a780faac6fbf501ce24a3e947c062cc0 Mon Sep 17 00:00:00 2001 From: bbaker Date: Mon, 27 May 2024 19:46:09 +1000 Subject: [PATCH 041/156] Tweaked readme --- README.md | 23 ++++++++++++++----- src/test/java/ReadmeExamples.java | 8 +++++-- .../org/dataloader/fixtures/UserManager.java | 9 ++++---- 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 9ffe265c..da5d063a 100644 --- a/README.md +++ b/README.md @@ -288,7 +288,7 @@ For example, let's assume you want to load users from a database, you could prob ### Returning a stream of results from your batch publisher -It may be that your batch loader function is a [Reactive Streams](https://www.reactive-streams.org/) [Publisher](https://www.reactive-streams.org/reactive-streams-1.0.3-javadoc/org/reactivestreams/Publisher.html), where values are emitted as an asynchronous stream. +It may be that your batch loader function can use a [Reactive Streams](https://www.reactive-streams.org/) [Publisher](https://www.reactive-streams.org/reactive-streams-1.0.3-javadoc/org/reactivestreams/Publisher.html), where values are emitted as an asynchronous stream. For example, let's say you wanted to load many users from a service without forcing the service to load all users into its memory (which may exert considerable pressure on it). @@ -299,7 +299,8 @@ A `org.dataloader.BatchPublisher` may be used to load this data: BatchPublisher batchPublisher = new BatchPublisher() { @Override public void load(List userIds, Subscriber userSubscriber) { - userManager.publishUsersById(userIds, userSubscriber); + Publisher userResults = userManager.publishUsersById(userIds); + userResults.subscribe(userSubscriber); } }; DataLoader userLoader = DataLoaderFactory.newPublisherDataLoader(batchPublisher); @@ -307,21 +308,28 @@ A `org.dataloader.BatchPublisher` may be used to load this data: // ... ``` -Rather than waiting for all values to be returned, this `DataLoader` will complete +Rather than waiting for all user values to be returned on one batch, this `DataLoader` will complete the `CompletableFuture` returned by `Dataloader#load(Long)` as each value is -processed. +published. + +This pattern means that data loader values can (in theory) be satisfied more quickly than if we wait for +all results in the batch to be retrieved and hence the overall result may finish more quickly. If an exception is thrown, the remaining futures yet to be completed are completed exceptionally. You *MUST* ensure that the values are streamed in the same order as the keys provided, with the same cardinality (i.e. the number of values must match the number of keys). + Failing to do so will result in incorrect data being returned from `DataLoader#load`. +`BatchPublisher` is the reactive version of `BatchLoader`. + ### Returning a mapped stream of results from your batch publisher -Your publisher may not necessarily return values in the same order in which it processes keys. +Your publisher may not necessarily return values in the same order in which it processes keys and it +may not be able to find a value for each key presented. For example, let's say your batch publisher function loads user data which is spread across shards, with some shards responding more quickly than others. @@ -332,7 +340,8 @@ In instances like these, `org.dataloader.MappedBatchPublisher` can be used. MappedBatchPublisher mappedBatchPublisher = new MappedBatchPublisher() { @Override public void load(Set userIds, Subscriber> userEntrySubscriber) { - userManager.publishUsersById(userIds, userEntrySubscriber); + Publisher> userEntries = userManager.publishUsersById(userIds); + userEntries.subscribe(userEntrySubscriber); } }; DataLoader userLoader = DataLoaderFactory.newMappedPublisherDataLoader(mappedBatchPublisher); @@ -346,6 +355,8 @@ exceptionally. Unlike the `BatchPublisher`, however, it is not necessary to return values in the same order as the provided keys, or even the same number of values. +`MappedBatchPublisher` is the reactive version of `MappedBatchLoader`. + ### Error object is not a thing in a type safe Java world In the reference JS implementation if the batch loader returns an `Error` object back from the `load()` promise is rejected diff --git a/src/test/java/ReadmeExamples.java b/src/test/java/ReadmeExamples.java index a20c0ea9..8baba6a3 100644 --- a/src/test/java/ReadmeExamples.java +++ b/src/test/java/ReadmeExamples.java @@ -17,7 +17,9 @@ import org.dataloader.scheduler.BatchLoaderScheduler; import org.dataloader.stats.Statistics; import org.dataloader.stats.ThreadLocalStatisticsCollector; +import org.reactivestreams.Publisher; import org.reactivestreams.Subscriber; +import reactor.core.publisher.Flux; import java.time.Duration; import java.util.ArrayList; @@ -194,7 +196,8 @@ private void batchPublisher() { BatchPublisher batchPublisher = new BatchPublisher() { @Override public void load(List userIds, Subscriber userSubscriber) { - userManager.publishUsersById(userIds, userSubscriber); + Publisher userResults = userManager.publishUsersById(userIds); + userResults.subscribe(userSubscriber); } }; DataLoader userLoader = DataLoaderFactory.newPublisherDataLoader(batchPublisher); @@ -204,7 +207,8 @@ private void mappedBatchPublisher() { MappedBatchPublisher mappedBatchPublisher = new MappedBatchPublisher() { @Override public void load(Set userIds, Subscriber> userEntrySubscriber) { - userManager.publishUsersById(userIds, userEntrySubscriber); + Publisher> userEntries = userManager.publishUsersById(userIds); + userEntries.subscribe(userEntrySubscriber); } }; DataLoader userLoader = DataLoaderFactory.newMappedPublisherDataLoader(mappedBatchPublisher); diff --git a/src/test/java/org/dataloader/fixtures/UserManager.java b/src/test/java/org/dataloader/fixtures/UserManager.java index 4fed3f71..6ce21691 100644 --- a/src/test/java/org/dataloader/fixtures/UserManager.java +++ b/src/test/java/org/dataloader/fixtures/UserManager.java @@ -1,5 +1,6 @@ package org.dataloader.fixtures; +import org.reactivestreams.Publisher; import org.reactivestreams.Subscriber; import reactor.core.publisher.Flux; @@ -55,12 +56,12 @@ public List loadUsersById(List userIds) { return userIds.stream().map(this::loadUserById).collect(Collectors.toList()); } - public void publishUsersById(List userIds, Subscriber userSubscriber) { - Flux.fromIterable(loadUsersById(userIds)).subscribe(userSubscriber); + public Publisher publishUsersById(List userIds) { + return Flux.fromIterable(loadUsersById(userIds)); } - public void publishUsersById(Set userIds, Subscriber> userEntrySubscriber) { - Flux.fromIterable(loadMapOfUsersByIds(null, userIds).entrySet()).subscribe(userEntrySubscriber); + public Publisher> publishUsersById(Set userIds) { + return Flux.fromIterable(loadMapOfUsersByIds(null, userIds).entrySet()); } public Map loadMapOfUsersByIds(SecurityCtx callCtx, Set userIds) { From f759b34afeffa4632c0c6414a2588aa3ed842bd9 Mon Sep 17 00:00:00 2001 From: bbaker Date: Tue, 28 May 2024 09:43:06 +1000 Subject: [PATCH 042/156] Tweaked readme - streamUsersById renamed --- README.md | 4 ++-- src/test/java/ReadmeExamples.java | 5 ++--- src/test/java/org/dataloader/fixtures/UserManager.java | 5 ++--- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index da5d063a..fb38e69e 100644 --- a/README.md +++ b/README.md @@ -299,7 +299,7 @@ A `org.dataloader.BatchPublisher` may be used to load this data: BatchPublisher batchPublisher = new BatchPublisher() { @Override public void load(List userIds, Subscriber userSubscriber) { - Publisher userResults = userManager.publishUsersById(userIds); + Publisher userResults = userManager.streamUsersById(userIds); userResults.subscribe(userSubscriber); } }; @@ -340,7 +340,7 @@ In instances like these, `org.dataloader.MappedBatchPublisher` can be used. MappedBatchPublisher mappedBatchPublisher = new MappedBatchPublisher() { @Override public void load(Set userIds, Subscriber> userEntrySubscriber) { - Publisher> userEntries = userManager.publishUsersById(userIds); + Publisher> userEntries = userManager.streamUsersById(userIds); userEntries.subscribe(userEntrySubscriber); } }; diff --git a/src/test/java/ReadmeExamples.java b/src/test/java/ReadmeExamples.java index 8baba6a3..9e30c903 100644 --- a/src/test/java/ReadmeExamples.java +++ b/src/test/java/ReadmeExamples.java @@ -19,7 +19,6 @@ import org.dataloader.stats.ThreadLocalStatisticsCollector; import org.reactivestreams.Publisher; import org.reactivestreams.Subscriber; -import reactor.core.publisher.Flux; import java.time.Duration; import java.util.ArrayList; @@ -196,7 +195,7 @@ private void batchPublisher() { BatchPublisher batchPublisher = new BatchPublisher() { @Override public void load(List userIds, Subscriber userSubscriber) { - Publisher userResults = userManager.publishUsersById(userIds); + Publisher userResults = userManager.streamUsersById(userIds); userResults.subscribe(userSubscriber); } }; @@ -207,7 +206,7 @@ private void mappedBatchPublisher() { MappedBatchPublisher mappedBatchPublisher = new MappedBatchPublisher() { @Override public void load(Set userIds, Subscriber> userEntrySubscriber) { - Publisher> userEntries = userManager.publishUsersById(userIds); + Publisher> userEntries = userManager.streamUsersById(userIds); userEntries.subscribe(userEntrySubscriber); } }; diff --git a/src/test/java/org/dataloader/fixtures/UserManager.java b/src/test/java/org/dataloader/fixtures/UserManager.java index 6ce21691..1d2ff1fe 100644 --- a/src/test/java/org/dataloader/fixtures/UserManager.java +++ b/src/test/java/org/dataloader/fixtures/UserManager.java @@ -1,7 +1,6 @@ package org.dataloader.fixtures; import org.reactivestreams.Publisher; -import org.reactivestreams.Subscriber; import reactor.core.publisher.Flux; import java.util.HashMap; @@ -56,11 +55,11 @@ public List loadUsersById(List userIds) { return userIds.stream().map(this::loadUserById).collect(Collectors.toList()); } - public Publisher publishUsersById(List userIds) { + public Publisher streamUsersById(List userIds) { return Flux.fromIterable(loadUsersById(userIds)); } - public Publisher> publishUsersById(Set userIds) { + public Publisher> streamUsersById(Set userIds) { return Flux.fromIterable(loadMapOfUsersByIds(null, userIds).entrySet()); } From 23a311013141f9862b1dfe36993bc814b6cfd3d4 Mon Sep 17 00:00:00 2001 From: dondonz <13839920+dondonz@users.noreply.github.com> Date: Thu, 12 Dec 2024 15:21:53 +1100 Subject: [PATCH 043/156] Update version in readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fb38e69e..4f5622d3 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ repositories { } dependencies { - compile 'com.graphql-java:java-dataloader: 3.1.0' + compile 'com.graphql-java:java-dataloader: 3.3.0' } ``` From 22413bed249295455c53830b438fa96ff6c19551 Mon Sep 17 00:00:00 2001 From: Harrison Cole Date: Thu, 12 Dec 2024 15:30:57 +1100 Subject: [PATCH 044/156] `DataLoader` and `CompletableFutureKit`: add support for `loadMany` given a Map of keys and context objects. --- src/main/java/org/dataloader/DataLoader.java | 34 ++++++++++++++++--- .../dataloader/impl/CompletableFutureKit.java | 15 +++++++- 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/dataloader/DataLoader.java b/src/main/java/org/dataloader/DataLoader.java index 1e4ce7da..dc4b726c 100644 --- a/src/main/java/org/dataloader/DataLoader.java +++ b/src/main/java/org/dataloader/DataLoader.java @@ -25,10 +25,7 @@ import java.time.Clock; import java.time.Duration; import java.time.Instant; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Optional; +import java.util.*; import java.util.concurrent.CompletableFuture; import java.util.function.BiConsumer; @@ -574,6 +571,35 @@ public CompletableFuture> loadMany(List keys, List keyContext } } + /** + * Requests to load the map of data provided by the specified keys asynchronously, and returns a composite future + * of the resulting values. + *

+ * If batching is enabled (the default), you'll have to call {@link DataLoader#dispatch()} at a later stage to + * start batch execution. If you forget this call the future will never be completed (unless already completed, + * and returned from cache). + *

+ * The key context object may be useful in the batch loader interfaces such as {@link org.dataloader.BatchLoaderWithContext} or + * {@link org.dataloader.MappedBatchLoaderWithContext} to help retrieve data. + * + * @param keysAndContexts the map of keys to their respective contexts + * + * @return the composite future of the map of keys and values + */ + public CompletableFuture> loadMany(Map keysAndContexts) { + nonNull(keysAndContexts); + + synchronized (this) { + Map> collect = new HashMap<>(keysAndContexts.size()); + for (Map.Entry entry : keysAndContexts.entrySet()) { + K key = entry.getKey(); + Object keyContext = entry.getValue(); + collect.put(key, load(key, keyContext)); + } + return CompletableFutureKit.allOf(collect); + } + } + /** * Dispatches the queued load requests to the batch execution function and returns a promise of the result. *

diff --git a/src/main/java/org/dataloader/impl/CompletableFutureKit.java b/src/main/java/org/dataloader/impl/CompletableFutureKit.java index 2b94d10b..ebc35eca 100644 --- a/src/main/java/org/dataloader/impl/CompletableFutureKit.java +++ b/src/main/java/org/dataloader/impl/CompletableFutureKit.java @@ -3,8 +3,10 @@ import org.dataloader.annotations.Internal; import java.util.List; +import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; import static java.util.stream.Collectors.toList; @@ -48,10 +50,21 @@ public static boolean failed(CompletableFuture future) { } public static CompletableFuture> allOf(List> cfs) { - return CompletableFuture.allOf(cfs.toArray(new CompletableFuture[0])) + return CompletableFuture.allOf(cfs.toArray(CompletableFuture[]::new)) .thenApply(v -> cfs.stream() .map(CompletableFuture::join) .collect(toList()) ); } + + public static CompletableFuture> allOf(Map> cfs) { + return CompletableFuture.allOf(cfs.values().toArray(CompletableFuture[]::new)) + .thenApply(v -> cfs.entrySet().stream() + .collect( + Collectors.toMap( + Map.Entry::getKey, + task -> task.getValue().join()) + ) + ); + } } From 2f437a75dac6a68de201e66d5f4f294e8a6b8589 Mon Sep 17 00:00:00 2001 From: Harrison Cole Date: Thu, 12 Dec 2024 17:16:55 +1100 Subject: [PATCH 045/156] `DataLoader` and `CompletableFutureKit`: add test cases for `loadMany` given a Map of keys and context objects. --- .../DataLoaderBatchLoaderEnvironmentTest.java | 56 ++++++++--- .../org/dataloader/DataLoaderStatsTest.java | 12 ++- .../java/org/dataloader/DataLoaderTest.java | 93 +++++++++++++++---- 3 files changed, 124 insertions(+), 37 deletions(-) diff --git a/src/test/java/org/dataloader/DataLoaderBatchLoaderEnvironmentTest.java b/src/test/java/org/dataloader/DataLoaderBatchLoaderEnvironmentTest.java index 6d3010e0..90adbc5d 100644 --- a/src/test/java/org/dataloader/DataLoaderBatchLoaderEnvironmentTest.java +++ b/src/test/java/org/dataloader/DataLoaderBatchLoaderEnvironmentTest.java @@ -2,10 +2,7 @@ import org.junit.jupiter.api.Test; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; @@ -50,10 +47,14 @@ public void context_is_passed_to_batch_loader_function() { loader.load("A"); loader.load("B"); loader.loadMany(asList("C", "D")); + Map keysAndContexts = new LinkedHashMap<>(); + keysAndContexts.put("E", null); + keysAndContexts.put("F", null); + loader.loadMany(keysAndContexts); List results = loader.dispatchAndJoin(); - assertThat(results, equalTo(asList("A-ctx", "B-ctx", "C-ctx", "D-ctx"))); + assertThat(results, equalTo(asList("A-ctx", "B-ctx", "C-ctx", "D-ctx", "E-ctx", "F-ctx"))); } @Test @@ -66,10 +67,14 @@ public void key_contexts_are_passed_to_batch_loader_function() { loader.load("A", "aCtx"); loader.load("B", "bCtx"); loader.loadMany(asList("C", "D"), asList("cCtx", "dCtx")); + Map keysAndContexts = new LinkedHashMap<>(); + keysAndContexts.put("E", "eCtx"); + keysAndContexts.put("F", "fCtx"); + loader.loadMany(keysAndContexts); List results = loader.dispatchAndJoin(); - assertThat(results, equalTo(asList("A-ctx-m:aCtx-l:aCtx", "B-ctx-m:bCtx-l:bCtx", "C-ctx-m:cCtx-l:cCtx", "D-ctx-m:dCtx-l:dCtx"))); + assertThat(results, equalTo(asList("A-ctx-m:aCtx-l:aCtx", "B-ctx-m:bCtx-l:bCtx", "C-ctx-m:cCtx-l:cCtx", "D-ctx-m:dCtx-l:dCtx", "E-ctx-m:eCtx-l:eCtx", "F-ctx-m:fCtx-l:fCtx"))); } @Test @@ -82,12 +87,17 @@ public void key_contexts_are_passed_to_batch_loader_function_when_batching_disab CompletableFuture aLoad = loader.load("A", "aCtx"); CompletableFuture bLoad = loader.load("B", "bCtx"); - CompletableFuture> canDLoad = loader.loadMany(asList("C", "D"), asList("cCtx", "dCtx")); + CompletableFuture> cAndDLoad = loader.loadMany(asList("C", "D"), asList("cCtx", "dCtx")); + Map keysAndContexts = new LinkedHashMap<>(); + keysAndContexts.put("E", "eCtx"); + keysAndContexts.put("F", "fCtx"); + CompletableFuture> eAndFLoad = loader.loadMany(keysAndContexts); List results = new ArrayList<>(asList(aLoad.join(), bLoad.join())); - results.addAll(canDLoad.join()); + results.addAll(cAndDLoad.join()); + results.addAll(eAndFLoad.join().values()); - assertThat(results, equalTo(asList("A-ctx-m:aCtx-l:aCtx", "B-ctx-m:bCtx-l:bCtx", "C-ctx-m:cCtx-l:cCtx", "D-ctx-m:dCtx-l:dCtx"))); + assertThat(results, equalTo(asList("A-ctx-m:aCtx-l:aCtx", "B-ctx-m:bCtx-l:bCtx", "C-ctx-m:cCtx-l:cCtx", "D-ctx-m:dCtx-l:dCtx", "E-ctx-m:eCtx-l:eCtx", "F-ctx-m:fCtx-l:fCtx"))); } @Test @@ -101,9 +111,14 @@ public void missing_key_contexts_are_passed_to_batch_loader_function() { loader.load("B"); loader.loadMany(asList("C", "D"), singletonList("cCtx")); + Map keysAndContexts = new LinkedHashMap<>(); + keysAndContexts.put("E", "eCtx"); + keysAndContexts.put("F", null); + loader.loadMany(keysAndContexts); + List results = loader.dispatchAndJoin(); - assertThat(results, equalTo(asList("A-ctx-m:aCtx-l:aCtx", "B-ctx-m:null-l:null", "C-ctx-m:cCtx-l:cCtx", "D-ctx-m:null-l:null"))); + assertThat(results, equalTo(asList("A-ctx-m:aCtx-l:aCtx", "B-ctx-m:null-l:null", "C-ctx-m:cCtx-l:cCtx", "D-ctx-m:null-l:null", "E-ctx-m:eCtx-l:eCtx", "F-ctx-m:null-l:null"))); } @Test @@ -125,9 +140,14 @@ public void context_is_passed_to_map_batch_loader_function() { loader.load("B"); loader.loadMany(asList("C", "D"), singletonList("cCtx")); + Map keysAndContexts = new LinkedHashMap<>(); + keysAndContexts.put("E", "eCtx"); + keysAndContexts.put("F", null); + loader.loadMany(keysAndContexts); + List results = loader.dispatchAndJoin(); - assertThat(results, equalTo(asList("A-ctx-aCtx", "B-ctx-null", "C-ctx-cCtx", "D-ctx-null"))); + assertThat(results, equalTo(asList("A-ctx-aCtx", "B-ctx-null", "C-ctx-cCtx", "D-ctx-null", "E-ctx-eCtx", "F-ctx-null"))); } @Test @@ -142,9 +162,14 @@ public void null_is_passed_as_context_if_you_do_nothing() { loader.load("B"); loader.loadMany(asList("C", "D")); + Map keysAndContexts = new LinkedHashMap<>(); + keysAndContexts.put("E", null); + keysAndContexts.put("F", null); + loader.loadMany(keysAndContexts); + List results = loader.dispatchAndJoin(); - assertThat(results, equalTo(asList("A-null", "B-null", "C-null", "D-null"))); + assertThat(results, equalTo(asList("A-null", "B-null", "C-null", "D-null", "E-null", "F-null"))); } @Test @@ -160,9 +185,14 @@ public void null_is_passed_as_context_to_map_loader_if_you_do_nothing() { loader.load("B"); loader.loadMany(asList("C", "D")); + Map keysAndContexts = new LinkedHashMap<>(); + keysAndContexts.put("E", null); + keysAndContexts.put("F", null); + loader.loadMany(keysAndContexts); + List results = loader.dispatchAndJoin(); - assertThat(results, equalTo(asList("A-null", "B-null", "C-null", "D-null"))); + assertThat(results, equalTo(asList("A-null", "B-null", "C-null", "D-null", "E-null", "F-null"))); } @Test diff --git a/src/test/java/org/dataloader/DataLoaderStatsTest.java b/src/test/java/org/dataloader/DataLoaderStatsTest.java index 06e8ae6b..b8393e63 100644 --- a/src/test/java/org/dataloader/DataLoaderStatsTest.java +++ b/src/test/java/org/dataloader/DataLoaderStatsTest.java @@ -13,6 +13,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.concurrent.CompletableFuture; import static java.util.Arrays.asList; @@ -118,9 +119,10 @@ public void stats_are_collected_with_caching_disabled() { loader.load("A"); loader.load("B"); loader.loadMany(asList("C", "D")); + loader.loadMany(Map.of("E", "E", "F", "F")); Statistics stats = loader.getStatistics(); - assertThat(stats.getLoadCount(), equalTo(4L)); + assertThat(stats.getLoadCount(), equalTo(6L)); assertThat(stats.getBatchInvokeCount(), equalTo(0L)); assertThat(stats.getBatchLoadCount(), equalTo(0L)); assertThat(stats.getCacheHitCount(), equalTo(0L)); @@ -128,9 +130,9 @@ public void stats_are_collected_with_caching_disabled() { loader.dispatch(); stats = loader.getStatistics(); - assertThat(stats.getLoadCount(), equalTo(4L)); + assertThat(stats.getLoadCount(), equalTo(6L)); assertThat(stats.getBatchInvokeCount(), equalTo(1L)); - assertThat(stats.getBatchLoadCount(), equalTo(4L)); + assertThat(stats.getBatchLoadCount(), equalTo(6L)); assertThat(stats.getCacheHitCount(), equalTo(0L)); loader.load("A"); @@ -139,9 +141,9 @@ public void stats_are_collected_with_caching_disabled() { loader.dispatch(); stats = loader.getStatistics(); - assertThat(stats.getLoadCount(), equalTo(6L)); + assertThat(stats.getLoadCount(), equalTo(8L)); assertThat(stats.getBatchInvokeCount(), equalTo(2L)); - assertThat(stats.getBatchLoadCount(), equalTo(6L)); + assertThat(stats.getBatchLoadCount(), equalTo(8L)); assertThat(stats.getCacheHitCount(), equalTo(0L)); } diff --git a/src/test/java/org/dataloader/DataLoaderTest.java b/src/test/java/org/dataloader/DataLoaderTest.java index 1ce34ea9..c4ae883e 100644 --- a/src/test/java/org/dataloader/DataLoaderTest.java +++ b/src/test/java/org/dataloader/DataLoaderTest.java @@ -35,24 +35,19 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; +import java.util.*; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; import static java.util.Arrays.asList; -import static java.util.Collections.emptyList; -import static java.util.Collections.singletonList; +import static java.util.Collections.*; import static java.util.concurrent.CompletableFuture.*; import static org.awaitility.Awaitility.await; import static org.dataloader.DataLoaderFactory.newDataLoader; @@ -66,10 +61,7 @@ import static org.dataloader.fixtures.TestKit.listFrom; import static org.dataloader.impl.CompletableFutureKit.cause; import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.empty; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.instanceOf; -import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.*; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.fail; @@ -116,19 +108,25 @@ public void basic_map_batch_loading() { }; DataLoader loader = DataLoaderFactory.newMappedDataLoader(evensOnlyMappedBatchLoader); + final List keys = asList("C", "D"); + final Map keysAndContexts = new LinkedHashMap<>(); + keysAndContexts.put("E", null); + keysAndContexts.put("F", null); + loader.load("A"); loader.load("B"); - loader.loadMany(asList("C", "D")); + loader.loadMany(keys); + loader.loadMany(keysAndContexts); List results = loader.dispatchAndJoin(); - assertThat(results.size(), equalTo(4)); - assertThat(results, equalTo(asList("A", null, "C", null))); + assertThat(results.size(), equalTo(6)); + assertThat(results, equalTo(asList("A", null, "C", null, "E", null))); } @ParameterizedTest @MethodSource("dataLoaderFactories") - public void should_Support_loading_multiple_keys_in_one_call(TestDataLoaderFactory factory) { + public void should_Support_loading_multiple_keys_in_one_call_via_list(TestDataLoaderFactory factory) { AtomicBoolean success = new AtomicBoolean(); DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), new ArrayList<>()); @@ -142,6 +140,26 @@ public void should_Support_loading_multiple_keys_in_one_call(TestDataLoaderFacto assertThat(futureAll.toCompletableFuture().join(), equalTo(asList(1, 2))); } + @ParameterizedTest + @MethodSource("dataLoaderFactories") + public void should_Support_loading_multiple_keys_in_one_call_via_map(TestDataLoaderFactory factory) { + AtomicBoolean success = new AtomicBoolean(); + DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), new ArrayList<>()); + + final Map keysAndContexts = new LinkedHashMap<>(); + keysAndContexts.put(1, null); + keysAndContexts.put(2, null); + + CompletionStage> futureAll = identityLoader.loadMany(keysAndContexts); + futureAll.thenAccept(promisedValues -> { + assertThat(promisedValues.size(), is(2)); + success.set(true); + }); + identityLoader.dispatch(); + await().untilAtomic(success, is(true)); + assertThat(futureAll.toCompletableFuture().join(), equalTo(Map.of(1, 1, 2, 2))); + } + @ParameterizedTest @MethodSource("dataLoaderFactories") public void should_Resolve_to_empty_list_when_no_keys_supplied(TestDataLoaderFactory factory) { @@ -159,7 +177,22 @@ public void should_Resolve_to_empty_list_when_no_keys_supplied(TestDataLoaderFac @ParameterizedTest @MethodSource("dataLoaderFactories") - public void should_Return_zero_entries_dispatched_when_no_keys_supplied(TestDataLoaderFactory factory) { + public void should_Resolve_to_empty_map_when_no_keys_supplied(TestDataLoaderFactory factory) { + AtomicBoolean success = new AtomicBoolean(); + DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), new ArrayList<>()); + CompletableFuture> futureEmpty = identityLoader.loadMany(emptyMap()); + futureEmpty.thenAccept(promisedValues -> { + assertThat(promisedValues.size(), is(0)); + success.set(true); + }); + identityLoader.dispatch(); + await().untilAtomic(success, is(true)); + assertThat(futureEmpty.join(), anEmptyMap()); + } + + @ParameterizedTest + @MethodSource("dataLoaderFactories") + public void should_Return_zero_entries_dispatched_when_no_keys_supplied_via_list(TestDataLoaderFactory factory) { AtomicBoolean success = new AtomicBoolean(); DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), new ArrayList<>()); CompletableFuture> futureEmpty = identityLoader.loadMany(emptyList()); @@ -172,6 +205,21 @@ public void should_Return_zero_entries_dispatched_when_no_keys_supplied(TestData assertThat(dispatchResult.getKeysCount(), equalTo(0)); } + @ParameterizedTest + @MethodSource("dataLoaderFactories") + public void should_Return_zero_entries_dispatched_when_no_keys_supplied_via_map(TestDataLoaderFactory factory) { + AtomicBoolean success = new AtomicBoolean(); + DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), new ArrayList<>()); + CompletableFuture> futureEmpty = identityLoader.loadMany(emptyMap()); + futureEmpty.thenAccept(promisedValues -> { + assertThat(promisedValues.size(), is(0)); + success.set(true); + }); + DispatchResult dispatchResult = identityLoader.dispatchWithCounts(); + await().untilAtomic(success, is(true)); + assertThat(dispatchResult.getKeysCount(), equalTo(0)); + } + @ParameterizedTest @MethodSource("dataLoaderFactories") public void should_Batch_multiple_requests(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { @@ -286,10 +334,17 @@ public void should_Cache_on_redispatch(TestDataLoaderFactory factory) throws Exe CompletableFuture> future2 = identityLoader.loadMany(asList("A", "B")); identityLoader.dispatch(); - await().until(() -> future1.isDone() && future2.isDone()); + Map keysAndContexts = new LinkedHashMap<>(); + keysAndContexts.put("A", null); + keysAndContexts.put("C", null); + CompletableFuture> future3 = identityLoader.loadMany(keysAndContexts); + identityLoader.dispatch(); + + await().until(() -> future1.isDone() && future2.isDone() && future3.isDone()); assertThat(future1.get(), equalTo("A")); assertThat(future2.get(), equalTo(asList("A", "B"))); - assertThat(loadCalls, equalTo(asList(singletonList("A"), singletonList("B")))); + assertThat(future3.get(), equalTo(keysAndContexts.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getKey)))); + assertThat(loadCalls, equalTo(asList(singletonList("A"), singletonList("B"), singletonList("C")))); } @ParameterizedTest From a3b25b08e391d8e3ca918653f682794393f18670 Mon Sep 17 00:00:00 2001 From: Harrison Cole Date: Fri, 13 Dec 2024 16:29:00 +1100 Subject: [PATCH 046/156] Enable `org.gradle.toolchains.foojay-resolver-convention` plugin. --- settings.gradle | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/settings.gradle b/settings.gradle index e69de29b..91f1c2b1 100644 --- a/settings.gradle +++ b/settings.gradle @@ -0,0 +1,10 @@ +plugins { + id 'org.gradle.toolchains.foojay-resolver-convention' version '0.9.0' +} + + +dependencyResolutionManagement { + repositories { + mavenCentral() + } +} \ No newline at end of file From 4cc8a791c30f6943a120ec29ac0b21bf7b24136c Mon Sep 17 00:00:00 2001 From: Harrison Cole Date: Fri, 13 Dec 2024 16:29:19 +1100 Subject: [PATCH 047/156] Enable `com.gradle.develocity` plugin. --- settings.gradle | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/settings.gradle b/settings.gradle index 91f1c2b1..47404e7c 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,7 +1,18 @@ plugins { + id 'com.gradle.develocity' version '3.19' id 'org.gradle.toolchains.foojay-resolver-convention' version '0.9.0' } +develocity { + buildScan { + final def isCI = System.getenv('CI') != null; + termsOfUseUrl = "https://gradle.com/help/legal-terms-of-use" + termsOfUseAgree = "yes" + publishing.onlyIf { true } + tag(isCI ? 'CI' : 'Local') + uploadInBackground = !isCI + } +} dependencyResolutionManagement { repositories { From dc86c97bfd2411fd270ebf0b69c3ef5533fb49ec Mon Sep 17 00:00:00 2001 From: Harrison Cole Date: Fri, 13 Dec 2024 16:29:46 +1100 Subject: [PATCH 048/156] Set `CI:true` in `master.yml`, `release.yml` and `pull_request.yml`. --- .github/workflows/master.yml | 2 ++ .github/workflows/pull_request.yml | 2 ++ .github/workflows/release.yml | 2 ++ 3 files changed, 6 insertions(+) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 6073cf37..221ce090 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -21,3 +21,5 @@ jobs: java-version: '11.0.23' - name: build test and publish run: ./gradlew assemble && ./gradlew check --info && ./gradlew publishToSonatype closeAndReleaseSonatypeStagingRepository -x check --info --stacktrace + env: + CI: true diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 007bfd22..1379be0b 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -21,3 +21,5 @@ jobs: java-version: '11.0.23' - name: build and test run: ./gradlew assemble && ./gradlew check --info --stacktrace + env: + CI: true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 090fc88c..fee61a30 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -25,3 +25,5 @@ jobs: java-version: '11.0.23' - name: build test and publish run: ./gradlew assemble && ./gradlew check --info && ./gradlew publishToSonatype closeAndReleaseSonatypeStagingRepository -x check --info --stacktrace + env: + CI: true From 2b7ce184e922cc41cba3883db8afab8f49ce15be Mon Sep 17 00:00:00 2001 From: Harrison Cole Date: Fri, 13 Dec 2024 16:30:05 +1100 Subject: [PATCH 049/156] Update `README.MD` to reference `Java 11` instead of `Java 8`. --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 4f5622d3..fbfd6205 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![Latest Release](https://maven-badges.herokuapp.com/maven-central/com.graphql-java/java-dataloader/badge.svg)](https://maven-badges.herokuapp.com/maven-central/com.graphql-java/java-dataloader/) [![Apache licensed](https://img.shields.io/hexpm/l/plug.svg?maxAge=2592000)](https://github.com/graphql-java/java-dataloader/blob/master/LICENSE) -This small and simple utility library is a pure Java 8 port of [Facebook DataLoader](https://github.com/facebook/dataloader). +This small and simple utility library is a pure Java 11 port of [Facebook DataLoader](https://github.com/facebook/dataloader). It can serve as integral part of your application's data layer to provide a consistent API over various back-ends and reduce message communication overhead through batching and caching. @@ -15,7 +15,7 @@ are resolved independently and, with a true graph of objects, you may be fetchin A naive implementation of graphql data fetchers can easily lead to the dreaded "n+1" fetch problem. Most of the code is ported directly from Facebook's reference implementation, with one IMPORTANT adaptation to make -it work for Java 8. ([more on this below](#manual-dispatching)). +it work for Java 11. ([more on this below](#manual-dispatching)). Before reading on, be sure to take a short dive into the [original documentation](https://github.com/facebook/dataloader/blob/master/README.md) provided by Lee Byron (@leebyron) @@ -774,10 +774,10 @@ This library was originally written for use within a [VertX world](http://vertx. itself. All the heavy lifting has been done by this project : [vertx-dataloader](https://github.com/engagingspaces/vertx-dataloader) including the extensive testing (which itself came from Facebook). -This particular port was done to reduce the dependency on Vertx and to write a pure Java 8 implementation with no dependencies and also +This particular port was done to reduce the dependency on Vertx and to write a pure Java 11 implementation with no dependencies and also to use the more normative Java CompletableFuture. -[vertx-core](http://vertx.io/docs/vertx-core/java/) is not a lightweight library by any means so having a pure Java 8 implementation is +[vertx-core](http://vertx.io/docs/vertx-core/java/) is not a lightweight library by any means so having a pure Java 11 implementation is very desirable. From 25832e34987418b936764d3718c185ca08997cbe Mon Sep 17 00:00:00 2001 From: Harrison Cole Date: Fri, 13 Dec 2024 16:30:36 +1100 Subject: [PATCH 050/156] Configure gradle project wide settings. --- gradle.properties | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/gradle.properties b/gradle.properties index 03949461..0db64ab7 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,2 +1,18 @@ +# Project-wide Gradle settings. + +# For more details on how to configure your build environment visit +# http://www.gradle.org/docs/current/userguide/build_environment.html + +# Specifies the JVM arguments used for the daemon process. +# The setting is particularly useful for tweaking memory settings. +org.gradle.jvmargs=-Xmx4096m + +# When configured, Gradle will run in parallel mode. +# This option should only be used with decoupled projects. More details, visit +# http://www.gradle.org/docs/current/userguide/multi_project_builds.html#sec:decoupled_projects +org.gradle.parallel=true +org.gradle.caching=true + +# Bespoke settings. projectTitle = Java Dataloader projectDescription = Port of Facebook Dataloader for Java \ No newline at end of file From 240f86ac2a3345fb5fa1d09a3d172d3d96009ee7 Mon Sep 17 00:00:00 2001 From: Harrison Cole Date: Fri, 13 Dec 2024 16:30:57 +1100 Subject: [PATCH 051/156] Update gradle wrapper to latest version (8.11.1). --- gradle/wrapper/gradle-wrapper.properties | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index d2880ba8..e2847c82 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.3.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip +networkTimeout=10000 +validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists From 9e593c5a6dfe1df18ddd66a9c626d8d8b1721456 Mon Sep 17 00:00:00 2001 From: Harrison Cole Date: Fri, 13 Dec 2024 16:31:18 +1100 Subject: [PATCH 052/156] Externalise dependency versions and update `junit` and `caffeine` to latest versions. --- gradle.properties | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 0db64ab7..22d56037 100644 --- a/gradle.properties +++ b/gradle.properties @@ -15,4 +15,13 @@ org.gradle.caching=true # Bespoke settings. projectTitle = Java Dataloader -projectDescription = Port of Facebook Dataloader for Java \ No newline at end of file +projectDescription = Port of Facebook Dataloader for Java + +# Dependency versions. +junit_version=5.11.3 +hamcrest_version=2.2 +slf4j_version=1.7.30 +awaitility_version=2.0.0 +reactor_core_version=3.6.6 +caffeine_version=3.1.8 +reactive_streams_version=1.0.3 \ No newline at end of file From 11ebae9156e2d4bb79201518ad41401347e911ef Mon Sep 17 00:00:00 2001 From: Harrison Cole Date: Fri, 13 Dec 2024 16:31:46 +1100 Subject: [PATCH 053/156] Modernise gradle build configuration. --- build.gradle | 86 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 50 insertions(+), 36 deletions(-) diff --git a/build.gradle b/build.gradle index ccbcec69..70d39181 100644 --- a/build.gradle +++ b/build.gradle @@ -3,10 +3,13 @@ import java.text.SimpleDateFormat plugins { id 'java' id 'java-library' + id 'jvm-test-suite' id 'maven-publish' id 'signing' - id "biz.aQute.bnd.builder" version "6.2.0" - id "io.github.gradle-nexus.publish-plugin" version "1.0.0" + id 'groovy' + id 'biz.aQute.bnd.builder' version '6.2.0' + id 'io.github.gradle-nexus.publish-plugin' version '1.0.0' + id 'com.github.ben-manes.versions' version '0.51.0' } java { @@ -53,60 +56,65 @@ repositories { mavenLocal() } -apply plugin: 'groovy' - jar { manifest { attributes('Automatic-Module-Name': 'org.dataloader', - '-exportcontents': 'org.dataloader.*', - '-removeheaders': 'Private-Package') + '-exportcontents': 'org.dataloader.*', + '-removeheaders': 'Private-Package') } } -def slf4jVersion = '1.7.30' -def reactiveStreamsVersion = '1.0.3' - dependencies { - api 'org.slf4j:slf4j-api:' + slf4jVersion - api 'org.reactivestreams:reactive-streams:' + reactiveStreamsVersion - - testImplementation 'org.slf4j:slf4j-simple:' + slf4jVersion - testImplementation 'org.awaitility:awaitility:2.0.0' - testImplementation "org.hamcrest:hamcrest:2.2" - testImplementation 'io.projectreactor:reactor-core:3.6.6' - testImplementation 'com.github.ben-manes.caffeine:caffeine:2.9.0' - testImplementation platform('org.junit:junit-bom:5.10.2') - testImplementation 'org.junit.jupiter:junit-jupiter-api' - testImplementation 'org.junit.jupiter:junit-jupiter-params' - testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine' - testImplementation 'io.projectreactor:reactor-core:3.6.6' + api "org.slf4j:slf4j-api:$slf4j_version" + api "org.reactivestreams:reactive-streams:$reactive_streams_version" } task sourcesJar(type: Jar) { dependsOn classes - classifier 'sources' + archiveClassifier.set('sources') from sourceSets.main.allSource } -task javadocJar(type: Jar, dependsOn: javadoc) { - classifier = 'javadoc' - from javadoc.destinationDir -} - javadoc { options.encoding = 'UTF-8' } +task javadocJar(type: Jar, dependsOn: javadoc) { + archiveClassifier.set('javadoc') + from javadoc.destinationDir +} + artifacts { archives sourcesJar archives javadocJar } -test { - testLogging { - exceptionFormat = 'full' +testing { + suites { + test { + useJUnitJupiter(junit_version) + dependencies { + // Testing dependencies + implementation platform("org.junit:junit-bom:$junit_version") + implementation 'org.junit.jupiter:junit-jupiter-api' + implementation 'org.junit.jupiter:junit-jupiter-params' + implementation 'org.junit.jupiter:junit-jupiter-engine' + implementation "org.slf4j:slf4j-simple:$slf4j_version" + implementation "org.awaitility:awaitility:$awaitility_version" + implementation "org.hamcrest:hamcrest:$hamcrest_version" + implementation "io.projectreactor:reactor-core:$reactor_core_version" + implementation "com.github.ben-manes.caffeine:caffeine:$caffeine_version" + } + + targets.configureEach { + testTask.configure { + testLogging { + exceptionFormat = 'full' + } + } + } + } } - useJUnitPlatform() } publishing { @@ -180,9 +188,15 @@ tasks.withType(PublishToMavenRepository) { dependsOn build } - -task myWrapper(type: Wrapper) { - gradleVersion = '6.6.1' - distributionUrl = "https://services.gradle.org/distributions/gradle-${gradleVersion}-all.zip" +def isNonStable = { String version -> + def stableKeyword = ['RELEASE', 'FINAL', 'GA'].any { it -> version.toUpperCase().contains(it) } + def regex = /^[0-9,.v-]+(-r)?$/ + return !stableKeyword && !(version ==~ regex) } +// https://github.com/ben-manes/gradle-versions-plugin +tasks.named("dependencyUpdates").configure { + rejectVersionIf { + isNonStable(it.candidate.version) + } +} \ No newline at end of file From a38f74189f4be8c77e1078d0f6b097f9cd938bd0 Mon Sep 17 00:00:00 2001 From: Harrison Cole Date: Fri, 13 Dec 2024 16:34:54 +1100 Subject: [PATCH 054/156] Use `actions/checkout@v4` instead of `actions/checkout@v1` --- .github/workflows/master.yml | 2 +- .github/workflows/pull_request.yml | 2 +- .github/workflows/release.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 221ce090..a8e622af 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -13,7 +13,7 @@ jobs: MAVEN_CENTRAL_PGP_KEY: ${{ secrets.MAVEN_CENTRAL_PGP_KEY }} steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v4 - uses: gradle/wrapper-validation-action@v1 - name: Set up JDK 11 uses: actions/setup-java@v1 diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 1379be0b..dc145966 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -13,7 +13,7 @@ jobs: buildAndTest: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v4 - uses: gradle/wrapper-validation-action@v1 - name: Set up JDK 11 uses: actions/setup-java@v1 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fee61a30..c37b2f93 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -17,7 +17,7 @@ jobs: RELEASE_VERSION: ${{ github.event.inputs.version }} steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v4 - uses: gradle/wrapper-validation-action@v1 - name: Set up JDK 11 uses: actions/setup-java@v1 From 5514c8cfffc31f165460f6c857d6b5fde4d4e32d Mon Sep 17 00:00:00 2001 From: Harrison Cole Date: Fri, 13 Dec 2024 16:36:24 +1100 Subject: [PATCH 055/156] Use `actions/setup-java@v4` instead of `actions/setup-java@v1` --- .github/workflows/master.yml | 6 ++++-- .github/workflows/pull_request.yml | 6 ++++-- .github/workflows/release.yml | 6 ++++-- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index a8e622af..f5339e20 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -16,9 +16,11 @@ jobs: - uses: actions/checkout@v4 - uses: gradle/wrapper-validation-action@v1 - name: Set up JDK 11 - uses: actions/setup-java@v1 + uses: actions/setup-java@v4 with: - java-version: '11.0.23' + java-version: '11' + distribution: 'temurin' + check-latest: true - name: build test and publish run: ./gradlew assemble && ./gradlew check --info && ./gradlew publishToSonatype closeAndReleaseSonatypeStagingRepository -x check --info --stacktrace env: diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index dc145966..6f11e7a9 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -16,9 +16,11 @@ jobs: - uses: actions/checkout@v4 - uses: gradle/wrapper-validation-action@v1 - name: Set up JDK 11 - uses: actions/setup-java@v1 + uses: actions/setup-java@v4 with: - java-version: '11.0.23' + java-version: '11' + distribution: 'temurin' + check-latest: true - name: build and test run: ./gradlew assemble && ./gradlew check --info --stacktrace env: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c37b2f93..ee351c49 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,9 +20,11 @@ jobs: - uses: actions/checkout@v4 - uses: gradle/wrapper-validation-action@v1 - name: Set up JDK 11 - uses: actions/setup-java@v1 + uses: actions/setup-java@v4 with: - java-version: '11.0.23' + java-version: '11' + distribution: 'temurin' + check-latest: true - name: build test and publish run: ./gradlew assemble && ./gradlew check --info && ./gradlew publishToSonatype closeAndReleaseSonatypeStagingRepository -x check --info --stacktrace env: From 691e48147f9085b41dbbb7b8a925086cb54c082d Mon Sep 17 00:00:00 2001 From: Harrison Cole Date: Fri, 13 Dec 2024 16:37:10 +1100 Subject: [PATCH 056/156] add `gradle/actions/setup-gradle@v4` to pipelines. --- .github/workflows/master.yml | 4 ++++ .github/workflows/pull_request.yml | 4 ++++ .github/workflows/release.yml | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index f5339e20..3febe71a 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -21,6 +21,10 @@ jobs: java-version: '11' distribution: 'temurin' check-latest: true + # Configure Gradle for optimal use in GiHub Actions, including caching of downloaded dependencies. + # See: https://github.com/gradle/actions/blob/main/setup-gradle/README.md + - name: Setup Gradle + uses: gradle/actions/setup-gradle@v4 - name: build test and publish run: ./gradlew assemble && ./gradlew check --info && ./gradlew publishToSonatype closeAndReleaseSonatypeStagingRepository -x check --info --stacktrace env: diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 6f11e7a9..d4828210 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -21,6 +21,10 @@ jobs: java-version: '11' distribution: 'temurin' check-latest: true + # Configure Gradle for optimal use in GiHub Actions, including caching of downloaded dependencies. + # See: https://github.com/gradle/actions/blob/main/setup-gradle/README.md + - name: Setup Gradle + uses: gradle/actions/setup-gradle@v4 - name: build and test run: ./gradlew assemble && ./gradlew check --info --stacktrace env: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ee351c49..5474744f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -25,6 +25,10 @@ jobs: java-version: '11' distribution: 'temurin' check-latest: true + # Configure Gradle for optimal use in GiHub Actions, including caching of downloaded dependencies. + # See: https://github.com/gradle/actions/blob/main/setup-gradle/README.md + - name: Setup Gradle + uses: gradle/actions/setup-gradle@v4 - name: build test and publish run: ./gradlew assemble && ./gradlew check --info && ./gradlew publishToSonatype closeAndReleaseSonatypeStagingRepository -x check --info --stacktrace env: From 85f38c2075c21d41503d4240019156ad246cabea Mon Sep 17 00:00:00 2001 From: Harrison Cole Date: Fri, 13 Dec 2024 16:44:16 +1100 Subject: [PATCH 057/156] Replace `gradle/wrapper-validation-action@v1` with `gradle/actions/wrapper-validation@v3` --- .github/workflows/master.yml | 2 +- .github/workflows/pull_request.yml | 2 +- .github/workflows/release.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 3febe71a..01b89bc4 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -14,7 +14,7 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: gradle/wrapper-validation-action@v1 + - uses: gradle/actions/wrapper-validation@v3 - name: Set up JDK 11 uses: actions/setup-java@v4 with: diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index d4828210..f16bf96b 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: gradle/wrapper-validation-action@v1 + - uses: gradle/actions/wrapper-validation@v3 - name: Set up JDK 11 uses: actions/setup-java@v4 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5474744f..a574a68f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -18,7 +18,7 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: gradle/wrapper-validation-action@v1 + - uses: gradle/actions/wrapper-validation@v3 - name: Set up JDK 11 uses: actions/setup-java@v4 with: From 8999240ce58c1dd7a7227f4820ec5135720c7fe2 Mon Sep 17 00:00:00 2001 From: dondonz <13839920+dondonz@users.noreply.github.com> Date: Tue, 17 Dec 2024 10:57:50 +1100 Subject: [PATCH 058/156] Add new version --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fbfd6205..2a0e08f7 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ repositories { } dependencies { - compile 'com.graphql-java:java-dataloader: 3.3.0' + compile 'com.graphql-java:java-dataloader: 3.4.0' } ``` From 8f7547935a74a036cf0b68af2670d95a86fb30ae Mon Sep 17 00:00:00 2001 From: dondonz <13839920+dondonz@users.noreply.github.com> Date: Wed, 1 Jan 2025 14:33:30 +1100 Subject: [PATCH 059/156] Add stale bot --- .github/workflows/stale-pr-issue.yml | 48 ++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 .github/workflows/stale-pr-issue.yml diff --git a/.github/workflows/stale-pr-issue.yml b/.github/workflows/stale-pr-issue.yml new file mode 100644 index 00000000..d9454022 --- /dev/null +++ b/.github/workflows/stale-pr-issue.yml @@ -0,0 +1,48 @@ +# Mark inactive issues and PRs as stale +# GitHub action based on https://github.com/actions/stale + +name: 'Close stale issues and PRs' +on: + schedule: + # Execute every day + - cron: '0 0 * * *' + +permissions: + actions: write + issues: write + pull-requests: write + +jobs: + close-pending: + runs-on: ubuntu-latest + steps: + - uses: actions/stale@v9 + with: + # GLOBAL ------------------------------------------------------------ + # Exempt any PRs or issues already added to a milestone + exempt-all-milestones: true + # Days until issues or pull requests are labelled as stale + days-before-stale: 60 + + # ISSUES ------------------------------------------------------------ + # Issues will be closed after 90 days of inactive (60 to mark as stale + 30 to close) + days-before-issue-close: 30 + stale-issue-message: > + Hello, this issue has been inactive for 60 days, so we're marking it as stale. + If you would like to continue this discussion, please comment within the next 30 days or we'll close the issue. + close-issue-message: > + Hello, as this issue has been inactive for 90 days, we're closing the issue. + If you would like to resume the discussion, please create a new issue. + exempt-issue-labels: keep-open + + # PULL REQUESTS ----------------------------------------------------- + # PRs will be closed after 90 days of inactive (60 to mark as stale + 30 to close) + days-before-pr-close: 30 + stale-pr-message: > + Hello, this pull request has been inactive for 60 days, so we're marking it as stale. + If you would like to continue working on this pull request, please make an update within the next 30 days, or we'll close the pull request. + close-pr-message: > + Hello, as this pull request has been inactive for 90 days, we're closing this pull request. + We always welcome contributions, and if you would like to continue, please open a new pull request. + exempt-pr-labels: keep-open + \ No newline at end of file From 3a784836378975557944c520bd173ffa88aafe59 Mon Sep 17 00:00:00 2001 From: dondonz <13839920+dondonz@users.noreply.github.com> Date: Wed, 1 Jan 2025 14:56:10 +1100 Subject: [PATCH 060/156] Removing unused slf4j references --- build.gradle | 2 -- gradle.properties | 1 - 2 files changed, 3 deletions(-) diff --git a/build.gradle b/build.gradle index 70d39181..0a585598 100644 --- a/build.gradle +++ b/build.gradle @@ -65,7 +65,6 @@ jar { } dependencies { - api "org.slf4j:slf4j-api:$slf4j_version" api "org.reactivestreams:reactive-streams:$reactive_streams_version" } @@ -99,7 +98,6 @@ testing { implementation 'org.junit.jupiter:junit-jupiter-api' implementation 'org.junit.jupiter:junit-jupiter-params' implementation 'org.junit.jupiter:junit-jupiter-engine' - implementation "org.slf4j:slf4j-simple:$slf4j_version" implementation "org.awaitility:awaitility:$awaitility_version" implementation "org.hamcrest:hamcrest:$hamcrest_version" implementation "io.projectreactor:reactor-core:$reactor_core_version" diff --git a/gradle.properties b/gradle.properties index 22d56037..428b6e29 100644 --- a/gradle.properties +++ b/gradle.properties @@ -20,7 +20,6 @@ projectDescription = Port of Facebook Dataloader for Java # Dependency versions. junit_version=5.11.3 hamcrest_version=2.2 -slf4j_version=1.7.30 awaitility_version=2.0.0 reactor_core_version=3.6.6 caffeine_version=3.1.8 From 47411a9b05ac15acc8757c510b9933662491107c Mon Sep 17 00:00:00 2001 From: Alexandre Carlton Date: Wed, 1 Jan 2025 22:04:35 +0100 Subject: [PATCH 061/156] Parameterise more tests across DataLoader implementations We parameterise: - `DataLoaderValueCacheTest` - `ScheduledDataLoaderRegistryTest` to provide increased coverage across all DataLoader implementations. Notably, `DataLoaderValueCacheTest` is _not_ parameterised across the Publisher DataLoaders (yet) as these do not appear to work when a `ValueCache` is provided (which is what prompted this pull request). It will be fixed in a future pull request, at which point we will restore coverage. --- .../java/org/dataloader/DataLoaderTest.java | 107 +++++++----------- .../dataloader/DataLoaderValueCacheTest.java | 105 +++++++++-------- .../java/org/dataloader/fixtures/TestKit.java | 30 ----- .../parameterized/ListDataLoaderFactory.java | 11 ++ .../MappedDataLoaderFactory.java | 16 +++ .../MappedPublisherDataLoaderFactory.java | 18 +++ .../PublisherDataLoaderFactory.java | 15 +++ .../TestDataLoaderFactories.java | 25 ++++ .../parameterized/TestDataLoaderFactory.java | 16 +++ .../ScheduledDataLoaderRegistryTest.java | 99 +++++++++------- 10 files changed, 259 insertions(+), 183 deletions(-) create mode 100644 src/test/java/org/dataloader/fixtures/parameterized/TestDataLoaderFactories.java diff --git a/src/test/java/org/dataloader/DataLoaderTest.java b/src/test/java/org/dataloader/DataLoaderTest.java index c4ae883e..9a595b45 100644 --- a/src/test/java/org/dataloader/DataLoaderTest.java +++ b/src/test/java/org/dataloader/DataLoaderTest.java @@ -29,10 +29,8 @@ import org.dataloader.fixtures.parameterized.TestReactiveDataLoaderFactory; import org.dataloader.impl.CompletableFutureKit; import org.dataloader.impl.DataLoaderAssertionException; -import org.junit.jupiter.api.Named; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import java.util.*; @@ -41,21 +39,14 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; -import java.util.stream.Stream; import static java.util.Arrays.asList; import static java.util.Collections.*; import static java.util.concurrent.CompletableFuture.*; import static org.awaitility.Awaitility.await; import static org.dataloader.DataLoaderFactory.newDataLoader; -import static org.dataloader.DataLoaderFactory.newMappedDataLoader; -import static org.dataloader.DataLoaderFactory.newMappedPublisherDataLoader; -import static org.dataloader.DataLoaderFactory.newMappedPublisherDataLoaderWithTry; -import static org.dataloader.DataLoaderFactory.newPublisherDataLoader; -import static org.dataloader.DataLoaderFactory.newPublisherDataLoaderWithTry; import static org.dataloader.DataLoaderOptions.newOptions; import static org.dataloader.fixtures.TestKit.areAllDone; import static org.dataloader.fixtures.TestKit.listFrom; @@ -63,7 +54,6 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.*; import static org.junit.jupiter.api.Assertions.assertArrayEquals; -import static org.junit.jupiter.api.Assertions.fail; /** * Tests for {@link DataLoader}. @@ -125,7 +115,7 @@ public void basic_map_batch_loading() { } @ParameterizedTest - @MethodSource("dataLoaderFactories") + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void should_Support_loading_multiple_keys_in_one_call_via_list(TestDataLoaderFactory factory) { AtomicBoolean success = new AtomicBoolean(); DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), new ArrayList<>()); @@ -141,7 +131,7 @@ public void should_Support_loading_multiple_keys_in_one_call_via_list(TestDataLo } @ParameterizedTest - @MethodSource("dataLoaderFactories") + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void should_Support_loading_multiple_keys_in_one_call_via_map(TestDataLoaderFactory factory) { AtomicBoolean success = new AtomicBoolean(); DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), new ArrayList<>()); @@ -161,7 +151,7 @@ public void should_Support_loading_multiple_keys_in_one_call_via_map(TestDataLoa } @ParameterizedTest - @MethodSource("dataLoaderFactories") + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void should_Resolve_to_empty_list_when_no_keys_supplied(TestDataLoaderFactory factory) { AtomicBoolean success = new AtomicBoolean(); DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), new ArrayList<>()); @@ -176,7 +166,7 @@ public void should_Resolve_to_empty_list_when_no_keys_supplied(TestDataLoaderFac } @ParameterizedTest - @MethodSource("dataLoaderFactories") + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void should_Resolve_to_empty_map_when_no_keys_supplied(TestDataLoaderFactory factory) { AtomicBoolean success = new AtomicBoolean(); DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), new ArrayList<>()); @@ -191,7 +181,7 @@ public void should_Resolve_to_empty_map_when_no_keys_supplied(TestDataLoaderFact } @ParameterizedTest - @MethodSource("dataLoaderFactories") + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void should_Return_zero_entries_dispatched_when_no_keys_supplied_via_list(TestDataLoaderFactory factory) { AtomicBoolean success = new AtomicBoolean(); DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), new ArrayList<>()); @@ -206,7 +196,7 @@ public void should_Return_zero_entries_dispatched_when_no_keys_supplied_via_list } @ParameterizedTest - @MethodSource("dataLoaderFactories") + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void should_Return_zero_entries_dispatched_when_no_keys_supplied_via_map(TestDataLoaderFactory factory) { AtomicBoolean success = new AtomicBoolean(); DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), new ArrayList<>()); @@ -221,7 +211,7 @@ public void should_Return_zero_entries_dispatched_when_no_keys_supplied_via_map( } @ParameterizedTest - @MethodSource("dataLoaderFactories") + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void should_Batch_multiple_requests(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), loadCalls); @@ -237,7 +227,7 @@ public void should_Batch_multiple_requests(TestDataLoaderFactory factory) throws } @ParameterizedTest - @MethodSource("dataLoaderFactories") + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void should_Return_number_of_batched_entries(TestDataLoaderFactory factory) { List> loadCalls = new ArrayList<>(); DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), loadCalls); @@ -252,7 +242,7 @@ public void should_Return_number_of_batched_entries(TestDataLoaderFactory factor } @ParameterizedTest - @MethodSource("dataLoaderFactories") + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void should_Coalesce_identical_requests(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), loadCalls); @@ -269,7 +259,7 @@ public void should_Coalesce_identical_requests(TestDataLoaderFactory factory) th } @ParameterizedTest - @MethodSource("dataLoaderFactories") + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void should_Cache_repeated_requests(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), loadCalls); @@ -305,7 +295,7 @@ public void should_Cache_repeated_requests(TestDataLoaderFactory factory) throws } @ParameterizedTest - @MethodSource("dataLoaderFactories") + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void should_Not_redispatch_previous_load(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), loadCalls); @@ -323,7 +313,7 @@ public void should_Not_redispatch_previous_load(TestDataLoaderFactory factory) t } @ParameterizedTest - @MethodSource("dataLoaderFactories") + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void should_Cache_on_redispatch(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), loadCalls); @@ -348,7 +338,7 @@ public void should_Cache_on_redispatch(TestDataLoaderFactory factory) throws Exe } @ParameterizedTest - @MethodSource("dataLoaderFactories") + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void should_Clear_single_value_in_loader(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), loadCalls); @@ -377,7 +367,7 @@ public void should_Clear_single_value_in_loader(TestDataLoaderFactory factory) t } @ParameterizedTest - @MethodSource("dataLoaderFactories") + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void should_Clear_all_values_in_loader(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), loadCalls); @@ -405,7 +395,7 @@ public void should_Clear_all_values_in_loader(TestDataLoaderFactory factory) thr } @ParameterizedTest - @MethodSource("dataLoaderFactories") + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void should_Allow_priming_the_cache(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), loadCalls); @@ -424,7 +414,7 @@ public void should_Allow_priming_the_cache(TestDataLoaderFactory factory) throws } @ParameterizedTest - @MethodSource("dataLoaderFactories") + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void should_Not_prime_keys_that_already_exist(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), loadCalls); @@ -453,7 +443,7 @@ public void should_Not_prime_keys_that_already_exist(TestDataLoaderFactory facto } @ParameterizedTest - @MethodSource("dataLoaderFactories") + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void should_Allow_to_forcefully_prime_the_cache(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), loadCalls); @@ -482,7 +472,7 @@ public void should_Allow_to_forcefully_prime_the_cache(TestDataLoaderFactory fac } @ParameterizedTest - @MethodSource("dataLoaderFactories") + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void should_Allow_priming_the_cache_with_a_future(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), loadCalls); @@ -501,7 +491,7 @@ public void should_Allow_priming_the_cache_with_a_future(TestDataLoaderFactory f } @ParameterizedTest - @MethodSource("dataLoaderFactories") + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void should_not_Cache_failed_fetches_on_complete_failure(TestDataLoaderFactory factory) { List> loadCalls = new ArrayList<>(); DataLoader errorLoader = factory.idLoaderBlowsUps(new DataLoaderOptions(), loadCalls); @@ -523,7 +513,7 @@ public void should_not_Cache_failed_fetches_on_complete_failure(TestDataLoaderFa } @ParameterizedTest - @MethodSource("dataLoaderFactories") + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void should_Resolve_to_error_to_indicate_failure(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); DataLoader evenLoader = factory.idLoaderOddEvenExceptions(new DataLoaderOptions(), loadCalls); @@ -546,7 +536,7 @@ public void should_Resolve_to_error_to_indicate_failure(TestDataLoaderFactory fa // Accept any kind of key. @ParameterizedTest - @MethodSource("dataLoaderFactories") + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void should_Represent_failures_and_successes_simultaneously(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { AtomicBoolean success = new AtomicBoolean(); List> loadCalls = new ArrayList<>(); @@ -573,7 +563,7 @@ public void should_Represent_failures_and_successes_simultaneously(TestDataLoade // Accepts options @ParameterizedTest - @MethodSource("dataLoaderFactories") + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void should_Cache_failed_fetches(TestDataLoaderFactory factory) { List> loadCalls = new ArrayList<>(); DataLoader errorLoader = factory.idLoaderAllExceptions(new DataLoaderOptions(), loadCalls); @@ -596,7 +586,7 @@ public void should_Cache_failed_fetches(TestDataLoaderFactory factory) { } @ParameterizedTest - @MethodSource("dataLoaderFactories") + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void should_NOT_Cache_failed_fetches_if_told_not_too(TestDataLoaderFactory factory) { DataLoaderOptions options = DataLoaderOptions.newOptions().setCachingExceptionsEnabled(false); List> loadCalls = new ArrayList<>(); @@ -623,7 +613,7 @@ public void should_NOT_Cache_failed_fetches_if_told_not_too(TestDataLoaderFactor // Accepts object key in custom cacheKey function @ParameterizedTest - @MethodSource("dataLoaderFactories") + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void should_Handle_priming_the_cache_with_an_error(TestDataLoaderFactory factory) { List> loadCalls = new ArrayList<>(); DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), loadCalls); @@ -640,7 +630,7 @@ public void should_Handle_priming_the_cache_with_an_error(TestDataLoaderFactory } @ParameterizedTest - @MethodSource("dataLoaderFactories") + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void should_Clear_values_from_cache_after_errors(TestDataLoaderFactory factory) { List> loadCalls = new ArrayList<>(); DataLoader errorLoader = factory.idLoaderBlowsUps(new DataLoaderOptions(), loadCalls); @@ -676,7 +666,7 @@ public void should_Clear_values_from_cache_after_errors(TestDataLoaderFactory fa } @ParameterizedTest - @MethodSource("dataLoaderFactories") + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void should_Propagate_error_to_all_loads(TestDataLoaderFactory factory) { List> loadCalls = new ArrayList<>(); DataLoader errorLoader = factory.idLoaderBlowsUps(new DataLoaderOptions(), loadCalls); @@ -700,7 +690,7 @@ public void should_Propagate_error_to_all_loads(TestDataLoaderFactory factory) { } @ParameterizedTest - @MethodSource("dataLoaderFactories") + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void should_Accept_objects_as_keys(TestDataLoaderFactory factory) { List> loadCalls = new ArrayList<>(); DataLoader identityLoader = factory.idLoader(new DataLoaderOptions(), loadCalls); @@ -742,7 +732,7 @@ public void should_Accept_objects_as_keys(TestDataLoaderFactory factory) { } @ParameterizedTest - @MethodSource("dataLoaderFactories") + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void should_Disable_caching(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); DataLoader identityLoader = @@ -780,7 +770,7 @@ public void should_Disable_caching(TestDataLoaderFactory factory) throws Executi } @ParameterizedTest - @MethodSource("dataLoaderFactories") + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void should_work_with_duplicate_keys_when_caching_disabled(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); DataLoader identityLoader = @@ -803,7 +793,7 @@ public void should_work_with_duplicate_keys_when_caching_disabled(TestDataLoader } @ParameterizedTest - @MethodSource("dataLoaderFactories") + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void should_work_with_duplicate_keys_when_caching_enabled(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); DataLoader identityLoader = @@ -824,7 +814,7 @@ public void should_work_with_duplicate_keys_when_caching_enabled(TestDataLoaderF // It is resilient to job queue ordering @ParameterizedTest - @MethodSource("dataLoaderFactories") + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void should_Accept_objects_with_a_complex_key(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); DataLoaderOptions options = newOptions().setCacheKeyFunction(getJsonObjectCacheMapFn()); @@ -846,7 +836,7 @@ public void should_Accept_objects_with_a_complex_key(TestDataLoaderFactory facto // Helper methods @ParameterizedTest - @MethodSource("dataLoaderFactories") + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void should_Clear_objects_with_complex_key(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); DataLoaderOptions options = newOptions().setCacheKeyFunction(getJsonObjectCacheMapFn()); @@ -871,7 +861,7 @@ public void should_Clear_objects_with_complex_key(TestDataLoaderFactory factory) } @ParameterizedTest - @MethodSource("dataLoaderFactories") + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void should_Accept_objects_with_different_order_of_keys(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); DataLoaderOptions options = newOptions().setCacheKeyFunction(getJsonObjectCacheMapFn()); @@ -894,7 +884,7 @@ public void should_Accept_objects_with_different_order_of_keys(TestDataLoaderFac } @ParameterizedTest - @MethodSource("dataLoaderFactories") + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void should_Allow_priming_the_cache_with_an_object_key(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); DataLoaderOptions options = newOptions().setCacheKeyFunction(getJsonObjectCacheMapFn()); @@ -916,7 +906,7 @@ public void should_Allow_priming_the_cache_with_an_object_key(TestDataLoaderFact } @ParameterizedTest - @MethodSource("dataLoaderFactories") + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void should_Accept_a_custom_cache_map_implementation(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { CustomCacheMap customMap = new CustomCacheMap(); List> loadCalls = new ArrayList<>(); @@ -968,7 +958,7 @@ public void should_Accept_a_custom_cache_map_implementation(TestDataLoaderFactor } @ParameterizedTest - @MethodSource("dataLoaderFactories") + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void should_degrade_gracefully_if_cache_get_throws(TestDataLoaderFactory factory) { CacheMap cache = new ThrowingCacheMap(); DataLoaderOptions options = newOptions().setCachingEnabled(true).setCacheMap(cache); @@ -983,7 +973,7 @@ public void should_degrade_gracefully_if_cache_get_throws(TestDataLoaderFactory } @ParameterizedTest - @MethodSource("dataLoaderFactories") + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void batching_disabled_should_dispatch_immediately(TestDataLoaderFactory factory) { List> loadCalls = new ArrayList<>(); DataLoaderOptions options = newOptions().setBatchingEnabled(false); @@ -1012,7 +1002,7 @@ public void batching_disabled_should_dispatch_immediately(TestDataLoaderFactory } @ParameterizedTest - @MethodSource("dataLoaderFactories") + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void batching_disabled_and_caching_disabled_should_dispatch_immediately_and_forget(TestDataLoaderFactory factory) { List> loadCalls = new ArrayList<>(); DataLoaderOptions options = newOptions().setBatchingEnabled(false).setCachingEnabled(false); @@ -1044,7 +1034,7 @@ public void batching_disabled_and_caching_disabled_should_dispatch_immediately_a } @ParameterizedTest - @MethodSource("dataLoaderFactories") + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void batches_multiple_requests_with_max_batch_size(TestDataLoaderFactory factory) { List> loadCalls = new ArrayList<>(); DataLoader identityLoader = factory.idLoader(newOptions().setMaxBatchSize(2), loadCalls); @@ -1066,7 +1056,7 @@ public void batches_multiple_requests_with_max_batch_size(TestDataLoaderFactory } @ParameterizedTest - @MethodSource("dataLoaderFactories") + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void can_split_max_batch_sizes_correctly(TestDataLoaderFactory factory) { List> loadCalls = new ArrayList<>(); DataLoader identityLoader = factory.idLoader(newOptions().setMaxBatchSize(5), loadCalls); @@ -1089,7 +1079,7 @@ public void can_split_max_batch_sizes_correctly(TestDataLoaderFactory factory) { } @ParameterizedTest - @MethodSource("dataLoaderFactories") + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void should_Batch_loads_occurring_within_futures(TestDataLoaderFactory factory) { List> loadCalls = new ArrayList<>(); DataLoader identityLoader = factory.idLoader(newOptions(), loadCalls); @@ -1122,7 +1112,7 @@ public void should_Batch_loads_occurring_within_futures(TestDataLoaderFactory fa } @ParameterizedTest - @MethodSource("dataLoaderFactories") + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void should_blowup_after_N_keys(TestDataLoaderFactory factory) { if (!(factory instanceof TestReactiveDataLoaderFactory)) { return; @@ -1148,7 +1138,7 @@ public void should_blowup_after_N_keys(TestDataLoaderFactory factory) { } @ParameterizedTest - @MethodSource("dataLoaderFactories") + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void when_values_size_are_less_then_key_size(TestDataLoaderFactory factory) { // // what happens if we want 4 values but are only given 2 back say @@ -1183,7 +1173,7 @@ public void when_values_size_are_less_then_key_size(TestDataLoaderFactory factor } @ParameterizedTest - @MethodSource("dataLoaderFactories") + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void when_values_size_are_more_then_key_size(TestDataLoaderFactory factory) { // // what happens if we want 4 values but only given 6 back say @@ -1307,14 +1297,5 @@ public CompletableFuture get(String key) { throw new RuntimeException("Cache implementation failed."); } } - - private static Stream dataLoaderFactories() { - return Stream.of( - Arguments.of(Named.of("List DataLoader", new ListDataLoaderFactory())), - Arguments.of(Named.of("Mapped DataLoader", new MappedDataLoaderFactory())), - Arguments.of(Named.of("Publisher DataLoader", new PublisherDataLoaderFactory())), - Arguments.of(Named.of("Mapped Publisher DataLoader", new MappedPublisherDataLoaderFactory())) - ); - } } diff --git a/src/test/java/org/dataloader/DataLoaderValueCacheTest.java b/src/test/java/org/dataloader/DataLoaderValueCacheTest.java index 1fb5ea2b..6c05b013 100644 --- a/src/test/java/org/dataloader/DataLoaderValueCacheTest.java +++ b/src/test/java/org/dataloader/DataLoaderValueCacheTest.java @@ -4,10 +4,13 @@ import com.github.benmanes.caffeine.cache.Caffeine; import org.dataloader.fixtures.CaffeineValueCache; import org.dataloader.fixtures.CustomValueCache; +import org.dataloader.fixtures.parameterized.TestDataLoaderFactory; import org.dataloader.impl.DataLoaderAssertionException; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; @@ -18,7 +21,6 @@ import static java.util.Collections.singletonList; import static org.awaitility.Awaitility.await; import static org.dataloader.DataLoaderOptions.newOptions; -import static org.dataloader.fixtures.TestKit.idLoader; import static org.dataloader.fixtures.TestKit.snooze; import static org.dataloader.fixtures.TestKit.sort; import static org.dataloader.impl.CompletableFutureKit.failedFuture; @@ -30,11 +32,12 @@ public class DataLoaderValueCacheTest { - @Test - public void test_by_default_we_have_no_value_caching() { - List> loadCalls = new ArrayList<>(); + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#getWithoutPublisher") + public void test_by_default_we_have_no_value_caching(TestDataLoaderFactory factory) { + List> loadCalls = new ArrayList<>(); DataLoaderOptions options = newOptions(); - DataLoader identityLoader = idLoader(options, loadCalls); + DataLoader identityLoader = factory.idLoader(options, loadCalls); CompletableFuture fA = identityLoader.load("a"); CompletableFuture fB = identityLoader.load("b"); @@ -64,12 +67,13 @@ public void test_by_default_we_have_no_value_caching() { assertThat(loadCalls, equalTo(emptyList())); } - @Test - public void should_accept_a_remote_value_store_for_caching() { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#getWithoutPublisher") + public void should_accept_a_remote_value_store_for_caching(TestDataLoaderFactory factory) { CustomValueCache customValueCache = new CustomValueCache(); - List> loadCalls = new ArrayList<>(); + List> loadCalls = new ArrayList<>(); DataLoaderOptions options = newOptions().setValueCache(customValueCache); - DataLoader identityLoader = idLoader(options, loadCalls); + DataLoader identityLoader = factory.idLoader(options, loadCalls); // Fetches as expected @@ -108,8 +112,9 @@ public void should_accept_a_remote_value_store_for_caching() { assertArrayEquals(customValueCache.store.keySet().toArray(), emptyList().toArray()); } - @Test - public void can_use_caffeine_for_caching() { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#getWithoutPublisher") + public void can_use_caffeine_for_caching(TestDataLoaderFactory factory) { // // Mostly to prove that some other CACHE library could be used // as the backing value cache. Not really Caffeine specific. @@ -121,9 +126,9 @@ public void can_use_caffeine_for_caching() { ValueCache caffeineValueCache = new CaffeineValueCache(caffeineCache); - List> loadCalls = new ArrayList<>(); + List> loadCalls = new ArrayList<>(); DataLoaderOptions options = newOptions().setValueCache(caffeineValueCache); - DataLoader identityLoader = idLoader(options, loadCalls); + DataLoader identityLoader = factory.idLoader(options, loadCalls); // Fetches as expected @@ -148,8 +153,9 @@ public void can_use_caffeine_for_caching() { assertArrayEquals(caffeineCache.asMap().keySet().toArray(), asList("a", "b", "c").toArray()); } - @Test - public void will_invoke_loader_if_CACHE_GET_call_throws_exception() { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#getWithoutPublisher") + public void will_invoke_loader_if_CACHE_GET_call_throws_exception(TestDataLoaderFactory factory) { CustomValueCache customValueCache = new CustomValueCache() { @Override @@ -163,9 +169,9 @@ public CompletableFuture get(String key) { customValueCache.set("a", "Not From Cache"); customValueCache.set("b", "From Cache"); - List> loadCalls = new ArrayList<>(); + List> loadCalls = new ArrayList<>(); DataLoaderOptions options = newOptions().setValueCache(customValueCache); - DataLoader identityLoader = idLoader(options, loadCalls); + DataLoader identityLoader = factory.idLoader(options, loadCalls); CompletableFuture fA = identityLoader.load("a"); CompletableFuture fB = identityLoader.load("b"); @@ -178,8 +184,9 @@ public CompletableFuture get(String key) { assertThat(loadCalls, equalTo(singletonList(singletonList("a")))); } - @Test - public void will_still_work_if_CACHE_SET_call_throws_exception() { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#getWithoutPublisher") + public void will_still_work_if_CACHE_SET_call_throws_exception(TestDataLoaderFactory factory) { CustomValueCache customValueCache = new CustomValueCache() { @Override public CompletableFuture set(String key, Object value) { @@ -190,9 +197,9 @@ public CompletableFuture set(String key, Object value) { } }; - List> loadCalls = new ArrayList<>(); + List> loadCalls = new ArrayList<>(); DataLoaderOptions options = newOptions().setValueCache(customValueCache); - DataLoader identityLoader = idLoader(options, loadCalls); + DataLoader identityLoader = factory.idLoader(options, loadCalls); CompletableFuture fA = identityLoader.load("a"); CompletableFuture fB = identityLoader.load("b"); @@ -206,8 +213,9 @@ public CompletableFuture set(String key, Object value) { assertArrayEquals(customValueCache.store.keySet().toArray(), singletonList("b").toArray()); } - @Test - public void caching_can_take_some_time_complete() { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#getWithoutPublisher") + public void caching_can_take_some_time_complete(TestDataLoaderFactory factory) { CustomValueCache customValueCache = new CustomValueCache() { @Override @@ -228,9 +236,9 @@ public CompletableFuture get(String key) { }; - List> loadCalls = new ArrayList<>(); + List> loadCalls = new ArrayList<>(); DataLoaderOptions options = newOptions().setValueCache(customValueCache); - DataLoader identityLoader = idLoader(options, loadCalls); + DataLoader identityLoader = factory.idLoader(options, loadCalls); CompletableFuture fA = identityLoader.load("a"); CompletableFuture fB = identityLoader.load("b"); @@ -247,8 +255,9 @@ public CompletableFuture get(String key) { assertThat(loadCalls, equalTo(singletonList(asList("missC", "missD")))); } - @Test - public void batch_caching_works_as_expected() { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#getWithoutPublisher") + public void batch_caching_works_as_expected(TestDataLoaderFactory factory) { CustomValueCache customValueCache = new CustomValueCache() { @Override @@ -269,9 +278,9 @@ public CompletableFuture>> getValues(List keys) { }; - List> loadCalls = new ArrayList<>(); + List> loadCalls = new ArrayList<>(); DataLoaderOptions options = newOptions().setValueCache(customValueCache); - DataLoader identityLoader = idLoader(options, loadCalls); + DataLoader identityLoader = factory.idLoader(options, loadCalls); CompletableFuture fA = identityLoader.load("a"); CompletableFuture fB = identityLoader.load("b"); @@ -293,8 +302,9 @@ public CompletableFuture>> getValues(List keys) { assertThat(values, equalTo(asList("missC", "missD"))); } - @Test - public void assertions_will_be_thrown_if_the_cache_does_not_follow_contract() { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#getWithoutPublisher") + public void assertions_will_be_thrown_if_the_cache_does_not_follow_contract(TestDataLoaderFactory factory) { CustomValueCache customValueCache = new CustomValueCache() { @Override @@ -312,9 +322,9 @@ public CompletableFuture>> getValues(List keys) { } }; - List> loadCalls = new ArrayList<>(); + List> loadCalls = new ArrayList<>(); DataLoaderOptions options = newOptions().setValueCache(customValueCache); - DataLoader identityLoader = idLoader(options, loadCalls); + DataLoader identityLoader = factory.idLoader(options, loadCalls); CompletableFuture fA = identityLoader.load("a"); CompletableFuture fB = identityLoader.load("b"); @@ -335,8 +345,9 @@ private boolean isAssertionException(CompletableFuture fA) { } - @Test - public void if_caching_is_off_its_never_hit() { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#getWithoutPublisher") + public void if_caching_is_off_its_never_hit(TestDataLoaderFactory factory) { AtomicInteger getCalls = new AtomicInteger(); CustomValueCache customValueCache = new CustomValueCache() { @@ -347,9 +358,9 @@ public CompletableFuture get(String key) { } }; - List> loadCalls = new ArrayList<>(); + List> loadCalls = new ArrayList<>(); DataLoaderOptions options = newOptions().setValueCache(customValueCache).setCachingEnabled(false); - DataLoader identityLoader = idLoader(options, loadCalls); + DataLoader identityLoader = factory.idLoader(options, loadCalls); CompletableFuture fA = identityLoader.load("a"); CompletableFuture fB = identityLoader.load("b"); @@ -368,8 +379,9 @@ public CompletableFuture get(String key) { assertTrue(customValueCache.asMap().isEmpty()); } - @Test - public void if_everything_is_cached_no_batching_happens() { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#getWithoutPublisher") + public void if_everything_is_cached_no_batching_happens(TestDataLoaderFactory factory) { AtomicInteger getCalls = new AtomicInteger(); AtomicInteger setCalls = new AtomicInteger(); CustomValueCache customValueCache = new CustomValueCache() { @@ -390,9 +402,9 @@ public CompletableFuture> setValues(List keys, List customValueCache.asMap().put("b", "cachedB"); customValueCache.asMap().put("c", "cachedC"); - List> loadCalls = new ArrayList<>(); + List> loadCalls = new ArrayList<>(); DataLoaderOptions options = newOptions().setValueCache(customValueCache).setCachingEnabled(true); - DataLoader identityLoader = idLoader(options, loadCalls); + DataLoader identityLoader = factory.idLoader(options, loadCalls); CompletableFuture fA = identityLoader.load("a"); CompletableFuture fB = identityLoader.load("b"); @@ -410,8 +422,9 @@ public CompletableFuture> setValues(List keys, List } - @Test - public void if_batching_is_off_it_still_can_cache() { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#getWithoutPublisher") + public void if_batching_is_off_it_still_can_cache(TestDataLoaderFactory factory) { AtomicInteger getCalls = new AtomicInteger(); AtomicInteger setCalls = new AtomicInteger(); CustomValueCache customValueCache = new CustomValueCache() { @@ -430,9 +443,9 @@ public CompletableFuture> setValues(List keys, List }; customValueCache.asMap().put("a", "cachedA"); - List> loadCalls = new ArrayList<>(); + List> loadCalls = new ArrayList<>(); DataLoaderOptions options = newOptions().setValueCache(customValueCache).setCachingEnabled(true).setBatchingEnabled(false); - DataLoader identityLoader = idLoader(options, loadCalls); + DataLoader identityLoader = factory.idLoader(options, loadCalls); CompletableFuture fA = identityLoader.load("a"); CompletableFuture fB = identityLoader.load("b"); diff --git a/src/test/java/org/dataloader/fixtures/TestKit.java b/src/test/java/org/dataloader/fixtures/TestKit.java index c22988d0..04ec5e50 100644 --- a/src/test/java/org/dataloader/fixtures/TestKit.java +++ b/src/test/java/org/dataloader/fixtures/TestKit.java @@ -8,7 +8,6 @@ import org.dataloader.MappedBatchLoader; import org.dataloader.MappedBatchLoaderWithContext; -import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -61,43 +60,14 @@ public static BatchLoader keysAsValues(List> loadCalls) { }; } - public static BatchLoader keysAsValuesAsync(Duration delay) { - return keysAsValuesAsync(new ArrayList<>(), delay); - } - - public static BatchLoader keysAsValuesAsync(List> loadCalls, Duration delay) { - return keys -> CompletableFuture.supplyAsync(() -> { - snooze(delay.toMillis()); - List ks = new ArrayList<>(keys); - loadCalls.add(ks); - @SuppressWarnings("unchecked") - List values = keys.stream() - .map(k -> (V) k) - .collect(toList()); - return values; - }); - } - public static DataLoader idLoader() { return idLoader(null, new ArrayList<>()); } - public static DataLoader idLoader(List> loadCalls) { - return idLoader(null, loadCalls); - } - public static DataLoader idLoader(DataLoaderOptions options, List> loadCalls) { return DataLoaderFactory.newDataLoader(keysAsValues(loadCalls), options); } - public static DataLoader idLoaderAsync(Duration delay) { - return idLoaderAsync(null, new ArrayList<>(), delay); - } - - public static DataLoader idLoaderAsync(DataLoaderOptions options, List> loadCalls, Duration delay) { - return DataLoaderFactory.newDataLoader(keysAsValuesAsync(loadCalls, delay), options); - } - public static Collection listFrom(int i, int max) { List ints = new ArrayList<>(); for (int j = i; j < max; j++) { diff --git a/src/test/java/org/dataloader/fixtures/parameterized/ListDataLoaderFactory.java b/src/test/java/org/dataloader/fixtures/parameterized/ListDataLoaderFactory.java index ee1f1d7d..0644d3ca 100644 --- a/src/test/java/org/dataloader/fixtures/parameterized/ListDataLoaderFactory.java +++ b/src/test/java/org/dataloader/fixtures/parameterized/ListDataLoaderFactory.java @@ -4,9 +4,11 @@ import org.dataloader.DataLoaderOptions; import org.dataloader.fixtures.TestKit; +import java.time.Duration; import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; import static java.util.concurrent.CompletableFuture.completedFuture; @@ -21,6 +23,15 @@ public DataLoader idLoader(DataLoaderOptions options, List DataLoader idLoaderDelayed(DataLoaderOptions options, List> loadCalls, Duration delay) { + return newDataLoader(keys -> CompletableFuture.supplyAsync(() -> { + TestKit.snooze(delay.toMillis()); + loadCalls.add(new ArrayList<>(keys)); + return keys; + })); + } + @Override public DataLoader idLoaderBlowsUps( DataLoaderOptions options, List> loadCalls) { diff --git a/src/test/java/org/dataloader/fixtures/parameterized/MappedDataLoaderFactory.java b/src/test/java/org/dataloader/fixtures/parameterized/MappedDataLoaderFactory.java index 8f41441a..e7c47ec4 100644 --- a/src/test/java/org/dataloader/fixtures/parameterized/MappedDataLoaderFactory.java +++ b/src/test/java/org/dataloader/fixtures/parameterized/MappedDataLoaderFactory.java @@ -2,15 +2,19 @@ import org.dataloader.DataLoader; import org.dataloader.DataLoaderOptions; +import org.dataloader.fixtures.TestKit; +import java.time.Duration; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; import static java.util.concurrent.CompletableFuture.completedFuture; +import static org.dataloader.DataLoaderFactory.newDataLoader; import static org.dataloader.DataLoaderFactory.newMappedDataLoader; import static org.dataloader.fixtures.TestKit.futureError; @@ -27,6 +31,18 @@ public DataLoader idLoader( }, options); } + @Override + public DataLoader idLoaderDelayed( + DataLoaderOptions options, List> loadCalls, Duration delay) { + return newMappedDataLoader(keys -> CompletableFuture.supplyAsync(() -> { + TestKit.snooze(delay.toMillis()); + loadCalls.add(new ArrayList<>(keys)); + Map map = new HashMap<>(); + keys.forEach(k -> map.put(k, k)); + return map; + })); + } + @Override public DataLoader idLoaderBlowsUps(DataLoaderOptions options, List> loadCalls) { return newMappedDataLoader((keys) -> { diff --git a/src/test/java/org/dataloader/fixtures/parameterized/MappedPublisherDataLoaderFactory.java b/src/test/java/org/dataloader/fixtures/parameterized/MappedPublisherDataLoaderFactory.java index 9c923304..fa920cf7 100644 --- a/src/test/java/org/dataloader/fixtures/parameterized/MappedPublisherDataLoaderFactory.java +++ b/src/test/java/org/dataloader/fixtures/parameterized/MappedPublisherDataLoaderFactory.java @@ -3,14 +3,17 @@ import org.dataloader.DataLoader; import org.dataloader.DataLoaderOptions; import org.dataloader.Try; +import org.dataloader.fixtures.TestKit; import reactor.core.publisher.Flux; +import java.time.Duration; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -18,6 +21,7 @@ import static java.util.stream.Collectors.toSet; import static org.dataloader.DataLoaderFactory.newMappedPublisherDataLoader; import static org.dataloader.DataLoaderFactory.newMappedPublisherDataLoaderWithTry; +import static org.dataloader.DataLoaderFactory.newPublisherDataLoader; public class MappedPublisherDataLoaderFactory implements TestDataLoaderFactory, TestReactiveDataLoaderFactory { @@ -32,6 +36,20 @@ public DataLoader idLoader( }, options); } + @Override + public DataLoader idLoaderDelayed( + DataLoaderOptions options, List> loadCalls, Duration delay) { + return newMappedPublisherDataLoader((keys, subscriber) -> { + CompletableFuture.runAsync(() -> { + TestKit.snooze(delay.toMillis()); + loadCalls.add(new ArrayList<>(keys)); + Map map = new HashMap<>(); + keys.forEach(k -> map.put(k, k)); + Flux.fromIterable(map.entrySet()).subscribe(subscriber); + }); + }, options); + } + @Override public DataLoader idLoaderBlowsUps(DataLoaderOptions options, List> loadCalls) { return newMappedPublisherDataLoader((keys, subscriber) -> { diff --git a/src/test/java/org/dataloader/fixtures/parameterized/PublisherDataLoaderFactory.java b/src/test/java/org/dataloader/fixtures/parameterized/PublisherDataLoaderFactory.java index d75ff385..20497192 100644 --- a/src/test/java/org/dataloader/fixtures/parameterized/PublisherDataLoaderFactory.java +++ b/src/test/java/org/dataloader/fixtures/parameterized/PublisherDataLoaderFactory.java @@ -3,13 +3,17 @@ import org.dataloader.DataLoader; import org.dataloader.DataLoaderOptions; import org.dataloader.Try; +import org.dataloader.fixtures.TestKit; import reactor.core.publisher.Flux; +import java.time.Duration; import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.concurrent.CompletableFuture; import java.util.stream.Stream; +import static org.dataloader.DataLoaderFactory.newDataLoader; import static org.dataloader.DataLoaderFactory.newPublisherDataLoader; import static org.dataloader.DataLoaderFactory.newPublisherDataLoaderWithTry; @@ -24,6 +28,17 @@ public DataLoader idLoader( }, options); } + @Override + public DataLoader idLoaderDelayed(DataLoaderOptions options, List> loadCalls, Duration delay) { + return newPublisherDataLoader((keys, subscriber) -> { + CompletableFuture.runAsync(() -> { + TestKit.snooze(delay.toMillis()); + loadCalls.add(new ArrayList<>(keys)); + Flux.fromIterable(keys).subscribe(subscriber); + }); + }, options); + } + @Override public DataLoader idLoaderBlowsUps(DataLoaderOptions options, List> loadCalls) { return newPublisherDataLoader((keys, subscriber) -> { diff --git a/src/test/java/org/dataloader/fixtures/parameterized/TestDataLoaderFactories.java b/src/test/java/org/dataloader/fixtures/parameterized/TestDataLoaderFactories.java new file mode 100644 index 00000000..dbdfd7d3 --- /dev/null +++ b/src/test/java/org/dataloader/fixtures/parameterized/TestDataLoaderFactories.java @@ -0,0 +1,25 @@ +package org.dataloader.fixtures.parameterized; + +import org.junit.jupiter.api.Named; +import org.junit.jupiter.params.provider.Arguments; + +import java.util.stream.Stream; + +public class TestDataLoaderFactories { + public static Stream get() { + return Stream.of( + Arguments.of(Named.of("List DataLoader", new ListDataLoaderFactory())), + Arguments.of(Named.of("Mapped DataLoader", new MappedDataLoaderFactory())), + Arguments.of(Named.of("Publisher DataLoader", new PublisherDataLoaderFactory())), + Arguments.of(Named.of("Mapped Publisher DataLoader", new MappedPublisherDataLoaderFactory())) + ); + } + + // TODO: Remove in favour of #get when ValueCache supports Publisher Factories. + public static Stream getWithoutPublisher() { + return Stream.of( + Arguments.of(Named.of("List DataLoader", new ListDataLoaderFactory())), + Arguments.of(Named.of("Mapped DataLoader", new MappedDataLoaderFactory())) + ); + } +} diff --git a/src/test/java/org/dataloader/fixtures/parameterized/TestDataLoaderFactory.java b/src/test/java/org/dataloader/fixtures/parameterized/TestDataLoaderFactory.java index 8c1bc22d..97e35a8c 100644 --- a/src/test/java/org/dataloader/fixtures/parameterized/TestDataLoaderFactory.java +++ b/src/test/java/org/dataloader/fixtures/parameterized/TestDataLoaderFactory.java @@ -3,6 +3,7 @@ import org.dataloader.DataLoader; import org.dataloader.DataLoaderOptions; +import java.time.Duration; import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -10,6 +11,8 @@ public interface TestDataLoaderFactory { DataLoader idLoader(DataLoaderOptions options, List> loadCalls); + DataLoader idLoaderDelayed(DataLoaderOptions options, List> loadCalls, Duration delay); + DataLoader idLoaderBlowsUps(DataLoaderOptions options, List> loadCalls); DataLoader idLoaderAllExceptions(DataLoaderOptions options, List> loadCalls); @@ -19,4 +22,17 @@ public interface TestDataLoaderFactory { DataLoader onlyReturnsNValues(int N, DataLoaderOptions options, ArrayList loadCalls); DataLoader idLoaderReturnsTooMany(int howManyMore, DataLoaderOptions options, ArrayList loadCalls); + + // Convenience methods + + default DataLoader idLoader(List> calls) { + return idLoader(null, calls); + } + default DataLoader idLoader() { + return idLoader(null, new ArrayList<>()); + } + + default DataLoader idLoaderDelayed(Duration delay) { + return idLoaderDelayed(null, new ArrayList<>(), delay); + } } diff --git a/src/test/java/org/dataloader/registries/ScheduledDataLoaderRegistryTest.java b/src/test/java/org/dataloader/registries/ScheduledDataLoaderRegistryTest.java index e82205d4..e89939ca 100644 --- a/src/test/java/org/dataloader/registries/ScheduledDataLoaderRegistryTest.java +++ b/src/test/java/org/dataloader/registries/ScheduledDataLoaderRegistryTest.java @@ -2,13 +2,15 @@ import org.awaitility.core.ConditionTimeoutException; import org.dataloader.DataLoader; -import org.dataloader.DataLoaderFactory; import org.dataloader.DataLoaderRegistry; -import org.dataloader.fixtures.TestKit; +import org.dataloader.fixtures.parameterized.TestDataLoaderFactory; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; import java.time.Duration; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executors; @@ -20,7 +22,6 @@ import static java.util.Collections.singletonList; import static org.awaitility.Awaitility.await; import static org.awaitility.Duration.TWO_SECONDS; -import static org.dataloader.fixtures.TestKit.keysAsValues; import static org.dataloader.fixtures.TestKit.snooze; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; @@ -36,17 +37,18 @@ public class ScheduledDataLoaderRegistryTest { DispatchPredicate neverDispatch = (key, dl) -> false; - @Test - public void basic_setup_works_like_a_normal_dlr() { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void basic_setup_works_like_a_normal_dlr(TestDataLoaderFactory factory) { - List> aCalls = new ArrayList<>(); - List> bCalls = new ArrayList<>(); + List> aCalls = new ArrayList<>(); + List> bCalls = new ArrayList<>(); - DataLoader dlA = TestKit.idLoader(aCalls); + DataLoader dlA = factory.idLoader(aCalls); dlA.load("AK1"); dlA.load("AK2"); - DataLoader dlB = TestKit.idLoader(bCalls); + DataLoader dlB = factory.idLoader(bCalls); dlB.load("BK1"); dlB.load("BK2"); @@ -68,11 +70,12 @@ public void basic_setup_works_like_a_normal_dlr() { assertThat(bCalls, equalTo(singletonList(asList("BK1", "BK2")))); } - @Test - public void predicate_always_false() { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void predicate_always_false(TestDataLoaderFactory factory) { - List> calls = new ArrayList<>(); - DataLoader dlA = DataLoaderFactory.newDataLoader(keysAsValues(calls)); + List> calls = new ArrayList<>(); + DataLoader dlA = factory.idLoader(calls); dlA.load("K1"); dlA.load("K2"); @@ -98,15 +101,16 @@ public void predicate_always_false() { assertThat(calls.size(), equalTo(0)); } - @Test - public void predicate_that_eventually_returns_true() { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void predicate_that_eventually_returns_true(TestDataLoaderFactory factory) { AtomicInteger counter = new AtomicInteger(); DispatchPredicate neverDispatch = (key, dl) -> counter.incrementAndGet() > 5; - List> calls = new ArrayList<>(); - DataLoader dlA = DataLoaderFactory.newDataLoader(keysAsValues(calls)); + List> calls = new ArrayList<>(); + DataLoader dlA = factory.idLoader(calls); CompletableFuture p1 = dlA.load("K1"); CompletableFuture p2 = dlA.load("K2"); @@ -130,10 +134,11 @@ public void predicate_that_eventually_returns_true() { assertTrue(p2.isDone()); } - @Test - public void dispatchAllWithCountImmediately() { - List> calls = new ArrayList<>(); - DataLoader dlA = DataLoaderFactory.newDataLoader(keysAsValues(calls)); + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void dispatchAllWithCountImmediately(TestDataLoaderFactory factory) { + List> calls = new ArrayList<>(); + DataLoader dlA = factory.idLoader(calls); dlA.load("K1"); dlA.load("K2"); @@ -148,10 +153,11 @@ public void dispatchAllWithCountImmediately() { assertThat(calls, equalTo(singletonList(asList("K1", "K2")))); } - @Test - public void dispatchAllImmediately() { - List> calls = new ArrayList<>(); - DataLoader dlA = DataLoaderFactory.newDataLoader(keysAsValues(calls)); + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void dispatchAllImmediately(TestDataLoaderFactory factory) { + List> calls = new ArrayList<>(); + DataLoader dlA = factory.idLoader(calls); dlA.load("K1"); dlA.load("K2"); @@ -165,13 +171,14 @@ public void dispatchAllImmediately() { assertThat(calls, equalTo(singletonList(asList("K1", "K2")))); } - @Test - public void rescheduleNow() { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void rescheduleNow(TestDataLoaderFactory factory) { AtomicInteger i = new AtomicInteger(); DispatchPredicate countingPredicate = (dataLoaderKey, dataLoader) -> i.incrementAndGet() > 5; - List> calls = new ArrayList<>(); - DataLoader dlA = DataLoaderFactory.newDataLoader(keysAsValues(calls)); + List> calls = new ArrayList<>(); + DataLoader dlA = factory.idLoader(calls); dlA.load("K1"); dlA.load("K2"); @@ -189,13 +196,14 @@ public void rescheduleNow() { assertThat(calls, equalTo(singletonList(asList("K1", "K2")))); } - @Test - public void it_will_take_out_the_schedule_once_it_dispatches() { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void it_will_take_out_the_schedule_once_it_dispatches(TestDataLoaderFactory factory) { AtomicInteger counter = new AtomicInteger(); DispatchPredicate countingPredicate = (dataLoaderKey, dataLoader) -> counter.incrementAndGet() > 5; - List> calls = new ArrayList<>(); - DataLoader dlA = DataLoaderFactory.newDataLoader(keysAsValues(calls)); + List> calls = new ArrayList<>(); + DataLoader dlA = factory.idLoader(calls); dlA.load("K1"); dlA.load("K2"); @@ -231,15 +239,16 @@ public void it_will_take_out_the_schedule_once_it_dispatches() { assertThat(calls, equalTo(asList(asList("K1", "K2"), asList("K3", "K4")))); } - @Test - public void close_is_a_one_way_door() { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void close_is_a_one_way_door(TestDataLoaderFactory factory) { AtomicInteger counter = new AtomicInteger(); DispatchPredicate countingPredicate = (dataLoaderKey, dataLoader) -> { counter.incrementAndGet(); return false; }; - DataLoader dlA = TestKit.idLoader(); + DataLoader dlA = factory.idLoader(); dlA.load("K1"); dlA.load("K2"); @@ -276,12 +285,13 @@ public void close_is_a_one_way_door() { assertEquals(counter.get(), countThen + 1); } - @Test - public void can_tick_after_first_dispatch_for_chain_data_loaders() { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void can_tick_after_first_dispatch_for_chain_data_loaders(TestDataLoaderFactory factory) { // delays much bigger than the tick rate will mean multiple calls to dispatch - DataLoader dlA = TestKit.idLoaderAsync(Duration.ofMillis(100)); - DataLoader dlB = TestKit.idLoaderAsync(Duration.ofMillis(200)); + DataLoader dlA = factory.idLoaderDelayed(Duration.ofMillis(100)); + DataLoader dlB = factory.idLoaderDelayed(Duration.ofMillis(200)); CompletableFuture chainedCF = dlA.load("AK1").thenCompose(dlB::load); @@ -306,12 +316,13 @@ public void can_tick_after_first_dispatch_for_chain_data_loaders() { registry.close(); } - @Test - public void chain_data_loaders_will_hang_if_not_in_ticker_mode() { + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void chain_data_loaders_will_hang_if_not_in_ticker_mode(TestDataLoaderFactory factory) { // delays much bigger than the tick rate will mean multiple calls to dispatch - DataLoader dlA = TestKit.idLoaderAsync(Duration.ofMillis(100)); - DataLoader dlB = TestKit.idLoaderAsync(Duration.ofMillis(200)); + DataLoader dlA = factory.idLoaderDelayed(Duration.ofMillis(100)); + DataLoader dlB = factory.idLoaderDelayed(Duration.ofMillis(200)); CompletableFuture chainedCF = dlA.load("AK1").thenCompose(dlB::load); From 9a20334effd34fc1dd31c9592b3ac278c6fbc464 Mon Sep 17 00:00:00 2001 From: Alexandre Carlton Date: Wed, 1 Jan 2025 23:44:59 +0100 Subject: [PATCH 062/156] Allow ValueCache to work with Publisher DataLoader We resolve two bugs in the interaction between Publisher `DataLoader`s and `ValueCache`s: - if the value was cached, we complete the corresponding queued futures immediately. - if the value was _not_ cached, we remember to provide the queued future to the invoker so that the future can eventually be completed. --- .../java/org/dataloader/DataLoaderHelper.java | 3 +++ .../dataloader/DataLoaderValueCacheTest.java | 22 +++++++++---------- .../TestDataLoaderFactories.java | 9 +------- 3 files changed, 15 insertions(+), 19 deletions(-) diff --git a/src/main/java/org/dataloader/DataLoaderHelper.java b/src/main/java/org/dataloader/DataLoaderHelper.java index 9cd38d6c..29701b59 100644 --- a/src/main/java/org/dataloader/DataLoaderHelper.java +++ b/src/main/java/org/dataloader/DataLoaderHelper.java @@ -390,6 +390,9 @@ CompletableFuture> invokeLoader(List keys, List keyContexts, missedKeyIndexes.add(i); missedKeys.add(keys.get(i)); missedKeyContexts.add(keyContexts.get(i)); + missedQueuedFutures.add(queuedFutures.get(i)); + } else { + queuedFutures.get(i).complete(cacheGet.get()); } } } diff --git a/src/test/java/org/dataloader/DataLoaderValueCacheTest.java b/src/test/java/org/dataloader/DataLoaderValueCacheTest.java index 6c05b013..732febe9 100644 --- a/src/test/java/org/dataloader/DataLoaderValueCacheTest.java +++ b/src/test/java/org/dataloader/DataLoaderValueCacheTest.java @@ -33,7 +33,7 @@ public class DataLoaderValueCacheTest { @ParameterizedTest - @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#getWithoutPublisher") + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void test_by_default_we_have_no_value_caching(TestDataLoaderFactory factory) { List> loadCalls = new ArrayList<>(); DataLoaderOptions options = newOptions(); @@ -68,7 +68,7 @@ public void test_by_default_we_have_no_value_caching(TestDataLoaderFactory facto } @ParameterizedTest - @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#getWithoutPublisher") + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void should_accept_a_remote_value_store_for_caching(TestDataLoaderFactory factory) { CustomValueCache customValueCache = new CustomValueCache(); List> loadCalls = new ArrayList<>(); @@ -113,7 +113,7 @@ public void should_accept_a_remote_value_store_for_caching(TestDataLoaderFactory } @ParameterizedTest - @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#getWithoutPublisher") + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void can_use_caffeine_for_caching(TestDataLoaderFactory factory) { // // Mostly to prove that some other CACHE library could be used @@ -154,7 +154,7 @@ public void can_use_caffeine_for_caching(TestDataLoaderFactory factory) { } @ParameterizedTest - @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#getWithoutPublisher") + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void will_invoke_loader_if_CACHE_GET_call_throws_exception(TestDataLoaderFactory factory) { CustomValueCache customValueCache = new CustomValueCache() { @@ -185,7 +185,7 @@ public CompletableFuture get(String key) { } @ParameterizedTest - @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#getWithoutPublisher") + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void will_still_work_if_CACHE_SET_call_throws_exception(TestDataLoaderFactory factory) { CustomValueCache customValueCache = new CustomValueCache() { @Override @@ -214,7 +214,7 @@ public CompletableFuture set(String key, Object value) { } @ParameterizedTest - @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#getWithoutPublisher") + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void caching_can_take_some_time_complete(TestDataLoaderFactory factory) { CustomValueCache customValueCache = new CustomValueCache() { @@ -256,7 +256,7 @@ public CompletableFuture get(String key) { } @ParameterizedTest - @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#getWithoutPublisher") + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void batch_caching_works_as_expected(TestDataLoaderFactory factory) { CustomValueCache customValueCache = new CustomValueCache() { @@ -303,7 +303,7 @@ public CompletableFuture>> getValues(List keys) { } @ParameterizedTest - @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#getWithoutPublisher") + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void assertions_will_be_thrown_if_the_cache_does_not_follow_contract(TestDataLoaderFactory factory) { CustomValueCache customValueCache = new CustomValueCache() { @@ -346,7 +346,7 @@ private boolean isAssertionException(CompletableFuture fA) { @ParameterizedTest - @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#getWithoutPublisher") + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void if_caching_is_off_its_never_hit(TestDataLoaderFactory factory) { AtomicInteger getCalls = new AtomicInteger(); CustomValueCache customValueCache = new CustomValueCache() { @@ -380,7 +380,7 @@ public CompletableFuture get(String key) { } @ParameterizedTest - @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#getWithoutPublisher") + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void if_everything_is_cached_no_batching_happens(TestDataLoaderFactory factory) { AtomicInteger getCalls = new AtomicInteger(); AtomicInteger setCalls = new AtomicInteger(); @@ -423,7 +423,7 @@ public CompletableFuture> setValues(List keys, List @ParameterizedTest - @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#getWithoutPublisher") + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void if_batching_is_off_it_still_can_cache(TestDataLoaderFactory factory) { AtomicInteger getCalls = new AtomicInteger(); AtomicInteger setCalls = new AtomicInteger(); diff --git a/src/test/java/org/dataloader/fixtures/parameterized/TestDataLoaderFactories.java b/src/test/java/org/dataloader/fixtures/parameterized/TestDataLoaderFactories.java index dbdfd7d3..6afd05c7 100644 --- a/src/test/java/org/dataloader/fixtures/parameterized/TestDataLoaderFactories.java +++ b/src/test/java/org/dataloader/fixtures/parameterized/TestDataLoaderFactories.java @@ -6,6 +6,7 @@ import java.util.stream.Stream; public class TestDataLoaderFactories { + public static Stream get() { return Stream.of( Arguments.of(Named.of("List DataLoader", new ListDataLoaderFactory())), @@ -14,12 +15,4 @@ public static Stream get() { Arguments.of(Named.of("Mapped Publisher DataLoader", new MappedPublisherDataLoaderFactory())) ); } - - // TODO: Remove in favour of #get when ValueCache supports Publisher Factories. - public static Stream getWithoutPublisher() { - return Stream.of( - Arguments.of(Named.of("List DataLoader", new ListDataLoaderFactory())), - Arguments.of(Named.of("Mapped DataLoader", new MappedDataLoaderFactory())) - ); - } } From 742620728889d7f35418c44d4fb472835a90d913 Mon Sep 17 00:00:00 2001 From: bbaker Date: Mon, 20 Jan 2025 13:53:23 +1100 Subject: [PATCH 063/156] Transform support for DataLoaders --- src/main/java/org/dataloader/DataLoader.java | 89 ++++++++--------- .../org/dataloader/DataLoaderFactory.java | 97 +++++++++++-------- .../org/dataloader/DataLoaderBuilderTest.java | 64 ++++++++++++ 3 files changed, 160 insertions(+), 90 deletions(-) create mode 100644 src/test/java/org/dataloader/DataLoaderBuilderTest.java diff --git a/src/main/java/org/dataloader/DataLoader.java b/src/main/java/org/dataloader/DataLoader.java index dc4b726c..62e80de4 100644 --- a/src/main/java/org/dataloader/DataLoader.java +++ b/src/main/java/org/dataloader/DataLoader.java @@ -25,9 +25,15 @@ import java.time.Clock; import java.time.Duration; import java.time.Instant; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.function.BiConsumer; +import java.util.function.Consumer; import static org.dataloader.impl.Assertions.nonNull; @@ -54,7 +60,6 @@ * * @param type parameter indicating the type of the data load keys * @param type parameter indicating the type of the data that is returned - * * @author Arnold Schrijver * @author Brad Baker */ @@ -65,6 +70,8 @@ public class DataLoader { private final StatisticsCollector stats; private final CacheMap futureCache; private final ValueCache valueCache; + private final DataLoaderOptions options; + private final Object batchLoadFunction; /** * Creates new DataLoader with the specified batch loader function and default options @@ -73,9 +80,7 @@ public class DataLoader { * @param batchLoadFunction the batch load function to use * @param the key type * @param the value type - * * @return a new DataLoader - * * @deprecated use {@link DataLoaderFactory} instead */ @Deprecated @@ -90,9 +95,7 @@ public static DataLoader newDataLoader(BatchLoader batchLoadF * @param options the options to use * @param the key type * @param the value type - * * @return a new DataLoader - * * @deprecated use {@link DataLoaderFactory} instead */ @Deprecated @@ -114,9 +117,7 @@ public static DataLoader newDataLoader(BatchLoader batchLoadF * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects * @param the key type * @param the value type - * * @return a new DataLoader - * * @deprecated use {@link DataLoaderFactory} instead */ @Deprecated @@ -133,9 +134,7 @@ public static DataLoader newDataLoaderWithTry(BatchLoader * @param options the options to use * @param the key type * @param the value type - * * @return a new DataLoader - * * @see DataLoaderFactory#newDataLoaderWithTry(BatchLoader) * @deprecated use {@link DataLoaderFactory} instead */ @@ -151,9 +150,7 @@ public static DataLoader newDataLoaderWithTry(BatchLoader * @param batchLoadFunction the batch load function to use * @param the key type * @param the value type - * * @return a new DataLoader - * * @deprecated use {@link DataLoaderFactory} instead */ @Deprecated @@ -168,9 +165,7 @@ public static DataLoader newDataLoader(BatchLoaderWithContext * @param options the options to use * @param the key type * @param the value type - * * @return a new DataLoader - * * @deprecated use {@link DataLoaderFactory} instead */ @Deprecated @@ -192,9 +187,7 @@ public static DataLoader newDataLoader(BatchLoaderWithContext * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects * @param the key type * @param the value type - * * @return a new DataLoader - * * @deprecated use {@link DataLoaderFactory} instead */ @Deprecated @@ -211,9 +204,7 @@ public static DataLoader newDataLoaderWithTry(BatchLoaderWithContex * @param options the options to use * @param the key type * @param the value type - * * @return a new DataLoader - * * @see DataLoaderFactory#newDataLoaderWithTry(BatchLoader) * @deprecated use {@link DataLoaderFactory} instead */ @@ -229,9 +220,7 @@ public static DataLoader newDataLoaderWithTry(BatchLoaderWithContex * @param batchLoadFunction the batch load function to use * @param the key type * @param the value type - * * @return a new DataLoader - * * @deprecated use {@link DataLoaderFactory} instead */ @Deprecated @@ -246,9 +235,7 @@ public static DataLoader newMappedDataLoader(MappedBatchLoader the key type * @param the value type - * * @return a new DataLoader - * * @deprecated use {@link DataLoaderFactory} instead */ @Deprecated @@ -271,9 +258,7 @@ public static DataLoader newMappedDataLoader(MappedBatchLoader the key type * @param the value type - * * @return a new DataLoader - * * @deprecated use {@link DataLoaderFactory} instead */ @Deprecated @@ -290,9 +275,7 @@ public static DataLoader newMappedDataLoaderWithTry(MappedBatchLoad * @param options the options to use * @param the key type * @param the value type - * * @return a new DataLoader - * * @see DataLoaderFactory#newDataLoaderWithTry(BatchLoader) * @deprecated use {@link DataLoaderFactory} instead */ @@ -308,9 +291,7 @@ public static DataLoader newMappedDataLoaderWithTry(MappedBatchLoad * @param batchLoadFunction the batch load function to use * @param the key type * @param the value type - * * @return a new DataLoader - * * @deprecated use {@link DataLoaderFactory} instead */ @Deprecated @@ -325,9 +306,7 @@ public static DataLoader newMappedDataLoader(MappedBatchLoaderWithC * @param options the options to use * @param the key type * @param the value type - * * @return a new DataLoader - * * @deprecated use {@link DataLoaderFactory} instead */ @Deprecated @@ -349,9 +328,7 @@ public static DataLoader newMappedDataLoader(MappedBatchLoaderWithC * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects * @param the key type * @param the value type - * * @return a new DataLoader - * * @deprecated use {@link DataLoaderFactory} instead */ @Deprecated @@ -368,9 +345,7 @@ public static DataLoader newMappedDataLoaderWithTry(MappedBatchLoad * @param options the options to use * @param the key type * @param the value type - * * @return a new DataLoader - * * @see DataLoaderFactory#newDataLoaderWithTry(BatchLoader) * @deprecated use {@link DataLoaderFactory} instead */ @@ -383,7 +358,6 @@ public static DataLoader newMappedDataLoaderWithTry(MappedBatchLoad * Creates a new data loader with the provided batch load function, and default options. * * @param batchLoadFunction the batch load function to use - * * @deprecated use {@link DataLoaderFactory} instead */ @Deprecated @@ -396,7 +370,6 @@ public DataLoader(BatchLoader batchLoadFunction) { * * @param batchLoadFunction the batch load function to use * @param options the batch load options - * * @deprecated use {@link DataLoaderFactory} instead */ @Deprecated @@ -416,6 +389,8 @@ public DataLoader(BatchLoader batchLoadFunction, DataLoaderOptions options this.valueCache = determineValueCache(loaderOptions); // order of keys matter in data loader this.stats = nonNull(loaderOptions.getStatisticsCollector()); + this.batchLoadFunction = nonNull(batchLoadFunction); + this.options = loaderOptions; this.helper = new DataLoaderHelper<>(this, batchLoadFunction, loaderOptions, this.futureCache, this.valueCache, this.stats, clock); } @@ -431,6 +406,32 @@ private ValueCache determineValueCache(DataLoaderOptions loaderOptions) { return (ValueCache) loaderOptions.valueCache().orElseGet(ValueCache::defaultValueCache); } + /** + * @return the options used to build this {@link DataLoader} + */ + public DataLoaderOptions getOptions() { + return options; + } + + /** + * @return the batch load interface used to build this {@link DataLoader} + */ + public Object getBatchLoadFunction() { + return batchLoadFunction; + } + + /** + * This allows you to change the current {@link DataLoader} and turn it into a new one + * + * @param builderConsumer the {@link DataLoaderFactory.Builder} consumer for changing the {@link DataLoader} + * @return a newly built {@link DataLoader} instance + */ + public DataLoader transform(Consumer> builderConsumer) { + DataLoaderFactory.Builder builder = DataLoaderFactory.builder(this); + builderConsumer.accept(builder); + return builder.build(); + } + /** * This returns the last instant the data loader was dispatched. When the data loader is created this value is set to now. * @@ -457,7 +458,6 @@ public Duration getTimeSinceDispatch() { * and returned from cache). * * @param key the key to load - * * @return the future of the value */ public CompletableFuture load(K key) { @@ -475,7 +475,6 @@ public CompletableFuture load(K key) { * NOTE : This will NOT cause a data load to happen. You must call {@link #load(Object)} for that to happen. * * @param key the key to check - * * @return an Optional to the future of the value */ public Optional> getIfPresent(K key) { @@ -494,7 +493,6 @@ public Optional> getIfPresent(K key) { * NOTE : This will NOT cause a data load to happen. You must call {@link #load(Object)} for that to happen. * * @param key the key to check - * * @return an Optional to the future of the value */ public Optional> getIfCompleted(K key) { @@ -514,7 +512,6 @@ public Optional> getIfCompleted(K key) { * * @param key the key to load * @param keyContext a context object that is specific to this key - * * @return the future of the value */ public CompletableFuture load(K key, Object keyContext) { @@ -530,7 +527,6 @@ public CompletableFuture load(K key, Object keyContext) { * and returned from cache). * * @param keys the list of keys to load - * * @return the composite future of the list of values */ public CompletableFuture> loadMany(List keys) { @@ -550,7 +546,6 @@ public CompletableFuture> loadMany(List keys) { * * @param keys the list of keys to load * @param keyContexts the list of key calling context objects - * * @return the composite future of the list of values */ public CompletableFuture> loadMany(List keys, List keyContexts) { @@ -583,7 +578,6 @@ public CompletableFuture> loadMany(List keys, List keyContext * {@link org.dataloader.MappedBatchLoaderWithContext} to help retrieve data. * * @param keysAndContexts the map of keys to their respective contexts - * * @return the composite future of the map of keys and values */ public CompletableFuture> loadMany(Map keysAndContexts) { @@ -656,7 +650,6 @@ public int dispatchDepth() { * on the next load request. * * @param key the key to remove - * * @return the data loader for fluent coding */ public DataLoader clear(K key) { @@ -670,7 +663,6 @@ public DataLoader clear(K key) { * * @param key the key to remove * @param handler a handler that will be called after the async remote clear completes - * * @return the data loader for fluent coding */ public DataLoader clear(K key, BiConsumer handler) { @@ -696,7 +688,6 @@ public DataLoader clearAll() { * Clears the entire cache map of the loader, and of the cached value store. * * @param handler a handler that will be called after the async remote clear all completes - * * @return the data loader for fluent coding */ public DataLoader clearAll(BiConsumer handler) { @@ -714,7 +705,6 @@ public DataLoader clearAll(BiConsumer handler) { * * @param key the key * @param value the value - * * @return the data loader for fluent coding */ public DataLoader prime(K key, V value) { @@ -726,7 +716,6 @@ public DataLoader prime(K key, V value) { * * @param key the key * @param error the exception to prime instead of a value - * * @return the data loader for fluent coding */ public DataLoader prime(K key, Exception error) { @@ -740,7 +729,6 @@ public DataLoader prime(K key, Exception error) { * * @param key the key * @param value the value - * * @return the data loader for fluent coding */ public DataLoader prime(K key, CompletableFuture value) { @@ -760,7 +748,6 @@ public DataLoader prime(K key, CompletableFuture value) { * If no cache key function is present in {@link DataLoaderOptions}, then the returned value equals the input key. * * @param key the input key - * * @return the cache key after the input is transformed with the cache key function */ public Object getCacheKey(K key) { @@ -779,6 +766,7 @@ public Statistics getStatistics() { /** * Gets the cacheMap associated with this data loader passed in via {@link DataLoaderOptions#cacheMap()} + * * @return the cacheMap of this data loader */ public CacheMap getCacheMap() { @@ -788,6 +776,7 @@ public CacheMap getCacheMap() { /** * Gets the valueCache associated with this data loader passed in via {@link DataLoaderOptions#valueCache()} + * * @return the valueCache of this data loader */ public ValueCache getValueCache() { diff --git a/src/main/java/org/dataloader/DataLoaderFactory.java b/src/main/java/org/dataloader/DataLoaderFactory.java index db14f2e8..0dc029a5 100644 --- a/src/main/java/org/dataloader/DataLoaderFactory.java +++ b/src/main/java/org/dataloader/DataLoaderFactory.java @@ -16,7 +16,6 @@ public class DataLoaderFactory { * @param batchLoadFunction the batch load function to use * @param the key type * @param the value type - * * @return a new DataLoader */ public static DataLoader newDataLoader(BatchLoader batchLoadFunction) { @@ -30,7 +29,6 @@ public static DataLoader newDataLoader(BatchLoader batchLoadF * @param options the options to use * @param the key type * @param the value type - * * @return a new DataLoader */ public static DataLoader newDataLoader(BatchLoader batchLoadFunction, DataLoaderOptions options) { @@ -51,7 +49,6 @@ public static DataLoader newDataLoader(BatchLoader batchLoadF * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects * @param the key type * @param the value type - * * @return a new DataLoader */ public static DataLoader newDataLoaderWithTry(BatchLoader> batchLoadFunction) { @@ -67,9 +64,7 @@ public static DataLoader newDataLoaderWithTry(BatchLoader * @param options the options to use * @param the key type * @param the value type - * * @return a new DataLoader - * * @see #newDataLoaderWithTry(BatchLoader) */ public static DataLoader newDataLoaderWithTry(BatchLoader> batchLoadFunction, DataLoaderOptions options) { @@ -83,7 +78,6 @@ public static DataLoader newDataLoaderWithTry(BatchLoader * @param batchLoadFunction the batch load function to use * @param the key type * @param the value type - * * @return a new DataLoader */ public static DataLoader newDataLoader(BatchLoaderWithContext batchLoadFunction) { @@ -97,7 +91,6 @@ public static DataLoader newDataLoader(BatchLoaderWithContext * @param options the options to use * @param the key type * @param the value type - * * @return a new DataLoader */ public static DataLoader newDataLoader(BatchLoaderWithContext batchLoadFunction, DataLoaderOptions options) { @@ -118,7 +111,6 @@ public static DataLoader newDataLoader(BatchLoaderWithContext * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects * @param the key type * @param the value type - * * @return a new DataLoader */ public static DataLoader newDataLoaderWithTry(BatchLoaderWithContext> batchLoadFunction) { @@ -134,9 +126,7 @@ public static DataLoader newDataLoaderWithTry(BatchLoaderWithContex * @param options the options to use * @param the key type * @param the value type - * * @return a new DataLoader - * * @see #newDataLoaderWithTry(BatchLoader) */ public static DataLoader newDataLoaderWithTry(BatchLoaderWithContext> batchLoadFunction, DataLoaderOptions options) { @@ -150,7 +140,6 @@ public static DataLoader newDataLoaderWithTry(BatchLoaderWithContex * @param batchLoadFunction the batch load function to use * @param the key type * @param the value type - * * @return a new DataLoader */ public static DataLoader newMappedDataLoader(MappedBatchLoader batchLoadFunction) { @@ -164,7 +153,6 @@ public static DataLoader newMappedDataLoader(MappedBatchLoader the key type * @param the value type - * * @return a new DataLoader */ public static DataLoader newMappedDataLoader(MappedBatchLoader batchLoadFunction, DataLoaderOptions options) { @@ -186,7 +174,6 @@ public static DataLoader newMappedDataLoader(MappedBatchLoader the key type * @param the value type - * * @return a new DataLoader */ public static DataLoader newMappedDataLoaderWithTry(MappedBatchLoader> batchLoadFunction) { @@ -202,9 +189,7 @@ public static DataLoader newMappedDataLoaderWithTry(MappedBatchLoad * @param options the options to use * @param the key type * @param the value type - * * @return a new DataLoader - * * @see #newDataLoaderWithTry(BatchLoader) */ public static DataLoader newMappedDataLoaderWithTry(MappedBatchLoader> batchLoadFunction, DataLoaderOptions options) { @@ -218,7 +203,6 @@ public static DataLoader newMappedDataLoaderWithTry(MappedBatchLoad * @param batchLoadFunction the batch load function to use * @param the key type * @param the value type - * * @return a new DataLoader */ public static DataLoader newMappedDataLoader(MappedBatchLoaderWithContext batchLoadFunction) { @@ -232,7 +216,6 @@ public static DataLoader newMappedDataLoader(MappedBatchLoaderWithC * @param options the options to use * @param the key type * @param the value type - * * @return a new DataLoader */ public static DataLoader newMappedDataLoader(MappedBatchLoaderWithContext batchLoadFunction, DataLoaderOptions options) { @@ -253,7 +236,6 @@ public static DataLoader newMappedDataLoader(MappedBatchLoaderWithC * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects * @param the key type * @param the value type - * * @return a new DataLoader */ public static DataLoader newMappedDataLoaderWithTry(MappedBatchLoaderWithContext> batchLoadFunction) { @@ -269,9 +251,7 @@ public static DataLoader newMappedDataLoaderWithTry(MappedBatchLoad * @param options the options to use * @param the key type * @param the value type - * * @return a new DataLoader - * * @see #newDataLoaderWithTry(BatchLoader) */ public static DataLoader newMappedDataLoaderWithTry(MappedBatchLoaderWithContext> batchLoadFunction, DataLoaderOptions options) { @@ -285,7 +265,6 @@ public static DataLoader newMappedDataLoaderWithTry(MappedBatchLoad * @param batchLoadFunction the batch load function to use * @param the key type * @param the value type - * * @return a new DataLoader */ public static DataLoader newPublisherDataLoader(BatchPublisher batchLoadFunction) { @@ -299,7 +278,6 @@ public static DataLoader newPublisherDataLoader(BatchPublisher the key type * @param the value type - * * @return a new DataLoader */ public static DataLoader newPublisherDataLoader(BatchPublisher batchLoadFunction, DataLoaderOptions options) { @@ -320,7 +298,6 @@ public static DataLoader newPublisherDataLoader(BatchPublisher the key type * @param the value type - * * @return a new DataLoader */ public static DataLoader newPublisherDataLoaderWithTry(BatchPublisher> batchLoadFunction) { @@ -336,9 +313,7 @@ public static DataLoader newPublisherDataLoaderWithTry(BatchPublish * @param options the options to use * @param the key type * @param the value type - * * @return a new DataLoader - * * @see #newDataLoaderWithTry(BatchLoader) */ public static DataLoader newPublisherDataLoaderWithTry(BatchPublisher> batchLoadFunction, DataLoaderOptions options) { @@ -352,7 +327,6 @@ public static DataLoader newPublisherDataLoaderWithTry(BatchPublish * @param batchLoadFunction the batch load function to use * @param the key type * @param the value type - * * @return a new DataLoader */ public static DataLoader newPublisherDataLoader(BatchPublisherWithContext batchLoadFunction) { @@ -366,7 +340,6 @@ public static DataLoader newPublisherDataLoader(BatchPublisherWithC * @param options the options to use * @param the key type * @param the value type - * * @return a new DataLoader */ public static DataLoader newPublisherDataLoader(BatchPublisherWithContext batchLoadFunction, DataLoaderOptions options) { @@ -387,7 +360,6 @@ public static DataLoader newPublisherDataLoader(BatchPublisherWithC * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects * @param the key type * @param the value type - * * @return a new DataLoader */ public static DataLoader newPublisherDataLoaderWithTry(BatchPublisherWithContext> batchLoadFunction) { @@ -403,9 +375,7 @@ public static DataLoader newPublisherDataLoaderWithTry(BatchPublish * @param options the options to use * @param the key type * @param the value type - * * @return a new DataLoader - * * @see #newPublisherDataLoaderWithTry(BatchPublisher) */ public static DataLoader newPublisherDataLoaderWithTry(BatchPublisherWithContext> batchLoadFunction, DataLoaderOptions options) { @@ -419,7 +389,6 @@ public static DataLoader newPublisherDataLoaderWithTry(BatchPublish * @param batchLoadFunction the batch load function to use * @param the key type * @param the value type - * * @return a new DataLoader */ public static DataLoader newMappedPublisherDataLoader(MappedBatchPublisher batchLoadFunction) { @@ -433,7 +402,6 @@ public static DataLoader newMappedPublisherDataLoader(MappedBatchPu * @param options the options to use * @param the key type * @param the value type - * * @return a new DataLoader */ public static DataLoader newMappedPublisherDataLoader(MappedBatchPublisher batchLoadFunction, DataLoaderOptions options) { @@ -454,7 +422,6 @@ public static DataLoader newMappedPublisherDataLoader(MappedBatchPu * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects * @param the key type * @param the value type - * * @return a new DataLoader */ public static DataLoader newMappedPublisherDataLoaderWithTry(MappedBatchPublisher> batchLoadFunction) { @@ -470,9 +437,7 @@ public static DataLoader newMappedPublisherDataLoaderWithTry(Mapped * @param options the options to use * @param the key type * @param the value type - * * @return a new DataLoader - * * @see #newDataLoaderWithTry(BatchLoader) */ public static DataLoader newMappedPublisherDataLoaderWithTry(MappedBatchPublisher> batchLoadFunction, DataLoaderOptions options) { @@ -486,7 +451,6 @@ public static DataLoader newMappedPublisherDataLoaderWithTry(Mapped * @param batchLoadFunction the batch load function to use * @param the key type * @param the value type - * * @return a new DataLoader */ public static DataLoader newMappedPublisherDataLoader(MappedBatchPublisherWithContext batchLoadFunction) { @@ -500,7 +464,6 @@ public static DataLoader newMappedPublisherDataLoader(MappedBatchPu * @param options the options to use * @param the key type * @param the value type - * * @return a new DataLoader */ public static DataLoader newMappedPublisherDataLoader(MappedBatchPublisherWithContext batchLoadFunction, DataLoaderOptions options) { @@ -521,7 +484,6 @@ public static DataLoader newMappedPublisherDataLoader(MappedBatchPu * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects * @param the key type * @param the value type - * * @return a new DataLoader */ public static DataLoader newMappedPublisherDataLoaderWithTry(MappedBatchPublisherWithContext> batchLoadFunction) { @@ -537,9 +499,7 @@ public static DataLoader newMappedPublisherDataLoaderWithTry(Mapped * @param options the options to use * @param the key type * @param the value type - * * @return a new DataLoader - * * @see #newMappedPublisherDataLoaderWithTry(MappedBatchPublisher) */ public static DataLoader newMappedPublisherDataLoaderWithTry(MappedBatchPublisherWithContext> batchLoadFunction, DataLoaderOptions options) { @@ -549,4 +509,61 @@ public static DataLoader newMappedPublisherDataLoaderWithTry(Mapped static DataLoader mkDataLoader(Object batchLoadFunction, DataLoaderOptions options) { return new DataLoader<>(batchLoadFunction, options); } + + /** + * Return a new {@link Builder} of a data loader. + * + * @param the key type + * @param the value type + * @return a new {@link Builder} of a data loader + */ + public static Builder builder() { + return new Builder<>(); + } + + /** + * Return a new {@link Builder} of a data loader using the specified one as a template. + * + * @param the key type + * @param the value type + * @param dataLoader the {@link DataLoader} to copy values from into the builder + * @return a new {@link Builder} of a data loader + */ + public static Builder builder(DataLoader dataLoader) { + return new Builder<>(dataLoader); + } + + /** + * A builder of {@link DataLoader}s + * + * @param the key type + * @param the value type + */ + public static class Builder { + Object batchLoadFunction; + DataLoaderOptions options = DataLoaderOptions.newOptions(); + + Builder() { + } + + Builder(DataLoader dataLoader) { + this.batchLoadFunction = dataLoader.getBatchLoadFunction(); + this.options = dataLoader.getOptions(); + } + + public Builder batchLoadFunction(Object batchLoadFunction) { + this.batchLoadFunction = batchLoadFunction; + return this; + } + + public Builder options(DataLoaderOptions options) { + this.options = options; + return this; + } + + DataLoader build() { + return mkDataLoader(batchLoadFunction, options); + } + } } + diff --git a/src/test/java/org/dataloader/DataLoaderBuilderTest.java b/src/test/java/org/dataloader/DataLoaderBuilderTest.java new file mode 100644 index 00000000..523b9a50 --- /dev/null +++ b/src/test/java/org/dataloader/DataLoaderBuilderTest.java @@ -0,0 +1,64 @@ +package org.dataloader; + +import org.junit.jupiter.api.Test; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; + +public class DataLoaderBuilderTest { + + BatchLoader batchLoader1 = keys -> null; + + BatchLoader batchLoader2 = keys -> null; + + DataLoaderOptions defaultOptions = DataLoaderOptions.newOptions(); + DataLoaderOptions differentOptions = DataLoaderOptions.newOptions().setCachingEnabled(false); + + @Test + void canBuildNewDataLoaders() { + DataLoaderFactory.Builder builder = DataLoaderFactory.builder(); + builder.options(differentOptions); + builder.batchLoadFunction(batchLoader1); + DataLoader dataLoader = builder.build(); + + assertThat(dataLoader.getOptions(), equalTo(differentOptions)); + assertThat(dataLoader.getBatchLoadFunction(), equalTo(batchLoader1)); + // + // and we can copy ok + // + builder = DataLoaderFactory.builder(dataLoader); + dataLoader = builder.build(); + + assertThat(dataLoader.getOptions(), equalTo(differentOptions)); + assertThat(dataLoader.getBatchLoadFunction(), equalTo(batchLoader1)); + // + // and we can copy and transform ok + // + builder = DataLoaderFactory.builder(dataLoader); + builder.options(defaultOptions); + builder.batchLoadFunction(batchLoader2); + dataLoader = builder.build(); + + assertThat(dataLoader.getOptions(), equalTo(defaultOptions)); + assertThat(dataLoader.getBatchLoadFunction(), equalTo(batchLoader2)); + } + + @Test + void theDataLoaderCanTransform() { + DataLoader dataLoader1 = DataLoaderFactory.newDataLoader(batchLoader1, defaultOptions); + assertThat(dataLoader1.getOptions(), equalTo(defaultOptions)); + assertThat(dataLoader1.getBatchLoadFunction(), equalTo(batchLoader1)); + // + // we can transform the data loader + // + DataLoader dataLoader2 = dataLoader1.transform(it -> { + it.options(differentOptions); + it.batchLoadFunction(batchLoader2); + }); + + assertThat(dataLoader2, not(equalTo(dataLoader1))); + assertThat(dataLoader2.getOptions(), equalTo(differentOptions)); + assertThat(dataLoader2.getBatchLoadFunction(), equalTo(batchLoader2)); + } +} From e55b6431fa9ce01dc6c6017da87b2d6ae95a7855 Mon Sep 17 00:00:00 2001 From: bbaker Date: Tue, 21 Jan 2025 10:11:57 +1100 Subject: [PATCH 064/156] Instrumentatin support for dataloader --- .../org/dataloader/DataLoaderFactory.java | 2 +- .../java/org/dataloader/DataLoaderHelper.java | 38 +++++- .../org/dataloader/DataLoaderOptions.java | 38 +++--- .../DataLoaderInstrumentation.java | 36 ++++++ .../DataLoaderInstrumentationContext.java | 28 +++++ .../DataLoaderInstrumentationHelper.java | 34 ++++++ .../DataLoaderInstrumentationTest.java | 108 ++++++++++++++++++ 7 files changed, 264 insertions(+), 20 deletions(-) create mode 100644 src/main/java/org/dataloader/instrumentation/DataLoaderInstrumentation.java create mode 100644 src/main/java/org/dataloader/instrumentation/DataLoaderInstrumentationContext.java create mode 100644 src/main/java/org/dataloader/instrumentation/DataLoaderInstrumentationHelper.java create mode 100644 src/test/java/org/dataloader/instrumentation/DataLoaderInstrumentationTest.java diff --git a/src/main/java/org/dataloader/DataLoaderFactory.java b/src/main/java/org/dataloader/DataLoaderFactory.java index 0dc029a5..a87e4ebd 100644 --- a/src/main/java/org/dataloader/DataLoaderFactory.java +++ b/src/main/java/org/dataloader/DataLoaderFactory.java @@ -561,7 +561,7 @@ public Builder options(DataLoaderOptions options) { return this; } - DataLoader build() { + public DataLoader build() { return mkDataLoader(batchLoadFunction, options); } } diff --git a/src/main/java/org/dataloader/DataLoaderHelper.java b/src/main/java/org/dataloader/DataLoaderHelper.java index 29701b59..9b5a59e8 100644 --- a/src/main/java/org/dataloader/DataLoaderHelper.java +++ b/src/main/java/org/dataloader/DataLoaderHelper.java @@ -3,6 +3,8 @@ import org.dataloader.annotations.GuardedBy; import org.dataloader.annotations.Internal; import org.dataloader.impl.CompletableFutureKit; +import org.dataloader.instrumentation.DataLoaderInstrumentation; +import org.dataloader.instrumentation.DataLoaderInstrumentationContext; import org.dataloader.reactive.ReactiveSupport; import org.dataloader.scheduler.BatchLoaderScheduler; import org.dataloader.stats.StatisticsCollector; @@ -34,6 +36,7 @@ import static java.util.stream.Collectors.toList; import static org.dataloader.impl.Assertions.assertState; import static org.dataloader.impl.Assertions.nonNull; +import static org.dataloader.instrumentation.DataLoaderInstrumentationHelper.ctxOrNoopCtx; /** * This helps break up the large DataLoader class functionality, and it contains the logic to dispatch the @@ -167,6 +170,8 @@ Object getCacheKeyWithContext(K key, Object context) { } DispatchResult dispatch() { + DataLoaderInstrumentationContext> instrCtx = ctxOrNoopCtx(instrumentation().beginDispatch(dataLoader)); + boolean batchingEnabled = loaderOptions.batchingEnabled(); final List keys; final List callContexts; @@ -175,7 +180,8 @@ DispatchResult dispatch() { int queueSize = loaderQueue.size(); if (queueSize == 0) { lastDispatchTime.set(now()); - return emptyDispatchResult(); + instrCtx.onDispatched(); + return endDispatchCtx(instrCtx, emptyDispatchResult()); } // we copy the pre-loaded set of futures ready for dispatch @@ -192,7 +198,8 @@ DispatchResult dispatch() { lastDispatchTime.set(now()); } if (!batchingEnabled) { - return emptyDispatchResult(); + instrCtx.onDispatched(); + return endDispatchCtx(instrCtx, emptyDispatchResult()); } final int totalEntriesHandled = keys.size(); // @@ -213,7 +220,15 @@ DispatchResult dispatch() { } else { futureList = dispatchQueueBatch(keys, callContexts, queuedFutures); } - return new DispatchResult<>(futureList, totalEntriesHandled); + instrCtx.onDispatched(); + return endDispatchCtx(instrCtx, new DispatchResult<>(futureList, totalEntriesHandled)); + } + + private DispatchResult endDispatchCtx(DataLoaderInstrumentationContext> instrCtx, DispatchResult dispatchResult) { + // once the CF completes, we can tell the instrumentation + dispatchResult.getPromisedResults() + .whenComplete((result, throwable) -> instrCtx.onCompleted(dispatchResult, throwable)); + return dispatchResult; } private CompletableFuture> sliceIntoBatchesOfBatches(List keys, List> queuedFutures, List callContexts, int maxBatchSize) { @@ -427,11 +442,14 @@ CompletableFuture> invokeLoader(List keys, List keyContexts, } CompletableFuture> invokeLoader(List keys, List keyContexts, List> queuedFutures) { + Object context = loaderOptions.getBatchLoaderContextProvider().getContext(); + BatchLoaderEnvironment environment = BatchLoaderEnvironment.newBatchLoaderEnvironment() + .context(context).keyContexts(keys, keyContexts).build(); + + DataLoaderInstrumentationContext> instrCtx = ctxOrNoopCtx(instrumentation().beginBatchLoader(dataLoader, keys, environment)); + CompletableFuture> batchLoad; try { - Object context = loaderOptions.getBatchLoaderContextProvider().getContext(); - BatchLoaderEnvironment environment = BatchLoaderEnvironment.newBatchLoaderEnvironment() - .context(context).keyContexts(keys, keyContexts).build(); if (isMapLoader()) { batchLoad = invokeMapBatchLoader(keys, environment); } else if (isPublisher()) { @@ -441,12 +459,16 @@ CompletableFuture> invokeLoader(List keys, List keyContexts, } else { batchLoad = invokeListBatchLoader(keys, environment); } + instrCtx.onDispatched(); } catch (Exception e) { + instrCtx.onDispatched(); batchLoad = CompletableFutureKit.failedFuture(e); } + batchLoad.whenComplete(instrCtx::onCompleted); return batchLoad; } + @SuppressWarnings("unchecked") private CompletableFuture> invokeListBatchLoader(List keys, BatchLoaderEnvironment environment) { CompletionStage> loadResult; @@ -575,6 +597,10 @@ private boolean isMappedPublisher() { return batchLoadFunction instanceof MappedBatchPublisher; } + private DataLoaderInstrumentation instrumentation() { + return loaderOptions.getInstrumentation(); + } + int dispatchDepth() { synchronized (dataLoader) { return loaderQueue.size(); diff --git a/src/main/java/org/dataloader/DataLoaderOptions.java b/src/main/java/org/dataloader/DataLoaderOptions.java index b96e7857..715bcc55 100644 --- a/src/main/java/org/dataloader/DataLoaderOptions.java +++ b/src/main/java/org/dataloader/DataLoaderOptions.java @@ -18,6 +18,8 @@ import org.dataloader.annotations.PublicApi; import org.dataloader.impl.Assertions; +import org.dataloader.instrumentation.DataLoaderInstrumentation; +import org.dataloader.instrumentation.DataLoaderInstrumentationHelper; import org.dataloader.scheduler.BatchLoaderScheduler; import org.dataloader.stats.NoOpStatisticsCollector; import org.dataloader.stats.StatisticsCollector; @@ -48,6 +50,7 @@ public class DataLoaderOptions { private BatchLoaderContextProvider environmentProvider; private ValueCacheOptions valueCacheOptions; private BatchLoaderScheduler batchLoaderScheduler; + private DataLoaderInstrumentation instrumentation; /** * Creates a new data loader options with default settings. @@ -61,6 +64,7 @@ public DataLoaderOptions() { environmentProvider = NULL_PROVIDER; valueCacheOptions = ValueCacheOptions.newOptions(); batchLoaderScheduler = null; + instrumentation = DataLoaderInstrumentationHelper.NOOP_INSTRUMENTATION; } /** @@ -80,7 +84,8 @@ public DataLoaderOptions(DataLoaderOptions other) { this.statisticsCollector = other.statisticsCollector; this.environmentProvider = other.environmentProvider; this.valueCacheOptions = other.valueCacheOptions; - batchLoaderScheduler = other.batchLoaderScheduler; + this.batchLoaderScheduler = other.batchLoaderScheduler; + this.instrumentation = other.instrumentation; } /** @@ -103,7 +108,6 @@ public boolean batchingEnabled() { * Sets the option that determines whether batch loading is enabled. * * @param batchingEnabled {@code true} to enable batch loading, {@code false} otherwise - * * @return the data loader options for fluent coding */ public DataLoaderOptions setBatchingEnabled(boolean batchingEnabled) { @@ -124,7 +128,6 @@ public boolean cachingEnabled() { * Sets the option that determines whether caching is enabled. * * @param cachingEnabled {@code true} to enable caching, {@code false} otherwise - * * @return the data loader options for fluent coding */ public DataLoaderOptions setCachingEnabled(boolean cachingEnabled) { @@ -134,7 +137,7 @@ public DataLoaderOptions setCachingEnabled(boolean cachingEnabled) { /** * Option that determines whether to cache exceptional values (the default), or not. - * + *

* For short-lived caches (that is request caches) it makes sense to cache exceptions since * it's likely the key is still poisoned. However, if you have long-lived caches, then it may make * sense to set this to false since the downstream system may have recovered from its failure @@ -150,7 +153,6 @@ public boolean cachingExceptionsEnabled() { * Sets the option that determines whether exceptional values are cache enabled. * * @param cachingExceptionsEnabled {@code true} to enable caching exceptional values, {@code false} otherwise - * * @return the data loader options for fluent coding */ public DataLoaderOptions setCachingExceptionsEnabled(boolean cachingExceptionsEnabled) { @@ -173,7 +175,6 @@ public Optional cacheKeyFunction() { * Sets the function to use for creating the cache key, if caching is enabled. * * @param cacheKeyFunction the cache key function to use - * * @return the data loader options for fluent coding */ public DataLoaderOptions setCacheKeyFunction(CacheKey cacheKeyFunction) { @@ -196,7 +197,6 @@ public DataLoaderOptions setCacheKeyFunction(CacheKey cacheKeyFunction) { * Sets the cache map implementation to use for caching, if caching is enabled. * * @param cacheMap the cache map instance - * * @return the data loader options for fluent coding */ public DataLoaderOptions setCacheMap(CacheMap cacheMap) { @@ -219,7 +219,6 @@ public int maxBatchSize() { * before they are split into multiple class * * @param maxBatchSize the maximum batch size - * * @return the data loader options for fluent coding */ public DataLoaderOptions setMaxBatchSize(int maxBatchSize) { @@ -240,7 +239,6 @@ public StatisticsCollector getStatisticsCollector() { * a common value * * @param statisticsCollector the statistics collector to use - * * @return the data loader options for fluent coding */ public DataLoaderOptions setStatisticsCollector(Supplier statisticsCollector) { @@ -259,7 +257,6 @@ public BatchLoaderContextProvider getBatchLoaderContextProvider() { * Sets the batch loader environment provider that will be used to give context to batch load functions * * @param contextProvider the batch loader context provider - * * @return the data loader options for fluent coding */ public DataLoaderOptions setBatchLoaderContextProvider(BatchLoaderContextProvider contextProvider) { @@ -282,7 +279,6 @@ public DataLoaderOptions setBatchLoaderContextProvider(BatchLoaderContextProvide * Sets the value cache implementation to use for caching values, if caching is enabled. * * @param valueCache the value cache instance - * * @return the data loader options for fluent coding */ public DataLoaderOptions setValueCache(ValueCache valueCache) { @@ -301,7 +297,6 @@ public ValueCacheOptions getValueCacheOptions() { * Sets the {@link ValueCacheOptions} that control how the {@link ValueCache} will be used * * @param valueCacheOptions the value cache options - * * @return the data loader options for fluent coding */ public DataLoaderOptions setValueCacheOptions(ValueCacheOptions valueCacheOptions) { @@ -321,11 +316,28 @@ public BatchLoaderScheduler getBatchLoaderScheduler() { * to some future time. * * @param batchLoaderScheduler the scheduler - * * @return the data loader options for fluent coding */ public DataLoaderOptions setBatchLoaderScheduler(BatchLoaderScheduler batchLoaderScheduler) { this.batchLoaderScheduler = batchLoaderScheduler; return this; } + + /** + * @return the {@link DataLoaderInstrumentation} to use + */ + public DataLoaderInstrumentation getInstrumentation() { + return instrumentation; + } + + /** + * Sets in a new {@link DataLoaderInstrumentation} + * + * @param instrumentation the new {@link DataLoaderInstrumentation} + * @return the data loader options for fluent coding + */ + public DataLoaderOptions setInstrumentation(DataLoaderInstrumentation instrumentation) { + this.instrumentation = nonNull(instrumentation); + return this; + } } diff --git a/src/main/java/org/dataloader/instrumentation/DataLoaderInstrumentation.java b/src/main/java/org/dataloader/instrumentation/DataLoaderInstrumentation.java new file mode 100644 index 00000000..f0da7887 --- /dev/null +++ b/src/main/java/org/dataloader/instrumentation/DataLoaderInstrumentation.java @@ -0,0 +1,36 @@ +package org.dataloader.instrumentation; + +import org.dataloader.BatchLoaderEnvironment; +import org.dataloader.DataLoader; +import org.dataloader.DispatchResult; + +import java.util.List; + +/** + * This interface is called when certain actions happen inside a data loader + */ +public interface DataLoaderInstrumentation { + /** + * This call back is done just before the {@link DataLoader#dispatch()} is invoked + * and it completes when the dispatch call promise is done. + * + * @param dataLoader the {@link DataLoader} in question + * @return a DataLoaderInstrumentationContext or null to be more performant + */ + default DataLoaderInstrumentationContext> beginDispatch(DataLoader dataLoader) { + return null; + } + + /** + * This call back is done just before the batch loader of a {@link DataLoader} is invoked. Remember a batch loader + * could be called multiple times during a dispatch event (because of max batch sizes) + * + * @param dataLoader the {@link DataLoader} in question + * @param keys the set of keys being fetched + * @param environment the {@link BatchLoaderEnvironment} + * @return a DataLoaderInstrumentationContext or null to be more performant + */ + default DataLoaderInstrumentationContext> beginBatchLoader(DataLoader dataLoader, List keys, BatchLoaderEnvironment environment) { + return null; + } +} diff --git a/src/main/java/org/dataloader/instrumentation/DataLoaderInstrumentationContext.java b/src/main/java/org/dataloader/instrumentation/DataLoaderInstrumentationContext.java new file mode 100644 index 00000000..ae0bbc12 --- /dev/null +++ b/src/main/java/org/dataloader/instrumentation/DataLoaderInstrumentationContext.java @@ -0,0 +1,28 @@ +package org.dataloader.instrumentation; + +import java.util.concurrent.CompletableFuture; + +/** + * When a {@link DataLoaderInstrumentation}.'beginXXX()' method is called then it must return a {@link DataLoaderInstrumentationContext} + * that will be invoked when the step is first dispatched and then when it completes. Sometimes this is effectively the same time + * whereas at other times it's when an asynchronous {@link CompletableFuture} completes. + *

+ * This pattern of construction of an object then call back is intended to allow "timers" to be created that can instrument what has + * just happened or "loggers" to be called to record what has happened. + */ +public interface DataLoaderInstrumentationContext { + /** + * This is invoked when the instrumentation step is initially dispatched + */ + default void onDispatched() { + } + + /** + * This is invoked when the instrumentation step is fully completed + * + * @param result the result of the step (which may be null) + * @param t this exception will be non-null if an exception was thrown during the step + */ + default void onCompleted(T result, Throwable t) { + } +} diff --git a/src/main/java/org/dataloader/instrumentation/DataLoaderInstrumentationHelper.java b/src/main/java/org/dataloader/instrumentation/DataLoaderInstrumentationHelper.java new file mode 100644 index 00000000..5ff545dc --- /dev/null +++ b/src/main/java/org/dataloader/instrumentation/DataLoaderInstrumentationHelper.java @@ -0,0 +1,34 @@ +package org.dataloader.instrumentation; + +public class DataLoaderInstrumentationHelper { + + private static final DataLoaderInstrumentationContext NOOP_CTX = new DataLoaderInstrumentationContext<>() { + @Override + public void onDispatched() { + } + + @Override + public void onCompleted(Object result, Throwable t) { + } + }; + + public static DataLoaderInstrumentationContext noOpCtx() { + //noinspection unchecked + return (DataLoaderInstrumentationContext) NOOP_CTX; + } + + public static final DataLoaderInstrumentation NOOP_INSTRUMENTATION = new DataLoaderInstrumentation() { + }; + + /** + * Check the {@link DataLoaderInstrumentationContext} to see if its null and returns a noop if it is or else the original + * context + * + * @param ic the context in play + * @param for two + * @return a non null context + */ + public static DataLoaderInstrumentationContext ctxOrNoopCtx(DataLoaderInstrumentationContext ic) { + return ic == null ? noOpCtx() : ic; + } +} diff --git a/src/test/java/org/dataloader/instrumentation/DataLoaderInstrumentationTest.java b/src/test/java/org/dataloader/instrumentation/DataLoaderInstrumentationTest.java new file mode 100644 index 00000000..2074aef1 --- /dev/null +++ b/src/test/java/org/dataloader/instrumentation/DataLoaderInstrumentationTest.java @@ -0,0 +1,108 @@ +package org.dataloader.instrumentation; + +import org.dataloader.BatchLoader; +import org.dataloader.BatchLoaderEnvironment; +import org.dataloader.DataLoader; +import org.dataloader.DataLoaderFactory; +import org.dataloader.DataLoaderOptions; +import org.dataloader.DispatchResult; +import org.dataloader.fixtures.TestKit; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; + +import static org.awaitility.Awaitility.await; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.is; + +class DataLoaderInstrumentationTest { + + BatchLoader snoozingBatchLoader = keys -> CompletableFuture.supplyAsync(() -> { + TestKit.snooze(100); + return keys; + }); + + @Test + void canMonitorDispatching() { + AtomicLong timer = new AtomicLong(); + AtomicReference> dlRef = new AtomicReference<>(); + + DataLoaderInstrumentation instrumentation = new DataLoaderInstrumentation() { + + @Override + public DataLoaderInstrumentationContext> beginDispatch(DataLoader dataLoader) { + dlRef.set(dataLoader); + + long then = System.currentTimeMillis(); + return new DataLoaderInstrumentationContext<>() { + @Override + public void onCompleted(DispatchResult result, Throwable t) { + timer.set(System.currentTimeMillis() - then); + } + }; + } + + @Override + public DataLoaderInstrumentationContext> beginBatchLoader(DataLoader dataLoader, List keys, BatchLoaderEnvironment environment) { + return DataLoaderInstrumentationHelper.noOpCtx(); + } + }; + + DataLoaderOptions options = new DataLoaderOptions().setInstrumentation(instrumentation).setMaxBatchSize(5); + + DataLoader dl = DataLoaderFactory.newDataLoader(snoozingBatchLoader, options); + + for (int i = 0; i < 20; i++) { + dl.load("X"+ i); + } + + CompletableFuture> dispatch = dl.dispatch(); + + await().until(dispatch::isDone); + assertThat(timer.get(), greaterThan(150L)); // we must have called batch load 4 times + assertThat(dlRef.get(), is(dl)); + } + + @Test + void canMonitorBatchLoading() { + AtomicLong timer = new AtomicLong(); + AtomicReference beRef = new AtomicReference<>(); + AtomicReference> dlRef = new AtomicReference<>(); + + DataLoaderInstrumentation instrumentation = new DataLoaderInstrumentation() { + + @Override + public DataLoaderInstrumentationContext> beginBatchLoader(DataLoader dataLoader, List keys, BatchLoaderEnvironment environment) { + dlRef.set(dataLoader); + beRef.set(environment); + + long then = System.currentTimeMillis(); + return new DataLoaderInstrumentationContext<>() { + @Override + public void onCompleted(List result, Throwable t) { + timer.set(System.currentTimeMillis() - then); + } + }; + } + }; + + DataLoaderOptions options = new DataLoaderOptions().setInstrumentation(instrumentation); + DataLoader dl = DataLoaderFactory.newDataLoader(snoozingBatchLoader, options); + + dl.load("A"); + dl.load("B"); + + CompletableFuture> dispatch = dl.dispatch(); + + await().until(dispatch::isDone); + assertThat(timer.get(), greaterThan(50L)); + assertThat(dlRef.get(), is(dl)); + assertThat(beRef.get().getKeyContexts().keySet(), equalTo(Set.of("A", "B"))); + } +} \ No newline at end of file From 97364ed880009fe09c5c76a1d1624275dd3ad2f8 Mon Sep 17 00:00:00 2001 From: bbaker Date: Tue, 21 Jan 2025 14:33:11 +1100 Subject: [PATCH 065/156] Instrumentatin support for dataloader - better tests --- .../org/dataloader/fixtures/Stopwatch.java | 57 +++++++++++++++++++ .../DataLoaderInstrumentationTest.java | 36 +++++++----- 2 files changed, 79 insertions(+), 14 deletions(-) create mode 100644 src/test/java/org/dataloader/fixtures/Stopwatch.java diff --git a/src/test/java/org/dataloader/fixtures/Stopwatch.java b/src/test/java/org/dataloader/fixtures/Stopwatch.java new file mode 100644 index 00000000..c815a8b9 --- /dev/null +++ b/src/test/java/org/dataloader/fixtures/Stopwatch.java @@ -0,0 +1,57 @@ +package org.dataloader.fixtures; + +import java.time.Duration; + +public class Stopwatch { + + public static Stopwatch stopwatchStarted() { + return new Stopwatch().start(); + } + + public static Stopwatch stopwatchUnStarted() { + return new Stopwatch(); + } + + private long started = -1; + private long stopped = -1; + + public Stopwatch start() { + synchronized (this) { + if (started != -1) { + throw new IllegalStateException("You have started it before"); + } + started = System.currentTimeMillis(); + } + return this; + } + + private Stopwatch() { + } + + public long elapsed() { + synchronized (this) { + if (started == -1) { + throw new IllegalStateException("You haven't started it"); + } + if (stopped == -1) { + return System.currentTimeMillis() - started; + } else { + return stopped - started; + } + } + } + + public Duration duration() { + return Duration.ofMillis(elapsed()); + } + + public Duration stop() { + synchronized (this) { + if (started != -1) { + throw new IllegalStateException("You have started it"); + } + stopped = System.currentTimeMillis(); + return duration(); + } + } +} diff --git a/src/test/java/org/dataloader/instrumentation/DataLoaderInstrumentationTest.java b/src/test/java/org/dataloader/instrumentation/DataLoaderInstrumentationTest.java index 2074aef1..4f43719e 100644 --- a/src/test/java/org/dataloader/instrumentation/DataLoaderInstrumentationTest.java +++ b/src/test/java/org/dataloader/instrumentation/DataLoaderInstrumentationTest.java @@ -6,13 +6,14 @@ import org.dataloader.DataLoaderFactory; import org.dataloader.DataLoaderOptions; import org.dataloader.DispatchResult; +import org.dataloader.fixtures.Stopwatch; import org.dataloader.fixtures.TestKit; import org.junit.jupiter.api.Test; +import java.util.ArrayList; import java.util.List; import java.util.Set; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import static org.awaitility.Awaitility.await; @@ -30,7 +31,7 @@ class DataLoaderInstrumentationTest { @Test void canMonitorDispatching() { - AtomicLong timer = new AtomicLong(); + Stopwatch stopwatch = Stopwatch.stopwatchUnStarted(); AtomicReference> dlRef = new AtomicReference<>(); DataLoaderInstrumentation instrumentation = new DataLoaderInstrumentation() { @@ -38,12 +39,11 @@ void canMonitorDispatching() { @Override public DataLoaderInstrumentationContext> beginDispatch(DataLoader dataLoader) { dlRef.set(dataLoader); - - long then = System.currentTimeMillis(); + stopwatch.start(); return new DataLoaderInstrumentationContext<>() { @Override public void onCompleted(DispatchResult result, Throwable t) { - timer.set(System.currentTimeMillis() - then); + stopwatch.stop(); } }; } @@ -54,24 +54,32 @@ public DataLoaderInstrumentationContext> beginBatchLoader(DataLoader dl = DataLoaderFactory.newDataLoader(snoozingBatchLoader, options); + List keys = new ArrayList<>(); for (int i = 0; i < 20; i++) { - dl.load("X"+ i); + String key = "X" + i; + keys.add(key); + dl.load(key); } CompletableFuture> dispatch = dl.dispatch(); await().until(dispatch::isDone); - assertThat(timer.get(), greaterThan(150L)); // we must have called batch load 4 times + // we must have called batch load 4 times at 100ms snooze per call + // but its in parallel via supplyAsync + assertThat(stopwatch.elapsed(), greaterThan(75L)); assertThat(dlRef.get(), is(dl)); + assertThat(dispatch.join(), equalTo(keys)); } @Test void canMonitorBatchLoading() { - AtomicLong timer = new AtomicLong(); + Stopwatch stopwatch = Stopwatch.stopwatchUnStarted(); AtomicReference beRef = new AtomicReference<>(); AtomicReference> dlRef = new AtomicReference<>(); @@ -82,11 +90,11 @@ public DataLoaderInstrumentationContext> beginBatchLoader(DataLoader() { @Override public void onCompleted(List result, Throwable t) { - timer.set(System.currentTimeMillis() - then); + stopwatch.stop(); } }; } @@ -95,13 +103,13 @@ public void onCompleted(List result, Throwable t) { DataLoaderOptions options = new DataLoaderOptions().setInstrumentation(instrumentation); DataLoader dl = DataLoaderFactory.newDataLoader(snoozingBatchLoader, options); - dl.load("A"); - dl.load("B"); + dl.load("A", "kcA"); + dl.load("B", "kcB"); CompletableFuture> dispatch = dl.dispatch(); await().until(dispatch::isDone); - assertThat(timer.get(), greaterThan(50L)); + assertThat(stopwatch.elapsed(), greaterThan(50L)); assertThat(dlRef.get(), is(dl)); assertThat(beRef.get().getKeyContexts().keySet(), equalTo(Set.of("A", "B"))); } From 411b9bde10a7fe118d52aa9297cee8e56b82b02c Mon Sep 17 00:00:00 2001 From: bbaker Date: Wed, 22 Jan 2025 11:58:29 +1100 Subject: [PATCH 066/156] Instrumentation support for dataloader - better tests and added ChainedDataLoaderInstrumentation --- .../ChainedDataLoaderInstrumentation.java | 86 +++++++++ .../DataLoaderInstrumentation.java | 4 +- .../DataLoaderInstrumentationContext.java | 6 +- .../parameterized/TestDataLoaderFactory.java | 4 + .../ChainedDataLoaderInstrumentationTest.java | 166 ++++++++++++++++++ 5 files changed, 262 insertions(+), 4 deletions(-) create mode 100644 src/main/java/org/dataloader/instrumentation/ChainedDataLoaderInstrumentation.java create mode 100644 src/test/java/org/dataloader/instrumentation/ChainedDataLoaderInstrumentationTest.java diff --git a/src/main/java/org/dataloader/instrumentation/ChainedDataLoaderInstrumentation.java b/src/main/java/org/dataloader/instrumentation/ChainedDataLoaderInstrumentation.java new file mode 100644 index 00000000..c80b5109 --- /dev/null +++ b/src/main/java/org/dataloader/instrumentation/ChainedDataLoaderInstrumentation.java @@ -0,0 +1,86 @@ +package org.dataloader.instrumentation; + +import org.dataloader.BatchLoaderEnvironment; +import org.dataloader.DataLoader; +import org.dataloader.DispatchResult; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * This {@link DataLoaderInstrumentation} can chain together multiple instrumentations and have them all called in + * the order of the provided list. + */ +public class ChainedDataLoaderInstrumentation implements DataLoaderInstrumentation { + private final List instrumentations; + + public ChainedDataLoaderInstrumentation() { + instrumentations = List.of(); + } + + public ChainedDataLoaderInstrumentation(List instrumentations) { + this.instrumentations = List.copyOf(instrumentations); + } + + /** + * Adds a new {@link DataLoaderInstrumentation} to the list and creates a new {@link ChainedDataLoaderInstrumentation} + * + * @param instrumentation the one to add + * @return a new ChainedDataLoaderInstrumentation object + */ + public ChainedDataLoaderInstrumentation add(DataLoaderInstrumentation instrumentation) { + ArrayList list = new ArrayList<>(instrumentations); + list.add(instrumentation); + return new ChainedDataLoaderInstrumentation(list); + } + + @Override + public DataLoaderInstrumentationContext> beginDispatch(DataLoader dataLoader) { + return chainedCtx(it -> it.beginDispatch(dataLoader)); + } + + @Override + public DataLoaderInstrumentationContext> beginBatchLoader(DataLoader dataLoader, List keys, BatchLoaderEnvironment environment) { + return chainedCtx(it -> it.beginBatchLoader(dataLoader, keys, environment)); + } + + private DataLoaderInstrumentationContext chainedCtx(Function> mapper) { + // if we have zero or 1 instrumentations (and 1 is the most common), then we can avoid an object allocation + // of the ChainedInstrumentationContext since it won't be needed + if (instrumentations.isEmpty()) { + return DataLoaderInstrumentationHelper.noOpCtx(); + } + if (instrumentations.size() == 1) { + return mapper.apply(instrumentations.get(0)); + } + return new ChainedInstrumentationContext<>(dropNullContexts(mapper)); + } + + private List> dropNullContexts(Function> mapper) { + return instrumentations.stream() + .map(mapper) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } + + private static class ChainedInstrumentationContext implements DataLoaderInstrumentationContext { + private final List> contexts; + + public ChainedInstrumentationContext(List> contexts) { + this.contexts = contexts; + } + + @Override + public void onDispatched() { + contexts.forEach(DataLoaderInstrumentationContext::onDispatched); + } + + @Override + public void onCompleted(T result, Throwable t) { + contexts.forEach(it -> it.onCompleted(result, t)); + } + } +} diff --git a/src/main/java/org/dataloader/instrumentation/DataLoaderInstrumentation.java b/src/main/java/org/dataloader/instrumentation/DataLoaderInstrumentation.java index f0da7887..5cce9db4 100644 --- a/src/main/java/org/dataloader/instrumentation/DataLoaderInstrumentation.java +++ b/src/main/java/org/dataloader/instrumentation/DataLoaderInstrumentation.java @@ -11,7 +11,7 @@ */ public interface DataLoaderInstrumentation { /** - * This call back is done just before the {@link DataLoader#dispatch()} is invoked + * This call back is done just before the {@link DataLoader#dispatch()} is invoked, * and it completes when the dispatch call promise is done. * * @param dataLoader the {@link DataLoader} in question @@ -22,7 +22,7 @@ default DataLoaderInstrumentationContext> beginDispatch(DataLo } /** - * This call back is done just before the batch loader of a {@link DataLoader} is invoked. Remember a batch loader + * This call back is done just before the `batch loader` of a {@link DataLoader} is invoked. Remember a batch loader * could be called multiple times during a dispatch event (because of max batch sizes) * * @param dataLoader the {@link DataLoader} in question diff --git a/src/main/java/org/dataloader/instrumentation/DataLoaderInstrumentationContext.java b/src/main/java/org/dataloader/instrumentation/DataLoaderInstrumentationContext.java index ae0bbc12..d327acd1 100644 --- a/src/main/java/org/dataloader/instrumentation/DataLoaderInstrumentationContext.java +++ b/src/main/java/org/dataloader/instrumentation/DataLoaderInstrumentationContext.java @@ -12,13 +12,15 @@ */ public interface DataLoaderInstrumentationContext { /** - * This is invoked when the instrumentation step is initially dispatched + * This is invoked when the instrumentation step is initially dispatched. Note this is NOT + * the same time as the {@link DataLoaderInstrumentation}`beginXXX()` starts, but rather after all the inner + * work has been done. */ default void onDispatched() { } /** - * This is invoked when the instrumentation step is fully completed + * This is invoked when the instrumentation step is fully completed. * * @param result the result of the step (which may be null) * @param t this exception will be non-null if an exception was thrown during the step diff --git a/src/test/java/org/dataloader/fixtures/parameterized/TestDataLoaderFactory.java b/src/test/java/org/dataloader/fixtures/parameterized/TestDataLoaderFactory.java index 97e35a8c..8cbe86cb 100644 --- a/src/test/java/org/dataloader/fixtures/parameterized/TestDataLoaderFactory.java +++ b/src/test/java/org/dataloader/fixtures/parameterized/TestDataLoaderFactory.java @@ -25,6 +25,10 @@ public interface TestDataLoaderFactory { // Convenience methods + default DataLoader idLoader(DataLoaderOptions options) { + return idLoader(options, new ArrayList<>()); + } + default DataLoader idLoader(List> calls) { return idLoader(null, calls); } diff --git a/src/test/java/org/dataloader/instrumentation/ChainedDataLoaderInstrumentationTest.java b/src/test/java/org/dataloader/instrumentation/ChainedDataLoaderInstrumentationTest.java new file mode 100644 index 00000000..93c0edc6 --- /dev/null +++ b/src/test/java/org/dataloader/instrumentation/ChainedDataLoaderInstrumentationTest.java @@ -0,0 +1,166 @@ +package org.dataloader.instrumentation; + +import org.dataloader.BatchLoaderEnvironment; +import org.dataloader.DataLoader; +import org.dataloader.DataLoaderFactory; +import org.dataloader.DataLoaderOptions; +import org.dataloader.DispatchResult; +import org.dataloader.fixtures.TestKit; +import org.dataloader.fixtures.parameterized.TestDataLoaderFactory; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +import static org.awaitility.Awaitility.await; +import static org.dataloader.DataLoaderOptions.newOptions; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; + +class ChainedDataLoaderInstrumentationTest { + + static class CapturingInstrumentation implements DataLoaderInstrumentation { + String name; + List methods = new ArrayList<>(); + + public CapturingInstrumentation(String name) { + this.name = name; + } + + @Override + public DataLoaderInstrumentationContext> beginDispatch(DataLoader dataLoader) { + methods.add(name + "_beginDispatch"); + return new DataLoaderInstrumentationContext<>() { + @Override + public void onDispatched() { + methods.add(name + "_beginDispatch_onDispatched"); + } + + @Override + public void onCompleted(DispatchResult result, Throwable t) { + methods.add(name + "_beginDispatch_onCompleted"); + } + }; + } + + @Override + public DataLoaderInstrumentationContext> beginBatchLoader(DataLoader dataLoader, List keys, BatchLoaderEnvironment environment) { + methods.add(name + "_beginBatchLoader"); + return new DataLoaderInstrumentationContext<>() { + @Override + public void onDispatched() { + methods.add(name + "_beginBatchLoader_onDispatched"); + } + + @Override + public void onCompleted(List result, Throwable t) { + methods.add(name + "_beginBatchLoader_onCompleted"); + } + }; + } + } + + + static class CapturingInstrumentationReturnsNull extends CapturingInstrumentation { + + public CapturingInstrumentationReturnsNull(String name) { + super(name); + } + + @Override + public DataLoaderInstrumentationContext> beginDispatch(DataLoader dataLoader) { + methods.add(name + "_beginDispatch"); + return null; + } + + @Override + public DataLoaderInstrumentationContext> beginBatchLoader(DataLoader dataLoader, List keys, BatchLoaderEnvironment environment) { + methods.add(name + "_beginBatchLoader"); + return null; + } + } + + @Test + void canChainTogetherZeroInstrumentation() { + // just to prove its useless but harmless + ChainedDataLoaderInstrumentation chainedItn = new ChainedDataLoaderInstrumentation(); + + DataLoaderOptions options = newOptions().setInstrumentation(chainedItn); + + DataLoader dl = DataLoaderFactory.newDataLoader(TestKit.keysAsValues(), options); + + dl.load("A"); + dl.load("B"); + + CompletableFuture> dispatch = dl.dispatch(); + + await().until(dispatch::isDone); + assertThat(dispatch.join(), equalTo(List.of("A", "B"))); + } + + @Test + void canChainTogetherOneInstrumentation() { + CapturingInstrumentation capturingA = new CapturingInstrumentation("A"); + + ChainedDataLoaderInstrumentation chainedItn = new ChainedDataLoaderInstrumentation() + .add(capturingA); + + DataLoaderOptions options = newOptions().setInstrumentation(chainedItn); + + DataLoader dl = DataLoaderFactory.newDataLoader(TestKit.keysAsValues(), options); + + dl.load("A"); + dl.load("B"); + + CompletableFuture> dispatch = dl.dispatch(); + + await().until(dispatch::isDone); + + assertThat(capturingA.methods, equalTo(List.of("A_beginDispatch", + "A_beginBatchLoader", "A_beginBatchLoader_onDispatched", "A_beginBatchLoader_onCompleted", + "A_beginDispatch_onDispatched", "A_beginDispatch_onCompleted"))); + } + + + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void canChainTogetherManyInstrumentationsWithDifferentBatchLoaders(TestDataLoaderFactory factory) { + CapturingInstrumentation capturingA = new CapturingInstrumentation("A"); + CapturingInstrumentation capturingB = new CapturingInstrumentation("B"); + CapturingInstrumentation capturingButReturnsNull = new CapturingInstrumentationReturnsNull("NULL"); + + ChainedDataLoaderInstrumentation chainedItn = new ChainedDataLoaderInstrumentation() + .add(capturingA) + .add(capturingB) + .add(capturingButReturnsNull); + + DataLoaderOptions options = newOptions().setInstrumentation(chainedItn); + + DataLoader dl = factory.idLoader(options); + + dl.load("A"); + dl.load("B"); + + CompletableFuture> dispatch = dl.dispatch(); + + await().until(dispatch::isDone); + + // + // A_beginBatchLoader happens before A_beginDispatch_onDispatched because these are sync + // and no async - a batch scheduler or async batch loader would change that + // + assertThat(capturingA.methods, equalTo(List.of("A_beginDispatch", + "A_beginBatchLoader", "A_beginBatchLoader_onDispatched", "A_beginBatchLoader_onCompleted", + "A_beginDispatch_onDispatched", "A_beginDispatch_onCompleted"))); + + assertThat(capturingB.methods, equalTo(List.of("B_beginDispatch", + "B_beginBatchLoader", "B_beginBatchLoader_onDispatched", "B_beginBatchLoader_onCompleted", + "B_beginDispatch_onDispatched", "B_beginDispatch_onCompleted"))); + + // it returned null on all its contexts - nothing to call back on + assertThat(capturingButReturnsNull.methods, equalTo(List.of("NULL_beginDispatch", "NULL_beginBatchLoader"))); + } +} \ No newline at end of file From 3a8adcd87402427f68bc6e3e93248d0cc1640c04 Mon Sep 17 00:00:00 2001 From: bbaker Date: Wed, 22 Jan 2025 14:33:08 +1100 Subject: [PATCH 067/156] Instrumentation support for dataloader -added registry instrumentation --- .../org/dataloader/DataLoaderRegistry.java | 82 ++++++- .../ChainedDataLoaderInstrumentation.java | 34 ++- .../DataLoaderInstrumentation.java | 2 + .../DataLoaderInstrumentationContext.java | 3 + .../DataLoaderInstrumentationHelper.java | 15 +- .../CapturingInstrumentation.java | 49 +++++ .../CapturingInstrumentationReturnsNull.java | 26 +++ .../ChainedDataLoaderInstrumentationTest.java | 80 ++----- ...DataLoaderRegistryInstrumentationTest.java | 205 ++++++++++++++++++ 9 files changed, 421 insertions(+), 75 deletions(-) create mode 100644 src/test/java/org/dataloader/instrumentation/CapturingInstrumentation.java create mode 100644 src/test/java/org/dataloader/instrumentation/CapturingInstrumentationReturnsNull.java create mode 100644 src/test/java/org/dataloader/instrumentation/DataLoaderRegistryInstrumentationTest.java diff --git a/src/main/java/org/dataloader/DataLoaderRegistry.java b/src/main/java/org/dataloader/DataLoaderRegistry.java index 5a3f90f1..5e00ee1b 100644 --- a/src/main/java/org/dataloader/DataLoaderRegistry.java +++ b/src/main/java/org/dataloader/DataLoaderRegistry.java @@ -1,6 +1,9 @@ package org.dataloader; import org.dataloader.annotations.PublicApi; +import org.dataloader.instrumentation.ChainedDataLoaderInstrumentation; +import org.dataloader.instrumentation.DataLoaderInstrumentation; +import org.dataloader.instrumentation.DataLoaderInstrumentationHelper; import org.dataloader.stats.Statistics; import java.util.ArrayList; @@ -21,25 +24,81 @@ @PublicApi public class DataLoaderRegistry { protected final Map> dataLoaders = new ConcurrentHashMap<>(); + protected final DataLoaderInstrumentation instrumentation; + public DataLoaderRegistry() { + instrumentation = null; } private DataLoaderRegistry(Builder builder) { - this.dataLoaders.putAll(builder.dataLoaders); + instrument(builder.instrumentation, builder.dataLoaders); + this.instrumentation = builder.instrumentation; + } + + private void instrument(DataLoaderInstrumentation registryInstrumentation, Map> incomingDataLoaders) { + this.dataLoaders.putAll(incomingDataLoaders); + if (registryInstrumentation != null) { + this.dataLoaders.replaceAll((k, existingDL) -> instrumentDL(registryInstrumentation, existingDL)); + } + } + + /** + * Can be called to tweak a {@link DataLoader} so that it has the registry {@link DataLoaderInstrumentation} added as the first one. + * + * @param registryInstrumentation the common registry {@link DataLoaderInstrumentation} + * @param existingDL the existing data loader + * @return a new {@link DataLoader} or the same one if there is nothing to change + */ + private static DataLoader instrumentDL(DataLoaderInstrumentation registryInstrumentation, DataLoader existingDL) { + if (registryInstrumentation == null) { + return existingDL; + } + DataLoaderOptions options = existingDL.getOptions(); + DataLoaderInstrumentation existingInstrumentation = options.getInstrumentation(); + // if they have any instrumentations then add to it + if (existingInstrumentation != null) { + if (existingInstrumentation == registryInstrumentation) { + // nothing to change + return existingDL; + } + if (existingInstrumentation == DataLoaderInstrumentationHelper.NOOP_INSTRUMENTATION) { + // replace it with the registry one + return mkInstrumentedDataLoader(existingDL, options, registryInstrumentation); + } + if (existingInstrumentation instanceof ChainedDataLoaderInstrumentation) { + // avoids calling a chained inside a chained + DataLoaderInstrumentation newInstrumentation = ((ChainedDataLoaderInstrumentation) existingInstrumentation).prepend(registryInstrumentation); + return mkInstrumentedDataLoader(existingDL, options, newInstrumentation); + } else { + DataLoaderInstrumentation newInstrumentation = new ChainedDataLoaderInstrumentation().add(registryInstrumentation).add(existingInstrumentation); + return mkInstrumentedDataLoader(existingDL, options, newInstrumentation); + } + } else { + return mkInstrumentedDataLoader(existingDL, options, registryInstrumentation); + } + } + + private static DataLoader mkInstrumentedDataLoader(DataLoader existingDL, DataLoaderOptions options, DataLoaderInstrumentation newInstrumentation) { + return existingDL.transform(builder -> { + options.setInstrumentation(newInstrumentation); + builder.options(options); + }); } + public DataLoaderInstrumentation getInstrumentation() { + return instrumentation; + } /** * This will register a new dataloader * * @param key the key to put the data loader under * @param dataLoader the data loader to register - * * @return this registry */ public DataLoaderRegistry register(String key, DataLoader dataLoader) { - dataLoaders.put(key, dataLoader); + dataLoaders.put(key, instrumentDL(instrumentation, dataLoader)); return this; } @@ -54,13 +113,15 @@ public DataLoaderRegistry register(String key, DataLoader dataLoader) { * @param mappingFunction the function to compute a data loader * @param the type of keys * @param the type of values - * * @return a data loader */ @SuppressWarnings("unchecked") public DataLoader computeIfAbsent(final String key, final Function> mappingFunction) { - return (DataLoader) dataLoaders.computeIfAbsent(key, mappingFunction); + return (DataLoader) dataLoaders.computeIfAbsent(key, (k) -> { + DataLoader dl = mappingFunction.apply(k); + return instrumentDL(instrumentation, dl); + }); } /** @@ -68,7 +129,6 @@ public DataLoader computeIfAbsent(final String key, * and return a new combined registry * * @param registry the registry to combine into this registry - * * @return a new combined registry */ public DataLoaderRegistry combine(DataLoaderRegistry registry) { @@ -97,7 +157,6 @@ public DataLoaderRegistry combine(DataLoaderRegistry registry) { * This will unregister a new dataloader * * @param key the key of the data loader to unregister - * * @return this registry */ public DataLoaderRegistry unregister(String key) { @@ -111,7 +170,6 @@ public DataLoaderRegistry unregister(String key) { * @param key the key of the data loader * @param the type of keys * @param the type of values - * * @return a data loader or null if its not present */ @SuppressWarnings("unchecked") @@ -182,13 +240,13 @@ public static Builder newRegistry() { public static class Builder { private final Map> dataLoaders = new HashMap<>(); + private DataLoaderInstrumentation instrumentation; /** * This will register a new dataloader * * @param key the key to put the data loader under * @param dataLoader the data loader to register - * * @return this builder for a fluent pattern */ public Builder register(String key, DataLoader dataLoader) { @@ -201,7 +259,6 @@ public Builder register(String key, DataLoader dataLoader) { * from a previous {@link DataLoaderRegistry} * * @param otherRegistry the previous {@link DataLoaderRegistry} - * * @return this builder for a fluent pattern */ public Builder registerAll(DataLoaderRegistry otherRegistry) { @@ -209,6 +266,11 @@ public Builder registerAll(DataLoaderRegistry otherRegistry) { return this; } + public Builder instrumentation(DataLoaderInstrumentation instrumentation) { + this.instrumentation = instrumentation; + return this; + } + /** * @return the newly built {@link DataLoaderRegistry} */ diff --git a/src/main/java/org/dataloader/instrumentation/ChainedDataLoaderInstrumentation.java b/src/main/java/org/dataloader/instrumentation/ChainedDataLoaderInstrumentation.java index c80b5109..eb9af0a6 100644 --- a/src/main/java/org/dataloader/instrumentation/ChainedDataLoaderInstrumentation.java +++ b/src/main/java/org/dataloader/instrumentation/ChainedDataLoaderInstrumentation.java @@ -3,8 +3,10 @@ import org.dataloader.BatchLoaderEnvironment; import org.dataloader.DataLoader; import org.dataloader.DispatchResult; +import org.dataloader.annotations.PublicApi; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.Objects; import java.util.function.Function; @@ -14,6 +16,7 @@ * This {@link DataLoaderInstrumentation} can chain together multiple instrumentations and have them all called in * the order of the provided list. */ +@PublicApi public class ChainedDataLoaderInstrumentation implements DataLoaderInstrumentation { private final List instrumentations; @@ -25,6 +28,10 @@ public ChainedDataLoaderInstrumentation(List instrume this.instrumentations = List.copyOf(instrumentations); } + public List getInstrumentations() { + return instrumentations; + } + /** * Adds a new {@link DataLoaderInstrumentation} to the list and creates a new {@link ChainedDataLoaderInstrumentation} * @@ -32,8 +39,33 @@ public ChainedDataLoaderInstrumentation(List instrume * @return a new ChainedDataLoaderInstrumentation object */ public ChainedDataLoaderInstrumentation add(DataLoaderInstrumentation instrumentation) { - ArrayList list = new ArrayList<>(instrumentations); + ArrayList list = new ArrayList<>(this.instrumentations); + list.add(instrumentation); + return new ChainedDataLoaderInstrumentation(list); + } + + /** + * Prepends a new {@link DataLoaderInstrumentation} to the list and creates a new {@link ChainedDataLoaderInstrumentation} + * + * @param instrumentation the one to add + * @return a new ChainedDataLoaderInstrumentation object + */ + public ChainedDataLoaderInstrumentation prepend(DataLoaderInstrumentation instrumentation) { + ArrayList list = new ArrayList<>(); list.add(instrumentation); + list.addAll(this.instrumentations); + return new ChainedDataLoaderInstrumentation(list); + } + + /** + * Adds a collection of {@link DataLoaderInstrumentation} to the list and creates a new {@link ChainedDataLoaderInstrumentation} + * + * @param instrumentations the new ones to add + * @return a new ChainedDataLoaderInstrumentation object + */ + public ChainedDataLoaderInstrumentation addAll(Collection instrumentations) { + ArrayList list = new ArrayList<>(this.instrumentations); + list.addAll(instrumentations); return new ChainedDataLoaderInstrumentation(list); } diff --git a/src/main/java/org/dataloader/instrumentation/DataLoaderInstrumentation.java b/src/main/java/org/dataloader/instrumentation/DataLoaderInstrumentation.java index 5cce9db4..78f2cf52 100644 --- a/src/main/java/org/dataloader/instrumentation/DataLoaderInstrumentation.java +++ b/src/main/java/org/dataloader/instrumentation/DataLoaderInstrumentation.java @@ -3,12 +3,14 @@ import org.dataloader.BatchLoaderEnvironment; import org.dataloader.DataLoader; import org.dataloader.DispatchResult; +import org.dataloader.annotations.PublicSpi; import java.util.List; /** * This interface is called when certain actions happen inside a data loader */ +@PublicSpi public interface DataLoaderInstrumentation { /** * This call back is done just before the {@link DataLoader#dispatch()} is invoked, diff --git a/src/main/java/org/dataloader/instrumentation/DataLoaderInstrumentationContext.java b/src/main/java/org/dataloader/instrumentation/DataLoaderInstrumentationContext.java index d327acd1..88b08efe 100644 --- a/src/main/java/org/dataloader/instrumentation/DataLoaderInstrumentationContext.java +++ b/src/main/java/org/dataloader/instrumentation/DataLoaderInstrumentationContext.java @@ -1,5 +1,7 @@ package org.dataloader.instrumentation; +import org.dataloader.annotations.PublicSpi; + import java.util.concurrent.CompletableFuture; /** @@ -10,6 +12,7 @@ * This pattern of construction of an object then call back is intended to allow "timers" to be created that can instrument what has * just happened or "loggers" to be called to record what has happened. */ +@PublicSpi public interface DataLoaderInstrumentationContext { /** * This is invoked when the instrumentation step is initially dispatched. Note this is NOT diff --git a/src/main/java/org/dataloader/instrumentation/DataLoaderInstrumentationHelper.java b/src/main/java/org/dataloader/instrumentation/DataLoaderInstrumentationHelper.java index 5ff545dc..844ec375 100644 --- a/src/main/java/org/dataloader/instrumentation/DataLoaderInstrumentationHelper.java +++ b/src/main/java/org/dataloader/instrumentation/DataLoaderInstrumentationHelper.java @@ -1,7 +1,11 @@ package org.dataloader.instrumentation; +import org.dataloader.annotations.PublicApi; + +@PublicApi public class DataLoaderInstrumentationHelper { + @SuppressWarnings("RedundantMethodOverride") private static final DataLoaderInstrumentationContext NOOP_CTX = new DataLoaderInstrumentationContext<>() { @Override public void onDispatched() { @@ -12,17 +16,26 @@ public void onCompleted(Object result, Throwable t) { } }; + /** + * Returns a noop {@link DataLoaderInstrumentationContext} of the right type + * + * @param for two + * @return a noop context + */ public static DataLoaderInstrumentationContext noOpCtx() { //noinspection unchecked return (DataLoaderInstrumentationContext) NOOP_CTX; } + /** + * A well known noop {@link DataLoaderInstrumentation} + */ public static final DataLoaderInstrumentation NOOP_INSTRUMENTATION = new DataLoaderInstrumentation() { }; /** * Check the {@link DataLoaderInstrumentationContext} to see if its null and returns a noop if it is or else the original - * context + * context. This is a bit of a helper method. * * @param ic the context in play * @param for two diff --git a/src/test/java/org/dataloader/instrumentation/CapturingInstrumentation.java b/src/test/java/org/dataloader/instrumentation/CapturingInstrumentation.java new file mode 100644 index 00000000..f5af683a --- /dev/null +++ b/src/test/java/org/dataloader/instrumentation/CapturingInstrumentation.java @@ -0,0 +1,49 @@ +package org.dataloader.instrumentation; + +import org.dataloader.BatchLoaderEnvironment; +import org.dataloader.DataLoader; +import org.dataloader.DispatchResult; + +import java.util.ArrayList; +import java.util.List; + +class CapturingInstrumentation implements DataLoaderInstrumentation { + String name; + List methods = new ArrayList<>(); + + public CapturingInstrumentation(String name) { + this.name = name; + } + + @Override + public DataLoaderInstrumentationContext> beginDispatch(DataLoader dataLoader) { + methods.add(name + "_beginDispatch"); + return new DataLoaderInstrumentationContext<>() { + @Override + public void onDispatched() { + methods.add(name + "_beginDispatch_onDispatched"); + } + + @Override + public void onCompleted(DispatchResult result, Throwable t) { + methods.add(name + "_beginDispatch_onCompleted"); + } + }; + } + + @Override + public DataLoaderInstrumentationContext> beginBatchLoader(DataLoader dataLoader, List keys, BatchLoaderEnvironment environment) { + methods.add(name + "_beginBatchLoader"); + return new DataLoaderInstrumentationContext<>() { + @Override + public void onDispatched() { + methods.add(name + "_beginBatchLoader_onDispatched"); + } + + @Override + public void onCompleted(List result, Throwable t) { + methods.add(name + "_beginBatchLoader_onCompleted"); + } + }; + } +} diff --git a/src/test/java/org/dataloader/instrumentation/CapturingInstrumentationReturnsNull.java b/src/test/java/org/dataloader/instrumentation/CapturingInstrumentationReturnsNull.java new file mode 100644 index 00000000..0c164295 --- /dev/null +++ b/src/test/java/org/dataloader/instrumentation/CapturingInstrumentationReturnsNull.java @@ -0,0 +1,26 @@ +package org.dataloader.instrumentation; + +import org.dataloader.BatchLoaderEnvironment; +import org.dataloader.DataLoader; +import org.dataloader.DispatchResult; + +import java.util.List; + +class CapturingInstrumentationReturnsNull extends CapturingInstrumentation { + + public CapturingInstrumentationReturnsNull(String name) { + super(name); + } + + @Override + public DataLoaderInstrumentationContext> beginDispatch(DataLoader dataLoader) { + methods.add(name + "_beginDispatch"); + return null; + } + + @Override + public DataLoaderInstrumentationContext> beginBatchLoader(DataLoader dataLoader, List keys, BatchLoaderEnvironment environment) { + methods.add(name + "_beginBatchLoader"); + return null; + } +} diff --git a/src/test/java/org/dataloader/instrumentation/ChainedDataLoaderInstrumentationTest.java b/src/test/java/org/dataloader/instrumentation/ChainedDataLoaderInstrumentationTest.java index 93c0edc6..31687349 100644 --- a/src/test/java/org/dataloader/instrumentation/ChainedDataLoaderInstrumentationTest.java +++ b/src/test/java/org/dataloader/instrumentation/ChainedDataLoaderInstrumentationTest.java @@ -1,17 +1,15 @@ package org.dataloader.instrumentation; -import org.dataloader.BatchLoaderEnvironment; import org.dataloader.DataLoader; import org.dataloader.DataLoaderFactory; import org.dataloader.DataLoaderOptions; -import org.dataloader.DispatchResult; import org.dataloader.fixtures.TestKit; import org.dataloader.fixtures.parameterized.TestDataLoaderFactory; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; -import java.util.ArrayList; import java.util.List; import java.util.concurrent.CompletableFuture; @@ -22,65 +20,16 @@ class ChainedDataLoaderInstrumentationTest { - static class CapturingInstrumentation implements DataLoaderInstrumentation { - String name; - List methods = new ArrayList<>(); - - public CapturingInstrumentation(String name) { - this.name = name; - } - - @Override - public DataLoaderInstrumentationContext> beginDispatch(DataLoader dataLoader) { - methods.add(name + "_beginDispatch"); - return new DataLoaderInstrumentationContext<>() { - @Override - public void onDispatched() { - methods.add(name + "_beginDispatch_onDispatched"); - } - - @Override - public void onCompleted(DispatchResult result, Throwable t) { - methods.add(name + "_beginDispatch_onCompleted"); - } - }; - } - - @Override - public DataLoaderInstrumentationContext> beginBatchLoader(DataLoader dataLoader, List keys, BatchLoaderEnvironment environment) { - methods.add(name + "_beginBatchLoader"); - return new DataLoaderInstrumentationContext<>() { - @Override - public void onDispatched() { - methods.add(name + "_beginBatchLoader_onDispatched"); - } - - @Override - public void onCompleted(List result, Throwable t) { - methods.add(name + "_beginBatchLoader_onCompleted"); - } - }; - } - } - - - static class CapturingInstrumentationReturnsNull extends CapturingInstrumentation { + CapturingInstrumentation capturingA; + CapturingInstrumentation capturingB; + CapturingInstrumentation capturingButReturnsNull; - public CapturingInstrumentationReturnsNull(String name) { - super(name); - } - @Override - public DataLoaderInstrumentationContext> beginDispatch(DataLoader dataLoader) { - methods.add(name + "_beginDispatch"); - return null; - } - - @Override - public DataLoaderInstrumentationContext> beginBatchLoader(DataLoader dataLoader, List keys, BatchLoaderEnvironment environment) { - methods.add(name + "_beginBatchLoader"); - return null; - } + @BeforeEach + void setUp() { + capturingA = new CapturingInstrumentation("A"); + capturingB = new CapturingInstrumentation("B"); + capturingButReturnsNull = new CapturingInstrumentationReturnsNull("NULL"); } @Test @@ -128,9 +77,6 @@ void canChainTogetherOneInstrumentation() { @ParameterizedTest @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void canChainTogetherManyInstrumentationsWithDifferentBatchLoaders(TestDataLoaderFactory factory) { - CapturingInstrumentation capturingA = new CapturingInstrumentation("A"); - CapturingInstrumentation capturingB = new CapturingInstrumentation("B"); - CapturingInstrumentation capturingButReturnsNull = new CapturingInstrumentationReturnsNull("NULL"); ChainedDataLoaderInstrumentation chainedItn = new ChainedDataLoaderInstrumentation() .add(capturingA) @@ -163,4 +109,12 @@ public void canChainTogetherManyInstrumentationsWithDifferentBatchLoaders(TestDa // it returned null on all its contexts - nothing to call back on assertThat(capturingButReturnsNull.methods, equalTo(List.of("NULL_beginDispatch", "NULL_beginBatchLoader"))); } + + @Test + void addition_works() { + ChainedDataLoaderInstrumentation chainedItn = new ChainedDataLoaderInstrumentation() + .add(capturingA).prepend(capturingB).addAll(List.of(capturingButReturnsNull)); + + assertThat(chainedItn.getInstrumentations(), equalTo(List.of(capturingB, capturingA, capturingButReturnsNull))); + } } \ No newline at end of file diff --git a/src/test/java/org/dataloader/instrumentation/DataLoaderRegistryInstrumentationTest.java b/src/test/java/org/dataloader/instrumentation/DataLoaderRegistryInstrumentationTest.java new file mode 100644 index 00000000..639ff487 --- /dev/null +++ b/src/test/java/org/dataloader/instrumentation/DataLoaderRegistryInstrumentationTest.java @@ -0,0 +1,205 @@ +package org.dataloader.instrumentation; + +import org.dataloader.DataLoader; +import org.dataloader.DataLoaderRegistry; +import org.dataloader.fixtures.TestKit; +import org.dataloader.fixtures.parameterized.TestDataLoaderFactory; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; + +import static org.awaitility.Awaitility.await; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; + +class DataLoaderRegistryInstrumentationTest { + DataLoader dlX; + DataLoader dlY; + DataLoader dlZ; + + CapturingInstrumentation instrA; + CapturingInstrumentation instrB; + ChainedDataLoaderInstrumentation chainedInstrA; + ChainedDataLoaderInstrumentation chainedInstrB; + + @BeforeEach + void setUp() { + dlX = TestKit.idLoader(); + dlY = TestKit.idLoader(); + dlZ = TestKit.idLoader(); + instrA = new CapturingInstrumentation("A"); + instrB = new CapturingInstrumentation("B"); + chainedInstrA = new ChainedDataLoaderInstrumentation().add(instrA); + chainedInstrB = new ChainedDataLoaderInstrumentation().add(instrB); + } + + @Test + void canInstrumentRegisteredDLsViaBuilder() { + + assertThat(dlX.getOptions().getInstrumentation(), equalTo(DataLoaderInstrumentationHelper.NOOP_INSTRUMENTATION)); + + DataLoaderRegistry registry = DataLoaderRegistry.newRegistry() + .instrumentation(chainedInstrA) + .register("X", dlX) + .register("Y", dlY) + .register("Z", dlZ) + .build(); + + assertThat(registry.getInstrumentation(), equalTo(chainedInstrA)); + + for (String key : List.of("X", "Y", "Z")) { + DataLoaderInstrumentation instrumentation = registry.getDataLoader(key).getOptions().getInstrumentation(); + assertThat(instrumentation, instanceOf(ChainedDataLoaderInstrumentation.class)); + List instrumentations = ((ChainedDataLoaderInstrumentation) instrumentation).getInstrumentations(); + assertThat(instrumentations, equalTo(List.of(instrA))); + } + } + + @Test + void canInstrumentRegisteredDLsViaBuilderCombined() { + + DataLoaderRegistry registry1 = DataLoaderRegistry.newRegistry() + .register("X", dlX) + .register("Y", dlY) + .build(); + + DataLoaderRegistry registry = DataLoaderRegistry.newRegistry() + .instrumentation(chainedInstrA) + .register("Z", dlZ) + .registerAll(registry1) + .build(); + + for (String key : List.of("X", "Y", "Z")) { + DataLoaderInstrumentation instrumentation = registry.getDataLoader(key).getOptions().getInstrumentation(); + assertThat(instrumentation, instanceOf(ChainedDataLoaderInstrumentation.class)); + List instrumentations = ((ChainedDataLoaderInstrumentation) instrumentation).getInstrumentations(); + assertThat(instrumentations, equalTo(List.of(instrA))); + } + } + + @Test + void canInstrumentViaMutativeRegistration() { + + DataLoaderRegistry registry = DataLoaderRegistry.newRegistry() + .instrumentation(chainedInstrA) + .build(); + + registry.register("X", dlX); + registry.computeIfAbsent("Y", l -> dlY); + registry.computeIfAbsent("Z", l -> dlZ); + + for (String key : List.of("X", "Y", "Z")) { + DataLoaderInstrumentation instrumentation = registry.getDataLoader(key).getOptions().getInstrumentation(); + assertThat(instrumentation, instanceOf(ChainedDataLoaderInstrumentation.class)); + List instrumentations = ((ChainedDataLoaderInstrumentation) instrumentation).getInstrumentations(); + assertThat(instrumentations, equalTo(List.of(instrA))); + } + } + + @Test + void wontDoAnyThingIfThereIsNoRegistryInstrumentation() { + DataLoaderRegistry registry = DataLoaderRegistry.newRegistry() + .register("X", dlX) + .register("Y", dlY) + .register("Z", dlZ) + .build(); + + for (String key : List.of("X", "Y", "Z")) { + DataLoaderInstrumentation instrumentation = registry.getDataLoader(key).getOptions().getInstrumentation(); + assertThat(instrumentation, equalTo(DataLoaderInstrumentationHelper.NOOP_INSTRUMENTATION)); + } + } + + @Test + void wontDoAnyThingIfThereTheyAreTheSameInstrumentationAlready() { + DataLoader newX = dlX.transform(builder -> dlX.getOptions().setInstrumentation(instrA)); + DataLoader newY = dlX.transform(builder -> dlY.getOptions().setInstrumentation(instrA)); + DataLoader newZ = dlX.transform(builder -> dlY.getOptions().setInstrumentation(instrA)); + DataLoaderRegistry registry = DataLoaderRegistry.newRegistry() + .instrumentation(instrA) + .register("X", newX) + .register("Y", newY) + .register("Z", newZ) + .build(); + + Map> dls = Map.of("X", newX, "Y", newY, "Z", newZ); + + assertThat(registry.getInstrumentation(), equalTo(instrA)); + + for (String key : List.of("X", "Y", "Z")) { + DataLoader dataLoader = registry.getDataLoader(key); + DataLoaderInstrumentation instrumentation = dataLoader.getOptions().getInstrumentation(); + assertThat(instrumentation, equalTo(instrA)); + // it's the same DL - it's not changed because it has the same instrumentation + assertThat(dls.get(key), equalTo(dataLoader)); + } + } + + @Test + void ifTheDLHasAInstrumentationThenItsTurnedIntoAChainedOne() { + DataLoader newX = dlX.transform(builder -> dlX.getOptions().setInstrumentation(instrA)); + + DataLoaderRegistry registry = DataLoaderRegistry.newRegistry() + .instrumentation(instrB) + .register("X", newX) + .build(); + + DataLoader dataLoader = registry.getDataLoader("X"); + DataLoaderInstrumentation instrumentation = dataLoader.getOptions().getInstrumentation(); + assertThat(instrumentation, instanceOf(ChainedDataLoaderInstrumentation.class)); + + List instrumentations = ((ChainedDataLoaderInstrumentation) instrumentation).getInstrumentations(); + // it gets turned into a chained one and the registry one goes first + assertThat(instrumentations, equalTo(List.of(instrB, instrA))); + } + + @Test + void chainedInstrumentationsWillBeCombined() { + DataLoader newX = dlX.transform(builder -> builder.options(dlX.getOptions().setInstrumentation(chainedInstrB))); + + DataLoaderRegistry registry = DataLoaderRegistry.newRegistry() + .instrumentation(instrA) + .register("X", newX) + .build(); + + DataLoader dataLoader = registry.getDataLoader("X"); + DataLoaderInstrumentation instrumentation = dataLoader.getOptions().getInstrumentation(); + assertThat(instrumentation, instanceOf(ChainedDataLoaderInstrumentation.class)); + + List instrumentations = ((ChainedDataLoaderInstrumentation) instrumentation).getInstrumentations(); + // it gets turned into a chained one and the registry one goes first + assertThat(instrumentations, equalTo(List.of(instrA, instrB))); + } + + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void endToEndIntegrationTest(TestDataLoaderFactory factory) { + DataLoader dl = factory.idLoader(); + + DataLoaderRegistry registry = DataLoaderRegistry.newRegistry() + .instrumentation(instrA) + .register("X", dl) + .build(); + + // since the data-loader changed when registered you MUST get the data loader from the registry + // not direct to the old one + DataLoader dataLoader = registry.getDataLoader("X"); + CompletableFuture loadA = dataLoader.load("A"); + + registry.dispatchAll(); + + await().until(loadA::isDone); + assertThat(loadA.join(), equalTo("A")); + + assertThat(instrA.methods, equalTo(List.of("A_beginDispatch", + "A_beginBatchLoader", "A_beginBatchLoader_onDispatched", "A_beginBatchLoader_onCompleted", + "A_beginDispatch_onDispatched", "A_beginDispatch_onCompleted"))); + + } +} \ No newline at end of file From 292b2835b78d745bb755b4222ae98ab4e7c84cd5 Mon Sep 17 00:00:00 2001 From: bbaker Date: Wed, 22 Jan 2025 14:50:45 +1100 Subject: [PATCH 068/156] Instrumentation support for dataloader - updated ScheduledDataLoaderRegistry --- .../org/dataloader/DataLoaderRegistry.java | 33 ++++++++++++++----- .../ScheduledDataLoaderRegistry.java | 11 +++++-- .../ChainedDataLoaderInstrumentationTest.java | 2 +- .../DataLoaderInstrumentationTest.java | 2 +- ...DataLoaderRegistryInstrumentationTest.java | 27 +++++++++++++-- 5 files changed, 61 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/dataloader/DataLoaderRegistry.java b/src/main/java/org/dataloader/DataLoaderRegistry.java index 5e00ee1b..d54ee531 100644 --- a/src/main/java/org/dataloader/DataLoaderRegistry.java +++ b/src/main/java/org/dataloader/DataLoaderRegistry.java @@ -19,28 +19,45 @@ /** * This allows data loaders to be registered together into a single place, so * they can be dispatched as one. It also allows you to retrieve data loaders by - * name from a central place + * name from a central place. + *

+ * Notes on {@link DataLoaderInstrumentation} : A {@link DataLoaderRegistry} can have an instrumentation + * associated with it. As each {@link DataLoader} is added to the registry, the {@link DataLoaderInstrumentation} + * of the registry is applied to that {@link DataLoader}. + *

+ * The {@link DataLoader} is changed and hence the object in the registry is not the + * same one as was originally registered. So you MUST get access to the {@link DataLoader} via {@link DataLoaderRegistry#getDataLoader(String)} methods + * and not use the original {@link DataLoader} object. + *

+ * If the {@link DataLoader} has no {@link DataLoaderInstrumentation} then the registry one is added to it. If it does have one already + * then a {@link ChainedDataLoaderInstrumentation} is created with the registry {@link DataLoaderInstrumentation} in it first and then any other + * {@link DataLoaderInstrumentation}s added after that. */ @PublicApi public class DataLoaderRegistry { - protected final Map> dataLoaders = new ConcurrentHashMap<>(); + protected final Map> dataLoaders; protected final DataLoaderInstrumentation instrumentation; public DataLoaderRegistry() { - instrumentation = null; + this(new ConcurrentHashMap<>(),null); } private DataLoaderRegistry(Builder builder) { - instrument(builder.instrumentation, builder.dataLoaders); - this.instrumentation = builder.instrumentation; + this(builder.dataLoaders,builder.instrumentation); } - private void instrument(DataLoaderInstrumentation registryInstrumentation, Map> incomingDataLoaders) { - this.dataLoaders.putAll(incomingDataLoaders); + protected DataLoaderRegistry(Map> dataLoaders, DataLoaderInstrumentation instrumentation ) { + this.dataLoaders = instrumentDLs(dataLoaders, instrumentation); + this.instrumentation = instrumentation; + } + + private Map> instrumentDLs(Map> incomingDataLoaders, DataLoaderInstrumentation registryInstrumentation) { + Map> dataLoaders = new ConcurrentHashMap<>(incomingDataLoaders); if (registryInstrumentation != null) { - this.dataLoaders.replaceAll((k, existingDL) -> instrumentDL(registryInstrumentation, existingDL)); + dataLoaders.replaceAll((k, existingDL) -> instrumentDL(registryInstrumentation, existingDL)); } + return dataLoaders; } /** diff --git a/src/main/java/org/dataloader/registries/ScheduledDataLoaderRegistry.java b/src/main/java/org/dataloader/registries/ScheduledDataLoaderRegistry.java index 6ea9425f..b6bc2573 100644 --- a/src/main/java/org/dataloader/registries/ScheduledDataLoaderRegistry.java +++ b/src/main/java/org/dataloader/registries/ScheduledDataLoaderRegistry.java @@ -3,6 +3,7 @@ import org.dataloader.DataLoader; import org.dataloader.DataLoaderRegistry; import org.dataloader.annotations.ExperimentalApi; +import org.dataloader.instrumentation.DataLoaderInstrumentation; import java.time.Duration; import java.util.LinkedHashMap; @@ -64,8 +65,7 @@ public class ScheduledDataLoaderRegistry extends DataLoaderRegistry implements A private volatile boolean closed; private ScheduledDataLoaderRegistry(Builder builder) { - super(); - this.dataLoaders.putAll(builder.dataLoaders); + super(builder.dataLoaders, builder.instrumentation); this.scheduledExecutorService = builder.scheduledExecutorService; this.defaultExecutorUsed = builder.defaultExecutorUsed; this.schedule = builder.schedule; @@ -271,6 +271,8 @@ public static class Builder { private boolean defaultExecutorUsed = false; private Duration schedule = Duration.ofMillis(10); private boolean tickerMode = false; + private DataLoaderInstrumentation instrumentation; + /** * If you provide a {@link ScheduledExecutorService} then it will NOT be shutdown when @@ -363,6 +365,11 @@ public Builder tickerMode(boolean tickerMode) { return this; } + public Builder instrumentation(DataLoaderInstrumentation instrumentation) { + this.instrumentation = instrumentation; + return this; + } + /** * @return the newly built {@link ScheduledDataLoaderRegistry} */ diff --git a/src/test/java/org/dataloader/instrumentation/ChainedDataLoaderInstrumentationTest.java b/src/test/java/org/dataloader/instrumentation/ChainedDataLoaderInstrumentationTest.java index 31687349..b3272ced 100644 --- a/src/test/java/org/dataloader/instrumentation/ChainedDataLoaderInstrumentationTest.java +++ b/src/test/java/org/dataloader/instrumentation/ChainedDataLoaderInstrumentationTest.java @@ -18,7 +18,7 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; -class ChainedDataLoaderInstrumentationTest { +public class ChainedDataLoaderInstrumentationTest { CapturingInstrumentation capturingA; CapturingInstrumentation capturingB; diff --git a/src/test/java/org/dataloader/instrumentation/DataLoaderInstrumentationTest.java b/src/test/java/org/dataloader/instrumentation/DataLoaderInstrumentationTest.java index 4f43719e..a35e13a9 100644 --- a/src/test/java/org/dataloader/instrumentation/DataLoaderInstrumentationTest.java +++ b/src/test/java/org/dataloader/instrumentation/DataLoaderInstrumentationTest.java @@ -22,7 +22,7 @@ import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.is; -class DataLoaderInstrumentationTest { +public class DataLoaderInstrumentationTest { BatchLoader snoozingBatchLoader = keys -> CompletableFuture.supplyAsync(() -> { TestKit.snooze(100); diff --git a/src/test/java/org/dataloader/instrumentation/DataLoaderRegistryInstrumentationTest.java b/src/test/java/org/dataloader/instrumentation/DataLoaderRegistryInstrumentationTest.java index 639ff487..5690bdc9 100644 --- a/src/test/java/org/dataloader/instrumentation/DataLoaderRegistryInstrumentationTest.java +++ b/src/test/java/org/dataloader/instrumentation/DataLoaderRegistryInstrumentationTest.java @@ -4,6 +4,7 @@ import org.dataloader.DataLoaderRegistry; import org.dataloader.fixtures.TestKit; import org.dataloader.fixtures.parameterized.TestDataLoaderFactory; +import org.dataloader.registries.ScheduledDataLoaderRegistry; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -18,7 +19,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; -class DataLoaderRegistryInstrumentationTest { +public class DataLoaderRegistryInstrumentationTest { DataLoader dlX; DataLoader dlY; DataLoader dlZ; @@ -177,6 +178,29 @@ void chainedInstrumentationsWillBeCombined() { assertThat(instrumentations, equalTo(List.of(instrA, instrB))); } + @SuppressWarnings("resource") + @Test + void canInstrumentScheduledRegistryViaBuilder() { + + assertThat(dlX.getOptions().getInstrumentation(), equalTo(DataLoaderInstrumentationHelper.NOOP_INSTRUMENTATION)); + + ScheduledDataLoaderRegistry registry = ScheduledDataLoaderRegistry.newScheduledRegistry() + .instrumentation(chainedInstrA) + .register("X", dlX) + .register("Y", dlY) + .register("Z", dlZ) + .build(); + + assertThat(registry.getInstrumentation(), equalTo(chainedInstrA)); + + for (String key : List.of("X", "Y", "Z")) { + DataLoaderInstrumentation instrumentation = registry.getDataLoader(key).getOptions().getInstrumentation(); + assertThat(instrumentation, instanceOf(ChainedDataLoaderInstrumentation.class)); + List instrumentations = ((ChainedDataLoaderInstrumentation) instrumentation).getInstrumentations(); + assertThat(instrumentations, equalTo(List.of(instrA))); + } + } + @ParameterizedTest @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void endToEndIntegrationTest(TestDataLoaderFactory factory) { @@ -200,6 +224,5 @@ public void endToEndIntegrationTest(TestDataLoaderFactory factory) { assertThat(instrA.methods, equalTo(List.of("A_beginDispatch", "A_beginBatchLoader", "A_beginBatchLoader_onDispatched", "A_beginBatchLoader_onCompleted", "A_beginDispatch_onDispatched", "A_beginDispatch_onCompleted"))); - } } \ No newline at end of file From 0b9fba57a316e01c0f6bfb56156e92913edf73bb Mon Sep 17 00:00:00 2001 From: bbaker Date: Wed, 22 Jan 2025 15:39:51 +1100 Subject: [PATCH 069/156] Instrumentation support for dataloader - Adde doco and SimpleDataLoaderInstrumentationContext --- README.md | 59 +++++++++++++++++++ .../DataLoaderInstrumentationHelper.java | 27 +++++++++ ...impleDataLoaderInstrumentationContext.java | 35 +++++++++++ src/test/java/ReadmeExamples.java | 47 +++++++++++++++ ...eDataLoaderInstrumentationContextTest.java | 49 +++++++++++++++ 5 files changed, 217 insertions(+) create mode 100644 src/main/java/org/dataloader/instrumentation/SimpleDataLoaderInstrumentationContext.java create mode 100644 src/test/java/org/dataloader/instrumentation/SimpleDataLoaderInstrumentationContextTest.java diff --git a/README.md b/README.md index 2a0e08f7..61180d9e 100644 --- a/README.md +++ b/README.md @@ -750,6 +750,65 @@ When ticker mode is **true** the `ScheduledDataLoaderRegistry` algorithm is as f * If it returns **true**, then `dataLoader.dispatch()` is called **and** a task is scheduled to re-evaluate this specific dataloader in the near future * The re-evaluation tasks are run periodically according to the `registry.getScheduleDuration()` +## Instrumenting the data loader code + +A `DataLoader` can have a `DataLoaderInstrumentation` associated with it. This callback interface is intended to provide +insight into working of the `DataLoader` such as how long it takes to run or to allow for logging of key events. + +You set the `DataLoaderInstrumentation` into the `DataLoaderOptions` at build time. + +```java + + + DataLoaderInstrumentation timingInstrumentation = new DataLoaderInstrumentation() { + @Override + public DataLoaderInstrumentationContext> beginDispatch(DataLoader dataLoader) { + long then = System.currentTimeMillis(); + return DataLoaderInstrumentationHelper.whenCompleted((result, err) -> { + long ms = System.currentTimeMillis() - then; + System.out.println(format("dispatch time: %d ms", ms)); + }); + } + + @Override + public DataLoaderInstrumentationContext> beginBatchLoader(DataLoader dataLoader, List keys, BatchLoaderEnvironment environment) { + long then = System.currentTimeMillis(); + return DataLoaderInstrumentationHelper.whenCompleted((result, err) -> { + long ms = System.currentTimeMillis() - then; + System.out.println(format("batch loader time: %d ms", ms)); + }); + } + }; + DataLoaderOptions options = DataLoaderOptions.newOptions().setInstrumentation(timingInstrumentation); + DataLoader userDataLoader = DataLoaderFactory.newDataLoader(userBatchLoader, options); + +``` + +The example shows how long the overall `DataLoader` dispatch takes or how long the batch loader takes to run. + +### Instrumenting the DataLoaderRegistry + +You can also associate a `DataLoaderInstrumentation` with a `DataLoaderRegistry`. Every `DataLoader` registered will be changed so that the registry +`DataLoaderInstrumentation` is associated with it. This allows you to set just the one `DataLoaderInstrumentation` in place and it applies to all +data loaders. + +```java + DataLoader userDataLoader = DataLoaderFactory.newDataLoader(userBatchLoader); + DataLoader teamsDataLoader = DataLoaderFactory.newDataLoader(teamsBatchLoader); + + DataLoaderRegistry registry = DataLoaderRegistry.newRegistry() + .instrumentation(timingInstrumentation) + .register("users", userDataLoader) + .register("teams", teamsDataLoader) + .build(); + + DataLoader changedUsersDataLoader = registry.getDataLoader("users"); +``` + +The `timingInstrumentation` here will be associated with the `DataLoader` under the key `users` and the key `teams`. Note that since +DataLoader is immutable, a new changed object is created so you must use the registry to get the `DataLoader`. + + ## Other information sources - [Facebook DataLoader Github repo](https://github.com/facebook/dataloader) diff --git a/src/main/java/org/dataloader/instrumentation/DataLoaderInstrumentationHelper.java b/src/main/java/org/dataloader/instrumentation/DataLoaderInstrumentationHelper.java index 844ec375..9e60060c 100644 --- a/src/main/java/org/dataloader/instrumentation/DataLoaderInstrumentationHelper.java +++ b/src/main/java/org/dataloader/instrumentation/DataLoaderInstrumentationHelper.java @@ -2,6 +2,8 @@ import org.dataloader.annotations.PublicApi; +import java.util.function.BiConsumer; + @PublicApi public class DataLoaderInstrumentationHelper { @@ -33,6 +35,31 @@ public static DataLoaderInstrumentationContext noOpCtx() { public static final DataLoaderInstrumentation NOOP_INSTRUMENTATION = new DataLoaderInstrumentation() { }; + /** + * Allows for the more fluent away to return an instrumentation context that runs the specified + * code on instrumentation step dispatch. + * + * @param codeToRun the code to run on dispatch + * @param the generic type + * @return an instrumentation context + */ + public static DataLoaderInstrumentationContext whenDispatched(Runnable codeToRun) { + return new SimpleDataLoaderInstrumentationContext<>(codeToRun, null); + } + + /** + * Allows for the more fluent away to return an instrumentation context that runs the specified + * code on instrumentation step completion. + * + * @param codeToRun the code to run on completion + * @param the generic type + * @return an instrumentation context + */ + public static DataLoaderInstrumentationContext whenCompleted(BiConsumer codeToRun) { + return new SimpleDataLoaderInstrumentationContext<>(null, codeToRun); + } + + /** * Check the {@link DataLoaderInstrumentationContext} to see if its null and returns a noop if it is or else the original * context. This is a bit of a helper method. diff --git a/src/main/java/org/dataloader/instrumentation/SimpleDataLoaderInstrumentationContext.java b/src/main/java/org/dataloader/instrumentation/SimpleDataLoaderInstrumentationContext.java new file mode 100644 index 00000000..f629a05a --- /dev/null +++ b/src/main/java/org/dataloader/instrumentation/SimpleDataLoaderInstrumentationContext.java @@ -0,0 +1,35 @@ +package org.dataloader.instrumentation; + + +import org.dataloader.annotations.Internal; + +import java.util.function.BiConsumer; + +/** + * A simple implementation of {@link DataLoaderInstrumentationContext} + */ +@Internal +class SimpleDataLoaderInstrumentationContext implements DataLoaderInstrumentationContext { + + private final BiConsumer codeToRunOnComplete; + private final Runnable codeToRunOnDispatch; + + SimpleDataLoaderInstrumentationContext(Runnable codeToRunOnDispatch, BiConsumer codeToRunOnComplete) { + this.codeToRunOnComplete = codeToRunOnComplete; + this.codeToRunOnDispatch = codeToRunOnDispatch; + } + + @Override + public void onDispatched() { + if (codeToRunOnDispatch != null) { + codeToRunOnDispatch.run(); + } + } + + @Override + public void onCompleted(T result, Throwable t) { + if (codeToRunOnComplete != null) { + codeToRunOnComplete.accept(result, t); + } + } +} diff --git a/src/test/java/ReadmeExamples.java b/src/test/java/ReadmeExamples.java index 9e30c903..3b8f57ed 100644 --- a/src/test/java/ReadmeExamples.java +++ b/src/test/java/ReadmeExamples.java @@ -6,12 +6,17 @@ import org.dataloader.DataLoader; import org.dataloader.DataLoaderFactory; import org.dataloader.DataLoaderOptions; +import org.dataloader.DataLoaderRegistry; +import org.dataloader.DispatchResult; import org.dataloader.MappedBatchLoaderWithContext; import org.dataloader.MappedBatchPublisher; import org.dataloader.Try; import org.dataloader.fixtures.SecurityCtx; import org.dataloader.fixtures.User; import org.dataloader.fixtures.UserManager; +import org.dataloader.instrumentation.DataLoaderInstrumentation; +import org.dataloader.instrumentation.DataLoaderInstrumentationContext; +import org.dataloader.instrumentation.DataLoaderInstrumentationHelper; import org.dataloader.registries.DispatchPredicate; import org.dataloader.registries.ScheduledDataLoaderRegistry; import org.dataloader.scheduler.BatchLoaderScheduler; @@ -228,6 +233,7 @@ private void clearCacheOnError() { } BatchLoader userBatchLoader; + BatchLoader teamsBatchLoader; private void disableCache() { DataLoaderFactory.newDataLoader(userBatchLoader, DataLoaderOptions.newOptions().setCachingEnabled(false)); @@ -380,4 +386,45 @@ private void ScheduledDispatcherChained() { .build(); } + + private DataLoaderInstrumentation timingInstrumentation = DataLoaderInstrumentationHelper.NOOP_INSTRUMENTATION; + + private void instrumentationExample() { + + DataLoaderInstrumentation timingInstrumentation = new DataLoaderInstrumentation() { + @Override + public DataLoaderInstrumentationContext> beginDispatch(DataLoader dataLoader) { + long then = System.currentTimeMillis(); + return DataLoaderInstrumentationHelper.whenCompleted((result, err) -> { + long ms = System.currentTimeMillis() - then; + System.out.println(format("dispatch time: %d ms", ms)); + }); + } + + @Override + public DataLoaderInstrumentationContext> beginBatchLoader(DataLoader dataLoader, List keys, BatchLoaderEnvironment environment) { + long then = System.currentTimeMillis(); + return DataLoaderInstrumentationHelper.whenCompleted((result, err) -> { + long ms = System.currentTimeMillis() - then; + System.out.println(format("batch loader time: %d ms", ms)); + }); + } + }; + DataLoaderOptions options = DataLoaderOptions.newOptions().setInstrumentation(timingInstrumentation); + DataLoader userDataLoader = DataLoaderFactory.newDataLoader(userBatchLoader, options); + } + + private void registryExample() { + DataLoader userDataLoader = DataLoaderFactory.newDataLoader(userBatchLoader); + DataLoader teamsDataLoader = DataLoaderFactory.newDataLoader(teamsBatchLoader); + + DataLoaderRegistry registry = DataLoaderRegistry.newRegistry() + .instrumentation(timingInstrumentation) + .register("users", userDataLoader) + .register("teams", teamsDataLoader) + .build(); + + DataLoader changedUsersDataLoader = registry.getDataLoader("users"); + + } } diff --git a/src/test/java/org/dataloader/instrumentation/SimpleDataLoaderInstrumentationContextTest.java b/src/test/java/org/dataloader/instrumentation/SimpleDataLoaderInstrumentationContextTest.java new file mode 100644 index 00000000..38328eb4 --- /dev/null +++ b/src/test/java/org/dataloader/instrumentation/SimpleDataLoaderInstrumentationContextTest.java @@ -0,0 +1,49 @@ +package org.dataloader.instrumentation; + +import org.hamcrest.Matchers; +import org.junit.jupiter.api.Test; + +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.nullValue; + +public class SimpleDataLoaderInstrumentationContextTest { + + @Test + void canRunCompletedCodeAsExpected() { + AtomicReference actual = new AtomicReference<>(); + AtomicReference actualErr = new AtomicReference<>(); + + DataLoaderInstrumentationContext ctx = DataLoaderInstrumentationHelper.whenCompleted((r, err) -> { + actualErr.set(err); + actual.set(r); + }); + + ctx.onDispatched(); // nothing happens + assertThat(actual.get(), nullValue()); + assertThat(actualErr.get(), nullValue()); + + ctx.onCompleted("X", null); + assertThat(actual.get(), Matchers.equalTo("X")); + assertThat(actualErr.get(), nullValue()); + + ctx.onCompleted(null, new RuntimeException()); + assertThat(actual.get(), nullValue()); + assertThat(actualErr.get(), Matchers.instanceOf(RuntimeException.class)); + } + + @Test + void canRunOnDispatchCodeAsExpected() { + AtomicBoolean dispatchedCalled = new AtomicBoolean(); + + DataLoaderInstrumentationContext ctx = DataLoaderInstrumentationHelper.whenDispatched(() -> dispatchedCalled.set(true)); + + ctx.onCompleted("X", null); // nothing happens + assertThat(dispatchedCalled.get(), Matchers.equalTo(false)); + + ctx.onDispatched(); + assertThat(dispatchedCalled.get(), Matchers.equalTo(true)); + } +} \ No newline at end of file From 610343f5dfeacee7aa6e294a0f903f4048d74e06 Mon Sep 17 00:00:00 2001 From: bbaker Date: Wed, 22 Jan 2025 15:58:31 +1100 Subject: [PATCH 070/156] Instrumentation support for dataloader - Adde more doco --- src/test/java/ReadmeExamples.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/test/java/ReadmeExamples.java b/src/test/java/ReadmeExamples.java index 3b8f57ed..1f718aa5 100644 --- a/src/test/java/ReadmeExamples.java +++ b/src/test/java/ReadmeExamples.java @@ -427,4 +427,22 @@ private void registryExample() { DataLoader changedUsersDataLoader = registry.getDataLoader("users"); } + + private void combiningRegistryExample() { + DataLoader userDataLoader = DataLoaderFactory.newDataLoader(userBatchLoader); + DataLoader teamsDataLoader = DataLoaderFactory.newDataLoader(teamsBatchLoader); + + DataLoaderRegistry registry = DataLoaderRegistry.newRegistry() + .register("users", userDataLoader) + .register("teams", teamsDataLoader) + .build(); + + DataLoaderRegistry registryCombined = DataLoaderRegistry.newRegistry() + .instrumentation(timingInstrumentation) + .registerAll(registry) + .build(); + + DataLoader changedUsersDataLoader = registryCombined.getDataLoader("users"); + + } } From a65cf3473d5d24819dcb59e22dd0f7e81dab23d2 Mon Sep 17 00:00:00 2001 From: bbaker Date: Thu, 30 Jan 2025 13:20:19 +1100 Subject: [PATCH 071/156] Making DataLoaderOptions immutable --- .../org/dataloader/DataLoaderOptions.java | 240 ++++++++++++++---- .../org/dataloader/DataLoaderOptionsTest.java | 187 ++++++++++++++ .../org/dataloader/ValueCacheOptionsTest.java | 19 ++ 3 files changed, 396 insertions(+), 50 deletions(-) create mode 100644 src/test/java/org/dataloader/DataLoaderOptionsTest.java create mode 100644 src/test/java/org/dataloader/ValueCacheOptionsTest.java diff --git a/src/main/java/org/dataloader/DataLoaderOptions.java b/src/main/java/org/dataloader/DataLoaderOptions.java index b96e7857..f8ea95cb 100644 --- a/src/main/java/org/dataloader/DataLoaderOptions.java +++ b/src/main/java/org/dataloader/DataLoaderOptions.java @@ -17,18 +17,20 @@ package org.dataloader; import org.dataloader.annotations.PublicApi; -import org.dataloader.impl.Assertions; import org.dataloader.scheduler.BatchLoaderScheduler; import org.dataloader.stats.NoOpStatisticsCollector; import org.dataloader.stats.StatisticsCollector; +import java.util.Objects; import java.util.Optional; +import java.util.function.Consumer; import java.util.function.Supplier; import static org.dataloader.impl.Assertions.nonNull; /** - * Configuration options for {@link DataLoader} instances. + * Configuration options for {@link DataLoader} instances. This is an immutable class so each time + * you change a value it returns a new object. * * @author Arnold Schrijver */ @@ -36,18 +38,20 @@ public class DataLoaderOptions { private static final BatchLoaderContextProvider NULL_PROVIDER = () -> null; - - private boolean batchingEnabled; - private boolean cachingEnabled; - private boolean cachingExceptionsEnabled; - private CacheKey cacheKeyFunction; - private CacheMap cacheMap; - private ValueCache valueCache; - private int maxBatchSize; - private Supplier statisticsCollector; - private BatchLoaderContextProvider environmentProvider; - private ValueCacheOptions valueCacheOptions; - private BatchLoaderScheduler batchLoaderScheduler; + private static final Supplier NOOP_COLLECTOR = NoOpStatisticsCollector::new; + private static final ValueCacheOptions DEFAULT_VALUE_CACHE_OPTIONS = ValueCacheOptions.newOptions(); + + private final boolean batchingEnabled; + private final boolean cachingEnabled; + private final boolean cachingExceptionsEnabled; + private final CacheKey cacheKeyFunction; + private final CacheMap cacheMap; + private final ValueCache valueCache; + private final int maxBatchSize; + private final Supplier statisticsCollector; + private final BatchLoaderContextProvider environmentProvider; + private final ValueCacheOptions valueCacheOptions; + private final BatchLoaderScheduler batchLoaderScheduler; /** * Creates a new data loader options with default settings. @@ -56,13 +60,30 @@ public DataLoaderOptions() { batchingEnabled = true; cachingEnabled = true; cachingExceptionsEnabled = true; + cacheKeyFunction = null; + cacheMap = null; + valueCache = null; maxBatchSize = -1; - statisticsCollector = NoOpStatisticsCollector::new; + statisticsCollector = NOOP_COLLECTOR; environmentProvider = NULL_PROVIDER; - valueCacheOptions = ValueCacheOptions.newOptions(); + valueCacheOptions = DEFAULT_VALUE_CACHE_OPTIONS; batchLoaderScheduler = null; } + private DataLoaderOptions(Builder builder) { + this.batchingEnabled = builder.batchingEnabled; + this.cachingEnabled = builder.cachingEnabled; + this.cachingExceptionsEnabled = builder.cachingExceptionsEnabled; + this.cacheKeyFunction = builder.cacheKeyFunction; + this.cacheMap = builder.cacheMap; + this.valueCache = builder.valueCache; + this.maxBatchSize = builder.maxBatchSize; + this.statisticsCollector = builder.statisticsCollector; + this.environmentProvider = builder.environmentProvider; + this.valueCacheOptions = builder.valueCacheOptions; + this.batchLoaderScheduler = builder.batchLoaderScheduler; + } + /** * Clones the provided data loader options. * @@ -90,6 +111,51 @@ public static DataLoaderOptions newOptions() { return new DataLoaderOptions(); } + /** + * @return a new default data loader options {@link Builder} that you can then customize + */ + public static DataLoaderOptions.Builder newOptionsBuilder() { + return new DataLoaderOptions.Builder(); + } + + /** + * @param otherOptions the options to copy + * @return a new default data loader options {@link Builder} from the specified one that you can then customize + */ + public static DataLoaderOptions.Builder newDataLoaderOptions(DataLoaderOptions otherOptions) { + return new DataLoaderOptions.Builder(otherOptions); + } + + /** + * Will transform the current options in to a builder ands allow you to build a new set of options + * + * @param builderConsumer the consumer of a builder that has this objects starting values + * @return a new {@link DataLoaderOptions} object + */ + public DataLoaderOptions transform(Consumer builderConsumer) { + Builder builder = newOptionsBuilder(); + builderConsumer.accept(builder); + return builder.build(); + } + + @Override + public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) return false; + DataLoaderOptions that = (DataLoaderOptions) o; + return batchingEnabled == that.batchingEnabled + && cachingEnabled == that.cachingEnabled + && cachingExceptionsEnabled == that.cachingExceptionsEnabled + && maxBatchSize == that.maxBatchSize + && Objects.equals(cacheKeyFunction, that.cacheKeyFunction) && + Objects.equals(cacheMap, that.cacheMap) && + Objects.equals(valueCache, that.valueCache) && + Objects.equals(statisticsCollector, that.statisticsCollector) && + Objects.equals(environmentProvider, that.environmentProvider) && + Objects.equals(valueCacheOptions, that.valueCacheOptions) && + Objects.equals(batchLoaderScheduler, that.batchLoaderScheduler); + } + + /** * Option that determines whether to use batching (the default), or not. * @@ -103,12 +169,10 @@ public boolean batchingEnabled() { * Sets the option that determines whether batch loading is enabled. * * @param batchingEnabled {@code true} to enable batch loading, {@code false} otherwise - * * @return the data loader options for fluent coding */ public DataLoaderOptions setBatchingEnabled(boolean batchingEnabled) { - this.batchingEnabled = batchingEnabled; - return this; + return builder().setBatchingEnabled(batchingEnabled).build(); } /** @@ -124,17 +188,15 @@ public boolean cachingEnabled() { * Sets the option that determines whether caching is enabled. * * @param cachingEnabled {@code true} to enable caching, {@code false} otherwise - * * @return the data loader options for fluent coding */ public DataLoaderOptions setCachingEnabled(boolean cachingEnabled) { - this.cachingEnabled = cachingEnabled; - return this; + return builder().setCachingEnabled(cachingEnabled).build(); } /** * Option that determines whether to cache exceptional values (the default), or not. - * + *

* For short-lived caches (that is request caches) it makes sense to cache exceptions since * it's likely the key is still poisoned. However, if you have long-lived caches, then it may make * sense to set this to false since the downstream system may have recovered from its failure @@ -150,12 +212,10 @@ public boolean cachingExceptionsEnabled() { * Sets the option that determines whether exceptional values are cache enabled. * * @param cachingExceptionsEnabled {@code true} to enable caching exceptional values, {@code false} otherwise - * * @return the data loader options for fluent coding */ public DataLoaderOptions setCachingExceptionsEnabled(boolean cachingExceptionsEnabled) { - this.cachingExceptionsEnabled = cachingExceptionsEnabled; - return this; + return builder().setCachingExceptionsEnabled(cachingExceptionsEnabled).build(); } /** @@ -173,12 +233,10 @@ public Optional cacheKeyFunction() { * Sets the function to use for creating the cache key, if caching is enabled. * * @param cacheKeyFunction the cache key function to use - * * @return the data loader options for fluent coding */ public DataLoaderOptions setCacheKeyFunction(CacheKey cacheKeyFunction) { - this.cacheKeyFunction = cacheKeyFunction; - return this; + return builder().setCacheKeyFunction(cacheKeyFunction).build(); } /** @@ -196,12 +254,10 @@ public DataLoaderOptions setCacheKeyFunction(CacheKey cacheKeyFunction) { * Sets the cache map implementation to use for caching, if caching is enabled. * * @param cacheMap the cache map instance - * * @return the data loader options for fluent coding */ public DataLoaderOptions setCacheMap(CacheMap cacheMap) { - this.cacheMap = cacheMap; - return this; + return builder().setCacheMap(cacheMap).build(); } /** @@ -219,12 +275,10 @@ public int maxBatchSize() { * before they are split into multiple class * * @param maxBatchSize the maximum batch size - * * @return the data loader options for fluent coding */ public DataLoaderOptions setMaxBatchSize(int maxBatchSize) { - this.maxBatchSize = maxBatchSize; - return this; + return builder().setMaxBatchSize(maxBatchSize).build(); } /** @@ -240,12 +294,10 @@ public StatisticsCollector getStatisticsCollector() { * a common value * * @param statisticsCollector the statistics collector to use - * * @return the data loader options for fluent coding */ public DataLoaderOptions setStatisticsCollector(Supplier statisticsCollector) { - this.statisticsCollector = nonNull(statisticsCollector); - return this; + return builder().setStatisticsCollector(nonNull(statisticsCollector)).build(); } /** @@ -259,12 +311,10 @@ public BatchLoaderContextProvider getBatchLoaderContextProvider() { * Sets the batch loader environment provider that will be used to give context to batch load functions * * @param contextProvider the batch loader context provider - * * @return the data loader options for fluent coding */ public DataLoaderOptions setBatchLoaderContextProvider(BatchLoaderContextProvider contextProvider) { - this.environmentProvider = nonNull(contextProvider); - return this; + return builder().setBatchLoaderContextProvider(nonNull(contextProvider)).build(); } /** @@ -282,12 +332,10 @@ public DataLoaderOptions setBatchLoaderContextProvider(BatchLoaderContextProvide * Sets the value cache implementation to use for caching values, if caching is enabled. * * @param valueCache the value cache instance - * * @return the data loader options for fluent coding */ public DataLoaderOptions setValueCache(ValueCache valueCache) { - this.valueCache = valueCache; - return this; + return builder().setValueCache(valueCache).build(); } /** @@ -301,12 +349,10 @@ public ValueCacheOptions getValueCacheOptions() { * Sets the {@link ValueCacheOptions} that control how the {@link ValueCache} will be used * * @param valueCacheOptions the value cache options - * * @return the data loader options for fluent coding */ public DataLoaderOptions setValueCacheOptions(ValueCacheOptions valueCacheOptions) { - this.valueCacheOptions = Assertions.nonNull(valueCacheOptions); - return this; + return builder().setValueCacheOptions(nonNull(valueCacheOptions)).build(); } /** @@ -321,11 +367,105 @@ public BatchLoaderScheduler getBatchLoaderScheduler() { * to some future time. * * @param batchLoaderScheduler the scheduler - * * @return the data loader options for fluent coding */ public DataLoaderOptions setBatchLoaderScheduler(BatchLoaderScheduler batchLoaderScheduler) { - this.batchLoaderScheduler = batchLoaderScheduler; - return this; + return builder().setBatchLoaderScheduler(batchLoaderScheduler).build(); + } + + private Builder builder() { + return new Builder(this); + } + + public static class Builder { + private boolean batchingEnabled; + private boolean cachingEnabled; + private boolean cachingExceptionsEnabled; + private CacheKey cacheKeyFunction; + private CacheMap cacheMap; + private ValueCache valueCache; + private int maxBatchSize; + private Supplier statisticsCollector; + private BatchLoaderContextProvider environmentProvider; + private ValueCacheOptions valueCacheOptions; + private BatchLoaderScheduler batchLoaderScheduler; + + public Builder() { + this(new DataLoaderOptions()); // use the defaults of the DataLoaderOptions for this builder + } + + Builder(DataLoaderOptions other) { + this.batchingEnabled = other.batchingEnabled; + this.cachingEnabled = other.cachingEnabled; + this.cachingExceptionsEnabled = other.cachingExceptionsEnabled; + this.cacheKeyFunction = other.cacheKeyFunction; + this.cacheMap = other.cacheMap; + this.valueCache = other.valueCache; + this.maxBatchSize = other.maxBatchSize; + this.statisticsCollector = other.statisticsCollector; + this.environmentProvider = other.environmentProvider; + this.valueCacheOptions = other.valueCacheOptions; + this.batchLoaderScheduler = other.batchLoaderScheduler; + } + + public Builder setBatchingEnabled(boolean batchingEnabled) { + this.batchingEnabled = batchingEnabled; + return this; + } + + public Builder setCachingEnabled(boolean cachingEnabled) { + this.cachingEnabled = cachingEnabled; + return this; + } + + public Builder setCachingExceptionsEnabled(boolean cachingExceptionsEnabled) { + this.cachingExceptionsEnabled = cachingExceptionsEnabled; + return this; + } + + public Builder setCacheKeyFunction(CacheKey cacheKeyFunction) { + this.cacheKeyFunction = cacheKeyFunction; + return this; + } + + public Builder setCacheMap(CacheMap cacheMap) { + this.cacheMap = cacheMap; + return this; + } + + public Builder setValueCache(ValueCache valueCache) { + this.valueCache = valueCache; + return this; + } + + public Builder setMaxBatchSize(int maxBatchSize) { + this.maxBatchSize = maxBatchSize; + return this; + } + + public Builder setStatisticsCollector(Supplier statisticsCollector) { + this.statisticsCollector = statisticsCollector; + return this; + } + + public Builder setBatchLoaderContextProvider(BatchLoaderContextProvider environmentProvider) { + this.environmentProvider = environmentProvider; + return this; + } + + public Builder setValueCacheOptions(ValueCacheOptions valueCacheOptions) { + this.valueCacheOptions = valueCacheOptions; + return this; + } + + public Builder setBatchLoaderScheduler(BatchLoaderScheduler batchLoaderScheduler) { + this.batchLoaderScheduler = batchLoaderScheduler; + return this; + } + + public DataLoaderOptions build() { + return new DataLoaderOptions(this); + } + } } diff --git a/src/test/java/org/dataloader/DataLoaderOptionsTest.java b/src/test/java/org/dataloader/DataLoaderOptionsTest.java new file mode 100644 index 00000000..f6e06e80 --- /dev/null +++ b/src/test/java/org/dataloader/DataLoaderOptionsTest.java @@ -0,0 +1,187 @@ +package org.dataloader; + +import org.dataloader.impl.DefaultCacheMap; +import org.dataloader.impl.NoOpValueCache; +import org.dataloader.scheduler.BatchLoaderScheduler; +import org.dataloader.stats.NoOpStatisticsCollector; +import org.dataloader.stats.StatisticsCollector; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletionStage; +import java.util.function.Supplier; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; + +@SuppressWarnings("OptionalGetWithoutIsPresent") +class DataLoaderOptionsTest { + + DataLoaderOptions optionsDefault = new DataLoaderOptions(); + + @Test + void canCreateDefaultOptions() { + + assertThat(optionsDefault.batchingEnabled(), equalTo(true)); + assertThat(optionsDefault.cachingEnabled(), equalTo(true)); + assertThat(optionsDefault.cachingExceptionsEnabled(), equalTo(true)); + assertThat(optionsDefault.maxBatchSize(), equalTo(-1)); + assertThat(optionsDefault.getBatchLoaderScheduler(), equalTo(null)); + + DataLoaderOptions builtOptions = DataLoaderOptions.newOptionsBuilder().build(); + assertThat(builtOptions, equalTo(optionsDefault)); + assertThat(builtOptions == optionsDefault, equalTo(false)); + + DataLoaderOptions transformedOptions = optionsDefault.transform(builder -> { + }); + assertThat(transformedOptions, equalTo(optionsDefault)); + assertThat(transformedOptions == optionsDefault, equalTo(false)); + } + + @Test + void canCopyOk() { + DataLoaderOptions optionsNext = new DataLoaderOptions(optionsDefault); + assertThat(optionsNext, equalTo(optionsDefault)); + assertThat(optionsNext == optionsDefault, equalTo(false)); + + optionsNext = DataLoaderOptions.newDataLoaderOptions(optionsDefault).build(); + assertThat(optionsNext, equalTo(optionsDefault)); + assertThat(optionsNext == optionsDefault, equalTo(false)); + } + + BatchLoaderScheduler testBatchLoaderScheduler = new BatchLoaderScheduler() { + @Override + public CompletionStage> scheduleBatchLoader(ScheduledBatchLoaderCall scheduledCall, List keys, BatchLoaderEnvironment environment) { + return null; + } + + @Override + public CompletionStage> scheduleMappedBatchLoader(ScheduledMappedBatchLoaderCall scheduledCall, List keys, BatchLoaderEnvironment environment) { + return null; + } + + @Override + public void scheduleBatchPublisher(ScheduledBatchPublisherCall scheduledCall, List keys, BatchLoaderEnvironment environment) { + + } + }; + + BatchLoaderContextProvider testBatchLoaderContextProvider = () -> null; + + CacheMap testCacheMap = new DefaultCacheMap<>(); + + ValueCache testValueCache = new NoOpValueCache<>(); + + CacheKey testCacheKey = new CacheKey() { + @Override + public Object getKey(Object input) { + return null; + } + }; + + ValueCacheOptions testValueCacheOptions = ValueCacheOptions.newOptions(); + + NoOpStatisticsCollector noOpStatisticsCollector = new NoOpStatisticsCollector(); + Supplier testStatisticsCollectorSupplier = () -> noOpStatisticsCollector; + + @Test + void canBuildOk() { + assertThat(optionsDefault.setBatchingEnabled(false).batchingEnabled(), + equalTo(false)); + assertThat(optionsDefault.setBatchLoaderScheduler(testBatchLoaderScheduler).getBatchLoaderScheduler(), + equalTo(testBatchLoaderScheduler)); + assertThat(optionsDefault.setBatchLoaderContextProvider(testBatchLoaderContextProvider).getBatchLoaderContextProvider(), + equalTo(testBatchLoaderContextProvider)); + assertThat(optionsDefault.setCacheMap(testCacheMap).cacheMap().get(), + equalTo(testCacheMap)); + assertThat(optionsDefault.setCachingEnabled(false).cachingEnabled(), + equalTo(false)); + assertThat(optionsDefault.setValueCacheOptions(testValueCacheOptions).getValueCacheOptions(), + equalTo(testValueCacheOptions)); + assertThat(optionsDefault.setCacheKeyFunction(testCacheKey).cacheKeyFunction().get(), + equalTo(testCacheKey)); + assertThat(optionsDefault.setValueCache(testValueCache).valueCache().get(), + equalTo(testValueCache)); + assertThat(optionsDefault.setMaxBatchSize(10).maxBatchSize(), + equalTo(10)); + assertThat(optionsDefault.setStatisticsCollector(testStatisticsCollectorSupplier).getStatisticsCollector(), + equalTo(testStatisticsCollectorSupplier.get())); + + DataLoaderOptions builtOptions = optionsDefault.transform(builder -> { + builder.setBatchingEnabled(false); + builder.setCachingExceptionsEnabled(false); + builder.setCachingEnabled(false); + builder.setBatchLoaderScheduler(testBatchLoaderScheduler); + builder.setBatchLoaderContextProvider(testBatchLoaderContextProvider); + builder.setCacheMap(testCacheMap); + builder.setValueCache(testValueCache); + builder.setCacheKeyFunction(testCacheKey); + builder.setValueCacheOptions(testValueCacheOptions); + builder.setMaxBatchSize(10); + builder.setStatisticsCollector(testStatisticsCollectorSupplier); + }); + + assertThat(builtOptions.batchingEnabled(), + equalTo(false)); + assertThat(builtOptions.getBatchLoaderScheduler(), + equalTo(testBatchLoaderScheduler)); + assertThat(builtOptions.getBatchLoaderContextProvider(), + equalTo(testBatchLoaderContextProvider)); + assertThat(builtOptions.cacheMap().get(), + equalTo(testCacheMap)); + assertThat(builtOptions.cachingEnabled(), + equalTo(false)); + assertThat(builtOptions.getValueCacheOptions(), + equalTo(testValueCacheOptions)); + assertThat(builtOptions.cacheKeyFunction().get(), + equalTo(testCacheKey)); + assertThat(builtOptions.valueCache().get(), + equalTo(testValueCache)); + assertThat(builtOptions.maxBatchSize(), + equalTo(10)); + assertThat(builtOptions.getStatisticsCollector(), + equalTo(testStatisticsCollectorSupplier.get())); + + } + + @Test + void canBuildViaBuilderOk() { + + DataLoaderOptions.Builder builder = DataLoaderOptions.newOptionsBuilder(); + builder.setBatchingEnabled(false); + builder.setCachingExceptionsEnabled(false); + builder.setCachingEnabled(false); + builder.setBatchLoaderScheduler(testBatchLoaderScheduler); + builder.setBatchLoaderContextProvider(testBatchLoaderContextProvider); + builder.setCacheMap(testCacheMap); + builder.setValueCache(testValueCache); + builder.setCacheKeyFunction(testCacheKey); + builder.setValueCacheOptions(testValueCacheOptions); + builder.setMaxBatchSize(10); + builder.setStatisticsCollector(testStatisticsCollectorSupplier); + + DataLoaderOptions builtOptions = builder.build(); + + assertThat(builtOptions.batchingEnabled(), + equalTo(false)); + assertThat(builtOptions.getBatchLoaderScheduler(), + equalTo(testBatchLoaderScheduler)); + assertThat(builtOptions.getBatchLoaderContextProvider(), + equalTo(testBatchLoaderContextProvider)); + assertThat(builtOptions.cacheMap().get(), + equalTo(testCacheMap)); + assertThat(builtOptions.cachingEnabled(), + equalTo(false)); + assertThat(builtOptions.getValueCacheOptions(), + equalTo(testValueCacheOptions)); + assertThat(builtOptions.cacheKeyFunction().get(), + equalTo(testCacheKey)); + assertThat(builtOptions.valueCache().get(), + equalTo(testValueCache)); + assertThat(builtOptions.maxBatchSize(), + equalTo(10)); + assertThat(builtOptions.getStatisticsCollector(), + equalTo(testStatisticsCollectorSupplier.get())); + } +} \ No newline at end of file diff --git a/src/test/java/org/dataloader/ValueCacheOptionsTest.java b/src/test/java/org/dataloader/ValueCacheOptionsTest.java new file mode 100644 index 00000000..469e291c --- /dev/null +++ b/src/test/java/org/dataloader/ValueCacheOptionsTest.java @@ -0,0 +1,19 @@ +package org.dataloader; + +import org.junit.jupiter.api.Test; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; + +class ValueCacheOptionsTest { + + @Test + void saneDefaults() { + ValueCacheOptions newOptions = ValueCacheOptions.newOptions(); + assertThat(newOptions.isCompleteValueAfterCacheSet(), equalTo(false)); + + ValueCacheOptions differentOptions = newOptions.setCompleteValueAfterCacheSet(true); + assertThat(differentOptions.isCompleteValueAfterCacheSet(), equalTo(true)); + assertThat(differentOptions == newOptions, equalTo(false)); + } +} \ No newline at end of file From 53799826e394aa69340e78d557a88f08ec0f0984 Mon Sep 17 00:00:00 2001 From: bbaker Date: Sat, 1 Mar 2025 09:20:39 +1100 Subject: [PATCH 072/156] Merged in master and tweaked immutable building --- .../org/dataloader/DataLoaderOptions.java | 65 ++++++++++--------- .../org/dataloader/DataLoaderRegistry.java | 21 +++--- .../ChainedDataLoaderInstrumentationTest.java | 8 +-- ...DataLoaderRegistryInstrumentationTest.java | 13 ++-- 4 files changed, 61 insertions(+), 46 deletions(-) diff --git a/src/main/java/org/dataloader/DataLoaderOptions.java b/src/main/java/org/dataloader/DataLoaderOptions.java index 408b32dd..e4b286ad 100644 --- a/src/main/java/org/dataloader/DataLoaderOptions.java +++ b/src/main/java/org/dataloader/DataLoaderOptions.java @@ -86,6 +86,7 @@ private DataLoaderOptions(Builder builder) { this.environmentProvider = builder.environmentProvider; this.valueCacheOptions = builder.valueCacheOptions; this.batchLoaderScheduler = builder.batchLoaderScheduler; + this.instrumentation = builder.instrumentation; } /** @@ -174,7 +175,7 @@ public boolean batchingEnabled() { * Sets the option that determines whether batch loading is enabled. * * @param batchingEnabled {@code true} to enable batch loading, {@code false} otherwise - * @return the data loader options for fluent coding + * @return a new data loader options instance for fluent coding */ public DataLoaderOptions setBatchingEnabled(boolean batchingEnabled) { return builder().setBatchingEnabled(batchingEnabled).build(); @@ -193,7 +194,7 @@ public boolean cachingEnabled() { * Sets the option that determines whether caching is enabled. * * @param cachingEnabled {@code true} to enable caching, {@code false} otherwise - * @return the data loader options for fluent coding + * @return a new data loader options instance for fluent coding */ public DataLoaderOptions setCachingEnabled(boolean cachingEnabled) { return builder().setCachingEnabled(cachingEnabled).build(); @@ -217,7 +218,7 @@ public boolean cachingExceptionsEnabled() { * Sets the option that determines whether exceptional values are cache enabled. * * @param cachingExceptionsEnabled {@code true} to enable caching exceptional values, {@code false} otherwise - * @return the data loader options for fluent coding + * @return a new data loader options instance for fluent coding */ public DataLoaderOptions setCachingExceptionsEnabled(boolean cachingExceptionsEnabled) { return builder().setCachingExceptionsEnabled(cachingExceptionsEnabled).build(); @@ -238,7 +239,7 @@ public Optional cacheKeyFunction() { * Sets the function to use for creating the cache key, if caching is enabled. * * @param cacheKeyFunction the cache key function to use - * @return the data loader options for fluent coding + * @return a new data loader options instance for fluent coding */ public DataLoaderOptions setCacheKeyFunction(CacheKey cacheKeyFunction) { return builder().setCacheKeyFunction(cacheKeyFunction).build(); @@ -259,7 +260,7 @@ public DataLoaderOptions setCacheKeyFunction(CacheKey cacheKeyFunction) { * Sets the cache map implementation to use for caching, if caching is enabled. * * @param cacheMap the cache map instance - * @return the data loader options for fluent coding + * @return a new data loader options instance for fluent coding */ public DataLoaderOptions setCacheMap(CacheMap cacheMap) { return builder().setCacheMap(cacheMap).build(); @@ -280,7 +281,7 @@ public int maxBatchSize() { * before they are split into multiple class * * @param maxBatchSize the maximum batch size - * @return the data loader options for fluent coding + * @return a new data loader options instance for fluent coding */ public DataLoaderOptions setMaxBatchSize(int maxBatchSize) { return builder().setMaxBatchSize(maxBatchSize).build(); @@ -299,7 +300,7 @@ public StatisticsCollector getStatisticsCollector() { * a common value * * @param statisticsCollector the statistics collector to use - * @return the data loader options for fluent coding + * @return a new data loader options instance for fluent coding */ public DataLoaderOptions setStatisticsCollector(Supplier statisticsCollector) { return builder().setStatisticsCollector(nonNull(statisticsCollector)).build(); @@ -316,7 +317,7 @@ public BatchLoaderContextProvider getBatchLoaderContextProvider() { * Sets the batch loader environment provider that will be used to give context to batch load functions * * @param contextProvider the batch loader context provider - * @return the data loader options for fluent coding + * @return a new data loader options instance for fluent coding */ public DataLoaderOptions setBatchLoaderContextProvider(BatchLoaderContextProvider contextProvider) { return builder().setBatchLoaderContextProvider(nonNull(contextProvider)).build(); @@ -337,7 +338,7 @@ public DataLoaderOptions setBatchLoaderContextProvider(BatchLoaderContextProvide * Sets the value cache implementation to use for caching values, if caching is enabled. * * @param valueCache the value cache instance - * @return the data loader options for fluent coding + * @return a new data loader options instance for fluent coding */ public DataLoaderOptions setValueCache(ValueCache valueCache) { return builder().setValueCache(valueCache).build(); @@ -354,7 +355,7 @@ public ValueCacheOptions getValueCacheOptions() { * Sets the {@link ValueCacheOptions} that control how the {@link ValueCache} will be used * * @param valueCacheOptions the value cache options - * @return the data loader options for fluent coding + * @return a new data loader options instance for fluent coding */ public DataLoaderOptions setValueCacheOptions(ValueCacheOptions valueCacheOptions) { return builder().setValueCacheOptions(nonNull(valueCacheOptions)).build(); @@ -372,12 +373,29 @@ public BatchLoaderScheduler getBatchLoaderScheduler() { * to some future time. * * @param batchLoaderScheduler the scheduler - * @return the data loader options for fluent coding + * @return a new data loader options instance for fluent coding */ public DataLoaderOptions setBatchLoaderScheduler(BatchLoaderScheduler batchLoaderScheduler) { return builder().setBatchLoaderScheduler(batchLoaderScheduler).build(); } + /** + * @return the {@link DataLoaderInstrumentation} to use + */ + public DataLoaderInstrumentation getInstrumentation() { + return instrumentation; + } + + /** + * Sets in a new {@link DataLoaderInstrumentation} + * + * @param instrumentation the new {@link DataLoaderInstrumentation} + * @return a new data loader options instance for fluent coding + */ + public DataLoaderOptions setInstrumentation(DataLoaderInstrumentation instrumentation) { + return builder().setInstrumentation(instrumentation).build(); + } + private Builder builder() { return new Builder(this); } @@ -394,6 +412,7 @@ public static class Builder { private BatchLoaderContextProvider environmentProvider; private ValueCacheOptions valueCacheOptions; private BatchLoaderScheduler batchLoaderScheduler; + private DataLoaderInstrumentation instrumentation; public Builder() { this(new DataLoaderOptions()); // use the defaults of the DataLoaderOptions for this builder @@ -411,6 +430,7 @@ public Builder() { this.environmentProvider = other.environmentProvider; this.valueCacheOptions = other.valueCacheOptions; this.batchLoaderScheduler = other.batchLoaderScheduler; + this.instrumentation = other.instrumentation; } public Builder setBatchingEnabled(boolean batchingEnabled) { @@ -468,27 +488,14 @@ public Builder setBatchLoaderScheduler(BatchLoaderScheduler batchLoaderScheduler return this; } + public Builder setInstrumentation(DataLoaderInstrumentation instrumentation) { + this.instrumentation = nonNull(instrumentation); + return this; + } + public DataLoaderOptions build() { return new DataLoaderOptions(this); } } - - /** - * @return the {@link DataLoaderInstrumentation} to use - */ - public DataLoaderInstrumentation getInstrumentation() { - return instrumentation; - } - - /** - * Sets in a new {@link DataLoaderInstrumentation} - * - * @param instrumentation the new {@link DataLoaderInstrumentation} - * @return the data loader options for fluent coding - */ - public DataLoaderOptions setInstrumentation(DataLoaderInstrumentation instrumentation) { - this.instrumentation = nonNull(instrumentation); - return this; - } } diff --git a/src/main/java/org/dataloader/DataLoaderRegistry.java b/src/main/java/org/dataloader/DataLoaderRegistry.java index d54ee531..06c93c41 100644 --- a/src/main/java/org/dataloader/DataLoaderRegistry.java +++ b/src/main/java/org/dataloader/DataLoaderRegistry.java @@ -31,7 +31,8 @@ *

* If the {@link DataLoader} has no {@link DataLoaderInstrumentation} then the registry one is added to it. If it does have one already * then a {@link ChainedDataLoaderInstrumentation} is created with the registry {@link DataLoaderInstrumentation} in it first and then any other - * {@link DataLoaderInstrumentation}s added after that. + * {@link DataLoaderInstrumentation}s added after that. If the registry {@link DataLoaderInstrumentation} instance and {@link DataLoader} {@link DataLoaderInstrumentation} instance + * are the same object, then nothing is changed, since the same instrumentation code is being run. */ @PublicApi public class DataLoaderRegistry { @@ -40,14 +41,14 @@ public class DataLoaderRegistry { public DataLoaderRegistry() { - this(new ConcurrentHashMap<>(),null); + this(new ConcurrentHashMap<>(), null); } private DataLoaderRegistry(Builder builder) { - this(builder.dataLoaders,builder.instrumentation); + this(builder.dataLoaders, builder.instrumentation); } - protected DataLoaderRegistry(Map> dataLoaders, DataLoaderInstrumentation instrumentation ) { + protected DataLoaderRegistry(Map> dataLoaders, DataLoaderInstrumentation instrumentation) { this.dataLoaders = instrumentDLs(dataLoaders, instrumentation); this.instrumentation = instrumentation; } @@ -97,12 +98,16 @@ protected DataLoaderRegistry(Map> dataLoaders, DataLoa } private static DataLoader mkInstrumentedDataLoader(DataLoader existingDL, DataLoaderOptions options, DataLoaderInstrumentation newInstrumentation) { - return existingDL.transform(builder -> { - options.setInstrumentation(newInstrumentation); - builder.options(options); - }); + return existingDL.transform(builder -> builder.options(setInInstrumentation(options, newInstrumentation))); + } + + private static DataLoaderOptions setInInstrumentation(DataLoaderOptions options, DataLoaderInstrumentation newInstrumentation) { + return options.transform(optionsBuilder -> optionsBuilder.setInstrumentation(newInstrumentation)); } + /** + * @return the {@link DataLoaderInstrumentation} associated with this registry which can be null + */ public DataLoaderInstrumentation getInstrumentation() { return instrumentation; } diff --git a/src/test/java/org/dataloader/instrumentation/ChainedDataLoaderInstrumentationTest.java b/src/test/java/org/dataloader/instrumentation/ChainedDataLoaderInstrumentationTest.java index b3272ced..d7917629 100644 --- a/src/test/java/org/dataloader/instrumentation/ChainedDataLoaderInstrumentationTest.java +++ b/src/test/java/org/dataloader/instrumentation/ChainedDataLoaderInstrumentationTest.java @@ -14,7 +14,7 @@ import java.util.concurrent.CompletableFuture; import static org.awaitility.Awaitility.await; -import static org.dataloader.DataLoaderOptions.newOptions; +import static org.dataloader.DataLoaderOptions.newOptionsBuilder; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; @@ -37,7 +37,7 @@ void canChainTogetherZeroInstrumentation() { // just to prove its useless but harmless ChainedDataLoaderInstrumentation chainedItn = new ChainedDataLoaderInstrumentation(); - DataLoaderOptions options = newOptions().setInstrumentation(chainedItn); + DataLoaderOptions options = newOptionsBuilder().setInstrumentation(chainedItn).build(); DataLoader dl = DataLoaderFactory.newDataLoader(TestKit.keysAsValues(), options); @@ -57,7 +57,7 @@ void canChainTogetherOneInstrumentation() { ChainedDataLoaderInstrumentation chainedItn = new ChainedDataLoaderInstrumentation() .add(capturingA); - DataLoaderOptions options = newOptions().setInstrumentation(chainedItn); + DataLoaderOptions options = newOptionsBuilder().setInstrumentation(chainedItn).build(); DataLoader dl = DataLoaderFactory.newDataLoader(TestKit.keysAsValues(), options); @@ -83,7 +83,7 @@ public void canChainTogetherManyInstrumentationsWithDifferentBatchLoaders(TestDa .add(capturingB) .add(capturingButReturnsNull); - DataLoaderOptions options = newOptions().setInstrumentation(chainedItn); + DataLoaderOptions options = newOptionsBuilder().setInstrumentation(chainedItn).build(); DataLoader dl = factory.idLoader(options); diff --git a/src/test/java/org/dataloader/instrumentation/DataLoaderRegistryInstrumentationTest.java b/src/test/java/org/dataloader/instrumentation/DataLoaderRegistryInstrumentationTest.java index 5690bdc9..465aa4dc 100644 --- a/src/test/java/org/dataloader/instrumentation/DataLoaderRegistryInstrumentationTest.java +++ b/src/test/java/org/dataloader/instrumentation/DataLoaderRegistryInstrumentationTest.java @@ -1,6 +1,7 @@ package org.dataloader.instrumentation; import org.dataloader.DataLoader; +import org.dataloader.DataLoaderOptions; import org.dataloader.DataLoaderRegistry; import org.dataloader.fixtures.TestKit; import org.dataloader.fixtures.parameterized.TestDataLoaderFactory; @@ -119,9 +120,9 @@ void wontDoAnyThingIfThereIsNoRegistryInstrumentation() { @Test void wontDoAnyThingIfThereTheyAreTheSameInstrumentationAlready() { - DataLoader newX = dlX.transform(builder -> dlX.getOptions().setInstrumentation(instrA)); - DataLoader newY = dlX.transform(builder -> dlY.getOptions().setInstrumentation(instrA)); - DataLoader newZ = dlX.transform(builder -> dlY.getOptions().setInstrumentation(instrA)); + DataLoader newX = dlX.transform(builder -> builder.options(dlX.getOptions().setInstrumentation(instrA))); + DataLoader newY = dlX.transform(builder -> builder.options(dlY.getOptions().setInstrumentation(instrA))); + DataLoader newZ = dlX.transform(builder -> builder.options(dlZ.getOptions().setInstrumentation(instrA))); DataLoaderRegistry registry = DataLoaderRegistry.newRegistry() .instrumentation(instrA) .register("X", newX) @@ -144,7 +145,8 @@ void wontDoAnyThingIfThereTheyAreTheSameInstrumentationAlready() { @Test void ifTheDLHasAInstrumentationThenItsTurnedIntoAChainedOne() { - DataLoader newX = dlX.transform(builder -> dlX.getOptions().setInstrumentation(instrA)); + DataLoaderOptions options = dlX.getOptions().setInstrumentation(instrA); + DataLoader newX = dlX.transform(builder -> builder.options(options)); DataLoaderRegistry registry = DataLoaderRegistry.newRegistry() .instrumentation(instrB) @@ -162,7 +164,8 @@ void ifTheDLHasAInstrumentationThenItsTurnedIntoAChainedOne() { @Test void chainedInstrumentationsWillBeCombined() { - DataLoader newX = dlX.transform(builder -> builder.options(dlX.getOptions().setInstrumentation(chainedInstrB))); + DataLoaderOptions options = dlX.getOptions().setInstrumentation(chainedInstrB); + DataLoader newX = dlX.transform(builder -> builder.options(options)); DataLoaderRegistry registry = DataLoaderRegistry.newRegistry() .instrumentation(instrA) From 325f82d5a38ae3583ab48b21df72022882c1d7e7 Mon Sep 17 00:00:00 2001 From: bbaker Date: Tue, 11 Mar 2025 12:46:29 +1100 Subject: [PATCH 073/156] Added a instrumentation of load calls --- .../java/org/dataloader/DataLoaderHelper.java | 10 +++- .../ChainedDataLoaderInstrumentation.java | 6 ++ .../DataLoaderInstrumentation.java | 15 +++++ .../dataloader/DataLoaderCacheMapTest.java | 4 +- .../CapturingInstrumentation.java | 38 ++++++++++++- .../CapturingInstrumentationReturnsNull.java | 6 ++ .../ChainedDataLoaderInstrumentationTest.java | 26 ++++++--- .../DataLoaderInstrumentationTest.java | 55 +++++++++++++++++++ ...DataLoaderRegistryInstrumentationTest.java | 2 +- 9 files changed, 146 insertions(+), 16 deletions(-) diff --git a/src/main/java/org/dataloader/DataLoaderHelper.java b/src/main/java/org/dataloader/DataLoaderHelper.java index 9b5a59e8..78587802 100644 --- a/src/main/java/org/dataloader/DataLoaderHelper.java +++ b/src/main/java/org/dataloader/DataLoaderHelper.java @@ -148,12 +148,16 @@ CompletableFuture load(K key, Object loadContext) { boolean cachingEnabled = loaderOptions.cachingEnabled(); stats.incrementLoadCount(new IncrementLoadCountStatisticsContext<>(key, loadContext)); - + DataLoaderInstrumentationContext ctx = ctxOrNoopCtx(instrumentation().beginLoad(dataLoader, key,loadContext)); + CompletableFuture cf; if (cachingEnabled) { - return loadFromCache(key, loadContext, batchingEnabled); + cf = loadFromCache(key, loadContext, batchingEnabled); } else { - return queueOrInvokeLoader(key, loadContext, batchingEnabled, false); + cf = queueOrInvokeLoader(key, loadContext, batchingEnabled, false); } + ctx.onDispatched(); + cf.whenComplete(ctx::onCompleted); + return cf; } } diff --git a/src/main/java/org/dataloader/instrumentation/ChainedDataLoaderInstrumentation.java b/src/main/java/org/dataloader/instrumentation/ChainedDataLoaderInstrumentation.java index eb9af0a6..bf8a40c2 100644 --- a/src/main/java/org/dataloader/instrumentation/ChainedDataLoaderInstrumentation.java +++ b/src/main/java/org/dataloader/instrumentation/ChainedDataLoaderInstrumentation.java @@ -69,6 +69,12 @@ public ChainedDataLoaderInstrumentation addAll(Collection beginLoad(DataLoader dataLoader, Object key, Object loadContext) { + return chainedCtx(it -> it.beginLoad(dataLoader, key, loadContext)); + } + @Override public DataLoaderInstrumentationContext> beginDispatch(DataLoader dataLoader) { return chainedCtx(it -> it.beginDispatch(dataLoader)); diff --git a/src/main/java/org/dataloader/instrumentation/DataLoaderInstrumentation.java b/src/main/java/org/dataloader/instrumentation/DataLoaderInstrumentation.java index 78f2cf52..bbdba873 100644 --- a/src/main/java/org/dataloader/instrumentation/DataLoaderInstrumentation.java +++ b/src/main/java/org/dataloader/instrumentation/DataLoaderInstrumentation.java @@ -12,6 +12,21 @@ */ @PublicSpi public interface DataLoaderInstrumentation { + /** + * This call back is done just before the {@link DataLoader#load(Object)} methods are invoked, + * and it completes when the load promise is completed. If the value is a cached {@link java.util.concurrent.CompletableFuture} + * then it might return almost immediately, otherwise it will return + * when the batch load function is invoked and values get returned + * + * @param dataLoader the {@link DataLoader} in question + * @param key the key used during the {@link DataLoader#load(Object)} call + * @param loadContext the load context used during the {@link DataLoader#load(Object, Object)} call + * @return a DataLoaderInstrumentationContext or null to be more performant + */ + default DataLoaderInstrumentationContext beginLoad(DataLoader dataLoader, Object key, Object loadContext) { + return null; + } + /** * This call back is done just before the {@link DataLoader#dispatch()} is invoked, * and it completes when the dispatch call promise is done. diff --git a/src/test/java/org/dataloader/DataLoaderCacheMapTest.java b/src/test/java/org/dataloader/DataLoaderCacheMapTest.java index a7b82b7c..df364a20 100644 --- a/src/test/java/org/dataloader/DataLoaderCacheMapTest.java +++ b/src/test/java/org/dataloader/DataLoaderCacheMapTest.java @@ -43,7 +43,7 @@ public void should_access_to_future_dependants() { Collection> futures = dataLoader.getCacheMap().getAll(); List> futuresList = new ArrayList<>(futures); - assertThat(futuresList.get(0).getNumberOfDependents(), equalTo(2)); - assertThat(futuresList.get(1).getNumberOfDependents(), equalTo(1)); + assertThat(futuresList.get(0).getNumberOfDependents(), equalTo(4)); // instrumentation is depending on the CF completing + assertThat(futuresList.get(1).getNumberOfDependents(), equalTo(2)); } } diff --git a/src/test/java/org/dataloader/instrumentation/CapturingInstrumentation.java b/src/test/java/org/dataloader/instrumentation/CapturingInstrumentation.java index f5af683a..b11bc278 100644 --- a/src/test/java/org/dataloader/instrumentation/CapturingInstrumentation.java +++ b/src/test/java/org/dataloader/instrumentation/CapturingInstrumentation.java @@ -6,15 +6,49 @@ import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; class CapturingInstrumentation implements DataLoaderInstrumentation { - String name; - List methods = new ArrayList<>(); + protected String name; + protected List methods = new ArrayList<>(); public CapturingInstrumentation(String name) { this.name = name; } + public String getName() { + return name; + } + + public List methods() { + return methods; + } + + public List notLoads() { + return methods.stream().filter(method -> !method.contains("beginLoad")).collect(Collectors.toList()); + } + + public List onlyLoads() { + return methods.stream().filter(method -> method.contains("beginLoad")).collect(Collectors.toList()); + } + + + @Override + public DataLoaderInstrumentationContext beginLoad(DataLoader dataLoader, Object key, Object loadContext) { + methods.add(name + "_beginLoad" +"_k:" + key); + return new DataLoaderInstrumentationContext<>() { + @Override + public void onDispatched() { + methods.add(name + "_beginLoad_onDispatched"+"_k:" + key); + } + + @Override + public void onCompleted(Object result, Throwable t) { + methods.add(name + "_beginLoad_onCompleted"+"_k:" + key); + } + }; + } + @Override public DataLoaderInstrumentationContext> beginDispatch(DataLoader dataLoader) { methods.add(name + "_beginDispatch"); diff --git a/src/test/java/org/dataloader/instrumentation/CapturingInstrumentationReturnsNull.java b/src/test/java/org/dataloader/instrumentation/CapturingInstrumentationReturnsNull.java index 0c164295..4d2f0f4f 100644 --- a/src/test/java/org/dataloader/instrumentation/CapturingInstrumentationReturnsNull.java +++ b/src/test/java/org/dataloader/instrumentation/CapturingInstrumentationReturnsNull.java @@ -12,6 +12,12 @@ public CapturingInstrumentationReturnsNull(String name) { super(name); } + @Override + public DataLoaderInstrumentationContext beginLoad(DataLoader dataLoader, Object key, Object loadContext) { + methods.add(name + "_beginLoad" +"_k:" + key); + return null; + } + @Override public DataLoaderInstrumentationContext> beginDispatch(DataLoader dataLoader) { methods.add(name + "_beginDispatch"); diff --git a/src/test/java/org/dataloader/instrumentation/ChainedDataLoaderInstrumentationTest.java b/src/test/java/org/dataloader/instrumentation/ChainedDataLoaderInstrumentationTest.java index d7917629..0d5ddb1f 100644 --- a/src/test/java/org/dataloader/instrumentation/ChainedDataLoaderInstrumentationTest.java +++ b/src/test/java/org/dataloader/instrumentation/ChainedDataLoaderInstrumentationTest.java @@ -61,16 +61,21 @@ void canChainTogetherOneInstrumentation() { DataLoader dl = DataLoaderFactory.newDataLoader(TestKit.keysAsValues(), options); - dl.load("A"); - dl.load("B"); + dl.load("X"); + dl.load("Y"); CompletableFuture> dispatch = dl.dispatch(); await().until(dispatch::isDone); - assertThat(capturingA.methods, equalTo(List.of("A_beginDispatch", + assertThat(capturingA.notLoads(), equalTo(List.of("A_beginDispatch", "A_beginBatchLoader", "A_beginBatchLoader_onDispatched", "A_beginBatchLoader_onCompleted", "A_beginDispatch_onDispatched", "A_beginDispatch_onCompleted"))); + + assertThat(capturingA.onlyLoads(), equalTo(List.of( + "A_beginLoad_k:X", "A_beginLoad_onDispatched_k:X", "A_beginLoad_k:Y", "A_beginLoad_onDispatched_k:Y", + "A_beginLoad_onCompleted_k:X", "A_beginLoad_onCompleted_k:Y" + ))); } @@ -87,8 +92,8 @@ public void canChainTogetherManyInstrumentationsWithDifferentBatchLoaders(TestDa DataLoader dl = factory.idLoader(options); - dl.load("A"); - dl.load("B"); + dl.load("X"); + dl.load("Y"); CompletableFuture> dispatch = dl.dispatch(); @@ -98,16 +103,21 @@ public void canChainTogetherManyInstrumentationsWithDifferentBatchLoaders(TestDa // A_beginBatchLoader happens before A_beginDispatch_onDispatched because these are sync // and no async - a batch scheduler or async batch loader would change that // - assertThat(capturingA.methods, equalTo(List.of("A_beginDispatch", + assertThat(capturingA.notLoads(), equalTo(List.of("A_beginDispatch", "A_beginBatchLoader", "A_beginBatchLoader_onDispatched", "A_beginBatchLoader_onCompleted", "A_beginDispatch_onDispatched", "A_beginDispatch_onCompleted"))); - assertThat(capturingB.methods, equalTo(List.of("B_beginDispatch", + assertThat(capturingA.onlyLoads(), equalTo(List.of( + "A_beginLoad_k:X", "A_beginLoad_onDispatched_k:X", "A_beginLoad_k:Y", "A_beginLoad_onDispatched_k:Y", + "A_beginLoad_onCompleted_k:X", "A_beginLoad_onCompleted_k:Y" + ))); + + assertThat(capturingB.notLoads(), equalTo(List.of("B_beginDispatch", "B_beginBatchLoader", "B_beginBatchLoader_onDispatched", "B_beginBatchLoader_onCompleted", "B_beginDispatch_onDispatched", "B_beginDispatch_onCompleted"))); // it returned null on all its contexts - nothing to call back on - assertThat(capturingButReturnsNull.methods, equalTo(List.of("NULL_beginDispatch", "NULL_beginBatchLoader"))); + assertThat(capturingButReturnsNull.notLoads(), equalTo(List.of("NULL_beginDispatch", "NULL_beginBatchLoader"))); } @Test diff --git a/src/test/java/org/dataloader/instrumentation/DataLoaderInstrumentationTest.java b/src/test/java/org/dataloader/instrumentation/DataLoaderInstrumentationTest.java index a35e13a9..97f21d34 100644 --- a/src/test/java/org/dataloader/instrumentation/DataLoaderInstrumentationTest.java +++ b/src/test/java/org/dataloader/instrumentation/DataLoaderInstrumentationTest.java @@ -29,6 +29,61 @@ public class DataLoaderInstrumentationTest { return keys; }); + @Test + void canMonitorLoading() { + AtomicReference> dlRef = new AtomicReference<>(); + + CapturingInstrumentation instrumentation = new CapturingInstrumentation("x") { + + @Override + public DataLoaderInstrumentationContext beginLoad(DataLoader dataLoader, Object key, Object loadContext) { + DataLoaderInstrumentationContext superCtx = super.beginLoad(dataLoader, key, loadContext); + dlRef.set(dataLoader); + return superCtx; + } + + @Override + public DataLoaderInstrumentationContext> beginBatchLoader(DataLoader dataLoader, List keys, BatchLoaderEnvironment environment) { + return DataLoaderInstrumentationHelper.noOpCtx(); + } + }; + + DataLoaderOptions options = new DataLoaderOptions() + .setInstrumentation(instrumentation) + .setMaxBatchSize(5); + + DataLoader dl = DataLoaderFactory.newDataLoader(snoozingBatchLoader, options); + + List keys = new ArrayList<>(); + for (int i = 0; i < 3; i++) { + String key = "X" + i; + keys.add(key); + dl.load(key); + } + + // load a key that is cached + dl.load("X0"); + + CompletableFuture> dispatch = dl.dispatch(); + + await().until(dispatch::isDone); + assertThat(dlRef.get(), is(dl)); + assertThat(dispatch.join(), equalTo(keys)); + + // the batch loading means they start and are instrumentation dispatched before they all end up completing + assertThat(instrumentation.onlyLoads(), + equalTo(List.of( + "x_beginLoad_k:X0", "x_beginLoad_onDispatched_k:X0", + "x_beginLoad_k:X1", "x_beginLoad_onDispatched_k:X1", + "x_beginLoad_k:X2", "x_beginLoad_onDispatched_k:X2", + "x_beginLoad_k:X0", "x_beginLoad_onDispatched_k:X0", // second cached call counts + "x_beginLoad_onCompleted_k:X0", + "x_beginLoad_onCompleted_k:X0", // each load call counts + "x_beginLoad_onCompleted_k:X1", "x_beginLoad_onCompleted_k:X2"))); + + } + + @Test void canMonitorDispatching() { Stopwatch stopwatch = Stopwatch.stopwatchUnStarted(); diff --git a/src/test/java/org/dataloader/instrumentation/DataLoaderRegistryInstrumentationTest.java b/src/test/java/org/dataloader/instrumentation/DataLoaderRegistryInstrumentationTest.java index 465aa4dc..49ccf0ee 100644 --- a/src/test/java/org/dataloader/instrumentation/DataLoaderRegistryInstrumentationTest.java +++ b/src/test/java/org/dataloader/instrumentation/DataLoaderRegistryInstrumentationTest.java @@ -224,7 +224,7 @@ public void endToEndIntegrationTest(TestDataLoaderFactory factory) { await().until(loadA::isDone); assertThat(loadA.join(), equalTo("A")); - assertThat(instrA.methods, equalTo(List.of("A_beginDispatch", + assertThat(instrA.notLoads(), equalTo(List.of("A_beginDispatch", "A_beginBatchLoader", "A_beginBatchLoader_onDispatched", "A_beginBatchLoader_onCompleted", "A_beginDispatch_onDispatched", "A_beginDispatch_onCompleted"))); } From 6cf5c7836c6f4ecdc0ace125ab54a46e5063dac8 Mon Sep 17 00:00:00 2001 From: Andreas Marek Date: Tue, 25 Mar 2025 11:26:19 +1000 Subject: [PATCH 074/156] add jspecify --- build.gradle | 3 ++- src/main/java/org/dataloader/BatchLoader.java | 3 +++ .../BatchLoaderContextProvider.java | 4 ++- .../dataloader/BatchLoaderEnvironment.java | 5 +++- .../BatchLoaderEnvironmentProvider.java | 4 ++- .../dataloader/BatchLoaderWithContext.java | 2 ++ .../java/org/dataloader/BatchPublisher.java | 5 ++++ .../dataloader/BatchPublisherWithContext.java | 4 +++ src/main/java/org/dataloader/CacheKey.java | 5 ++++ src/main/java/org/dataloader/CacheMap.java | 5 +++- src/main/java/org/dataloader/DataLoader.java | 25 +++++++++++-------- .../org/dataloader/DataLoaderFactory.java | 3 ++- .../java/org/dataloader/DispatchResult.java | 2 ++ .../org/dataloader/MappedBatchLoader.java | 5 ++++ .../MappedBatchLoaderWithContext.java | 5 ++++ .../org/dataloader/MappedBatchPublisher.java | 4 +++ .../MappedBatchPublisherWithContext.java | 4 +++ src/main/java/org/dataloader/ValueCache.java | 4 ++- .../org/dataloader/ValueCacheOptions.java | 5 ++++ 19 files changed, 79 insertions(+), 18 deletions(-) diff --git a/build.gradle b/build.gradle index 0a585598..9b943f33 100644 --- a/build.gradle +++ b/build.gradle @@ -66,6 +66,7 @@ jar { dependencies { api "org.reactivestreams:reactive-streams:$reactive_streams_version" + api "org.jspecify:jspecify:1.0.0" } task sourcesJar(type: Jar) { @@ -197,4 +198,4 @@ tasks.named("dependencyUpdates").configure { rejectVersionIf { isNonStable(it.candidate.version) } -} \ No newline at end of file +} diff --git a/src/main/java/org/dataloader/BatchLoader.java b/src/main/java/org/dataloader/BatchLoader.java index c1916e3e..2b0c3c53 100644 --- a/src/main/java/org/dataloader/BatchLoader.java +++ b/src/main/java/org/dataloader/BatchLoader.java @@ -17,6 +17,8 @@ package org.dataloader; import org.dataloader.annotations.PublicSpi; +import org.jspecify.annotations.NonNull; +import org.jspecify.annotations.NullMarked; import java.util.List; import java.util.concurrent.CompletionStage; @@ -74,6 +76,7 @@ */ @FunctionalInterface @PublicSpi +@NullMarked public interface BatchLoader { /** diff --git a/src/main/java/org/dataloader/BatchLoaderContextProvider.java b/src/main/java/org/dataloader/BatchLoaderContextProvider.java index d1eb1fea..702fd66d 100644 --- a/src/main/java/org/dataloader/BatchLoaderContextProvider.java +++ b/src/main/java/org/dataloader/BatchLoaderContextProvider.java @@ -1,6 +1,7 @@ package org.dataloader; import org.dataloader.annotations.PublicSpi; +import org.jspecify.annotations.NullMarked; /** * A BatchLoaderContextProvider is used by the {@link org.dataloader.DataLoader} code to @@ -8,9 +9,10 @@ * case is for propagating user security credentials or database connection parameters for example. */ @PublicSpi +@NullMarked public interface BatchLoaderContextProvider { /** * @return a context object that may be needed in batch load calls */ Object getContext(); -} \ No newline at end of file +} diff --git a/src/main/java/org/dataloader/BatchLoaderEnvironment.java b/src/main/java/org/dataloader/BatchLoaderEnvironment.java index 6039a4af..6b84e709 100644 --- a/src/main/java/org/dataloader/BatchLoaderEnvironment.java +++ b/src/main/java/org/dataloader/BatchLoaderEnvironment.java @@ -2,6 +2,8 @@ import org.dataloader.annotations.PublicApi; import org.dataloader.impl.Assertions; +import org.jspecify.annotations.NullMarked; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.Collections; @@ -14,6 +16,7 @@ * of the calling users for example or database parameters that allow the data layer call to succeed. */ @PublicApi +@NullMarked public class BatchLoaderEnvironment { private final Object context; @@ -34,7 +37,7 @@ private BatchLoaderEnvironment(Object context, List keyContextsList, Map * @return a context object or null if there isn't one */ @SuppressWarnings("unchecked") - public T getContext() { + public @Nullable T getContext() { return (T) context; } diff --git a/src/main/java/org/dataloader/BatchLoaderEnvironmentProvider.java b/src/main/java/org/dataloader/BatchLoaderEnvironmentProvider.java index fd60a149..dae7c924 100644 --- a/src/main/java/org/dataloader/BatchLoaderEnvironmentProvider.java +++ b/src/main/java/org/dataloader/BatchLoaderEnvironmentProvider.java @@ -1,6 +1,7 @@ package org.dataloader; import org.dataloader.annotations.PublicSpi; +import org.jspecify.annotations.NullMarked; /** * A BatchLoaderEnvironmentProvider is used by the {@link org.dataloader.DataLoader} code to @@ -9,9 +10,10 @@ * case is for propagating user security credentials or database connection parameters. */ @PublicSpi +@NullMarked public interface BatchLoaderEnvironmentProvider { /** * @return a {@link org.dataloader.BatchLoaderEnvironment} that may be needed in batch calls */ BatchLoaderEnvironment get(); -} \ No newline at end of file +} diff --git a/src/main/java/org/dataloader/BatchLoaderWithContext.java b/src/main/java/org/dataloader/BatchLoaderWithContext.java index fbe66b01..eba26e4b 100644 --- a/src/main/java/org/dataloader/BatchLoaderWithContext.java +++ b/src/main/java/org/dataloader/BatchLoaderWithContext.java @@ -1,6 +1,7 @@ package org.dataloader; import org.dataloader.annotations.PublicSpi; +import org.jspecify.annotations.NullMarked; import java.util.List; import java.util.concurrent.CompletionStage; @@ -14,6 +15,7 @@ * use this interface. */ @PublicSpi +@NullMarked public interface BatchLoaderWithContext { /** * Called to batch load the provided keys and return a promise to a list of values. This default diff --git a/src/main/java/org/dataloader/BatchPublisher.java b/src/main/java/org/dataloader/BatchPublisher.java index c4992266..943becf4 100644 --- a/src/main/java/org/dataloader/BatchPublisher.java +++ b/src/main/java/org/dataloader/BatchPublisher.java @@ -1,5 +1,8 @@ package org.dataloader; +import org.dataloader.annotations.PublicSpi; +import org.jspecify.annotations.NullMarked; +import org.jspecify.annotations.Nullable; import org.reactivestreams.Subscriber; import java.util.List; @@ -18,6 +21,8 @@ * @param type parameter indicating the type of values returned * @see BatchLoader for the non-reactive version */ +@NullMarked +@PublicSpi public interface BatchPublisher { /** * Called to batch the provided keys into a stream of values. You must provide diff --git a/src/main/java/org/dataloader/BatchPublisherWithContext.java b/src/main/java/org/dataloader/BatchPublisherWithContext.java index 4eadfe95..9ee010b8 100644 --- a/src/main/java/org/dataloader/BatchPublisherWithContext.java +++ b/src/main/java/org/dataloader/BatchPublisherWithContext.java @@ -1,5 +1,7 @@ package org.dataloader; +import org.dataloader.annotations.PublicSpi; +import org.jspecify.annotations.NullMarked; import org.reactivestreams.Subscriber; import java.util.List; @@ -12,6 +14,8 @@ * See {@link BatchPublisher} for more details on the design invariants that you must implement in order to * use this interface. */ +@NullMarked +@PublicSpi public interface BatchPublisherWithContext { /** * Called to batch the provided keys into a stream of values. You must provide diff --git a/src/main/java/org/dataloader/CacheKey.java b/src/main/java/org/dataloader/CacheKey.java index 88b5f974..c5641b1d 100644 --- a/src/main/java/org/dataloader/CacheKey.java +++ b/src/main/java/org/dataloader/CacheKey.java @@ -16,6 +16,9 @@ package org.dataloader; +import org.dataloader.annotations.PublicSpi; +import org.jspecify.annotations.NullMarked; + /** * Function that is invoked on input keys of type {@code K} to derive keys that are required by the {@link CacheMap} * implementation. @@ -25,6 +28,8 @@ * @author Arnold Schrijver */ @FunctionalInterface +@NullMarked +@PublicSpi public interface CacheKey { /** diff --git a/src/main/java/org/dataloader/CacheMap.java b/src/main/java/org/dataloader/CacheMap.java index 1a4a4551..54b1b497 100644 --- a/src/main/java/org/dataloader/CacheMap.java +++ b/src/main/java/org/dataloader/CacheMap.java @@ -18,6 +18,8 @@ import org.dataloader.annotations.PublicSpi; import org.dataloader.impl.DefaultCacheMap; +import org.jspecify.annotations.NullMarked; +import org.jspecify.annotations.Nullable; import java.util.Collection; import java.util.concurrent.CompletableFuture; @@ -39,6 +41,7 @@ * @author Brad Baker */ @PublicSpi +@NullMarked public interface CacheMap { /** @@ -71,7 +74,7 @@ static CacheMap simpleMap() { * * @return the cached value, or {@code null} if not found (depends on cache implementation) */ - CompletableFuture get(K key); + @Nullable CompletableFuture get(K key); /** * Gets a collection of CompletableFutures from the cache map. diff --git a/src/main/java/org/dataloader/DataLoader.java b/src/main/java/org/dataloader/DataLoader.java index 62e80de4..fb15d444 100644 --- a/src/main/java/org/dataloader/DataLoader.java +++ b/src/main/java/org/dataloader/DataLoader.java @@ -21,6 +21,8 @@ import org.dataloader.impl.CompletableFutureKit; import org.dataloader.stats.Statistics; import org.dataloader.stats.StatisticsCollector; +import org.jspecify.annotations.NullMarked; +import org.jspecify.annotations.Nullable; import java.time.Clock; import java.time.Duration; @@ -64,6 +66,7 @@ * @author Brad Baker */ @PublicApi +@NullMarked public class DataLoader { private final DataLoaderHelper helper; @@ -99,7 +102,7 @@ public static DataLoader newDataLoader(BatchLoader batchLoadF * @deprecated use {@link DataLoaderFactory} instead */ @Deprecated - public static DataLoader newDataLoader(BatchLoader batchLoadFunction, DataLoaderOptions options) { + public static DataLoader newDataLoader(BatchLoader batchLoadFunction, @Nullable DataLoaderOptions options) { return DataLoaderFactory.mkDataLoader(batchLoadFunction, options); } @@ -139,7 +142,7 @@ public static DataLoader newDataLoaderWithTry(BatchLoader * @deprecated use {@link DataLoaderFactory} instead */ @Deprecated - public static DataLoader newDataLoaderWithTry(BatchLoader> batchLoadFunction, DataLoaderOptions options) { + public static DataLoader newDataLoaderWithTry(BatchLoader> batchLoadFunction, @Nullable DataLoaderOptions options) { return DataLoaderFactory.mkDataLoader(batchLoadFunction, options); } @@ -169,7 +172,7 @@ public static DataLoader newDataLoader(BatchLoaderWithContext * @deprecated use {@link DataLoaderFactory} instead */ @Deprecated - public static DataLoader newDataLoader(BatchLoaderWithContext batchLoadFunction, DataLoaderOptions options) { + public static DataLoader newDataLoader(BatchLoaderWithContext batchLoadFunction, @Nullable DataLoaderOptions options) { return DataLoaderFactory.mkDataLoader(batchLoadFunction, options); } @@ -209,7 +212,7 @@ public static DataLoader newDataLoaderWithTry(BatchLoaderWithContex * @deprecated use {@link DataLoaderFactory} instead */ @Deprecated - public static DataLoader newDataLoaderWithTry(BatchLoaderWithContext> batchLoadFunction, DataLoaderOptions options) { + public static DataLoader newDataLoaderWithTry(BatchLoaderWithContext> batchLoadFunction, @Nullable DataLoaderOptions options) { return DataLoaderFactory.mkDataLoader(batchLoadFunction, options); } @@ -239,7 +242,7 @@ public static DataLoader newMappedDataLoader(MappedBatchLoader DataLoader newMappedDataLoader(MappedBatchLoader batchLoadFunction, DataLoaderOptions options) { + public static DataLoader newMappedDataLoader(MappedBatchLoader batchLoadFunction, @Nullable DataLoaderOptions options) { return DataLoaderFactory.mkDataLoader(batchLoadFunction, options); } @@ -280,7 +283,7 @@ public static DataLoader newMappedDataLoaderWithTry(MappedBatchLoad * @deprecated use {@link DataLoaderFactory} instead */ @Deprecated - public static DataLoader newMappedDataLoaderWithTry(MappedBatchLoader> batchLoadFunction, DataLoaderOptions options) { + public static DataLoader newMappedDataLoaderWithTry(MappedBatchLoader> batchLoadFunction, @Nullable DataLoaderOptions options) { return DataLoaderFactory.mkDataLoader(batchLoadFunction, options); } @@ -310,7 +313,7 @@ public static DataLoader newMappedDataLoader(MappedBatchLoaderWithC * @deprecated use {@link DataLoaderFactory} instead */ @Deprecated - public static DataLoader newMappedDataLoader(MappedBatchLoaderWithContext batchLoadFunction, DataLoaderOptions options) { + public static DataLoader newMappedDataLoader(MappedBatchLoaderWithContext batchLoadFunction, @Nullable DataLoaderOptions options) { return DataLoaderFactory.mkDataLoader(batchLoadFunction, options); } @@ -350,7 +353,7 @@ public static DataLoader newMappedDataLoaderWithTry(MappedBatchLoad * @deprecated use {@link DataLoaderFactory} instead */ @Deprecated - public static DataLoader newMappedDataLoaderWithTry(MappedBatchLoaderWithContext> batchLoadFunction, DataLoaderOptions options) { + public static DataLoader newMappedDataLoaderWithTry(MappedBatchLoaderWithContext> batchLoadFunction, @Nullable DataLoaderOptions options) { return DataLoaderFactory.mkDataLoader(batchLoadFunction, options); } @@ -373,17 +376,17 @@ public DataLoader(BatchLoader batchLoadFunction) { * @deprecated use {@link DataLoaderFactory} instead */ @Deprecated - public DataLoader(BatchLoader batchLoadFunction, DataLoaderOptions options) { + public DataLoader(BatchLoader batchLoadFunction, @Nullable DataLoaderOptions options) { this((Object) batchLoadFunction, options); } @VisibleForTesting - DataLoader(Object batchLoadFunction, DataLoaderOptions options) { + DataLoader(Object batchLoadFunction, @Nullable DataLoaderOptions options) { this(batchLoadFunction, options, Clock.systemUTC()); } @VisibleForTesting - DataLoader(Object batchLoadFunction, DataLoaderOptions options, Clock clock) { + DataLoader(Object batchLoadFunction, @Nullable DataLoaderOptions options, Clock clock) { DataLoaderOptions loaderOptions = options == null ? new DataLoaderOptions() : options; this.futureCache = determineFutureCache(loaderOptions); this.valueCache = determineValueCache(loaderOptions); diff --git a/src/main/java/org/dataloader/DataLoaderFactory.java b/src/main/java/org/dataloader/DataLoaderFactory.java index a87e4ebd..ef1a2870 100644 --- a/src/main/java/org/dataloader/DataLoaderFactory.java +++ b/src/main/java/org/dataloader/DataLoaderFactory.java @@ -1,6 +1,7 @@ package org.dataloader; import org.dataloader.annotations.PublicApi; +import org.jspecify.annotations.Nullable; /** * A factory class to create {@link DataLoader}s @@ -155,7 +156,7 @@ public static DataLoader newMappedDataLoader(MappedBatchLoader the value type * @return a new DataLoader */ - public static DataLoader newMappedDataLoader(MappedBatchLoader batchLoadFunction, DataLoaderOptions options) { + public static DataLoader newMappedDataLoader(MappedBatchLoader batchLoadFunction, @Nullable DataLoaderOptions options) { return mkDataLoader(batchLoadFunction, options); } diff --git a/src/main/java/org/dataloader/DispatchResult.java b/src/main/java/org/dataloader/DispatchResult.java index 97711daa..7305c78e 100644 --- a/src/main/java/org/dataloader/DispatchResult.java +++ b/src/main/java/org/dataloader/DispatchResult.java @@ -1,6 +1,7 @@ package org.dataloader; import org.dataloader.annotations.PublicApi; +import org.jspecify.annotations.NullMarked; import java.util.List; import java.util.concurrent.CompletableFuture; @@ -12,6 +13,7 @@ * @param for two */ @PublicApi +@NullMarked public class DispatchResult { private final CompletableFuture> futureList; private final int keysCount; diff --git a/src/main/java/org/dataloader/MappedBatchLoader.java b/src/main/java/org/dataloader/MappedBatchLoader.java index 5a7a1a63..1ad4c79a 100644 --- a/src/main/java/org/dataloader/MappedBatchLoader.java +++ b/src/main/java/org/dataloader/MappedBatchLoader.java @@ -16,6 +16,9 @@ package org.dataloader; +import org.dataloader.annotations.PublicSpi; +import org.jspecify.annotations.NullMarked; + import java.util.Map; import java.util.Set; import java.util.concurrent.CompletionStage; @@ -54,6 +57,8 @@ * @param type parameter indicating the type of values returned * */ +@PublicSpi +@NullMarked public interface MappedBatchLoader { /** diff --git a/src/main/java/org/dataloader/MappedBatchLoaderWithContext.java b/src/main/java/org/dataloader/MappedBatchLoaderWithContext.java index 7438d204..95592607 100644 --- a/src/main/java/org/dataloader/MappedBatchLoaderWithContext.java +++ b/src/main/java/org/dataloader/MappedBatchLoaderWithContext.java @@ -16,6 +16,9 @@ package org.dataloader; +import org.dataloader.annotations.PublicSpi; +import org.jspecify.annotations.NullMarked; + import java.util.Map; import java.util.Set; import java.util.concurrent.CompletionStage; @@ -28,6 +31,8 @@ * See {@link MappedBatchLoader} for more details on the design invariants that you must implement in order to * use this interface. */ +@PublicSpi +@NullMarked public interface MappedBatchLoaderWithContext { /** * Called to batch load the provided keys and return a promise to a map of values. diff --git a/src/main/java/org/dataloader/MappedBatchPublisher.java b/src/main/java/org/dataloader/MappedBatchPublisher.java index 754ee52c..493401f9 100644 --- a/src/main/java/org/dataloader/MappedBatchPublisher.java +++ b/src/main/java/org/dataloader/MappedBatchPublisher.java @@ -1,5 +1,7 @@ package org.dataloader; +import org.dataloader.annotations.PublicSpi; +import org.jspecify.annotations.NullMarked; import org.reactivestreams.Subscriber; import java.util.Map; @@ -16,6 +18,8 @@ * @param type parameter indicating the type of values returned * @see MappedBatchLoader for the non-reactive version */ +@PublicSpi +@NullMarked public interface MappedBatchPublisher { /** * Called to batch the provided keys into a stream of map entries of keys and values. diff --git a/src/main/java/org/dataloader/MappedBatchPublisherWithContext.java b/src/main/java/org/dataloader/MappedBatchPublisherWithContext.java index 2e94152f..7b862caa 100644 --- a/src/main/java/org/dataloader/MappedBatchPublisherWithContext.java +++ b/src/main/java/org/dataloader/MappedBatchPublisherWithContext.java @@ -1,5 +1,7 @@ package org.dataloader; +import org.dataloader.annotations.PublicSpi; +import org.jspecify.annotations.NullMarked; import org.reactivestreams.Subscriber; import java.util.List; @@ -13,6 +15,8 @@ * See {@link MappedBatchPublisher} for more details on the design invariants that you must implement in order to * use this interface. */ +@PublicSpi +@NullMarked public interface MappedBatchPublisherWithContext { /** diff --git a/src/main/java/org/dataloader/ValueCache.java b/src/main/java/org/dataloader/ValueCache.java index a8dabb10..80c8402e 100644 --- a/src/main/java/org/dataloader/ValueCache.java +++ b/src/main/java/org/dataloader/ValueCache.java @@ -3,6 +3,7 @@ import org.dataloader.annotations.PublicSpi; import org.dataloader.impl.CompletableFutureKit; import org.dataloader.impl.NoOpValueCache; +import org.jspecify.annotations.NullMarked; import java.util.ArrayList; import java.util.List; @@ -38,6 +39,7 @@ * @author Brad Baker */ @PublicSpi +@NullMarked public interface ValueCache { /** @@ -158,4 +160,4 @@ public Throwable fillInStackTrace() { return this; } } -} \ No newline at end of file +} diff --git a/src/main/java/org/dataloader/ValueCacheOptions.java b/src/main/java/org/dataloader/ValueCacheOptions.java index 7e2f0255..b681ddaf 100644 --- a/src/main/java/org/dataloader/ValueCacheOptions.java +++ b/src/main/java/org/dataloader/ValueCacheOptions.java @@ -1,10 +1,15 @@ package org.dataloader; +import org.dataloader.annotations.PublicSpi; +import org.jspecify.annotations.NullMarked; + /** * Options that control how the {@link ValueCache} is used by {@link DataLoader} * * @author Brad Baker */ +@PublicSpi +@NullMarked public class ValueCacheOptions { private final boolean completeValueAfterCacheSet; From 893bb0d4b1ae1c918ac94637f53236b699b327ad Mon Sep 17 00:00:00 2001 From: bbaker Date: Tue, 1 Apr 2025 18:09:00 +1100 Subject: [PATCH 075/156] Fixed a bug that the Spring team found --- .../org/dataloader/DataLoaderOptions.java | 2 +- .../org/dataloader/DataLoaderOptionsTest.java | 37 +++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/dataloader/DataLoaderOptions.java b/src/main/java/org/dataloader/DataLoaderOptions.java index e4b286ad..86679432 100644 --- a/src/main/java/org/dataloader/DataLoaderOptions.java +++ b/src/main/java/org/dataloader/DataLoaderOptions.java @@ -139,7 +139,7 @@ public static DataLoaderOptions.Builder newDataLoaderOptions(DataLoaderOptions o * @return a new {@link DataLoaderOptions} object */ public DataLoaderOptions transform(Consumer builderConsumer) { - Builder builder = newOptionsBuilder(); + Builder builder = newDataLoaderOptions(this); builderConsumer.accept(builder); return builder.build(); } diff --git a/src/test/java/org/dataloader/DataLoaderOptionsTest.java b/src/test/java/org/dataloader/DataLoaderOptionsTest.java index f6e06e80..6c012d8a 100644 --- a/src/test/java/org/dataloader/DataLoaderOptionsTest.java +++ b/src/test/java/org/dataloader/DataLoaderOptionsTest.java @@ -2,9 +2,11 @@ import org.dataloader.impl.DefaultCacheMap; import org.dataloader.impl.NoOpValueCache; +import org.dataloader.instrumentation.DataLoaderInstrumentation; import org.dataloader.scheduler.BatchLoaderScheduler; import org.dataloader.stats.NoOpStatisticsCollector; import org.dataloader.stats.StatisticsCollector; +import org.hamcrest.CoreMatchers; import org.junit.jupiter.api.Test; import java.util.List; @@ -184,4 +186,39 @@ void canBuildViaBuilderOk() { assertThat(builtOptions.getStatisticsCollector(), equalTo(testStatisticsCollectorSupplier.get())); } + + @Test + void canCopyExistingOptionValuesOnTransform() { + + DataLoaderInstrumentation instrumentation1 = new DataLoaderInstrumentation() { + }; + BatchLoaderContextProvider contextProvider1 = () -> null; + + DataLoaderOptions startingOptions = DataLoaderOptions.newOptionsBuilder().setBatchingEnabled(false) + .setCachingEnabled(false) + .setInstrumentation(instrumentation1) + .setBatchLoaderContextProvider(contextProvider1) + .build(); + + assertThat(startingOptions.batchingEnabled(), equalTo(false)); + assertThat(startingOptions.cachingEnabled(), equalTo(false)); + assertThat(startingOptions.getInstrumentation(), equalTo(instrumentation1)); + assertThat(startingOptions.getBatchLoaderContextProvider(), equalTo(contextProvider1)); + + DataLoaderOptions newOptions = startingOptions.transform(builder -> builder.setBatchingEnabled(true)); + + + // immutable + assertThat(newOptions, CoreMatchers.not(startingOptions)); + assertThat(startingOptions.batchingEnabled(), equalTo(false)); + assertThat(startingOptions.cachingEnabled(), equalTo(false)); + assertThat(startingOptions.getInstrumentation(), equalTo(instrumentation1)); + assertThat(startingOptions.getBatchLoaderContextProvider(), equalTo(contextProvider1)); + + // copied values + assertThat(newOptions.batchingEnabled(), equalTo(true)); + assertThat(newOptions.cachingEnabled(), equalTo(false)); + assertThat(newOptions.getInstrumentation(), equalTo(instrumentation1)); + assertThat(newOptions.getBatchLoaderContextProvider(), equalTo(contextProvider1)); + } } \ No newline at end of file From 5d0a2023caf3bd0e469f9a029ae4910cc7643a25 Mon Sep 17 00:00:00 2001 From: bbaker Date: Tue, 1 Apr 2025 18:11:29 +1100 Subject: [PATCH 076/156] Fixed a bug that the Spring team found - tweak --- .../java/org/dataloader/DataLoaderOptionsTest.java | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/test/java/org/dataloader/DataLoaderOptionsTest.java b/src/test/java/org/dataloader/DataLoaderOptionsTest.java index 6c012d8a..b4ebb9e3 100644 --- a/src/test/java/org/dataloader/DataLoaderOptionsTest.java +++ b/src/test/java/org/dataloader/DataLoaderOptionsTest.java @@ -192,6 +192,8 @@ void canCopyExistingOptionValuesOnTransform() { DataLoaderInstrumentation instrumentation1 = new DataLoaderInstrumentation() { }; + DataLoaderInstrumentation instrumentation2 = new DataLoaderInstrumentation() { + }; BatchLoaderContextProvider contextProvider1 = () -> null; DataLoaderOptions startingOptions = DataLoaderOptions.newOptionsBuilder().setBatchingEnabled(false) @@ -205,7 +207,8 @@ void canCopyExistingOptionValuesOnTransform() { assertThat(startingOptions.getInstrumentation(), equalTo(instrumentation1)); assertThat(startingOptions.getBatchLoaderContextProvider(), equalTo(contextProvider1)); - DataLoaderOptions newOptions = startingOptions.transform(builder -> builder.setBatchingEnabled(true)); + DataLoaderOptions newOptions = startingOptions.transform(builder -> + builder.setBatchingEnabled(true).setInstrumentation(instrumentation2)); // immutable @@ -215,10 +218,13 @@ void canCopyExistingOptionValuesOnTransform() { assertThat(startingOptions.getInstrumentation(), equalTo(instrumentation1)); assertThat(startingOptions.getBatchLoaderContextProvider(), equalTo(contextProvider1)); - // copied values - assertThat(newOptions.batchingEnabled(), equalTo(true)); + // stayed the same assertThat(newOptions.cachingEnabled(), equalTo(false)); - assertThat(newOptions.getInstrumentation(), equalTo(instrumentation1)); assertThat(newOptions.getBatchLoaderContextProvider(), equalTo(contextProvider1)); + + // was changed + assertThat(newOptions.batchingEnabled(), equalTo(true)); + assertThat(newOptions.getInstrumentation(), equalTo(instrumentation2)); + } } \ No newline at end of file From f54faf92f7683c0ad9188cfa262f62f99fe91d48 Mon Sep 17 00:00:00 2001 From: bbaker Date: Tue, 1 Apr 2025 18:14:47 +1100 Subject: [PATCH 077/156] Fixed a bug that the Spring team found - tweak - extra test --- .../org/dataloader/DataLoaderBuilderTest.java | 26 ++++++++++++++----- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/src/test/java/org/dataloader/DataLoaderBuilderTest.java b/src/test/java/org/dataloader/DataLoaderBuilderTest.java index 523b9a50..f38ff82b 100644 --- a/src/test/java/org/dataloader/DataLoaderBuilderTest.java +++ b/src/test/java/org/dataloader/DataLoaderBuilderTest.java @@ -46,19 +46,31 @@ void canBuildNewDataLoaders() { @Test void theDataLoaderCanTransform() { - DataLoader dataLoader1 = DataLoaderFactory.newDataLoader(batchLoader1, defaultOptions); - assertThat(dataLoader1.getOptions(), equalTo(defaultOptions)); - assertThat(dataLoader1.getBatchLoadFunction(), equalTo(batchLoader1)); + DataLoader dataLoaderOrig = DataLoaderFactory.newDataLoader(batchLoader1, defaultOptions); + assertThat(dataLoaderOrig.getOptions(), equalTo(defaultOptions)); + assertThat(dataLoaderOrig.getBatchLoadFunction(), equalTo(batchLoader1)); // // we can transform the data loader // - DataLoader dataLoader2 = dataLoader1.transform(it -> { + DataLoader dataLoaderTransformed = dataLoaderOrig.transform(it -> { it.options(differentOptions); it.batchLoadFunction(batchLoader2); }); - assertThat(dataLoader2, not(equalTo(dataLoader1))); - assertThat(dataLoader2.getOptions(), equalTo(differentOptions)); - assertThat(dataLoader2.getBatchLoadFunction(), equalTo(batchLoader2)); + assertThat(dataLoaderTransformed, not(equalTo(dataLoaderOrig))); + assertThat(dataLoaderTransformed.getOptions(), equalTo(differentOptions)); + assertThat(dataLoaderTransformed.getBatchLoadFunction(), equalTo(batchLoader2)); + + // can copy values + dataLoaderOrig = DataLoaderFactory.newDataLoader(batchLoader1, defaultOptions); + + dataLoaderTransformed = dataLoaderOrig.transform(it -> { + it.batchLoadFunction(batchLoader2); + }); + + assertThat(dataLoaderTransformed, not(equalTo(dataLoaderOrig))); + assertThat(dataLoaderTransformed.getOptions(), equalTo(defaultOptions)); + assertThat(dataLoaderTransformed.getBatchLoadFunction(), equalTo(batchLoader2)); + } } From 38bca5d640d8d79b2838254721d3c8221bd6940e Mon Sep 17 00:00:00 2001 From: bbaker Date: Tue, 1 Apr 2025 20:23:48 +1100 Subject: [PATCH 078/156] Added support for a delegating data loader --- .../org/dataloader/DelegatingDataLoader.java | 171 ++++++++++++++++++ .../java/org/dataloader/DataLoaderTest.java | 8 +- .../dataloader/DelegatingDataLoaderTest.java | 61 +++++++ .../DelegatingDataLoaderFactory.java | 71 ++++++++ .../TestDataLoaderFactories.java | 15 +- .../parameterized/TestDataLoaderFactory.java | 4 + 6 files changed, 322 insertions(+), 8 deletions(-) create mode 100644 src/main/java/org/dataloader/DelegatingDataLoader.java create mode 100644 src/test/java/org/dataloader/DelegatingDataLoaderTest.java create mode 100644 src/test/java/org/dataloader/fixtures/parameterized/DelegatingDataLoaderFactory.java diff --git a/src/main/java/org/dataloader/DelegatingDataLoader.java b/src/main/java/org/dataloader/DelegatingDataLoader.java new file mode 100644 index 00000000..f8f48988 --- /dev/null +++ b/src/main/java/org/dataloader/DelegatingDataLoader.java @@ -0,0 +1,171 @@ +package org.dataloader; + +import org.dataloader.annotations.PublicApi; +import org.dataloader.stats.Statistics; + +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.function.BiConsumer; +import java.util.function.Consumer; + +/** + * This delegating {@link DataLoader} makes it easier to create wrappers of {@link DataLoader}s in case you want to change how + * values are returned for example + * + * @param type parameter indicating the type of the data load keys + * @param type parameter indicating the type of the data that is returned + */ +@PublicApi +public class DelegatingDataLoader extends DataLoader { + + protected final DataLoader delegate; + + /** + * This can be called to unwrap a given {@link DataLoader} such that if it's a {@link DelegatingDataLoader} the underlying + * {@link DataLoader} is returned otherwise it's just passed in data loader + * + * @param dataLoader the dataLoader to unwrap + * @param type parameter indicating the type of the data load keys + * @param type parameter indicating the type of the data that is returned + * @return the delegate dataLoader OR just this current one if it's not wrapped + */ + public static DataLoader unwrap(DataLoader dataLoader) { + if (dataLoader instanceof DelegatingDataLoader) { + return ((DelegatingDataLoader) dataLoader).getDelegate(); + } + return dataLoader; + } + + public DelegatingDataLoader(DataLoader delegate) { + super(delegate.getBatchLoadFunction(), delegate.getOptions()); + this.delegate = delegate; + } + + public DataLoader getDelegate() { + return delegate; + } + + /** + * The {@link DataLoader#load(Object)} and {@link DataLoader#loadMany(List)} type methods all call back + * to the {@link DataLoader#load(Object, Object)} and hence we don't override them. + * + * @param key the key to load + * @param keyContext a context object that is specific to this key + * @return the future of the value + */ + @Override + public CompletableFuture load(K key, Object keyContext) { + return delegate.load(key, keyContext); + } + + + @Override + public DataLoader transform(Consumer> builderConsumer) { + return delegate.transform(builderConsumer); + } + + @Override + public Instant getLastDispatchTime() { + return delegate.getLastDispatchTime(); + } + + @Override + public Duration getTimeSinceDispatch() { + return delegate.getTimeSinceDispatch(); + } + + @Override + public Optional> getIfPresent(K key) { + return delegate.getIfPresent(key); + } + + @Override + public Optional> getIfCompleted(K key) { + return delegate.getIfCompleted(key); + } + + @Override + public CompletableFuture> dispatch() { + return delegate.dispatch(); + } + + @Override + public DispatchResult dispatchWithCounts() { + return delegate.dispatchWithCounts(); + } + + @Override + public List dispatchAndJoin() { + return delegate.dispatchAndJoin(); + } + + @Override + public int dispatchDepth() { + return delegate.dispatchDepth(); + } + + @Override + public Object getCacheKey(K key) { + return delegate.getCacheKey(key); + } + + @Override + public Statistics getStatistics() { + return delegate.getStatistics(); + } + + @Override + public CacheMap getCacheMap() { + return delegate.getCacheMap(); + } + + @Override + public ValueCache getValueCache() { + return delegate.getValueCache(); + } + + @Override + public DataLoader clear(K key) { + delegate.clear(key); + return this; + } + + @Override + public DataLoader clear(K key, BiConsumer handler) { + delegate.clear(key, handler); + return this; + } + + @Override + public DataLoader clearAll() { + delegate.clearAll(); + return this; + } + + @Override + public DataLoader clearAll(BiConsumer handler) { + delegate.clearAll(handler); + return this; + } + + @Override + public DataLoader prime(K key, V value) { + delegate.prime(key, value); + return this; + } + + @Override + public DataLoader prime(K key, Exception error) { + delegate.prime(key, error); + return this; + } + + @Override + public DataLoader prime(K key, CompletableFuture value) { + delegate.prime(key, value); + return this; + } +} diff --git a/src/test/java/org/dataloader/DataLoaderTest.java b/src/test/java/org/dataloader/DataLoaderTest.java index 9a595b45..069d390a 100644 --- a/src/test/java/org/dataloader/DataLoaderTest.java +++ b/src/test/java/org/dataloader/DataLoaderTest.java @@ -785,7 +785,7 @@ public void should_work_with_duplicate_keys_when_caching_disabled(TestDataLoader assertThat(future1.get(), equalTo("A")); assertThat(future2.get(), equalTo("B")); assertThat(future3.get(), equalTo("A")); - if (factory instanceof MappedDataLoaderFactory || factory instanceof MappedPublisherDataLoaderFactory) { + if (factory.unwrap() instanceof MappedDataLoaderFactory || factory.unwrap() instanceof MappedPublisherDataLoaderFactory) { assertThat(loadCalls, equalTo(singletonList(asList("A", "B")))); } else { assertThat(loadCalls, equalTo(singletonList(asList("A", "B", "A")))); @@ -1152,12 +1152,12 @@ public void when_values_size_are_less_then_key_size(TestDataLoaderFactory factor await().atMost(Duration.FIVE_SECONDS).until(() -> areAllDone(cf1, cf2, cf3, cf4)); - if (factory instanceof ListDataLoaderFactory) { + if (factory.unwrap() instanceof ListDataLoaderFactory) { assertThat(cause(cf1), instanceOf(DataLoaderAssertionException.class)); assertThat(cause(cf2), instanceOf(DataLoaderAssertionException.class)); assertThat(cause(cf3), instanceOf(DataLoaderAssertionException.class)); assertThat(cause(cf4), instanceOf(DataLoaderAssertionException.class)); - } else if (factory instanceof PublisherDataLoaderFactory) { + } else if (factory.unwrap() instanceof PublisherDataLoaderFactory) { // some have completed progressively but the other never did assertThat(cf1.join(), equalTo("A")); assertThat(cf2.join(), equalTo("B")); @@ -1187,7 +1187,7 @@ public void when_values_size_are_more_then_key_size(TestDataLoaderFactory factor await().atMost(Duration.FIVE_SECONDS).until(() -> areAllDone(cf1, cf2, cf3, cf4)); - if (factory instanceof ListDataLoaderFactory) { + if (factory.unwrap() instanceof ListDataLoaderFactory) { assertThat(cause(cf1), instanceOf(DataLoaderAssertionException.class)); assertThat(cause(cf2), instanceOf(DataLoaderAssertionException.class)); assertThat(cause(cf3), instanceOf(DataLoaderAssertionException.class)); diff --git a/src/test/java/org/dataloader/DelegatingDataLoaderTest.java b/src/test/java/org/dataloader/DelegatingDataLoaderTest.java new file mode 100644 index 00000000..c81a5832 --- /dev/null +++ b/src/test/java/org/dataloader/DelegatingDataLoaderTest.java @@ -0,0 +1,61 @@ +package org.dataloader; + +import org.dataloader.fixtures.TestKit; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.concurrent.CompletableFuture; + +import static org.awaitility.Awaitility.await; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; + +/** + * There are WAY more tests via the {@link org.dataloader.fixtures.parameterized.DelegatingDataLoaderFactory} + * parameterized tests. All the basic {@link DataLoader} tests pass when wrapped in a {@link DelegatingDataLoader} + */ +public class DelegatingDataLoaderTest { + + @Test + void canUnwrapDataLoaders() { + DataLoader rawLoader = TestKit.idLoader(); + DataLoader delegateLoader = new DelegatingDataLoader<>(rawLoader); + + assertThat(DelegatingDataLoader.unwrap(rawLoader), is(rawLoader)); + assertThat(DelegatingDataLoader.unwrap(delegateLoader), is(rawLoader)); + } + + @Test + void canCreateAClassOk() { + DataLoader rawLoader = TestKit.idLoader(); + DelegatingDataLoader delegatingDataLoader = new DelegatingDataLoader<>(rawLoader) { + @Override + public CompletableFuture load(String key, Object keyContext) { + CompletableFuture cf = super.load(key, keyContext); + return cf.thenApply(v -> "|" + v + "|"); + } + }; + + assertThat(delegatingDataLoader.getDelegate(), is(rawLoader)); + + + CompletableFuture cfA = delegatingDataLoader.load("A"); + CompletableFuture cfB = delegatingDataLoader.load("B"); + CompletableFuture> cfCD = delegatingDataLoader.loadMany(List.of("C", "D")); + + CompletableFuture> dispatch = delegatingDataLoader.dispatch(); + + await().until(dispatch::isDone); + + assertThat(cfA.join(), equalTo("|A|")); + assertThat(cfB.join(), equalTo("|B|")); + assertThat(cfCD.join(), equalTo(List.of("|C|", "|D|"))); + + assertThat(delegatingDataLoader.getIfPresent("A").isEmpty(), equalTo(false)); + assertThat(delegatingDataLoader.getIfPresent("X").isEmpty(), equalTo(true)); + + assertThat(delegatingDataLoader.getIfCompleted("A").isEmpty(), equalTo(false)); + assertThat(delegatingDataLoader.getIfCompleted("X").isEmpty(), equalTo(true)); + } +} \ No newline at end of file diff --git a/src/test/java/org/dataloader/fixtures/parameterized/DelegatingDataLoaderFactory.java b/src/test/java/org/dataloader/fixtures/parameterized/DelegatingDataLoaderFactory.java new file mode 100644 index 00000000..0cbd3f34 --- /dev/null +++ b/src/test/java/org/dataloader/fixtures/parameterized/DelegatingDataLoaderFactory.java @@ -0,0 +1,71 @@ +package org.dataloader.fixtures.parameterized; + +import org.dataloader.DataLoader; +import org.dataloader.DataLoaderOptions; +import org.dataloader.DelegatingDataLoader; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +public class DelegatingDataLoaderFactory implements TestDataLoaderFactory { + // its delegates all the way down to the turtles + private final TestDataLoaderFactory delegateFactory; + + public DelegatingDataLoaderFactory(TestDataLoaderFactory delegateFactory) { + this.delegateFactory = delegateFactory; + } + + @Override + public String toString() { + return "DelegatingDataLoaderFactory{" + + "delegateFactory=" + delegateFactory + + '}'; + } + + @Override + public TestDataLoaderFactory unwrap() { + return delegateFactory.unwrap(); + } + + private DataLoader mkDelegateDataLoader(DataLoader dataLoader) { + return new DelegatingDataLoader<>(dataLoader); + } + + @Override + public DataLoader idLoader(DataLoaderOptions options, List> loadCalls) { + return mkDelegateDataLoader(delegateFactory.idLoader(options, loadCalls)); + } + + @Override + public DataLoader idLoaderDelayed(DataLoaderOptions options, List> loadCalls, Duration delay) { + return mkDelegateDataLoader(delegateFactory.idLoaderDelayed(options, loadCalls, delay)); + } + + @Override + public DataLoader idLoaderBlowsUps( + DataLoaderOptions options, List> loadCalls) { + return mkDelegateDataLoader(delegateFactory.idLoaderBlowsUps(options, loadCalls)); + } + + @Override + public DataLoader idLoaderAllExceptions(DataLoaderOptions options, List> loadCalls) { + return mkDelegateDataLoader(delegateFactory.idLoaderAllExceptions(options, loadCalls)); + } + + @Override + public DataLoader idLoaderOddEvenExceptions(DataLoaderOptions options, List> loadCalls) { + return mkDelegateDataLoader(delegateFactory.idLoaderOddEvenExceptions(options, loadCalls)); + } + + @Override + public DataLoader onlyReturnsNValues(int N, DataLoaderOptions options, ArrayList loadCalls) { + return mkDelegateDataLoader(delegateFactory.onlyReturnsNValues(N, options, loadCalls)); + } + + @Override + public DataLoader idLoaderReturnsTooMany(int howManyMore, DataLoaderOptions options, ArrayList loadCalls) { + return mkDelegateDataLoader(delegateFactory.idLoaderReturnsTooMany(howManyMore, options, loadCalls)); + } +} diff --git a/src/test/java/org/dataloader/fixtures/parameterized/TestDataLoaderFactories.java b/src/test/java/org/dataloader/fixtures/parameterized/TestDataLoaderFactories.java index 6afd05c7..48678c45 100644 --- a/src/test/java/org/dataloader/fixtures/parameterized/TestDataLoaderFactories.java +++ b/src/test/java/org/dataloader/fixtures/parameterized/TestDataLoaderFactories.java @@ -5,14 +5,21 @@ import java.util.stream.Stream; +@SuppressWarnings("unused") public class TestDataLoaderFactories { public static Stream get() { return Stream.of( - Arguments.of(Named.of("List DataLoader", new ListDataLoaderFactory())), - Arguments.of(Named.of("Mapped DataLoader", new MappedDataLoaderFactory())), - Arguments.of(Named.of("Publisher DataLoader", new PublisherDataLoaderFactory())), - Arguments.of(Named.of("Mapped Publisher DataLoader", new MappedPublisherDataLoaderFactory())) + Arguments.of(Named.of("List DataLoader", new ListDataLoaderFactory())), + Arguments.of(Named.of("Mapped DataLoader", new MappedDataLoaderFactory())), + Arguments.of(Named.of("Publisher DataLoader", new PublisherDataLoaderFactory())), + Arguments.of(Named.of("Mapped Publisher DataLoader", new MappedPublisherDataLoaderFactory())), + + // runs all the above via a DelegateDataLoader + Arguments.of(Named.of("Delegate List DataLoader", new DelegatingDataLoaderFactory(new ListDataLoaderFactory()))), + Arguments.of(Named.of("Delegate Mapped DataLoader", new DelegatingDataLoaderFactory(new MappedDataLoaderFactory()))), + Arguments.of(Named.of("Delegate Publisher DataLoader", new DelegatingDataLoaderFactory(new PublisherDataLoaderFactory()))), + Arguments.of(Named.of("Delegate Mapped Publisher DataLoader", new DelegatingDataLoaderFactory(new MappedPublisherDataLoaderFactory()))) ); } } diff --git a/src/test/java/org/dataloader/fixtures/parameterized/TestDataLoaderFactory.java b/src/test/java/org/dataloader/fixtures/parameterized/TestDataLoaderFactory.java index 8cbe86cb..789b136d 100644 --- a/src/test/java/org/dataloader/fixtures/parameterized/TestDataLoaderFactory.java +++ b/src/test/java/org/dataloader/fixtures/parameterized/TestDataLoaderFactory.java @@ -39,4 +39,8 @@ default DataLoader idLoader() { default DataLoader idLoaderDelayed(Duration delay) { return idLoaderDelayed(null, new ArrayList<>(), delay); } + + default TestDataLoaderFactory unwrap() { + return this; + } } From 3060a30e60871f50eb685ada5337aa7a9e7e5112 Mon Sep 17 00:00:00 2001 From: bbaker Date: Tue, 1 Apr 2025 20:46:11 +1100 Subject: [PATCH 079/156] Added support for a delegating data loader - jspecify annotations --- src/main/java/org/dataloader/DataLoader.java | 5 +++-- src/main/java/org/dataloader/DelegatingDataLoader.java | 6 +++++- src/test/java/org/dataloader/DelegatingDataLoaderTest.java | 4 +++- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/dataloader/DataLoader.java b/src/main/java/org/dataloader/DataLoader.java index fb15d444..d03e5acd 100644 --- a/src/main/java/org/dataloader/DataLoader.java +++ b/src/main/java/org/dataloader/DataLoader.java @@ -21,6 +21,7 @@ import org.dataloader.impl.CompletableFutureKit; import org.dataloader.stats.Statistics; import org.dataloader.stats.StatisticsCollector; +import org.jspecify.annotations.NonNull; import org.jspecify.annotations.NullMarked; import org.jspecify.annotations.Nullable; @@ -517,8 +518,8 @@ public Optional> getIfCompleted(K key) { * @param keyContext a context object that is specific to this key * @return the future of the value */ - public CompletableFuture load(K key, Object keyContext) { - return helper.load(key, keyContext); + public CompletableFuture load(@NonNull K key, @Nullable Object keyContext) { + return helper.load(nonNull(key), keyContext); } /** diff --git a/src/main/java/org/dataloader/DelegatingDataLoader.java b/src/main/java/org/dataloader/DelegatingDataLoader.java index f8f48988..2b4a0427 100644 --- a/src/main/java/org/dataloader/DelegatingDataLoader.java +++ b/src/main/java/org/dataloader/DelegatingDataLoader.java @@ -2,6 +2,9 @@ import org.dataloader.annotations.PublicApi; import org.dataloader.stats.Statistics; +import org.jspecify.annotations.NonNull; +import org.jspecify.annotations.NullMarked; +import org.jspecify.annotations.Nullable; import java.time.Duration; import java.time.Instant; @@ -19,6 +22,7 @@ * @param type parameter indicating the type of the data that is returned */ @PublicApi +@NullMarked public class DelegatingDataLoader extends DataLoader { protected final DataLoader delegate; @@ -57,7 +61,7 @@ public DataLoader getDelegate() { * @return the future of the value */ @Override - public CompletableFuture load(K key, Object keyContext) { + public CompletableFuture load(@NonNull K key, @Nullable Object keyContext) { return delegate.load(key, keyContext); } diff --git a/src/test/java/org/dataloader/DelegatingDataLoaderTest.java b/src/test/java/org/dataloader/DelegatingDataLoaderTest.java index c81a5832..62785033 100644 --- a/src/test/java/org/dataloader/DelegatingDataLoaderTest.java +++ b/src/test/java/org/dataloader/DelegatingDataLoaderTest.java @@ -1,6 +1,8 @@ package org.dataloader; import org.dataloader.fixtures.TestKit; +import org.jspecify.annotations.NonNull; +import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.Test; import java.util.List; @@ -31,7 +33,7 @@ void canCreateAClassOk() { DataLoader rawLoader = TestKit.idLoader(); DelegatingDataLoader delegatingDataLoader = new DelegatingDataLoader<>(rawLoader) { @Override - public CompletableFuture load(String key, Object keyContext) { + public CompletableFuture load(@NonNull String key, @Nullable Object keyContext) { CompletableFuture cf = super.load(key, keyContext); return cf.thenApply(v -> "|" + v + "|"); } From 3c30b816f2849411b51fee3c6760e7acd666d2a0 Mon Sep 17 00:00:00 2001 From: Andreas Marek Date: Wed, 2 Apr 2025 10:34:45 +1000 Subject: [PATCH 080/156] make local publishing easier by preventing signing --- build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/build.gradle b/build.gradle index 9b943f33..7d98bc38 100644 --- a/build.gradle +++ b/build.gradle @@ -176,6 +176,7 @@ nexusPublishing { } signing { + required { !project.hasProperty('publishToMavenLocal') } def signingKey = System.env.MAVEN_CENTRAL_PGP_KEY useInMemoryPgpKeys(signingKey, "") sign publishing.publications From 70b891545754eb811479d14d0ef6178cafef9761 Mon Sep 17 00:00:00 2001 From: bbaker Date: Wed, 2 Apr 2025 12:51:57 +1100 Subject: [PATCH 081/156] Added support for a delegating data loader - tweaked javadoc --- .../org/dataloader/DelegatingDataLoader.java | 21 ++++++++++++++++--- .../dataloader/DelegatingDataLoaderTest.java | 3 ++- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/dataloader/DelegatingDataLoader.java b/src/main/java/org/dataloader/DelegatingDataLoader.java index 2b4a0427..53fab86a 100644 --- a/src/main/java/org/dataloader/DelegatingDataLoader.java +++ b/src/main/java/org/dataloader/DelegatingDataLoader.java @@ -16,8 +16,24 @@ /** * This delegating {@link DataLoader} makes it easier to create wrappers of {@link DataLoader}s in case you want to change how - * values are returned for example - * + * values are returned for example. + *

+ * The most common way would be to make a new {@link DelegatingDataLoader} subclass that overloads the {@link DelegatingDataLoader#load(Object, Object)} + * method. + *

+ * For example the following allows you to change the returned value in some way : + *

+ * {@code
+ *         DataLoader rawLoader = createDataLoader();
+ *         DelegatingDataLoader delegatingDataLoader = new DelegatingDataLoader<>(rawLoader) {
+ *             @Override
+ *             public CompletableFuture load(@NonNull String key, @Nullable Object keyContext) {
+ *                 CompletableFuture cf = super.load(key, keyContext);
+ *                 return cf.thenApply(v -> "|" + v + "|");
+ *             }
+ *         };
+ * }
+ * 
* @param type parameter indicating the type of the data load keys * @param type parameter indicating the type of the data that is returned */ @@ -65,7 +81,6 @@ public CompletableFuture load(@NonNull K key, @Nullable Object keyContext) { return delegate.load(key, keyContext); } - @Override public DataLoader transform(Consumer> builderConsumer) { return delegate.transform(builderConsumer); diff --git a/src/test/java/org/dataloader/DelegatingDataLoaderTest.java b/src/test/java/org/dataloader/DelegatingDataLoaderTest.java index 62785033..9103ecaf 100644 --- a/src/test/java/org/dataloader/DelegatingDataLoaderTest.java +++ b/src/test/java/org/dataloader/DelegatingDataLoaderTest.java @@ -1,6 +1,7 @@ package org.dataloader; import org.dataloader.fixtures.TestKit; +import org.dataloader.fixtures.parameterized.DelegatingDataLoaderFactory; import org.jspecify.annotations.NonNull; import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.Test; @@ -14,7 +15,7 @@ import static org.hamcrest.MatcherAssert.assertThat; /** - * There are WAY more tests via the {@link org.dataloader.fixtures.parameterized.DelegatingDataLoaderFactory} + * There are WAY more tests via the {@link DelegatingDataLoaderFactory} * parameterized tests. All the basic {@link DataLoader} tests pass when wrapped in a {@link DelegatingDataLoader} */ public class DelegatingDataLoaderTest { From 6d217cc636ea864dd3ee2586964a9e007ad7e642 Mon Sep 17 00:00:00 2001 From: bbaker Date: Wed, 2 Apr 2025 13:07:58 +1100 Subject: [PATCH 082/156] Added support for a delegating data loader - tweaked javadoc more --- .../org/dataloader/DelegatingDataLoader.java | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/dataloader/DelegatingDataLoader.java b/src/main/java/org/dataloader/DelegatingDataLoader.java index 53fab86a..c54a7317 100644 --- a/src/main/java/org/dataloader/DelegatingDataLoader.java +++ b/src/main/java/org/dataloader/DelegatingDataLoader.java @@ -22,18 +22,16 @@ * method. *

* For example the following allows you to change the returned value in some way : - *

- * {@code
- *         DataLoader rawLoader = createDataLoader();
- *         DelegatingDataLoader delegatingDataLoader = new DelegatingDataLoader<>(rawLoader) {
- *             @Override
- *             public CompletableFuture load(@NonNull String key, @Nullable Object keyContext) {
- *                 CompletableFuture cf = super.load(key, keyContext);
- *                 return cf.thenApply(v -> "|" + v + "|");
- *             }
- *         };
- * }
- * 
+ *
{@code
+ * DataLoader rawLoader = createDataLoader();
+ * DelegatingDataLoader delegatingDataLoader = new DelegatingDataLoader<>(rawLoader) {
+ *    public CompletableFuture load(@NonNull String key, @Nullable Object keyContext) {
+ *       CompletableFuture cf = super.load(key, keyContext);
+ *       return cf.thenApply(v -> "|" + v + "|");
+ *    }
+ *};
+ *}
+ * * @param type parameter indicating the type of the data load keys * @param type parameter indicating the type of the data that is returned */ From 2b1fff1d3e617b3d8f65d95c3ddb526f3e1c9bfd Mon Sep 17 00:00:00 2001 From: dondonz <13839920+dondonz@users.noreply.github.com> Date: Sat, 5 Apr 2025 10:27:14 +1100 Subject: [PATCH 083/156] Update documentation ahead of release --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 61180d9e..9f47b152 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,8 @@ # java-dataloader [![Build](https://github.com/graphql-java/java-dataloader/actions/workflows/master.yml/badge.svg)](https://github.com/graphql-java/java-dataloader/actions/workflows/master.yml) -[![Latest Release](https://maven-badges.herokuapp.com/maven-central/com.graphql-java/java-dataloader/badge.svg)](https://maven-badges.herokuapp.com/maven-central/com.graphql-java/java-dataloader/) +[![Latest Release](https://img.shields.io/maven-central/v/com.graphql-java/java-dataloader?versionPrefix=4.)](https://maven-badges.herokuapp.com/maven-central/com.graphql-java/graphql-java/) +[![Latest Snapshot](https://img.shields.io/maven-central/v/com.graphql-java/java-dataloader?label=maven-central%20snapshot&versionPrefix=0)](https://maven-badges.herokuapp.com/maven-central/com.graphql-java/graphql-java/) [![Apache licensed](https://img.shields.io/hexpm/l/plug.svg?maxAge=2592000)](https://github.com/graphql-java/java-dataloader/blob/master/LICENSE) This small and simple utility library is a pure Java 11 port of [Facebook DataLoader](https://github.com/facebook/dataloader). @@ -67,7 +68,7 @@ repositories { } dependencies { - compile 'com.graphql-java:java-dataloader: 3.4.0' + compile 'com.graphql-java:java-dataloader: 4.0.0' } ``` From 0afdbab5e9a9cde90fee1604798bde1d3d5efea9 Mon Sep 17 00:00:00 2001 From: dondonz <13839920+dondonz@users.noreply.github.com> Date: Tue, 8 Apr 2025 09:46:03 +1000 Subject: [PATCH 084/156] Remove jcenter reference after sunset --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9f47b152..c7c6fe99 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ Gradle users configure the `java-dataloader` dependency in `build.gradle`: ``` repositories { - jcenter() + mavenCentral() } dependencies { From 6942cc1feff913ec2a52e2569cd57f78c73a7864 Mon Sep 17 00:00:00 2001 From: Davide Angelocola Date: Mon, 14 Apr 2025 15:50:42 +0200 Subject: [PATCH 085/156] Breaking change for StatisticsCollector Use `LongAdder` instead of `AtomicLong`, that is more suitable for high contention (see the javadoc). --- .../stats/DelegatingStatisticsCollector.java | 40 +++---- .../stats/NoOpStatisticsCollector.java | 36 +++--- .../stats/SimpleStatisticsCollector.java | 59 +++++----- .../dataloader/stats/StatisticsCollector.java | 44 +++---- .../stats/ThreadLocalStatisticsCollector.java | 40 +++---- .../org/dataloader/DataLoaderStatsTest.java | 32 ++--- .../dataloader/performance/AtomicVsAdder.java | 111 ++++++++++++++++++ 7 files changed, 224 insertions(+), 138 deletions(-) create mode 100644 src/test/java/org/dataloader/performance/AtomicVsAdder.java diff --git a/src/main/java/org/dataloader/stats/DelegatingStatisticsCollector.java b/src/main/java/org/dataloader/stats/DelegatingStatisticsCollector.java index 563d37b5..36ed69a3 100644 --- a/src/main/java/org/dataloader/stats/DelegatingStatisticsCollector.java +++ b/src/main/java/org/dataloader/stats/DelegatingStatisticsCollector.java @@ -26,63 +26,63 @@ public DelegatingStatisticsCollector(StatisticsCollector delegateCollector) { } @Override - public long incrementLoadCount(IncrementLoadCountStatisticsContext context) { + public void incrementLoadCount(IncrementLoadCountStatisticsContext context) { delegateCollector.incrementLoadCount(context); - return collector.incrementLoadCount(context); + collector.incrementLoadCount(context); } @Deprecated @Override - public long incrementLoadCount() { - return incrementLoadCount(null); + public void incrementLoadCount() { + incrementLoadCount(null); } @Override - public long incrementLoadErrorCount(IncrementLoadErrorCountStatisticsContext context) { + public void incrementLoadErrorCount(IncrementLoadErrorCountStatisticsContext context) { delegateCollector.incrementLoadErrorCount(context); - return collector.incrementLoadErrorCount(context); + collector.incrementLoadErrorCount(context); } @Deprecated @Override - public long incrementLoadErrorCount() { - return incrementLoadErrorCount(null); + public void incrementLoadErrorCount() { + incrementLoadErrorCount(null); } @Override - public long incrementBatchLoadCountBy(long delta, IncrementBatchLoadCountByStatisticsContext context) { + public void incrementBatchLoadCountBy(long delta, IncrementBatchLoadCountByStatisticsContext context) { delegateCollector.incrementBatchLoadCountBy(delta, context); - return collector.incrementBatchLoadCountBy(delta, context); + collector.incrementBatchLoadCountBy(delta, context); } @Deprecated @Override - public long incrementBatchLoadCountBy(long delta) { - return incrementBatchLoadCountBy(delta, null); + public void incrementBatchLoadCountBy(long delta) { + incrementBatchLoadCountBy(delta, null); } @Override - public long incrementBatchLoadExceptionCount(IncrementBatchLoadExceptionCountStatisticsContext context) { + public void incrementBatchLoadExceptionCount(IncrementBatchLoadExceptionCountStatisticsContext context) { delegateCollector.incrementBatchLoadExceptionCount(context); - return collector.incrementBatchLoadExceptionCount(context); + collector.incrementBatchLoadExceptionCount(context); } @Deprecated @Override - public long incrementBatchLoadExceptionCount() { - return incrementBatchLoadExceptionCount(null); + public void incrementBatchLoadExceptionCount() { + incrementBatchLoadExceptionCount(null); } @Override - public long incrementCacheHitCount(IncrementCacheHitCountStatisticsContext context) { + public void incrementCacheHitCount(IncrementCacheHitCountStatisticsContext context) { delegateCollector.incrementCacheHitCount(context); - return collector.incrementCacheHitCount(context); + collector.incrementCacheHitCount(context); } @Deprecated @Override - public long incrementCacheHitCount() { - return incrementCacheHitCount(null); + public void incrementCacheHitCount() { + incrementCacheHitCount(null); } /** diff --git a/src/main/java/org/dataloader/stats/NoOpStatisticsCollector.java b/src/main/java/org/dataloader/stats/NoOpStatisticsCollector.java index e7267b3f..6397d8f5 100644 --- a/src/main/java/org/dataloader/stats/NoOpStatisticsCollector.java +++ b/src/main/java/org/dataloader/stats/NoOpStatisticsCollector.java @@ -14,58 +14,54 @@ public class NoOpStatisticsCollector implements StatisticsCollector { private static final Statistics ZERO_STATS = new Statistics(); @Override - public long incrementLoadCount(IncrementLoadCountStatisticsContext context) { - return 0; + public void incrementLoadCount(IncrementLoadCountStatisticsContext context) { } @Deprecated @Override - public long incrementLoadCount() { - return incrementLoadCount(null); + public void incrementLoadCount() { + incrementLoadCount(null); } @Override - public long incrementLoadErrorCount(IncrementLoadErrorCountStatisticsContext context) { - return 0; + public void incrementLoadErrorCount(IncrementLoadErrorCountStatisticsContext context) { } @Deprecated @Override - public long incrementLoadErrorCount() { - return incrementLoadErrorCount(null); + public void incrementLoadErrorCount() { + incrementLoadErrorCount(null); } @Override - public long incrementBatchLoadCountBy(long delta, IncrementBatchLoadCountByStatisticsContext context) { - return 0; + public void incrementBatchLoadCountBy(long delta, IncrementBatchLoadCountByStatisticsContext context) { } @Deprecated @Override - public long incrementBatchLoadCountBy(long delta) { - return incrementBatchLoadCountBy(delta, null); + public void incrementBatchLoadCountBy(long delta) { + incrementBatchLoadCountBy(delta, null); } @Override - public long incrementBatchLoadExceptionCount(IncrementBatchLoadExceptionCountStatisticsContext context) { - return 0; + public void incrementBatchLoadExceptionCount(IncrementBatchLoadExceptionCountStatisticsContext context) { + } @Deprecated @Override - public long incrementBatchLoadExceptionCount() { - return incrementBatchLoadExceptionCount(null); + public void incrementBatchLoadExceptionCount() { + incrementBatchLoadExceptionCount(null); } @Override - public long incrementCacheHitCount(IncrementCacheHitCountStatisticsContext context) { - return 0; + public void incrementCacheHitCount(IncrementCacheHitCountStatisticsContext context) { } @Deprecated @Override - public long incrementCacheHitCount() { - return incrementCacheHitCount(null); + public void incrementCacheHitCount() { + incrementCacheHitCount(null); } @Override diff --git a/src/main/java/org/dataloader/stats/SimpleStatisticsCollector.java b/src/main/java/org/dataloader/stats/SimpleStatisticsCollector.java index 22b36628..2c2898a3 100644 --- a/src/main/java/org/dataloader/stats/SimpleStatisticsCollector.java +++ b/src/main/java/org/dataloader/stats/SimpleStatisticsCollector.java @@ -6,7 +6,7 @@ import org.dataloader.stats.context.IncrementLoadCountStatisticsContext; import org.dataloader.stats.context.IncrementLoadErrorCountStatisticsContext; -import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.LongAdder; /** * This simple collector uses {@link java.util.concurrent.atomic.AtomicLong}s to collect @@ -15,72 +15,73 @@ * @see org.dataloader.stats.StatisticsCollector */ public class SimpleStatisticsCollector implements StatisticsCollector { - private final AtomicLong loadCount = new AtomicLong(); - private final AtomicLong batchInvokeCount = new AtomicLong(); - private final AtomicLong batchLoadCount = new AtomicLong(); - private final AtomicLong cacheHitCount = new AtomicLong(); - private final AtomicLong batchLoadExceptionCount = new AtomicLong(); - private final AtomicLong loadErrorCount = new AtomicLong(); + + private final LongAdder loadCount = new LongAdder(); + private final LongAdder batchInvokeCount = new LongAdder(); + private final LongAdder batchLoadCount = new LongAdder(); + private final LongAdder cacheHitCount = new LongAdder(); + private final LongAdder batchLoadExceptionCount = new LongAdder(); + private final LongAdder loadErrorCount = new LongAdder(); @Override - public long incrementLoadCount(IncrementLoadCountStatisticsContext context) { - return loadCount.incrementAndGet(); + public void incrementLoadCount(IncrementLoadCountStatisticsContext context) { + loadCount.increment(); } @Deprecated @Override - public long incrementLoadCount() { - return incrementLoadCount(null); + public void incrementLoadCount() { + incrementLoadCount(null); } @Override - public long incrementLoadErrorCount(IncrementLoadErrorCountStatisticsContext context) { - return loadErrorCount.incrementAndGet(); + public void incrementLoadErrorCount(IncrementLoadErrorCountStatisticsContext context) { + loadErrorCount.increment(); } @Deprecated @Override - public long incrementLoadErrorCount() { - return incrementLoadErrorCount(null); + public void incrementLoadErrorCount() { + incrementLoadErrorCount(null); } @Override - public long incrementBatchLoadCountBy(long delta, IncrementBatchLoadCountByStatisticsContext context) { - batchInvokeCount.incrementAndGet(); - return batchLoadCount.addAndGet(delta); + public void incrementBatchLoadCountBy(long delta, IncrementBatchLoadCountByStatisticsContext context) { + batchInvokeCount.increment(); + batchLoadCount.add(delta); } @Deprecated @Override - public long incrementBatchLoadCountBy(long delta) { - return incrementBatchLoadCountBy(delta, null); + public void incrementBatchLoadCountBy(long delta) { + incrementBatchLoadCountBy(delta, null); } @Override - public long incrementBatchLoadExceptionCount(IncrementBatchLoadExceptionCountStatisticsContext context) { - return batchLoadExceptionCount.incrementAndGet(); + public void incrementBatchLoadExceptionCount(IncrementBatchLoadExceptionCountStatisticsContext context) { + batchLoadExceptionCount.increment(); } @Deprecated @Override - public long incrementBatchLoadExceptionCount() { - return incrementBatchLoadExceptionCount(null); + public void incrementBatchLoadExceptionCount() { + incrementBatchLoadExceptionCount(null); } @Override - public long incrementCacheHitCount(IncrementCacheHitCountStatisticsContext context) { - return cacheHitCount.incrementAndGet(); + public void incrementCacheHitCount(IncrementCacheHitCountStatisticsContext context) { + cacheHitCount.increment(); } @Deprecated @Override - public long incrementCacheHitCount() { - return incrementCacheHitCount(null); + public void incrementCacheHitCount() { + incrementCacheHitCount(null); } @Override public Statistics getStatistics() { - return new Statistics(loadCount.get(), loadErrorCount.get(), batchInvokeCount.get(), batchLoadCount.get(), batchLoadExceptionCount.get(), cacheHitCount.get()); + return new Statistics(loadCount.sum(), loadErrorCount.sum(), batchInvokeCount.sum(), batchLoadCount.sum(), batchLoadExceptionCount.sum(), cacheHitCount.sum()); } @Override diff --git a/src/main/java/org/dataloader/stats/StatisticsCollector.java b/src/main/java/org/dataloader/stats/StatisticsCollector.java index 33e417fd..7b14eabb 100644 --- a/src/main/java/org/dataloader/stats/StatisticsCollector.java +++ b/src/main/java/org/dataloader/stats/StatisticsCollector.java @@ -18,21 +18,18 @@ public interface StatisticsCollector { * * @param the class of the key in the data loader * @param context the context containing metadata of the data loader invocation - * - * @return the current value after increment */ - default long incrementLoadCount(IncrementLoadCountStatisticsContext context) { - return incrementLoadCount(); + default void incrementLoadCount(IncrementLoadCountStatisticsContext context) { + incrementLoadCount(); } /** * Called to increment the number of loads * * @deprecated use {@link #incrementLoadCount(IncrementLoadCountStatisticsContext)} - * @return the current value after increment */ @Deprecated - long incrementLoadCount(); + void incrementLoadCount(); /** * Called to increment the number of loads that resulted in an object deemed in error @@ -40,20 +37,18 @@ default long incrementLoadCount(IncrementLoadCountStatisticsContext conte * @param the class of the key in the data loader * @param context the context containing metadata of the data loader invocation * - * @return the current value after increment */ - default long incrementLoadErrorCount(IncrementLoadErrorCountStatisticsContext context) { - return incrementLoadErrorCount(); + default void incrementLoadErrorCount(IncrementLoadErrorCountStatisticsContext context) { + incrementLoadErrorCount(); } /** * Called to increment the number of loads that resulted in an object deemed in error * * @deprecated use {@link #incrementLoadErrorCount(IncrementLoadErrorCountStatisticsContext)} - * @return the current value after increment */ @Deprecated - long incrementLoadErrorCount(); + void incrementLoadErrorCount(); /** * Called to increment the number of batch loads @@ -61,11 +56,9 @@ default long incrementLoadErrorCount(IncrementLoadErrorCountStatisticsContex * @param the class of the key in the data loader * @param delta how much to add to the count * @param context the context containing metadata of the data loader invocation - * - * @return the current value after increment */ - default long incrementBatchLoadCountBy(long delta, IncrementBatchLoadCountByStatisticsContext context) { - return incrementBatchLoadCountBy(delta); + default void incrementBatchLoadCountBy(long delta, IncrementBatchLoadCountByStatisticsContext context) { + incrementBatchLoadCountBy(delta); } /** @@ -74,52 +67,45 @@ default long incrementBatchLoadCountBy(long delta, IncrementBatchLoadCountBy * @param delta how much to add to the count * * @deprecated use {@link #incrementBatchLoadCountBy(long, IncrementBatchLoadCountByStatisticsContext)} - * @return the current value after increment */ @Deprecated - long incrementBatchLoadCountBy(long delta); + void incrementBatchLoadCountBy(long delta); /** * Called to increment the number of batch loads exceptions * * @param the class of the key in the data loader * @param context the context containing metadata of the data loader invocation - * - * @return the current value after increment */ - default long incrementBatchLoadExceptionCount(IncrementBatchLoadExceptionCountStatisticsContext context) { - return incrementBatchLoadExceptionCount(); + default void incrementBatchLoadExceptionCount(IncrementBatchLoadExceptionCountStatisticsContext context) { + incrementBatchLoadExceptionCount(); } /** * Called to increment the number of batch loads exceptions * * @deprecated use {@link #incrementBatchLoadExceptionCount(IncrementBatchLoadExceptionCountStatisticsContext)} - * @return the current value after increment */ @Deprecated - long incrementBatchLoadExceptionCount(); + void incrementBatchLoadExceptionCount(); /** * Called to increment the number of cache hits * * @param the class of the key in the data loader * @param context the context containing metadata of the data loader invocation - * - * @return the current value after increment */ - default long incrementCacheHitCount(IncrementCacheHitCountStatisticsContext context) { - return incrementCacheHitCount(); + default void incrementCacheHitCount(IncrementCacheHitCountStatisticsContext context) { + incrementCacheHitCount(); } /** * Called to increment the number of cache hits * * @deprecated use {@link #incrementCacheHitCount(IncrementCacheHitCountStatisticsContext)} - * @return the current value after increment */ @Deprecated - long incrementCacheHitCount(); + void incrementCacheHitCount(); /** * @return the statistics that have been gathered to this point in time diff --git a/src/main/java/org/dataloader/stats/ThreadLocalStatisticsCollector.java b/src/main/java/org/dataloader/stats/ThreadLocalStatisticsCollector.java index d091c5a4..cab6d0df 100644 --- a/src/main/java/org/dataloader/stats/ThreadLocalStatisticsCollector.java +++ b/src/main/java/org/dataloader/stats/ThreadLocalStatisticsCollector.java @@ -35,63 +35,63 @@ public ThreadLocalStatisticsCollector resetThread() { } @Override - public long incrementLoadCount(IncrementLoadCountStatisticsContext context) { + public void incrementLoadCount(IncrementLoadCountStatisticsContext context) { overallCollector.incrementLoadCount(context); - return collector.get().incrementLoadCount(context); + collector.get().incrementLoadCount(context); } @Deprecated @Override - public long incrementLoadCount() { - return incrementLoadCount(null); + public void incrementLoadCount() { + incrementLoadCount(null); } @Override - public long incrementLoadErrorCount(IncrementLoadErrorCountStatisticsContext context) { + public void incrementLoadErrorCount(IncrementLoadErrorCountStatisticsContext context) { overallCollector.incrementLoadErrorCount(context); - return collector.get().incrementLoadErrorCount(context); + collector.get().incrementLoadErrorCount(context); } @Deprecated @Override - public long incrementLoadErrorCount() { - return incrementLoadErrorCount(null); + public void incrementLoadErrorCount() { + incrementLoadErrorCount(null); } @Override - public long incrementBatchLoadCountBy(long delta, IncrementBatchLoadCountByStatisticsContext context) { + public void incrementBatchLoadCountBy(long delta, IncrementBatchLoadCountByStatisticsContext context) { overallCollector.incrementBatchLoadCountBy(delta, context); - return collector.get().incrementBatchLoadCountBy(delta, context); + collector.get().incrementBatchLoadCountBy(delta, context); } @Deprecated @Override - public long incrementBatchLoadCountBy(long delta) { - return incrementBatchLoadCountBy(delta, null); + public void incrementBatchLoadCountBy(long delta) { + incrementBatchLoadCountBy(delta, null); } @Override - public long incrementBatchLoadExceptionCount(IncrementBatchLoadExceptionCountStatisticsContext context) { + public void incrementBatchLoadExceptionCount(IncrementBatchLoadExceptionCountStatisticsContext context) { overallCollector.incrementBatchLoadExceptionCount(context); - return collector.get().incrementBatchLoadExceptionCount(context); + collector.get().incrementBatchLoadExceptionCount(context); } @Deprecated @Override - public long incrementBatchLoadExceptionCount() { - return incrementBatchLoadExceptionCount(null); + public void incrementBatchLoadExceptionCount() { + incrementBatchLoadExceptionCount(null); } @Override - public long incrementCacheHitCount(IncrementCacheHitCountStatisticsContext context) { + public void incrementCacheHitCount(IncrementCacheHitCountStatisticsContext context) { overallCollector.incrementCacheHitCount(context); - return collector.get().incrementCacheHitCount(context); + collector.get().incrementCacheHitCount(context); } @Deprecated @Override - public long incrementCacheHitCount() { - return incrementCacheHitCount(null); + public void incrementCacheHitCount() { + incrementCacheHitCount(null); } /** diff --git a/src/test/java/org/dataloader/DataLoaderStatsTest.java b/src/test/java/org/dataloader/DataLoaderStatsTest.java index b8393e63..87814d19 100644 --- a/src/test/java/org/dataloader/DataLoaderStatsTest.java +++ b/src/test/java/org/dataloader/DataLoaderStatsTest.java @@ -221,63 +221,55 @@ private static class ContextPassingStatisticsCollector implements StatisticsColl public List> incrementCacheHitCountStatisticsContexts = new ArrayList<>(); @Override - public long incrementLoadCount(IncrementLoadCountStatisticsContext context) { + public void incrementLoadCount(IncrementLoadCountStatisticsContext context) { incrementLoadCountStatisticsContexts.add(context); - return 0; } @Deprecated @Override - public long incrementLoadCount() { - return 0; + public void incrementLoadCount() { + } @Override - public long incrementLoadErrorCount(IncrementLoadErrorCountStatisticsContext context) { + public void incrementLoadErrorCount(IncrementLoadErrorCountStatisticsContext context) { incrementLoadErrorCountStatisticsContexts.add(context); - return 0; } @Deprecated @Override - public long incrementLoadErrorCount() { - return 0; + public void incrementLoadErrorCount() { + } @Override - public long incrementBatchLoadCountBy(long delta, IncrementBatchLoadCountByStatisticsContext context) { + public void incrementBatchLoadCountBy(long delta, IncrementBatchLoadCountByStatisticsContext context) { incrementBatchLoadCountByStatisticsContexts.add(context); - return 0; } @Deprecated @Override - public long incrementBatchLoadCountBy(long delta) { - return 0; + public void incrementBatchLoadCountBy(long delta) { } @Override - public long incrementBatchLoadExceptionCount(IncrementBatchLoadExceptionCountStatisticsContext context) { + public void incrementBatchLoadExceptionCount(IncrementBatchLoadExceptionCountStatisticsContext context) { incrementBatchLoadExceptionCountStatisticsContexts.add(context); - return 0; } @Deprecated @Override - public long incrementBatchLoadExceptionCount() { - return 0; + public void incrementBatchLoadExceptionCount() { } @Override - public long incrementCacheHitCount(IncrementCacheHitCountStatisticsContext context) { + public void incrementCacheHitCount(IncrementCacheHitCountStatisticsContext context) { incrementCacheHitCountStatisticsContexts.add(context); - return 0; } @Deprecated @Override - public long incrementCacheHitCount() { - return 0; + public void incrementCacheHitCount() { } @Override diff --git a/src/test/java/org/dataloader/performance/AtomicVsAdder.java b/src/test/java/org/dataloader/performance/AtomicVsAdder.java new file mode 100644 index 00000000..ee5e02a3 --- /dev/null +++ b/src/test/java/org/dataloader/performance/AtomicVsAdder.java @@ -0,0 +1,111 @@ +package org.dataloader.performance; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.LongAdder; + +public class AtomicVsAdder { + + private static final ExecutorService EXECUTOR = Executors.newCachedThreadPool(); + + public static void main(final String[] args) throws Exception { + // knobs + final var iterationsList = List.of(1 << 20L, 1 << 24L); + final var numberOfThreadsList = List.of(1, 2, 4, 8, 16); + final var strategies = List.of(new LongAdderStrategy(), new AtomicLongStrategy()); + + // test + System.out.println("testing with #cpu=" + Runtime.getRuntime().availableProcessors()); + for (int iterations : iterationsList) { + for (int numberOfThreads : numberOfThreadsList) { + for (Strategy strategy : strategies) { + performTest(iterations, numberOfThreads, strategy); + } + } + } + + EXECUTOR.shutdownNow(); + + } + + private static void performTest(final long iterations, final int numberOfThreads, Strategy strategy) throws Exception { + final List> futures = new ArrayList<>(); + System.out.println("start test with " + iterations + " iterations using " + numberOfThreads + " threads and strategy " + strategy.getClass().getSimpleName()); + final long start = System.nanoTime(); + + for (int i = 0; i < numberOfThreads; i++) { + Future submit = EXECUTOR.submit(() -> concurrentWork(strategy, iterations)); + futures.add(submit); + } + for (final Future future : futures) { + future.get(); // wait for all + } + final long end = System.nanoTime(); + System.out.println("done in " + Duration.ofNanos(end - start).toMillis() + "ms => result " + strategy.get()); + System.out.println("----"); + strategy.reset(); + } + + @SuppressWarnings("SameParameterValue") + private static void concurrentWork(final Strategy strategy, final long iterations) { + long work = iterations; + while (work-- > 0) { + strategy.increment(); + } + } + + interface Strategy { + void increment(); + + long get(); + + void reset(); + } + + static class LongAdderStrategy implements Strategy { + + private LongAdder longAdder = new LongAdder(); + + @Override + public void increment() { + longAdder.increment(); + } + + @Override + public long get() { + return longAdder.sum(); + } + + @Override + public void reset() { + longAdder = new LongAdder(); + } + } + + static class AtomicLongStrategy implements Strategy { + + private final AtomicLong atomicLong = new AtomicLong(0); + + @Override + public void increment() { + atomicLong.incrementAndGet(); + } + + @Override + public long get() { + return atomicLong.get(); + } + + @Override + public void reset() { + atomicLong.set(0); + } + } + +} \ No newline at end of file From 8f46b37235a57344420c7ca501a1b583aedfe0e2 Mon Sep 17 00:00:00 2001 From: Andreas Marek Date: Tue, 15 Apr 2025 09:51:46 +1000 Subject: [PATCH 086/156] add jmh testing with one initial performance test --- build.gradle | 1 + .../DataLoaderDispatchPerformance.java | 309 ++++++++++++++++++ .../performance/PerformanceTestingUtils.java | 84 +++++ 3 files changed, 394 insertions(+) create mode 100644 src/jmh/java/performance/DataLoaderDispatchPerformance.java create mode 100644 src/jmh/java/performance/PerformanceTestingUtils.java diff --git a/build.gradle b/build.gradle index 7d98bc38..d7bef7af 100644 --- a/build.gradle +++ b/build.gradle @@ -10,6 +10,7 @@ plugins { id 'biz.aQute.bnd.builder' version '6.2.0' id 'io.github.gradle-nexus.publish-plugin' version '1.0.0' id 'com.github.ben-manes.versions' version '0.51.0' + id "me.champeau.jmh" version "0.7.3" } java { diff --git a/src/jmh/java/performance/DataLoaderDispatchPerformance.java b/src/jmh/java/performance/DataLoaderDispatchPerformance.java new file mode 100644 index 00000000..0b4696d5 --- /dev/null +++ b/src/jmh/java/performance/DataLoaderDispatchPerformance.java @@ -0,0 +1,309 @@ +package performance; + +import org.dataloader.BatchLoader; +import org.dataloader.DataLoader; +import org.dataloader.DataLoaderFactory; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +@State(Scope.Benchmark) +@Warmup(iterations = 2, time = 5) +@Measurement(iterations = 4) +@Fork(1) +public class DataLoaderDispatchPerformance { + + static Owner o1 = new Owner("O-1", "Andi", List.of("P-1", "P-2", "P-3")); + static Owner o2 = new Owner("O-2", "George", List.of("P-4", "P-5", "P-6")); + static Owner o3 = new Owner("O-3", "Peppa", List.of("P-7", "P-8", "P-9", "P-10")); + static Owner o4 = new Owner("O-4", "Alice", List.of("P-11", "P-12")); + static Owner o5 = new Owner("O-5", "Bob", List.of("P-13")); + static Owner o6 = new Owner("O-6", "Catherine", List.of("P-14", "P-15", "P-16")); + static Owner o7 = new Owner("O-7", "David", List.of("P-17")); + static Owner o8 = new Owner("O-8", "Emma", List.of("P-18", "P-19", "P-20", "P-21")); + static Owner o9 = new Owner("O-9", "Frank", List.of("P-22")); + static Owner o10 = new Owner("O-10", "Grace", List.of("P-23", "P-24")); + static Owner o11 = new Owner("O-11", "Hannah", List.of("P-25", "P-26", "P-27")); + static Owner o12 = new Owner("O-12", "Ian", List.of("P-28")); + static Owner o13 = new Owner("O-13", "Jane", List.of("P-29", "P-30")); + static Owner o14 = new Owner("O-14", "Kevin", List.of("P-31", "P-32", "P-33")); + static Owner o15 = new Owner("O-15", "Laura", List.of("P-34")); + static Owner o16 = new Owner("O-16", "Michael", List.of("P-35", "P-36")); + static Owner o17 = new Owner("O-17", "Nina", List.of("P-37", "P-38", "P-39", "P-40")); + static Owner o18 = new Owner("O-18", "Oliver", List.of("P-41")); + static Owner o19 = new Owner("O-19", "Paula", List.of("P-42", "P-43")); + static Owner o20 = new Owner("O-20", "Quinn", List.of("P-44", "P-45", "P-46")); + static Owner o21 = new Owner("O-21", "Rachel", List.of("P-47")); + static Owner o22 = new Owner("O-22", "Steve", List.of("P-48", "P-49")); + static Owner o23 = new Owner("O-23", "Tina", List.of("P-50", "P-51", "P-52")); + static Owner o24 = new Owner("O-24", "Uma", List.of("P-53")); + static Owner o25 = new Owner("O-25", "Victor", List.of("P-54", "P-55")); + static Owner o26 = new Owner("O-26", "Wendy", List.of("P-56", "P-57", "P-58")); + static Owner o27 = new Owner("O-27", "Xander", List.of("P-59")); + static Owner o28 = new Owner("O-28", "Yvonne", List.of("P-60", "P-61")); + static Owner o29 = new Owner("O-29", "Zach", List.of("P-62", "P-63", "P-64")); + static Owner o30 = new Owner("O-30", "Willy", List.of("P-65", "P-66", "P-67")); + + + static Pet p1 = new Pet("P-1", "Bella", "O-1", List.of("P-2", "P-3", "P-4")); + static Pet p2 = new Pet("P-2", "Charlie", "O-2", List.of("P-1", "P-5", "P-6")); + static Pet p3 = new Pet("P-3", "Luna", "O-3", List.of("P-1", "P-2", "P-7", "P-8")); + static Pet p4 = new Pet("P-4", "Max", "O-1", List.of("P-1", "P-9", "P-10")); + static Pet p5 = new Pet("P-5", "Lucy", "O-2", List.of("P-2", "P-6")); + static Pet p6 = new Pet("P-6", "Cooper", "O-3", List.of("P-3", "P-5", "P-7")); + static Pet p7 = new Pet("P-7", "Daisy", "O-1", List.of("P-4", "P-6", "P-8")); + static Pet p8 = new Pet("P-8", "Milo", "O-2", List.of("P-3", "P-7", "P-9")); + static Pet p9 = new Pet("P-9", "Lola", "O-3", List.of("P-4", "P-8", "P-10")); + static Pet p10 = new Pet("P-10", "Rocky", "O-1", List.of("P-4", "P-9")); + static Pet p11 = new Pet("P-11", "Buddy", "O-4", List.of("P-12")); + static Pet p12 = new Pet("P-12", "Bailey", "O-4", List.of("P-11", "P-13")); + static Pet p13 = new Pet("P-13", "Sadie", "O-5", List.of("P-12")); + static Pet p14 = new Pet("P-14", "Maggie", "O-6", List.of("P-15")); + static Pet p15 = new Pet("P-15", "Sophie", "O-6", List.of("P-14", "P-16")); + static Pet p16 = new Pet("P-16", "Chloe", "O-6", List.of("P-15")); + static Pet p17 = new Pet("P-17", "Duke", "O-7", List.of("P-18")); + static Pet p18 = new Pet("P-18", "Riley", "O-8", List.of("P-17", "P-19")); + static Pet p19 = new Pet("P-19", "Lilly", "O-8", List.of("P-18", "P-20")); + static Pet p20 = new Pet("P-20", "Zoey", "O-8", List.of("P-19")); + static Pet p21 = new Pet("P-21", "Oscar", "O-8", List.of("P-22")); + static Pet p22 = new Pet("P-22", "Toby", "O-9", List.of("P-21", "P-23")); + static Pet p23 = new Pet("P-23", "Ruby", "O-10", List.of("P-22")); + static Pet p24 = new Pet("P-24", "Milo", "O-10", List.of("P-25")); + static Pet p25 = new Pet("P-25", "Finn", "O-11", List.of("P-24", "P-26")); + static Pet p26 = new Pet("P-26", "Luna", "O-11", List.of("P-25")); + static Pet p27 = new Pet("P-27", "Ellie", "O-11", List.of("P-28")); + static Pet p28 = new Pet("P-28", "Harley", "O-12", List.of("P-27", "P-29")); + static Pet p29 = new Pet("P-29", "Penny", "O-13", List.of("P-28")); + static Pet p30 = new Pet("P-30", "Hazel", "O-13", List.of("P-31")); + static Pet p31 = new Pet("P-31", "Gus", "O-14", List.of("P-30", "P-32")); + static Pet p32 = new Pet("P-32", "Dexter", "O-14", List.of("P-31")); + static Pet p33 = new Pet("P-33", "Winnie", "O-14", List.of("P-34")); + static Pet p34 = new Pet("P-34", "Murphy", "O-15", List.of("P-33", "P-35")); + static Pet p35 = new Pet("P-35", "Moose", "O-16", List.of("P-34")); + static Pet p36 = new Pet("P-36", "Scout", "O-16", List.of("P-37")); + static Pet p37 = new Pet("P-37", "Rex", "O-17", List.of("P-36", "P-38")); + static Pet p38 = new Pet("P-38", "Coco", "O-17", List.of("P-37")); + static Pet p39 = new Pet("P-39", "Maddie", "O-17", List.of("P-40")); + static Pet p40 = new Pet("P-40", "Archie", "O-17", List.of("P-39", "P-41")); + static Pet p41 = new Pet("P-41", "Buster", "O-18", List.of("P-40")); + static Pet p42 = new Pet("P-42", "Rosie", "O-19", List.of("P-43")); + static Pet p43 = new Pet("P-43", "Molly", "O-19", List.of("P-42", "P-44")); + static Pet p44 = new Pet("P-44", "Henry", "O-20", List.of("P-43")); + static Pet p45 = new Pet("P-45", "Leo", "O-20", List.of("P-46")); + static Pet p46 = new Pet("P-46", "Jack", "O-20", List.of("P-45", "P-47")); + static Pet p47 = new Pet("P-47", "Zoe", "O-21", List.of("P-46")); + static Pet p48 = new Pet("P-48", "Lulu", "O-22", List.of("P-49")); + static Pet p49 = new Pet("P-49", "Mimi", "O-22", List.of("P-48", "P-50")); + static Pet p50 = new Pet("P-50", "Nala", "O-23", List.of("P-49")); + static Pet p51 = new Pet("P-51", "Simba", "O-23", List.of("P-52")); + static Pet p52 = new Pet("P-52", "Teddy", "O-23", List.of("P-51", "P-53")); + static Pet p53 = new Pet("P-53", "Mochi", "O-24", List.of("P-52")); + static Pet p54 = new Pet("P-54", "Oreo", "O-25", List.of("P-55")); + static Pet p55 = new Pet("P-55", "Peanut", "O-25", List.of("P-54", "P-56")); + static Pet p56 = new Pet("P-56", "Pumpkin", "O-26", List.of("P-55")); + static Pet p57 = new Pet("P-57", "Shadow", "O-26", List.of("P-58")); + static Pet p58 = new Pet("P-58", "Sunny", "O-26", List.of("P-57", "P-59")); + static Pet p59 = new Pet("P-59", "Thor", "O-27", List.of("P-58")); + static Pet p60 = new Pet("P-60", "Willow", "O-28", List.of("P-61")); + static Pet p61 = new Pet("P-61", "Zeus", "O-28", List.of("P-60", "P-62")); + static Pet p62 = new Pet("P-62", "Ace", "O-29", List.of("P-61")); + static Pet p63 = new Pet("P-63", "Blue", "O-29", List.of("P-64")); + static Pet p64 = new Pet("P-64", "Cleo", "O-29", List.of("P-63", "P-65")); + static Pet p65 = new Pet("P-65", "Dolly", "O-30", List.of("P-64")); + static Pet p66 = new Pet("P-66", "Ella", "O-30", List.of("P-67")); + static Pet p67 = new Pet("P-67", "Freddy", "O-30", List.of("P-66")); + + + static Map owners = Map.ofEntries( + Map.entry(o1.id, o1), + Map.entry(o2.id, o2), + Map.entry(o3.id, o3), + Map.entry(o4.id, o4), + Map.entry(o5.id, o5), + Map.entry(o6.id, o6), + Map.entry(o7.id, o7), + Map.entry(o8.id, o8), + Map.entry(o9.id, o9), + Map.entry(o10.id, o10), + Map.entry(o11.id, o11), + Map.entry(o12.id, o12), + Map.entry(o13.id, o13), + Map.entry(o14.id, o14), + Map.entry(o15.id, o15), + Map.entry(o16.id, o16), + Map.entry(o17.id, o17), + Map.entry(o18.id, o18), + Map.entry(o19.id, o19), + Map.entry(o20.id, o20), + Map.entry(o21.id, o21), + Map.entry(o22.id, o22), + Map.entry(o23.id, o23), + Map.entry(o24.id, o24), + Map.entry(o25.id, o25), + Map.entry(o26.id, o26), + Map.entry(o27.id, o27), + Map.entry(o28.id, o28), + Map.entry(o29.id, o29), + Map.entry(o30.id, o30) + ); + static Map pets = Map.ofEntries( + Map.entry(p1.id, p1), + Map.entry(p2.id, p2), + Map.entry(p3.id, p3), + Map.entry(p4.id, p4), + Map.entry(p5.id, p5), + Map.entry(p6.id, p6), + Map.entry(p7.id, p7), + Map.entry(p8.id, p8), + Map.entry(p9.id, p9), + Map.entry(p10.id, p10), + Map.entry(p11.id, p11), + Map.entry(p12.id, p12), + Map.entry(p13.id, p13), + Map.entry(p14.id, p14), + Map.entry(p15.id, p15), + Map.entry(p16.id, p16), + Map.entry(p17.id, p17), + Map.entry(p18.id, p18), + Map.entry(p19.id, p19), + Map.entry(p20.id, p20), + Map.entry(p21.id, p21), + Map.entry(p22.id, p22), + Map.entry(p23.id, p23), + Map.entry(p24.id, p24), + Map.entry(p25.id, p25), + Map.entry(p26.id, p26), + Map.entry(p27.id, p27), + Map.entry(p28.id, p28), + Map.entry(p29.id, p29), + Map.entry(p30.id, p30), + Map.entry(p31.id, p31), + Map.entry(p32.id, p32), + Map.entry(p33.id, p33), + Map.entry(p34.id, p34), + Map.entry(p35.id, p35), + Map.entry(p36.id, p36), + Map.entry(p37.id, p37), + Map.entry(p38.id, p38), + Map.entry(p39.id, p39), + Map.entry(p40.id, p40), + Map.entry(p41.id, p41), + Map.entry(p42.id, p42), + Map.entry(p43.id, p43), + Map.entry(p44.id, p44), + Map.entry(p45.id, p45), + Map.entry(p46.id, p46), + Map.entry(p47.id, p47), + Map.entry(p48.id, p48), + Map.entry(p49.id, p49), + Map.entry(p50.id, p50), + Map.entry(p51.id, p51), + Map.entry(p52.id, p52), + Map.entry(p53.id, p53), + Map.entry(p54.id, p54), + Map.entry(p55.id, p55), + Map.entry(p56.id, p56), + Map.entry(p57.id, p57), + Map.entry(p58.id, p58), + Map.entry(p59.id, p59), + Map.entry(p60.id, p60), + Map.entry(p61.id, p61), + Map.entry(p62.id, p62), + Map.entry(p63.id, p63), + Map.entry(p64.id, p64), + Map.entry(p65.id, p65), + Map.entry(p66.id, p66), + Map.entry(p67.id, p67) + ); + + static class Owner { + public Owner(String id, String name, List petIds) { + this.id = id; + this.name = name; + this.petIds = petIds; + } + + String id; + String name; + List petIds; + } + + static class Pet { + public Pet(String id, String name, String ownerId, List friendsIds) { + this.id = id; + this.name = name; + this.ownerId = ownerId; + this.friendsIds = friendsIds; + } + + String id; + String name; + String ownerId; + List friendsIds; + } + + + static BatchLoader ownerBatchLoader = list -> { + List collect = list.stream().map(key -> { + Owner owner = owners.get(key); + return owner; + }).collect(Collectors.toList()); + return CompletableFuture.completedFuture(collect); + }; + static BatchLoader petBatchLoader = list -> { + List collect = list.stream().map(key -> { + Pet owner = pets.get(key); + return owner; + }).collect(Collectors.toList()); + return CompletableFuture.completedFuture(collect); + }; + + + @State(Scope.Benchmark) + public static class MyState { + @Setup + public void setup() { + + } + + } + + + @Benchmark + @BenchmarkMode(Mode.AverageTime) + @OutputTimeUnit(TimeUnit.NANOSECONDS) + public void loadAndDispatch(MyState myState, Blackhole blackhole) { + DataLoader ownerDL = DataLoaderFactory.newDataLoader(ownerBatchLoader); + DataLoader petDL = DataLoaderFactory.newDataLoader(petBatchLoader); + + for (Owner owner : owners.values()) { + ownerDL.load(owner.id); + for (String petId : owner.petIds) { + petDL.load(petId); + for (String friendId : pets.get(petId).friendsIds) { + petDL.load(friendId); + } + } + } + + CompletableFuture cf1 = ownerDL.dispatch(); + CompletableFuture cf2 = petDL.dispatch(); + blackhole.consume(CompletableFuture.allOf(cf1, cf2).join()); + } + + +} diff --git a/src/jmh/java/performance/PerformanceTestingUtils.java b/src/jmh/java/performance/PerformanceTestingUtils.java new file mode 100644 index 00000000..9e05fd66 --- /dev/null +++ b/src/jmh/java/performance/PerformanceTestingUtils.java @@ -0,0 +1,84 @@ +package performance; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.URL; +import java.nio.charset.Charset; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.concurrent.Callable; + +public class PerformanceTestingUtils { + + @SuppressWarnings("UnstableApiUsage") + static String loadResource(String name) { + return asRTE(() -> { + URL resource = PerformanceTestingUtils.class.getClassLoader().getResource(name); + if (resource == null) { + throw new IllegalArgumentException("missing resource: " + name); + } + byte[] bytes; + try (InputStream inputStream = resource.openStream()) { + bytes = inputStream.readAllBytes(); + } + return new String(bytes, Charset.defaultCharset()); + }); + } + + static T asRTE(Callable callable) { + try { + return callable.call(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + public static void runInToolingForSomeTimeThenExit(Runnable setup, Runnable r, Runnable tearDown) { + int runForMillis = getRunForMillis(); + if (runForMillis <= 0) { + System.out.print("'runForMillis' environment var is not set - continuing \n"); + return; + } + System.out.printf("Running initial code in some tooling - runForMillis=%d \n", runForMillis); + System.out.print("Get your tooling in order and press enter..."); + readLine(); + System.out.print("Lets go...\n"); + setup.run(); + + DateTimeFormatter dtf = DateTimeFormatter.ofPattern("HH:mm:ss"); + long now, then = System.currentTimeMillis(); + do { + now = System.currentTimeMillis(); + long msLeft = runForMillis - (now - then); + System.out.printf("\t%s Running in loop... %s ms left\n", dtf.format(LocalDateTime.now()), msLeft); + r.run(); + now = System.currentTimeMillis(); + } while ((now - then) < runForMillis); + + tearDown.run(); + + System.out.printf("This ran for %d millis. Exiting...\n", System.currentTimeMillis() - then); + System.exit(0); + } + + private static void readLine() { + BufferedReader br = new BufferedReader(new InputStreamReader(System.in)); + try { + br.readLine(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private static int getRunForMillis() { + String runFor = System.getenv("runForMillis"); + try { + return Integer.parseInt(runFor); + } catch (NumberFormatException e) { + return -1; + } + } + +} From 4569c4324a022ab6c220ee6cf6e045fc9d0c2905 Mon Sep 17 00:00:00 2001 From: bbaker Date: Thu, 1 May 2025 12:48:45 +1000 Subject: [PATCH 087/156] Breaking change - renaming old mutable setXX methods --- README.md | 12 ++++---- .../org/dataloader/DataLoaderOptions.java | 24 ++++++++-------- src/test/java/ReadmeExamples.java | 12 ++++---- .../DataLoaderBatchLoaderEnvironmentTest.java | 16 +++++------ .../org/dataloader/DataLoaderBuilderTest.java | 2 +- .../org/dataloader/DataLoaderOptionsTest.java | 20 ++++++------- .../dataloader/DataLoaderRegistryTest.java | 6 ++-- .../org/dataloader/DataLoaderStatsTest.java | 10 +++---- .../java/org/dataloader/DataLoaderTest.java | 28 +++++++++---------- .../dataloader/DataLoaderValueCacheTest.java | 20 ++++++------- .../DataLoaderInstrumentationTest.java | 10 +++---- ...DataLoaderRegistryInstrumentationTest.java | 10 +++---- .../scheduler/BatchLoaderSchedulerTest.java | 12 ++++---- 13 files changed, 91 insertions(+), 91 deletions(-) diff --git a/README.md b/README.md index c7c6fe99..a5835dee 100644 --- a/README.md +++ b/README.md @@ -205,7 +205,7 @@ for the context object. ```java DataLoaderOptions options = DataLoaderOptions.newOptions() - .setBatchLoaderContextProvider(() -> SecurityCtx.getCallingUserCtx()); + .withBatchLoaderContextProvider(() -> SecurityCtx.getCallingUserCtx()); BatchLoaderWithContext batchLoader = new BatchLoaderWithContext() { @Override @@ -227,7 +227,7 @@ You can gain access to them as a map by key or as the original list of context o ```java DataLoaderOptions options = DataLoaderOptions.newOptions() - .setBatchLoaderContextProvider(() -> SecurityCtx.getCallingUserCtx()); + .withBatchLoaderContextProvider(() -> SecurityCtx.getCallingUserCtx()); BatchLoaderWithContext batchLoader = new BatchLoaderWithContext() { @Override @@ -433,7 +433,7 @@ However, you can create your own custom future cache and supply it to the data l ```java MyCustomCache customCache = new MyCustomCache(); - DataLoaderOptions options = DataLoaderOptions.newOptions().setCacheMap(customCache); + DataLoaderOptions options = DataLoaderOptions.newOptions().withsetCacheMap(customCache); DataLoaderFactory.newDataLoader(userBatchLoader, options); ``` @@ -467,7 +467,7 @@ The tests have an example based on [Caffeine](https://github.com/ben-manes/caffe In certain uncommon cases, a DataLoader which does not cache may be desirable. ```java - DataLoaderFactory.newDataLoader(userBatchLoader, DataLoaderOptions.newOptions().setCachingEnabled(false)); + DataLoaderFactory.newDataLoader(userBatchLoader, DataLoaderOptions.newOptions().withCachingEnabled(false)); ``` Calling the above will ensure that every call to `.load()` will produce a new promise, and requested keys will not be saved in memory. @@ -533,7 +533,7 @@ Knowing what the behaviour of your data is important for you to understand how e You can configure the statistics collector used when you build the data loader ```java - DataLoaderOptions options = DataLoaderOptions.newOptions().setStatisticsCollector(() -> new ThreadLocalStatisticsCollector()); + DataLoaderOptions options = DataLoaderOptions.newOptions().withStatisticsCollector(() -> new ThreadLocalStatisticsCollector()); DataLoader userDataLoader = DataLoaderFactory.newDataLoader(userBatchLoader,options); ``` @@ -780,7 +780,7 @@ You set the `DataLoaderInstrumentation` into the `DataLoaderOptions` at build ti }); } }; - DataLoaderOptions options = DataLoaderOptions.newOptions().setInstrumentation(timingInstrumentation); + DataLoaderOptions options = DataLoaderOptions.newOptions().withInstrumentation(timingInstrumentation); DataLoader userDataLoader = DataLoaderFactory.newDataLoader(userBatchLoader, options); ``` diff --git a/src/main/java/org/dataloader/DataLoaderOptions.java b/src/main/java/org/dataloader/DataLoaderOptions.java index 86679432..0679fbd6 100644 --- a/src/main/java/org/dataloader/DataLoaderOptions.java +++ b/src/main/java/org/dataloader/DataLoaderOptions.java @@ -177,7 +177,7 @@ public boolean batchingEnabled() { * @param batchingEnabled {@code true} to enable batch loading, {@code false} otherwise * @return a new data loader options instance for fluent coding */ - public DataLoaderOptions setBatchingEnabled(boolean batchingEnabled) { + public DataLoaderOptions withBatchingEnabled(boolean batchingEnabled) { return builder().setBatchingEnabled(batchingEnabled).build(); } @@ -196,7 +196,7 @@ public boolean cachingEnabled() { * @param cachingEnabled {@code true} to enable caching, {@code false} otherwise * @return a new data loader options instance for fluent coding */ - public DataLoaderOptions setCachingEnabled(boolean cachingEnabled) { + public DataLoaderOptions withCachingEnabled(boolean cachingEnabled) { return builder().setCachingEnabled(cachingEnabled).build(); } @@ -220,7 +220,7 @@ public boolean cachingExceptionsEnabled() { * @param cachingExceptionsEnabled {@code true} to enable caching exceptional values, {@code false} otherwise * @return a new data loader options instance for fluent coding */ - public DataLoaderOptions setCachingExceptionsEnabled(boolean cachingExceptionsEnabled) { + public DataLoaderOptions withCachingExceptionsEnabled(boolean cachingExceptionsEnabled) { return builder().setCachingExceptionsEnabled(cachingExceptionsEnabled).build(); } @@ -241,7 +241,7 @@ public Optional cacheKeyFunction() { * @param cacheKeyFunction the cache key function to use * @return a new data loader options instance for fluent coding */ - public DataLoaderOptions setCacheKeyFunction(CacheKey cacheKeyFunction) { + public DataLoaderOptions withCacheKeyFunction(CacheKey cacheKeyFunction) { return builder().setCacheKeyFunction(cacheKeyFunction).build(); } @@ -262,7 +262,7 @@ public DataLoaderOptions setCacheKeyFunction(CacheKey cacheKeyFunction) { * @param cacheMap the cache map instance * @return a new data loader options instance for fluent coding */ - public DataLoaderOptions setCacheMap(CacheMap cacheMap) { + public DataLoaderOptions withCacheMap(CacheMap cacheMap) { return builder().setCacheMap(cacheMap).build(); } @@ -283,7 +283,7 @@ public int maxBatchSize() { * @param maxBatchSize the maximum batch size * @return a new data loader options instance for fluent coding */ - public DataLoaderOptions setMaxBatchSize(int maxBatchSize) { + public DataLoaderOptions withMaxBatchSize(int maxBatchSize) { return builder().setMaxBatchSize(maxBatchSize).build(); } @@ -302,7 +302,7 @@ public StatisticsCollector getStatisticsCollector() { * @param statisticsCollector the statistics collector to use * @return a new data loader options instance for fluent coding */ - public DataLoaderOptions setStatisticsCollector(Supplier statisticsCollector) { + public DataLoaderOptions withStatisticsCollector(Supplier statisticsCollector) { return builder().setStatisticsCollector(nonNull(statisticsCollector)).build(); } @@ -319,7 +319,7 @@ public BatchLoaderContextProvider getBatchLoaderContextProvider() { * @param contextProvider the batch loader context provider * @return a new data loader options instance for fluent coding */ - public DataLoaderOptions setBatchLoaderContextProvider(BatchLoaderContextProvider contextProvider) { + public DataLoaderOptions withBatchLoaderContextProvider(BatchLoaderContextProvider contextProvider) { return builder().setBatchLoaderContextProvider(nonNull(contextProvider)).build(); } @@ -340,7 +340,7 @@ public DataLoaderOptions setBatchLoaderContextProvider(BatchLoaderContextProvide * @param valueCache the value cache instance * @return a new data loader options instance for fluent coding */ - public DataLoaderOptions setValueCache(ValueCache valueCache) { + public DataLoaderOptions withValueCache(ValueCache valueCache) { return builder().setValueCache(valueCache).build(); } @@ -357,7 +357,7 @@ public ValueCacheOptions getValueCacheOptions() { * @param valueCacheOptions the value cache options * @return a new data loader options instance for fluent coding */ - public DataLoaderOptions setValueCacheOptions(ValueCacheOptions valueCacheOptions) { + public DataLoaderOptions withValueCacheOptions(ValueCacheOptions valueCacheOptions) { return builder().setValueCacheOptions(nonNull(valueCacheOptions)).build(); } @@ -375,7 +375,7 @@ public BatchLoaderScheduler getBatchLoaderScheduler() { * @param batchLoaderScheduler the scheduler * @return a new data loader options instance for fluent coding */ - public DataLoaderOptions setBatchLoaderScheduler(BatchLoaderScheduler batchLoaderScheduler) { + public DataLoaderOptions withBatchLoaderScheduler(BatchLoaderScheduler batchLoaderScheduler) { return builder().setBatchLoaderScheduler(batchLoaderScheduler).build(); } @@ -392,7 +392,7 @@ public DataLoaderInstrumentation getInstrumentation() { * @param instrumentation the new {@link DataLoaderInstrumentation} * @return a new data loader options instance for fluent coding */ - public DataLoaderOptions setInstrumentation(DataLoaderInstrumentation instrumentation) { + public DataLoaderOptions withInstrumentation(DataLoaderInstrumentation instrumentation) { return builder().setInstrumentation(instrumentation).build(); } diff --git a/src/test/java/ReadmeExamples.java b/src/test/java/ReadmeExamples.java index 1f718aa5..327073f6 100644 --- a/src/test/java/ReadmeExamples.java +++ b/src/test/java/ReadmeExamples.java @@ -105,7 +105,7 @@ public CompletionStage> load(List userIds) { private void callContextExample() { DataLoaderOptions options = DataLoaderOptions.newOptions() - .setBatchLoaderContextProvider(() -> SecurityCtx.getCallingUserCtx()); + .withBatchLoaderContextProvider(() -> SecurityCtx.getCallingUserCtx()); BatchLoaderWithContext batchLoader = new BatchLoaderWithContext() { @Override @@ -120,7 +120,7 @@ public CompletionStage> load(List keys, BatchLoaderEnvironm private void keyContextExample() { DataLoaderOptions options = DataLoaderOptions.newOptions() - .setBatchLoaderContextProvider(() -> SecurityCtx.getCallingUserCtx()); + .withBatchLoaderContextProvider(() -> SecurityCtx.getCallingUserCtx()); BatchLoaderWithContext batchLoader = new BatchLoaderWithContext() { @Override @@ -236,7 +236,7 @@ private void clearCacheOnError() { BatchLoader teamsBatchLoader; private void disableCache() { - DataLoaderFactory.newDataLoader(userBatchLoader, DataLoaderOptions.newOptions().setCachingEnabled(false)); + DataLoaderFactory.newDataLoader(userBatchLoader, DataLoaderOptions.newOptions().withCachingEnabled(false)); userDataLoader.load("A"); @@ -283,7 +283,7 @@ public CacheMap clear() { private void customCache() { MyCustomCache customCache = new MyCustomCache(); - DataLoaderOptions options = DataLoaderOptions.newOptions().setCacheMap(customCache); + DataLoaderOptions options = DataLoaderOptions.newOptions().withCacheMap(customCache); DataLoaderFactory.newDataLoader(userBatchLoader, options); } @@ -311,7 +311,7 @@ private void statsExample() { private void statsConfigExample() { - DataLoaderOptions options = DataLoaderOptions.newOptions().setStatisticsCollector(() -> new ThreadLocalStatisticsCollector()); + DataLoaderOptions options = DataLoaderOptions.newOptions().withStatisticsCollector(() -> new ThreadLocalStatisticsCollector()); DataLoader userDataLoader = DataLoaderFactory.newDataLoader(userBatchLoader, options); } @@ -410,7 +410,7 @@ public DataLoaderInstrumentationContext> beginBatchLoader(DataLoader userDataLoader = DataLoaderFactory.newDataLoader(userBatchLoader, options); } diff --git a/src/test/java/org/dataloader/DataLoaderBatchLoaderEnvironmentTest.java b/src/test/java/org/dataloader/DataLoaderBatchLoaderEnvironmentTest.java index 90adbc5d..435610fc 100644 --- a/src/test/java/org/dataloader/DataLoaderBatchLoaderEnvironmentTest.java +++ b/src/test/java/org/dataloader/DataLoaderBatchLoaderEnvironmentTest.java @@ -41,7 +41,7 @@ public void context_is_passed_to_batch_loader_function() { return CompletableFuture.completedFuture(list); }; DataLoaderOptions options = DataLoaderOptions.newOptions() - .setBatchLoaderContextProvider(() -> "ctx"); + .withBatchLoaderContextProvider(() -> "ctx"); DataLoader loader = newDataLoader(batchLoader, options); loader.load("A"); @@ -61,7 +61,7 @@ public void context_is_passed_to_batch_loader_function() { public void key_contexts_are_passed_to_batch_loader_function() { BatchLoaderWithContext batchLoader = contextBatchLoader(); DataLoaderOptions options = DataLoaderOptions.newOptions() - .setBatchLoaderContextProvider(() -> "ctx"); + .withBatchLoaderContextProvider(() -> "ctx"); DataLoader loader = newDataLoader(batchLoader, options); loader.load("A", "aCtx"); @@ -81,8 +81,8 @@ public void key_contexts_are_passed_to_batch_loader_function() { public void key_contexts_are_passed_to_batch_loader_function_when_batching_disabled() { BatchLoaderWithContext batchLoader = contextBatchLoader(); DataLoaderOptions options = DataLoaderOptions.newOptions() - .setBatchingEnabled(false) - .setBatchLoaderContextProvider(() -> "ctx"); + .withBatchingEnabled(false) + .withBatchLoaderContextProvider(() -> "ctx"); DataLoader loader = newDataLoader(batchLoader, options); CompletableFuture aLoad = loader.load("A", "aCtx"); @@ -104,7 +104,7 @@ public void key_contexts_are_passed_to_batch_loader_function_when_batching_disab public void missing_key_contexts_are_passed_to_batch_loader_function() { BatchLoaderWithContext batchLoader = contextBatchLoader(); DataLoaderOptions options = DataLoaderOptions.newOptions() - .setBatchLoaderContextProvider(() -> "ctx"); + .withBatchLoaderContextProvider(() -> "ctx"); DataLoader loader = newDataLoader(batchLoader, options); loader.load("A", "aCtx"); @@ -133,7 +133,7 @@ public void context_is_passed_to_map_batch_loader_function() { return CompletableFuture.completedFuture(map); }; DataLoaderOptions options = DataLoaderOptions.newOptions() - .setBatchLoaderContextProvider(() -> "ctx"); + .withBatchLoaderContextProvider(() -> "ctx"); DataLoader loader = newMappedDataLoader(mapBatchLoader, options); loader.load("A", "aCtx"); @@ -199,8 +199,8 @@ public void null_is_passed_as_context_to_map_loader_if_you_do_nothing() { public void mmap_semantics_apply_to_batch_loader_context() { BatchLoaderWithContext batchLoader = contextBatchLoader(); DataLoaderOptions options = DataLoaderOptions.newOptions() - .setBatchLoaderContextProvider(() -> "ctx") - .setCachingEnabled(false); + .withBatchLoaderContextProvider(() -> "ctx") + .withCachingEnabled(false); DataLoader loader = newDataLoader(batchLoader, options); loader.load("A", "aCtx"); diff --git a/src/test/java/org/dataloader/DataLoaderBuilderTest.java b/src/test/java/org/dataloader/DataLoaderBuilderTest.java index f38ff82b..b9014b19 100644 --- a/src/test/java/org/dataloader/DataLoaderBuilderTest.java +++ b/src/test/java/org/dataloader/DataLoaderBuilderTest.java @@ -13,7 +13,7 @@ public class DataLoaderBuilderTest { BatchLoader batchLoader2 = keys -> null; DataLoaderOptions defaultOptions = DataLoaderOptions.newOptions(); - DataLoaderOptions differentOptions = DataLoaderOptions.newOptions().setCachingEnabled(false); + DataLoaderOptions differentOptions = DataLoaderOptions.newOptions().withCachingEnabled(false); @Test void canBuildNewDataLoaders() { diff --git a/src/test/java/org/dataloader/DataLoaderOptionsTest.java b/src/test/java/org/dataloader/DataLoaderOptionsTest.java index b4ebb9e3..c331942a 100644 --- a/src/test/java/org/dataloader/DataLoaderOptionsTest.java +++ b/src/test/java/org/dataloader/DataLoaderOptionsTest.java @@ -89,25 +89,25 @@ public Object getKey(Object input) { @Test void canBuildOk() { - assertThat(optionsDefault.setBatchingEnabled(false).batchingEnabled(), + assertThat(optionsDefault.withBatchingEnabled(false).batchingEnabled(), equalTo(false)); - assertThat(optionsDefault.setBatchLoaderScheduler(testBatchLoaderScheduler).getBatchLoaderScheduler(), + assertThat(optionsDefault.withBatchLoaderScheduler(testBatchLoaderScheduler).getBatchLoaderScheduler(), equalTo(testBatchLoaderScheduler)); - assertThat(optionsDefault.setBatchLoaderContextProvider(testBatchLoaderContextProvider).getBatchLoaderContextProvider(), + assertThat(optionsDefault.withBatchLoaderContextProvider(testBatchLoaderContextProvider).getBatchLoaderContextProvider(), equalTo(testBatchLoaderContextProvider)); - assertThat(optionsDefault.setCacheMap(testCacheMap).cacheMap().get(), + assertThat(optionsDefault.withCacheMap(testCacheMap).cacheMap().get(), equalTo(testCacheMap)); - assertThat(optionsDefault.setCachingEnabled(false).cachingEnabled(), + assertThat(optionsDefault.withCachingEnabled(false).cachingEnabled(), equalTo(false)); - assertThat(optionsDefault.setValueCacheOptions(testValueCacheOptions).getValueCacheOptions(), + assertThat(optionsDefault.withValueCacheOptions(testValueCacheOptions).getValueCacheOptions(), equalTo(testValueCacheOptions)); - assertThat(optionsDefault.setCacheKeyFunction(testCacheKey).cacheKeyFunction().get(), + assertThat(optionsDefault.withCacheKeyFunction(testCacheKey).cacheKeyFunction().get(), equalTo(testCacheKey)); - assertThat(optionsDefault.setValueCache(testValueCache).valueCache().get(), + assertThat(optionsDefault.withValueCache(testValueCache).valueCache().get(), equalTo(testValueCache)); - assertThat(optionsDefault.setMaxBatchSize(10).maxBatchSize(), + assertThat(optionsDefault.withMaxBatchSize(10).maxBatchSize(), equalTo(10)); - assertThat(optionsDefault.setStatisticsCollector(testStatisticsCollectorSupplier).getStatisticsCollector(), + assertThat(optionsDefault.withStatisticsCollector(testStatisticsCollectorSupplier).getStatisticsCollector(), equalTo(testStatisticsCollectorSupplier.get())); DataLoaderOptions builtOptions = optionsDefault.transform(builder -> { diff --git a/src/test/java/org/dataloader/DataLoaderRegistryTest.java b/src/test/java/org/dataloader/DataLoaderRegistryTest.java index bd1534dd..34190eb7 100644 --- a/src/test/java/org/dataloader/DataLoaderRegistryTest.java +++ b/src/test/java/org/dataloader/DataLoaderRegistryTest.java @@ -79,13 +79,13 @@ public void stats_can_be_collected() { DataLoaderRegistry registry = new DataLoaderRegistry(); DataLoader dlA = newDataLoader(identityBatchLoader, - DataLoaderOptions.newOptions().setStatisticsCollector(SimpleStatisticsCollector::new) + DataLoaderOptions.newOptions().withStatisticsCollector(SimpleStatisticsCollector::new) ); DataLoader dlB = newDataLoader(identityBatchLoader, - DataLoaderOptions.newOptions().setStatisticsCollector(SimpleStatisticsCollector::new) + DataLoaderOptions.newOptions().withStatisticsCollector(SimpleStatisticsCollector::new) ); DataLoader dlC = newDataLoader(identityBatchLoader, - DataLoaderOptions.newOptions().setStatisticsCollector(SimpleStatisticsCollector::new) + DataLoaderOptions.newOptions().withStatisticsCollector(SimpleStatisticsCollector::new) ); registry.register("a", dlA).register("b", dlB).register("c", dlC); diff --git a/src/test/java/org/dataloader/DataLoaderStatsTest.java b/src/test/java/org/dataloader/DataLoaderStatsTest.java index b8393e63..a236169e 100644 --- a/src/test/java/org/dataloader/DataLoaderStatsTest.java +++ b/src/test/java/org/dataloader/DataLoaderStatsTest.java @@ -33,7 +33,7 @@ public class DataLoaderStatsTest { public void stats_are_collected_by_default() { BatchLoader batchLoader = CompletableFuture::completedFuture; DataLoader loader = newDataLoader(batchLoader, - DataLoaderOptions.newOptions().setStatisticsCollector(SimpleStatisticsCollector::new) + DataLoaderOptions.newOptions().withStatisticsCollector(SimpleStatisticsCollector::new) ); loader.load("A"); @@ -75,7 +75,7 @@ public void stats_are_collected_with_specified_collector() { collector.incrementBatchLoadCountBy(1, new IncrementBatchLoadCountByStatisticsContext<>(1, null)); BatchLoader batchLoader = CompletableFuture::completedFuture; - DataLoaderOptions loaderOptions = DataLoaderOptions.newOptions().setStatisticsCollector(() -> collector); + DataLoaderOptions loaderOptions = DataLoaderOptions.newOptions().withStatisticsCollector(() -> collector); DataLoader loader = newDataLoader(batchLoader, loaderOptions); loader.load("A"); @@ -113,7 +113,7 @@ public void stats_are_collected_with_caching_disabled() { StatisticsCollector collector = new SimpleStatisticsCollector(); BatchLoader batchLoader = CompletableFuture::completedFuture; - DataLoaderOptions loaderOptions = DataLoaderOptions.newOptions().setStatisticsCollector(() -> collector).setCachingEnabled(false); + DataLoaderOptions loaderOptions = DataLoaderOptions.newOptions().withStatisticsCollector(() -> collector).withCachingEnabled(false); DataLoader loader = newDataLoader(batchLoader, loaderOptions); loader.load("A"); @@ -166,7 +166,7 @@ public void stats_are_collected_with_caching_disabled() { @Test public void stats_are_collected_on_exceptions() { DataLoader loader = DataLoaderFactory.newDataLoaderWithTry(batchLoaderThatBlows, - DataLoaderOptions.newOptions().setStatisticsCollector(SimpleStatisticsCollector::new) + DataLoaderOptions.newOptions().withStatisticsCollector(SimpleStatisticsCollector::new) ); loader.load("A"); @@ -290,7 +290,7 @@ public Statistics getStatistics() { public void context_is_passed_through_to_collector() { ContextPassingStatisticsCollector statisticsCollector = new ContextPassingStatisticsCollector(); DataLoader> loader = newDataLoader(batchLoaderThatBlows, - DataLoaderOptions.newOptions().setStatisticsCollector(() -> statisticsCollector) + DataLoaderOptions.newOptions().withStatisticsCollector(() -> statisticsCollector) ); loader.load("key", "keyContext"); diff --git a/src/test/java/org/dataloader/DataLoaderTest.java b/src/test/java/org/dataloader/DataLoaderTest.java index 069d390a..cb6adad7 100644 --- a/src/test/java/org/dataloader/DataLoaderTest.java +++ b/src/test/java/org/dataloader/DataLoaderTest.java @@ -588,7 +588,7 @@ public void should_Cache_failed_fetches(TestDataLoaderFactory factory) { @ParameterizedTest @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void should_NOT_Cache_failed_fetches_if_told_not_too(TestDataLoaderFactory factory) { - DataLoaderOptions options = DataLoaderOptions.newOptions().setCachingExceptionsEnabled(false); + DataLoaderOptions options = DataLoaderOptions.newOptions().withCachingExceptionsEnabled(false); List> loadCalls = new ArrayList<>(); DataLoader errorLoader = factory.idLoaderAllExceptions(options, loadCalls); @@ -736,7 +736,7 @@ public void should_Accept_objects_as_keys(TestDataLoaderFactory factory) { public void should_Disable_caching(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); DataLoader identityLoader = - factory.idLoader(newOptions().setCachingEnabled(false), loadCalls); + factory.idLoader(newOptions().withCachingEnabled(false), loadCalls); CompletableFuture future1 = identityLoader.load("A"); CompletableFuture future2 = identityLoader.load("B"); @@ -774,7 +774,7 @@ public void should_Disable_caching(TestDataLoaderFactory factory) throws Executi public void should_work_with_duplicate_keys_when_caching_disabled(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); DataLoader identityLoader = - factory.idLoader(newOptions().setCachingEnabled(false), loadCalls); + factory.idLoader(newOptions().withCachingEnabled(false), loadCalls); CompletableFuture future1 = identityLoader.load("A"); CompletableFuture future2 = identityLoader.load("B"); @@ -797,7 +797,7 @@ public void should_work_with_duplicate_keys_when_caching_disabled(TestDataLoader public void should_work_with_duplicate_keys_when_caching_enabled(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); DataLoader identityLoader = - factory.idLoader(newOptions().setCachingEnabled(true), loadCalls); + factory.idLoader(newOptions().withCachingEnabled(true), loadCalls); CompletableFuture future1 = identityLoader.load("A"); CompletableFuture future2 = identityLoader.load("B"); @@ -817,7 +817,7 @@ public void should_work_with_duplicate_keys_when_caching_enabled(TestDataLoaderF @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void should_Accept_objects_with_a_complex_key(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().setCacheKeyFunction(getJsonObjectCacheMapFn()); + DataLoaderOptions options = newOptions().withCacheKeyFunction(getJsonObjectCacheMapFn()); DataLoader identityLoader = factory.idLoader(options, loadCalls); JsonObject key1 = new JsonObject().put("id", 123); @@ -839,7 +839,7 @@ public void should_Accept_objects_with_a_complex_key(TestDataLoaderFactory facto @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void should_Clear_objects_with_complex_key(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().setCacheKeyFunction(getJsonObjectCacheMapFn()); + DataLoaderOptions options = newOptions().withCacheKeyFunction(getJsonObjectCacheMapFn()); DataLoader identityLoader = factory.idLoader(options, loadCalls); JsonObject key1 = new JsonObject().put("id", 123); @@ -864,7 +864,7 @@ public void should_Clear_objects_with_complex_key(TestDataLoaderFactory factory) @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void should_Accept_objects_with_different_order_of_keys(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().setCacheKeyFunction(getJsonObjectCacheMapFn()); + DataLoaderOptions options = newOptions().withCacheKeyFunction(getJsonObjectCacheMapFn()); DataLoader identityLoader = factory.idLoader(options, loadCalls); JsonObject key1 = new JsonObject().put("a", 123).put("b", 321); @@ -887,7 +887,7 @@ public void should_Accept_objects_with_different_order_of_keys(TestDataLoaderFac @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void should_Allow_priming_the_cache_with_an_object_key(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().setCacheKeyFunction(getJsonObjectCacheMapFn()); + DataLoaderOptions options = newOptions().withCacheKeyFunction(getJsonObjectCacheMapFn()); DataLoader identityLoader = factory.idLoader(options, loadCalls); JsonObject key1 = new JsonObject().put("id", 123); @@ -910,7 +910,7 @@ public void should_Allow_priming_the_cache_with_an_object_key(TestDataLoaderFact public void should_Accept_a_custom_cache_map_implementation(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { CustomCacheMap customMap = new CustomCacheMap(); List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().setCacheMap(customMap); + DataLoaderOptions options = newOptions().withCacheMap(customMap); DataLoader identityLoader = factory.idLoader(options, loadCalls); // Fetches as expected @@ -961,7 +961,7 @@ public void should_Accept_a_custom_cache_map_implementation(TestDataLoaderFactor @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void should_degrade_gracefully_if_cache_get_throws(TestDataLoaderFactory factory) { CacheMap cache = new ThrowingCacheMap(); - DataLoaderOptions options = newOptions().setCachingEnabled(true).setCacheMap(cache); + DataLoaderOptions options = newOptions().withCachingEnabled(true).withCacheMap(cache); List> loadCalls = new ArrayList<>(); DataLoader identityLoader = factory.idLoader(options, loadCalls); @@ -976,7 +976,7 @@ public void should_degrade_gracefully_if_cache_get_throws(TestDataLoaderFactory @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void batching_disabled_should_dispatch_immediately(TestDataLoaderFactory factory) { List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().setBatchingEnabled(false); + DataLoaderOptions options = newOptions().withBatchingEnabled(false); DataLoader identityLoader = factory.idLoader(options, loadCalls); CompletableFuture fa = identityLoader.load("A"); @@ -1005,7 +1005,7 @@ public void batching_disabled_should_dispatch_immediately(TestDataLoaderFactory @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void batching_disabled_and_caching_disabled_should_dispatch_immediately_and_forget(TestDataLoaderFactory factory) { List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().setBatchingEnabled(false).setCachingEnabled(false); + DataLoaderOptions options = newOptions().withBatchingEnabled(false).withCachingEnabled(false); DataLoader identityLoader = factory.idLoader(options, loadCalls); CompletableFuture fa = identityLoader.load("A"); @@ -1037,7 +1037,7 @@ public void batching_disabled_and_caching_disabled_should_dispatch_immediately_a @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void batches_multiple_requests_with_max_batch_size(TestDataLoaderFactory factory) { List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = factory.idLoader(newOptions().setMaxBatchSize(2), loadCalls); + DataLoader identityLoader = factory.idLoader(newOptions().withMaxBatchSize(2), loadCalls); CompletableFuture f1 = identityLoader.load(1); CompletableFuture f2 = identityLoader.load(2); @@ -1059,7 +1059,7 @@ public void batches_multiple_requests_with_max_batch_size(TestDataLoaderFactory @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void can_split_max_batch_sizes_correctly(TestDataLoaderFactory factory) { List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = factory.idLoader(newOptions().setMaxBatchSize(5), loadCalls); + DataLoader identityLoader = factory.idLoader(newOptions().withMaxBatchSize(5), loadCalls); for (int i = 0; i < 21; i++) { identityLoader.load(i); diff --git a/src/test/java/org/dataloader/DataLoaderValueCacheTest.java b/src/test/java/org/dataloader/DataLoaderValueCacheTest.java index 732febe9..8217690f 100644 --- a/src/test/java/org/dataloader/DataLoaderValueCacheTest.java +++ b/src/test/java/org/dataloader/DataLoaderValueCacheTest.java @@ -72,7 +72,7 @@ public void test_by_default_we_have_no_value_caching(TestDataLoaderFactory facto public void should_accept_a_remote_value_store_for_caching(TestDataLoaderFactory factory) { CustomValueCache customValueCache = new CustomValueCache(); List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().setValueCache(customValueCache); + DataLoaderOptions options = newOptions().withValueCache(customValueCache); DataLoader identityLoader = factory.idLoader(options, loadCalls); // Fetches as expected @@ -127,7 +127,7 @@ public void can_use_caffeine_for_caching(TestDataLoaderFactory factory) { ValueCache caffeineValueCache = new CaffeineValueCache(caffeineCache); List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().setValueCache(caffeineValueCache); + DataLoaderOptions options = newOptions().withValueCache(caffeineValueCache); DataLoader identityLoader = factory.idLoader(options, loadCalls); // Fetches as expected @@ -170,7 +170,7 @@ public CompletableFuture get(String key) { customValueCache.set("b", "From Cache"); List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().setValueCache(customValueCache); + DataLoaderOptions options = newOptions().withValueCache(customValueCache); DataLoader identityLoader = factory.idLoader(options, loadCalls); CompletableFuture fA = identityLoader.load("a"); @@ -198,7 +198,7 @@ public CompletableFuture set(String key, Object value) { }; List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().setValueCache(customValueCache); + DataLoaderOptions options = newOptions().withValueCache(customValueCache); DataLoader identityLoader = factory.idLoader(options, loadCalls); CompletableFuture fA = identityLoader.load("a"); @@ -237,7 +237,7 @@ public CompletableFuture get(String key) { List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().setValueCache(customValueCache); + DataLoaderOptions options = newOptions().withValueCache(customValueCache); DataLoader identityLoader = factory.idLoader(options, loadCalls); CompletableFuture fA = identityLoader.load("a"); @@ -279,7 +279,7 @@ public CompletableFuture>> getValues(List keys) { List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().setValueCache(customValueCache); + DataLoaderOptions options = newOptions().withValueCache(customValueCache); DataLoader identityLoader = factory.idLoader(options, loadCalls); CompletableFuture fA = identityLoader.load("a"); @@ -323,7 +323,7 @@ public CompletableFuture>> getValues(List keys) { }; List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().setValueCache(customValueCache); + DataLoaderOptions options = newOptions().withValueCache(customValueCache); DataLoader identityLoader = factory.idLoader(options, loadCalls); CompletableFuture fA = identityLoader.load("a"); @@ -359,7 +359,7 @@ public CompletableFuture get(String key) { }; List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().setValueCache(customValueCache).setCachingEnabled(false); + DataLoaderOptions options = newOptions().withValueCache(customValueCache).withCachingEnabled(false); DataLoader identityLoader = factory.idLoader(options, loadCalls); CompletableFuture fA = identityLoader.load("a"); @@ -403,7 +403,7 @@ public CompletableFuture> setValues(List keys, List customValueCache.asMap().put("c", "cachedC"); List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().setValueCache(customValueCache).setCachingEnabled(true); + DataLoaderOptions options = newOptions().withValueCache(customValueCache).withCachingEnabled(true); DataLoader identityLoader = factory.idLoader(options, loadCalls); CompletableFuture fA = identityLoader.load("a"); @@ -444,7 +444,7 @@ public CompletableFuture> setValues(List keys, List customValueCache.asMap().put("a", "cachedA"); List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().setValueCache(customValueCache).setCachingEnabled(true).setBatchingEnabled(false); + DataLoaderOptions options = newOptions().withValueCache(customValueCache).withCachingEnabled(true).withBatchingEnabled(false); DataLoader identityLoader = factory.idLoader(options, loadCalls); CompletableFuture fA = identityLoader.load("a"); diff --git a/src/test/java/org/dataloader/instrumentation/DataLoaderInstrumentationTest.java b/src/test/java/org/dataloader/instrumentation/DataLoaderInstrumentationTest.java index 97f21d34..a0fc1086 100644 --- a/src/test/java/org/dataloader/instrumentation/DataLoaderInstrumentationTest.java +++ b/src/test/java/org/dataloader/instrumentation/DataLoaderInstrumentationTest.java @@ -49,8 +49,8 @@ public DataLoaderInstrumentationContext> beginBatchLoader(DataLoader dl = DataLoaderFactory.newDataLoader(snoozingBatchLoader, options); @@ -110,8 +110,8 @@ public DataLoaderInstrumentationContext> beginBatchLoader(DataLoader dl = DataLoaderFactory.newDataLoader(snoozingBatchLoader, options); @@ -155,7 +155,7 @@ public void onCompleted(List result, Throwable t) { } }; - DataLoaderOptions options = new DataLoaderOptions().setInstrumentation(instrumentation); + DataLoaderOptions options = new DataLoaderOptions().withInstrumentation(instrumentation); DataLoader dl = DataLoaderFactory.newDataLoader(snoozingBatchLoader, options); dl.load("A", "kcA"); diff --git a/src/test/java/org/dataloader/instrumentation/DataLoaderRegistryInstrumentationTest.java b/src/test/java/org/dataloader/instrumentation/DataLoaderRegistryInstrumentationTest.java index 49ccf0ee..8d919345 100644 --- a/src/test/java/org/dataloader/instrumentation/DataLoaderRegistryInstrumentationTest.java +++ b/src/test/java/org/dataloader/instrumentation/DataLoaderRegistryInstrumentationTest.java @@ -120,9 +120,9 @@ void wontDoAnyThingIfThereIsNoRegistryInstrumentation() { @Test void wontDoAnyThingIfThereTheyAreTheSameInstrumentationAlready() { - DataLoader newX = dlX.transform(builder -> builder.options(dlX.getOptions().setInstrumentation(instrA))); - DataLoader newY = dlX.transform(builder -> builder.options(dlY.getOptions().setInstrumentation(instrA))); - DataLoader newZ = dlX.transform(builder -> builder.options(dlZ.getOptions().setInstrumentation(instrA))); + DataLoader newX = dlX.transform(builder -> builder.options(dlX.getOptions().withInstrumentation(instrA))); + DataLoader newY = dlX.transform(builder -> builder.options(dlY.getOptions().withInstrumentation(instrA))); + DataLoader newZ = dlX.transform(builder -> builder.options(dlZ.getOptions().withInstrumentation(instrA))); DataLoaderRegistry registry = DataLoaderRegistry.newRegistry() .instrumentation(instrA) .register("X", newX) @@ -145,7 +145,7 @@ void wontDoAnyThingIfThereTheyAreTheSameInstrumentationAlready() { @Test void ifTheDLHasAInstrumentationThenItsTurnedIntoAChainedOne() { - DataLoaderOptions options = dlX.getOptions().setInstrumentation(instrA); + DataLoaderOptions options = dlX.getOptions().withInstrumentation(instrA); DataLoader newX = dlX.transform(builder -> builder.options(options)); DataLoaderRegistry registry = DataLoaderRegistry.newRegistry() @@ -164,7 +164,7 @@ void ifTheDLHasAInstrumentationThenItsTurnedIntoAChainedOne() { @Test void chainedInstrumentationsWillBeCombined() { - DataLoaderOptions options = dlX.getOptions().setInstrumentation(chainedInstrB); + DataLoaderOptions options = dlX.getOptions().withInstrumentation(chainedInstrB); DataLoader newX = dlX.transform(builder -> builder.options(options)); DataLoaderRegistry registry = DataLoaderRegistry.newRegistry() diff --git a/src/test/java/org/dataloader/scheduler/BatchLoaderSchedulerTest.java b/src/test/java/org/dataloader/scheduler/BatchLoaderSchedulerTest.java index ff9ec8e3..fd590afc 100644 --- a/src/test/java/org/dataloader/scheduler/BatchLoaderSchedulerTest.java +++ b/src/test/java/org/dataloader/scheduler/BatchLoaderSchedulerTest.java @@ -83,7 +83,7 @@ private static void commonSetupAndSimpleAsserts(DataLoader ide @Test public void can_allow_a_simple_scheduler() { - DataLoaderOptions options = DataLoaderOptions.newOptions().setBatchLoaderScheduler(immediateScheduling); + DataLoaderOptions options = DataLoaderOptions.newOptions().withBatchLoaderScheduler(immediateScheduling); DataLoader identityLoader = newDataLoader(keysAsValues(), options); @@ -92,7 +92,7 @@ public void can_allow_a_simple_scheduler() { @Test public void can_allow_a_simple_scheduler_with_context() { - DataLoaderOptions options = DataLoaderOptions.newOptions().setBatchLoaderScheduler(immediateScheduling); + DataLoaderOptions options = DataLoaderOptions.newOptions().withBatchLoaderScheduler(immediateScheduling); DataLoader identityLoader = newDataLoader(keysAsValuesWithContext(), options); @@ -101,7 +101,7 @@ public void can_allow_a_simple_scheduler_with_context() { @Test public void can_allow_a_simple_scheduler_with_mapped_batch_load() { - DataLoaderOptions options = DataLoaderOptions.newOptions().setBatchLoaderScheduler(immediateScheduling); + DataLoaderOptions options = DataLoaderOptions.newOptions().withBatchLoaderScheduler(immediateScheduling); DataLoader identityLoader = newMappedDataLoader(keysAsMapOfValues(), options); @@ -110,7 +110,7 @@ public void can_allow_a_simple_scheduler_with_mapped_batch_load() { @Test public void can_allow_a_simple_scheduler_with_mapped_batch_load_with_context() { - DataLoaderOptions options = DataLoaderOptions.newOptions().setBatchLoaderScheduler(immediateScheduling); + DataLoaderOptions options = DataLoaderOptions.newOptions().withBatchLoaderScheduler(immediateScheduling); DataLoader identityLoader = newMappedDataLoader(keysAsMapOfValuesWithContext(), options); @@ -119,7 +119,7 @@ public void can_allow_a_simple_scheduler_with_mapped_batch_load_with_context() { @Test public void can_allow_an_async_scheduler() { - DataLoaderOptions options = DataLoaderOptions.newOptions().setBatchLoaderScheduler(delayedScheduling(50)); + DataLoaderOptions options = DataLoaderOptions.newOptions().withBatchLoaderScheduler(delayedScheduling(50)); DataLoader identityLoader = newDataLoader(keysAsValues(), options); @@ -160,7 +160,7 @@ public void scheduleBatchPublisher(ScheduledBatchPublisherCall scheduledCall }); } }; - DataLoaderOptions options = DataLoaderOptions.newOptions().setBatchLoaderScheduler(funkyScheduler); + DataLoaderOptions options = DataLoaderOptions.newOptions().withBatchLoaderScheduler(funkyScheduler); DataLoader identityLoader = newDataLoader(keysAsValues(), options); From abe2e202c83c27c6aac0afd797a2b6fd4e1eb62b Mon Sep 17 00:00:00 2001 From: Andreas Marek Date: Thu, 1 May 2025 13:59:03 +1000 Subject: [PATCH 088/156] adds explicit jmh dependency for the idea jmh plugin --- build.gradle | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/build.gradle b/build.gradle index d7bef7af..6072f4f6 100644 --- a/build.gradle +++ b/build.gradle @@ -68,6 +68,10 @@ jar { dependencies { api "org.reactivestreams:reactive-streams:$reactive_streams_version" api "org.jspecify:jspecify:1.0.0" + + // this is needed for the idea jmh plugin to work correctly + jmh 'org.openjdk.jmh:jmh-core:1.37' + jmh 'org.openjdk.jmh:jmh-generator-annprocess:1.37' } task sourcesJar(type: Jar) { From 533065bc9e6a0e9a255d9a95fba1347a089333f0 Mon Sep 17 00:00:00 2001 From: bbaker Date: Fri, 2 May 2025 12:25:37 +1000 Subject: [PATCH 089/156] Breaking change - renaming old mutable setXX methods and going to builder pattern only --- README.md | 12 +- .../org/dataloader/DataLoaderFactory.java | 2 +- .../org/dataloader/DataLoaderOptions.java | 242 +++++++----------- src/test/java/ReadmeExamples.java | 12 +- .../DataLoaderBatchLoaderEnvironmentTest.java | 20 +- .../org/dataloader/DataLoaderBuilderTest.java | 4 +- .../org/dataloader/DataLoaderOptionsTest.java | 32 +-- .../dataloader/DataLoaderRegistryTest.java | 6 +- .../org/dataloader/DataLoaderStatsTest.java | 10 +- .../java/org/dataloader/DataLoaderTest.java | 31 +-- .../dataloader/DataLoaderValueCacheTest.java | 22 +- .../ChainedDataLoaderInstrumentationTest.java | 8 +- .../DataLoaderInstrumentationTest.java | 16 +- ...DataLoaderRegistryInstrumentationTest.java | 10 +- .../scheduler/BatchLoaderSchedulerTest.java | 12 +- 15 files changed, 186 insertions(+), 253 deletions(-) diff --git a/README.md b/README.md index a5835dee..a53e7668 100644 --- a/README.md +++ b/README.md @@ -205,7 +205,7 @@ for the context object. ```java DataLoaderOptions options = DataLoaderOptions.newOptions() - .withBatchLoaderContextProvider(() -> SecurityCtx.getCallingUserCtx()); + .setBatchLoaderContextProvider(() -> SecurityCtx.getCallingUserCtx()).build(); BatchLoaderWithContext batchLoader = new BatchLoaderWithContext() { @Override @@ -227,7 +227,7 @@ You can gain access to them as a map by key or as the original list of context o ```java DataLoaderOptions options = DataLoaderOptions.newOptions() - .withBatchLoaderContextProvider(() -> SecurityCtx.getCallingUserCtx()); + .setBatchLoaderContextProvider(() -> SecurityCtx.getCallingUserCtx()).build(); BatchLoaderWithContext batchLoader = new BatchLoaderWithContext() { @Override @@ -433,7 +433,7 @@ However, you can create your own custom future cache and supply it to the data l ```java MyCustomCache customCache = new MyCustomCache(); - DataLoaderOptions options = DataLoaderOptions.newOptions().withsetCacheMap(customCache); + DataLoaderOptions options = DataLoaderOptions.newOptions().setCacheMap(customCache).build(); DataLoaderFactory.newDataLoader(userBatchLoader, options); ``` @@ -467,7 +467,7 @@ The tests have an example based on [Caffeine](https://github.com/ben-manes/caffe In certain uncommon cases, a DataLoader which does not cache may be desirable. ```java - DataLoaderFactory.newDataLoader(userBatchLoader, DataLoaderOptions.newOptions().withCachingEnabled(false)); + DataLoaderFactory.newDataLoader(userBatchLoader, DataLoaderOptions.newOptions().setCachingEnabled(false).build()); ``` Calling the above will ensure that every call to `.load()` will produce a new promise, and requested keys will not be saved in memory. @@ -533,7 +533,7 @@ Knowing what the behaviour of your data is important for you to understand how e You can configure the statistics collector used when you build the data loader ```java - DataLoaderOptions options = DataLoaderOptions.newOptions().withStatisticsCollector(() -> new ThreadLocalStatisticsCollector()); + DataLoaderOptions options = DataLoaderOptions.newOptions().setStatisticsCollector(() -> new ThreadLocalStatisticsCollector()).build(); DataLoader userDataLoader = DataLoaderFactory.newDataLoader(userBatchLoader,options); ``` @@ -780,7 +780,7 @@ You set the `DataLoaderInstrumentation` into the `DataLoaderOptions` at build ti }); } }; - DataLoaderOptions options = DataLoaderOptions.newOptions().withInstrumentation(timingInstrumentation); + DataLoaderOptions options = DataLoaderOptions.newOptions().setInstrumentation(timingInstrumentation).build(); DataLoader userDataLoader = DataLoaderFactory.newDataLoader(userBatchLoader, options); ``` diff --git a/src/main/java/org/dataloader/DataLoaderFactory.java b/src/main/java/org/dataloader/DataLoaderFactory.java index ef1a2870..8daf3ca0 100644 --- a/src/main/java/org/dataloader/DataLoaderFactory.java +++ b/src/main/java/org/dataloader/DataLoaderFactory.java @@ -542,7 +542,7 @@ public static Builder builder(DataLoader dataLoader) { */ public static class Builder { Object batchLoadFunction; - DataLoaderOptions options = DataLoaderOptions.newOptions(); + DataLoaderOptions options = DataLoaderOptions.newDefaultOptions(); Builder() { } diff --git a/src/main/java/org/dataloader/DataLoaderOptions.java b/src/main/java/org/dataloader/DataLoaderOptions.java index 0679fbd6..f7c006fa 100644 --- a/src/main/java/org/dataloader/DataLoaderOptions.java +++ b/src/main/java/org/dataloader/DataLoaderOptions.java @@ -89,47 +89,27 @@ private DataLoaderOptions(Builder builder) { this.instrumentation = builder.instrumentation; } - /** - * Clones the provided data loader options. - * - * @param other the other options instance - */ - public DataLoaderOptions(DataLoaderOptions other) { - nonNull(other); - this.batchingEnabled = other.batchingEnabled; - this.cachingEnabled = other.cachingEnabled; - this.cachingExceptionsEnabled = other.cachingExceptionsEnabled; - this.cacheKeyFunction = other.cacheKeyFunction; - this.cacheMap = other.cacheMap; - this.valueCache = other.valueCache; - this.maxBatchSize = other.maxBatchSize; - this.statisticsCollector = other.statisticsCollector; - this.environmentProvider = other.environmentProvider; - this.valueCacheOptions = other.valueCacheOptions; - this.batchLoaderScheduler = other.batchLoaderScheduler; - this.instrumentation = other.instrumentation; - } - /** * @return a new default data loader options that you can then customize */ - public static DataLoaderOptions newOptions() { + public static DataLoaderOptions newDefaultOptions() { return new DataLoaderOptions(); } /** - * @return a new default data loader options {@link Builder} that you can then customize + * @return a new default data loader options builder that you can then customize */ - public static DataLoaderOptions.Builder newOptionsBuilder() { - return new DataLoaderOptions.Builder(); + public static DataLoaderOptions.Builder newOptions() { + return new Builder(); } /** - * @param otherOptions the options to copy - * @return a new default data loader options {@link Builder} from the specified one that you can then customize + * Copies the options into a new builder + * + * @return a new default data loader options builder that you can then customize */ - public static DataLoaderOptions.Builder newDataLoaderOptions(DataLoaderOptions otherOptions) { - return new DataLoaderOptions.Builder(otherOptions); + public static DataLoaderOptions.Builder newOptions(DataLoaderOptions otherOptions) { + return new Builder(otherOptions); } /** @@ -139,7 +119,7 @@ public static DataLoaderOptions.Builder newDataLoaderOptions(DataLoaderOptions o * @return a new {@link DataLoaderOptions} object */ public DataLoaderOptions transform(Consumer builderConsumer) { - Builder builder = newDataLoaderOptions(this); + Builder builder = new Builder(this); builderConsumer.accept(builder); return builder.build(); } @@ -171,16 +151,6 @@ public boolean batchingEnabled() { return batchingEnabled; } - /** - * Sets the option that determines whether batch loading is enabled. - * - * @param batchingEnabled {@code true} to enable batch loading, {@code false} otherwise - * @return a new data loader options instance for fluent coding - */ - public DataLoaderOptions withBatchingEnabled(boolean batchingEnabled) { - return builder().setBatchingEnabled(batchingEnabled).build(); - } - /** * Option that determines whether to use caching of futures (the default), or not. * @@ -190,16 +160,6 @@ public boolean cachingEnabled() { return cachingEnabled; } - /** - * Sets the option that determines whether caching is enabled. - * - * @param cachingEnabled {@code true} to enable caching, {@code false} otherwise - * @return a new data loader options instance for fluent coding - */ - public DataLoaderOptions withCachingEnabled(boolean cachingEnabled) { - return builder().setCachingEnabled(cachingEnabled).build(); - } - /** * Option that determines whether to cache exceptional values (the default), or not. *

@@ -214,16 +174,6 @@ public boolean cachingExceptionsEnabled() { return cachingExceptionsEnabled; } - /** - * Sets the option that determines whether exceptional values are cache enabled. - * - * @param cachingExceptionsEnabled {@code true} to enable caching exceptional values, {@code false} otherwise - * @return a new data loader options instance for fluent coding - */ - public DataLoaderOptions withCachingExceptionsEnabled(boolean cachingExceptionsEnabled) { - return builder().setCachingExceptionsEnabled(cachingExceptionsEnabled).build(); - } - /** * Gets an (optional) function to invoke for creation of the cache key, if caching is enabled. *

@@ -235,16 +185,6 @@ public Optional cacheKeyFunction() { return Optional.ofNullable(cacheKeyFunction); } - /** - * Sets the function to use for creating the cache key, if caching is enabled. - * - * @param cacheKeyFunction the cache key function to use - * @return a new data loader options instance for fluent coding - */ - public DataLoaderOptions withCacheKeyFunction(CacheKey cacheKeyFunction) { - return builder().setCacheKeyFunction(cacheKeyFunction).build(); - } - /** * Gets the (optional) cache map implementation that is used for caching, if caching is enabled. *

@@ -256,15 +196,6 @@ public DataLoaderOptions withCacheKeyFunction(CacheKey cacheKeyFunction) { return Optional.ofNullable(cacheMap); } - /** - * Sets the cache map implementation to use for caching, if caching is enabled. - * - * @param cacheMap the cache map instance - * @return a new data loader options instance for fluent coding - */ - public DataLoaderOptions withCacheMap(CacheMap cacheMap) { - return builder().setCacheMap(cacheMap).build(); - } /** * Gets the maximum number of keys that will be presented to the {@link BatchLoader} function @@ -276,17 +207,6 @@ public int maxBatchSize() { return maxBatchSize; } - /** - * Sets the maximum number of keys that will be presented to the {@link BatchLoader} function - * before they are split into multiple class - * - * @param maxBatchSize the maximum batch size - * @return a new data loader options instance for fluent coding - */ - public DataLoaderOptions withMaxBatchSize(int maxBatchSize) { - return builder().setMaxBatchSize(maxBatchSize).build(); - } - /** * @return the statistics collector to use with these options */ @@ -294,18 +214,6 @@ public StatisticsCollector getStatisticsCollector() { return nonNull(this.statisticsCollector.get()); } - /** - * Sets the statistics collector supplier that will be used with these data loader options. Since it uses - * the supplier pattern, you can create a new statistics collector on each call, or you can reuse - * a common value - * - * @param statisticsCollector the statistics collector to use - * @return a new data loader options instance for fluent coding - */ - public DataLoaderOptions withStatisticsCollector(Supplier statisticsCollector) { - return builder().setStatisticsCollector(nonNull(statisticsCollector)).build(); - } - /** * @return the batch environment provider that will be used to give context to batch load functions */ @@ -313,16 +221,6 @@ public BatchLoaderContextProvider getBatchLoaderContextProvider() { return environmentProvider; } - /** - * Sets the batch loader environment provider that will be used to give context to batch load functions - * - * @param contextProvider the batch loader context provider - * @return a new data loader options instance for fluent coding - */ - public DataLoaderOptions withBatchLoaderContextProvider(BatchLoaderContextProvider contextProvider) { - return builder().setBatchLoaderContextProvider(nonNull(contextProvider)).build(); - } - /** * Gets the (optional) cache store implementation that is used for value caching, if caching is enabled. *

@@ -334,15 +232,6 @@ public DataLoaderOptions withBatchLoaderContextProvider(BatchLoaderContextProvid return Optional.ofNullable(valueCache); } - /** - * Sets the value cache implementation to use for caching values, if caching is enabled. - * - * @param valueCache the value cache instance - * @return a new data loader options instance for fluent coding - */ - public DataLoaderOptions withValueCache(ValueCache valueCache) { - return builder().setValueCache(valueCache).build(); - } /** * @return the {@link ValueCacheOptions} that control how the {@link ValueCache} will be used @@ -351,16 +240,6 @@ public ValueCacheOptions getValueCacheOptions() { return valueCacheOptions; } - /** - * Sets the {@link ValueCacheOptions} that control how the {@link ValueCache} will be used - * - * @param valueCacheOptions the value cache options - * @return a new data loader options instance for fluent coding - */ - public DataLoaderOptions withValueCacheOptions(ValueCacheOptions valueCacheOptions) { - return builder().setValueCacheOptions(nonNull(valueCacheOptions)).build(); - } - /** * @return the {@link BatchLoaderScheduler} to use, which can be null */ @@ -368,17 +247,6 @@ public BatchLoaderScheduler getBatchLoaderScheduler() { return batchLoaderScheduler; } - /** - * Sets in a new {@link BatchLoaderScheduler} that allows the call to a {@link BatchLoader} function to be scheduled - * to some future time. - * - * @param batchLoaderScheduler the scheduler - * @return a new data loader options instance for fluent coding - */ - public DataLoaderOptions withBatchLoaderScheduler(BatchLoaderScheduler batchLoaderScheduler) { - return builder().setBatchLoaderScheduler(batchLoaderScheduler).build(); - } - /** * @return the {@link DataLoaderInstrumentation} to use */ @@ -386,20 +254,6 @@ public DataLoaderInstrumentation getInstrumentation() { return instrumentation; } - /** - * Sets in a new {@link DataLoaderInstrumentation} - * - * @param instrumentation the new {@link DataLoaderInstrumentation} - * @return a new data loader options instance for fluent coding - */ - public DataLoaderOptions withInstrumentation(DataLoaderInstrumentation instrumentation) { - return builder().setInstrumentation(instrumentation).build(); - } - - private Builder builder() { - return new Builder(this); - } - public static class Builder { private boolean batchingEnabled; private boolean cachingEnabled; @@ -433,61 +287,137 @@ public Builder() { this.instrumentation = other.instrumentation; } + /** + * Sets the option that determines whether batch loading is enabled. + * + * @param batchingEnabled {@code true} to enable batch loading, {@code false} otherwise + * @return this builder for fluent coding + */ public Builder setBatchingEnabled(boolean batchingEnabled) { this.batchingEnabled = batchingEnabled; return this; } + /** + * Sets the option that determines whether caching is enabled. + * + * @param cachingEnabled {@code true} to enable caching, {@code false} otherwise + * @return this builder for fluent coding + */ public Builder setCachingEnabled(boolean cachingEnabled) { this.cachingEnabled = cachingEnabled; return this; } + /** + * Sets the option that determines whether exceptional values are cache enabled. + * + * @param cachingExceptionsEnabled {@code true} to enable caching exceptional values, {@code false} otherwise + * @return this builder for fluent coding + */ public Builder setCachingExceptionsEnabled(boolean cachingExceptionsEnabled) { this.cachingExceptionsEnabled = cachingExceptionsEnabled; return this; } + /** + * Sets the function to use for creating the cache key, if caching is enabled. + * + * @param cacheKeyFunction the cache key function to use + * @return this builder for fluent coding + */ public Builder setCacheKeyFunction(CacheKey cacheKeyFunction) { this.cacheKeyFunction = cacheKeyFunction; return this; } + /** + * Sets the cache map implementation to use for caching, if caching is enabled. + * + * @param cacheMap the cache map instance + * @return this builder for fluent coding + */ public Builder setCacheMap(CacheMap cacheMap) { this.cacheMap = cacheMap; return this; } + /** + * Sets the value cache implementation to use for caching values, if caching is enabled. + * + * @param valueCache the value cache instance + * @return this builder for fluent coding + */ public Builder setValueCache(ValueCache valueCache) { this.valueCache = valueCache; return this; } + /** + * Sets the maximum number of keys that will be presented to the {@link BatchLoader} function + * before they are split into multiple class + * + * @param maxBatchSize the maximum batch size + * @return this builder for fluent coding + */ public Builder setMaxBatchSize(int maxBatchSize) { this.maxBatchSize = maxBatchSize; return this; } + /** + * Sets the statistics collector supplier that will be used with these data loader options. Since it uses + * the supplier pattern, you can create a new statistics collector on each call, or you can reuse + * a common value + * + * @param statisticsCollector the statistics collector to use + * @return this builder for fluent coding + */ public Builder setStatisticsCollector(Supplier statisticsCollector) { this.statisticsCollector = statisticsCollector; return this; } + /** + * Sets the batch loader environment provider that will be used to give context to batch load functions + * + * @param environmentProvider the batch loader context provider + * @return this builder for fluent coding + */ public Builder setBatchLoaderContextProvider(BatchLoaderContextProvider environmentProvider) { this.environmentProvider = environmentProvider; return this; } + /** + * Sets the {@link ValueCacheOptions} that control how the {@link ValueCache} will be used + * + * @param valueCacheOptions the value cache options + * @return this builder for fluent coding + */ public Builder setValueCacheOptions(ValueCacheOptions valueCacheOptions) { this.valueCacheOptions = valueCacheOptions; return this; } + /** + * Sets in a new {@link BatchLoaderScheduler} that allows the call to a {@link BatchLoader} function to be scheduled + * to some future time. + * + * @param batchLoaderScheduler the scheduler + * @return this builder for fluent coding + */ public Builder setBatchLoaderScheduler(BatchLoaderScheduler batchLoaderScheduler) { this.batchLoaderScheduler = batchLoaderScheduler; return this; } + /** + * Sets in a new {@link DataLoaderInstrumentation} + * + * @param instrumentation the new {@link DataLoaderInstrumentation} + * @return this builder for fluent coding + */ public Builder setInstrumentation(DataLoaderInstrumentation instrumentation) { this.instrumentation = nonNull(instrumentation); return this; diff --git a/src/test/java/ReadmeExamples.java b/src/test/java/ReadmeExamples.java index 327073f6..f391b80d 100644 --- a/src/test/java/ReadmeExamples.java +++ b/src/test/java/ReadmeExamples.java @@ -105,7 +105,7 @@ public CompletionStage> load(List userIds) { private void callContextExample() { DataLoaderOptions options = DataLoaderOptions.newOptions() - .withBatchLoaderContextProvider(() -> SecurityCtx.getCallingUserCtx()); + .setBatchLoaderContextProvider(() -> SecurityCtx.getCallingUserCtx()).build(); BatchLoaderWithContext batchLoader = new BatchLoaderWithContext() { @Override @@ -120,7 +120,7 @@ public CompletionStage> load(List keys, BatchLoaderEnvironm private void keyContextExample() { DataLoaderOptions options = DataLoaderOptions.newOptions() - .withBatchLoaderContextProvider(() -> SecurityCtx.getCallingUserCtx()); + .setBatchLoaderContextProvider(() -> SecurityCtx.getCallingUserCtx()).build(); BatchLoaderWithContext batchLoader = new BatchLoaderWithContext() { @Override @@ -236,7 +236,7 @@ private void clearCacheOnError() { BatchLoader teamsBatchLoader; private void disableCache() { - DataLoaderFactory.newDataLoader(userBatchLoader, DataLoaderOptions.newOptions().withCachingEnabled(false)); + DataLoaderFactory.newDataLoader(userBatchLoader, DataLoaderOptions.newOptions().setCachingEnabled(false).build()); userDataLoader.load("A"); @@ -283,7 +283,7 @@ public CacheMap clear() { private void customCache() { MyCustomCache customCache = new MyCustomCache(); - DataLoaderOptions options = DataLoaderOptions.newOptions().withCacheMap(customCache); + DataLoaderOptions options = DataLoaderOptions.newOptions().setCacheMap(customCache).build(); DataLoaderFactory.newDataLoader(userBatchLoader, options); } @@ -311,7 +311,7 @@ private void statsExample() { private void statsConfigExample() { - DataLoaderOptions options = DataLoaderOptions.newOptions().withStatisticsCollector(() -> new ThreadLocalStatisticsCollector()); + DataLoaderOptions options = DataLoaderOptions.newOptions().setStatisticsCollector(() -> new ThreadLocalStatisticsCollector()).build(); DataLoader userDataLoader = DataLoaderFactory.newDataLoader(userBatchLoader, options); } @@ -410,7 +410,7 @@ public DataLoaderInstrumentationContext> beginBatchLoader(DataLoader userDataLoader = DataLoaderFactory.newDataLoader(userBatchLoader, options); } diff --git a/src/test/java/org/dataloader/DataLoaderBatchLoaderEnvironmentTest.java b/src/test/java/org/dataloader/DataLoaderBatchLoaderEnvironmentTest.java index 435610fc..274820d0 100644 --- a/src/test/java/org/dataloader/DataLoaderBatchLoaderEnvironmentTest.java +++ b/src/test/java/org/dataloader/DataLoaderBatchLoaderEnvironmentTest.java @@ -41,7 +41,7 @@ public void context_is_passed_to_batch_loader_function() { return CompletableFuture.completedFuture(list); }; DataLoaderOptions options = DataLoaderOptions.newOptions() - .withBatchLoaderContextProvider(() -> "ctx"); + .setBatchLoaderContextProvider(() -> "ctx").build(); DataLoader loader = newDataLoader(batchLoader, options); loader.load("A"); @@ -61,7 +61,7 @@ public void context_is_passed_to_batch_loader_function() { public void key_contexts_are_passed_to_batch_loader_function() { BatchLoaderWithContext batchLoader = contextBatchLoader(); DataLoaderOptions options = DataLoaderOptions.newOptions() - .withBatchLoaderContextProvider(() -> "ctx"); + .setBatchLoaderContextProvider(() -> "ctx").build(); DataLoader loader = newDataLoader(batchLoader, options); loader.load("A", "aCtx"); @@ -81,8 +81,9 @@ public void key_contexts_are_passed_to_batch_loader_function() { public void key_contexts_are_passed_to_batch_loader_function_when_batching_disabled() { BatchLoaderWithContext batchLoader = contextBatchLoader(); DataLoaderOptions options = DataLoaderOptions.newOptions() - .withBatchingEnabled(false) - .withBatchLoaderContextProvider(() -> "ctx"); + .setBatchingEnabled(false) + .setBatchLoaderContextProvider(() -> "ctx") + .build(); DataLoader loader = newDataLoader(batchLoader, options); CompletableFuture aLoad = loader.load("A", "aCtx"); @@ -104,7 +105,8 @@ public void key_contexts_are_passed_to_batch_loader_function_when_batching_disab public void missing_key_contexts_are_passed_to_batch_loader_function() { BatchLoaderWithContext batchLoader = contextBatchLoader(); DataLoaderOptions options = DataLoaderOptions.newOptions() - .withBatchLoaderContextProvider(() -> "ctx"); + .setBatchLoaderContextProvider(() -> "ctx") + .build(); DataLoader loader = newDataLoader(batchLoader, options); loader.load("A", "aCtx"); @@ -133,7 +135,8 @@ public void context_is_passed_to_map_batch_loader_function() { return CompletableFuture.completedFuture(map); }; DataLoaderOptions options = DataLoaderOptions.newOptions() - .withBatchLoaderContextProvider(() -> "ctx"); + .setBatchLoaderContextProvider(() -> "ctx") + .build(); DataLoader loader = newMappedDataLoader(mapBatchLoader, options); loader.load("A", "aCtx"); @@ -199,8 +202,9 @@ public void null_is_passed_as_context_to_map_loader_if_you_do_nothing() { public void mmap_semantics_apply_to_batch_loader_context() { BatchLoaderWithContext batchLoader = contextBatchLoader(); DataLoaderOptions options = DataLoaderOptions.newOptions() - .withBatchLoaderContextProvider(() -> "ctx") - .withCachingEnabled(false); + .setBatchLoaderContextProvider(() -> "ctx") + .setCachingEnabled(false) + .build(); DataLoader loader = newDataLoader(batchLoader, options); loader.load("A", "aCtx"); diff --git a/src/test/java/org/dataloader/DataLoaderBuilderTest.java b/src/test/java/org/dataloader/DataLoaderBuilderTest.java index b9014b19..bf8b7627 100644 --- a/src/test/java/org/dataloader/DataLoaderBuilderTest.java +++ b/src/test/java/org/dataloader/DataLoaderBuilderTest.java @@ -12,8 +12,8 @@ public class DataLoaderBuilderTest { BatchLoader batchLoader2 = keys -> null; - DataLoaderOptions defaultOptions = DataLoaderOptions.newOptions(); - DataLoaderOptions differentOptions = DataLoaderOptions.newOptions().withCachingEnabled(false); + DataLoaderOptions defaultOptions = DataLoaderOptions.newOptions().build(); + DataLoaderOptions differentOptions = DataLoaderOptions.newOptions().setCachingEnabled(false).build(); @Test void canBuildNewDataLoaders() { diff --git a/src/test/java/org/dataloader/DataLoaderOptionsTest.java b/src/test/java/org/dataloader/DataLoaderOptionsTest.java index c331942a..81a7126f 100644 --- a/src/test/java/org/dataloader/DataLoaderOptionsTest.java +++ b/src/test/java/org/dataloader/DataLoaderOptionsTest.java @@ -31,7 +31,7 @@ void canCreateDefaultOptions() { assertThat(optionsDefault.maxBatchSize(), equalTo(-1)); assertThat(optionsDefault.getBatchLoaderScheduler(), equalTo(null)); - DataLoaderOptions builtOptions = DataLoaderOptions.newOptionsBuilder().build(); + DataLoaderOptions builtOptions = DataLoaderOptions.newDefaultOptions(); assertThat(builtOptions, equalTo(optionsDefault)); assertThat(builtOptions == optionsDefault, equalTo(false)); @@ -43,11 +43,7 @@ void canCreateDefaultOptions() { @Test void canCopyOk() { - DataLoaderOptions optionsNext = new DataLoaderOptions(optionsDefault); - assertThat(optionsNext, equalTo(optionsDefault)); - assertThat(optionsNext == optionsDefault, equalTo(false)); - - optionsNext = DataLoaderOptions.newDataLoaderOptions(optionsDefault).build(); + DataLoaderOptions optionsNext = DataLoaderOptions.newOptions(optionsDefault).build(); assertThat(optionsNext, equalTo(optionsDefault)); assertThat(optionsNext == optionsDefault, equalTo(false)); } @@ -89,25 +85,25 @@ public Object getKey(Object input) { @Test void canBuildOk() { - assertThat(optionsDefault.withBatchingEnabled(false).batchingEnabled(), + assertThat(optionsDefault.transform(b -> b.setBatchingEnabled(false)).batchingEnabled(), equalTo(false)); - assertThat(optionsDefault.withBatchLoaderScheduler(testBatchLoaderScheduler).getBatchLoaderScheduler(), + assertThat(optionsDefault.transform(b -> b.setBatchLoaderScheduler(testBatchLoaderScheduler)).getBatchLoaderScheduler(), equalTo(testBatchLoaderScheduler)); - assertThat(optionsDefault.withBatchLoaderContextProvider(testBatchLoaderContextProvider).getBatchLoaderContextProvider(), + assertThat(optionsDefault.transform(b -> b.setBatchLoaderContextProvider(testBatchLoaderContextProvider)).getBatchLoaderContextProvider(), equalTo(testBatchLoaderContextProvider)); - assertThat(optionsDefault.withCacheMap(testCacheMap).cacheMap().get(), + assertThat(optionsDefault.transform(b -> b.setCacheMap(testCacheMap)).cacheMap().get(), equalTo(testCacheMap)); - assertThat(optionsDefault.withCachingEnabled(false).cachingEnabled(), + assertThat(optionsDefault.transform(b -> b.setCachingEnabled(false)).cachingEnabled(), equalTo(false)); - assertThat(optionsDefault.withValueCacheOptions(testValueCacheOptions).getValueCacheOptions(), + assertThat(optionsDefault.transform(b -> b.setValueCacheOptions(testValueCacheOptions)).getValueCacheOptions(), equalTo(testValueCacheOptions)); - assertThat(optionsDefault.withCacheKeyFunction(testCacheKey).cacheKeyFunction().get(), + assertThat(optionsDefault.transform(b -> b.setCacheKeyFunction(testCacheKey)).cacheKeyFunction().get(), equalTo(testCacheKey)); - assertThat(optionsDefault.withValueCache(testValueCache).valueCache().get(), + assertThat(optionsDefault.transform(b -> b.setValueCache(testValueCache)).valueCache().get(), equalTo(testValueCache)); - assertThat(optionsDefault.withMaxBatchSize(10).maxBatchSize(), + assertThat(optionsDefault.transform(b -> b.setMaxBatchSize(10)).maxBatchSize(), equalTo(10)); - assertThat(optionsDefault.withStatisticsCollector(testStatisticsCollectorSupplier).getStatisticsCollector(), + assertThat(optionsDefault.transform(b -> b.setStatisticsCollector(testStatisticsCollectorSupplier)).getStatisticsCollector(), equalTo(testStatisticsCollectorSupplier.get())); DataLoaderOptions builtOptions = optionsDefault.transform(builder -> { @@ -150,7 +146,7 @@ void canBuildOk() { @Test void canBuildViaBuilderOk() { - DataLoaderOptions.Builder builder = DataLoaderOptions.newOptionsBuilder(); + DataLoaderOptions.Builder builder = DataLoaderOptions.newOptions(); builder.setBatchingEnabled(false); builder.setCachingExceptionsEnabled(false); builder.setCachingEnabled(false); @@ -196,7 +192,7 @@ void canCopyExistingOptionValuesOnTransform() { }; BatchLoaderContextProvider contextProvider1 = () -> null; - DataLoaderOptions startingOptions = DataLoaderOptions.newOptionsBuilder().setBatchingEnabled(false) + DataLoaderOptions startingOptions = DataLoaderOptions.newOptions().setBatchingEnabled(false) .setCachingEnabled(false) .setInstrumentation(instrumentation1) .setBatchLoaderContextProvider(contextProvider1) diff --git a/src/test/java/org/dataloader/DataLoaderRegistryTest.java b/src/test/java/org/dataloader/DataLoaderRegistryTest.java index 34190eb7..67bd343c 100644 --- a/src/test/java/org/dataloader/DataLoaderRegistryTest.java +++ b/src/test/java/org/dataloader/DataLoaderRegistryTest.java @@ -79,13 +79,13 @@ public void stats_can_be_collected() { DataLoaderRegistry registry = new DataLoaderRegistry(); DataLoader dlA = newDataLoader(identityBatchLoader, - DataLoaderOptions.newOptions().withStatisticsCollector(SimpleStatisticsCollector::new) + DataLoaderOptions.newOptions().setStatisticsCollector(SimpleStatisticsCollector::new).build() ); DataLoader dlB = newDataLoader(identityBatchLoader, - DataLoaderOptions.newOptions().withStatisticsCollector(SimpleStatisticsCollector::new) + DataLoaderOptions.newOptions().setStatisticsCollector(SimpleStatisticsCollector::new).build() ); DataLoader dlC = newDataLoader(identityBatchLoader, - DataLoaderOptions.newOptions().withStatisticsCollector(SimpleStatisticsCollector::new) + DataLoaderOptions.newOptions().setStatisticsCollector(SimpleStatisticsCollector::new).build() ); registry.register("a", dlA).register("b", dlB).register("c", dlC); diff --git a/src/test/java/org/dataloader/DataLoaderStatsTest.java b/src/test/java/org/dataloader/DataLoaderStatsTest.java index a236169e..c7d9daab 100644 --- a/src/test/java/org/dataloader/DataLoaderStatsTest.java +++ b/src/test/java/org/dataloader/DataLoaderStatsTest.java @@ -33,7 +33,7 @@ public class DataLoaderStatsTest { public void stats_are_collected_by_default() { BatchLoader batchLoader = CompletableFuture::completedFuture; DataLoader loader = newDataLoader(batchLoader, - DataLoaderOptions.newOptions().withStatisticsCollector(SimpleStatisticsCollector::new) + DataLoaderOptions.newOptions().setStatisticsCollector(SimpleStatisticsCollector::new).build() ); loader.load("A"); @@ -75,7 +75,7 @@ public void stats_are_collected_with_specified_collector() { collector.incrementBatchLoadCountBy(1, new IncrementBatchLoadCountByStatisticsContext<>(1, null)); BatchLoader batchLoader = CompletableFuture::completedFuture; - DataLoaderOptions loaderOptions = DataLoaderOptions.newOptions().withStatisticsCollector(() -> collector); + DataLoaderOptions loaderOptions = DataLoaderOptions.newOptions().setStatisticsCollector(() -> collector).build(); DataLoader loader = newDataLoader(batchLoader, loaderOptions); loader.load("A"); @@ -113,7 +113,7 @@ public void stats_are_collected_with_caching_disabled() { StatisticsCollector collector = new SimpleStatisticsCollector(); BatchLoader batchLoader = CompletableFuture::completedFuture; - DataLoaderOptions loaderOptions = DataLoaderOptions.newOptions().withStatisticsCollector(() -> collector).withCachingEnabled(false); + DataLoaderOptions loaderOptions = DataLoaderOptions.newOptions().setStatisticsCollector(() -> collector).setCachingEnabled(false).build(); DataLoader loader = newDataLoader(batchLoader, loaderOptions); loader.load("A"); @@ -166,7 +166,7 @@ public void stats_are_collected_with_caching_disabled() { @Test public void stats_are_collected_on_exceptions() { DataLoader loader = DataLoaderFactory.newDataLoaderWithTry(batchLoaderThatBlows, - DataLoaderOptions.newOptions().withStatisticsCollector(SimpleStatisticsCollector::new) + DataLoaderOptions.newOptions().setStatisticsCollector(SimpleStatisticsCollector::new).build() ); loader.load("A"); @@ -290,7 +290,7 @@ public Statistics getStatistics() { public void context_is_passed_through_to_collector() { ContextPassingStatisticsCollector statisticsCollector = new ContextPassingStatisticsCollector(); DataLoader> loader = newDataLoader(batchLoaderThatBlows, - DataLoaderOptions.newOptions().withStatisticsCollector(() -> statisticsCollector) + DataLoaderOptions.newOptions().setStatisticsCollector(() -> statisticsCollector).build() ); loader.load("key", "keyContext"); diff --git a/src/test/java/org/dataloader/DataLoaderTest.java b/src/test/java/org/dataloader/DataLoaderTest.java index cb6adad7..1b44712a 100644 --- a/src/test/java/org/dataloader/DataLoaderTest.java +++ b/src/test/java/org/dataloader/DataLoaderTest.java @@ -47,6 +47,7 @@ import static java.util.concurrent.CompletableFuture.*; import static org.awaitility.Awaitility.await; import static org.dataloader.DataLoaderFactory.newDataLoader; +import static org.dataloader.DataLoaderOptions.newDefaultOptions; import static org.dataloader.DataLoaderOptions.newOptions; import static org.dataloader.fixtures.TestKit.areAllDone; import static org.dataloader.fixtures.TestKit.listFrom; @@ -588,7 +589,7 @@ public void should_Cache_failed_fetches(TestDataLoaderFactory factory) { @ParameterizedTest @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void should_NOT_Cache_failed_fetches_if_told_not_too(TestDataLoaderFactory factory) { - DataLoaderOptions options = DataLoaderOptions.newOptions().withCachingExceptionsEnabled(false); + DataLoaderOptions options = DataLoaderOptions.newOptions().setCachingExceptionsEnabled(false).build(); List> loadCalls = new ArrayList<>(); DataLoader errorLoader = factory.idLoaderAllExceptions(options, loadCalls); @@ -736,7 +737,7 @@ public void should_Accept_objects_as_keys(TestDataLoaderFactory factory) { public void should_Disable_caching(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); DataLoader identityLoader = - factory.idLoader(newOptions().withCachingEnabled(false), loadCalls); + factory.idLoader(newOptions().setCachingEnabled(false).build(), loadCalls); CompletableFuture future1 = identityLoader.load("A"); CompletableFuture future2 = identityLoader.load("B"); @@ -774,7 +775,7 @@ public void should_Disable_caching(TestDataLoaderFactory factory) throws Executi public void should_work_with_duplicate_keys_when_caching_disabled(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); DataLoader identityLoader = - factory.idLoader(newOptions().withCachingEnabled(false), loadCalls); + factory.idLoader(newOptions().setCachingEnabled(false).build(), loadCalls); CompletableFuture future1 = identityLoader.load("A"); CompletableFuture future2 = identityLoader.load("B"); @@ -797,7 +798,7 @@ public void should_work_with_duplicate_keys_when_caching_disabled(TestDataLoader public void should_work_with_duplicate_keys_when_caching_enabled(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); DataLoader identityLoader = - factory.idLoader(newOptions().withCachingEnabled(true), loadCalls); + factory.idLoader(newOptions().setCachingEnabled(true).build(), loadCalls); CompletableFuture future1 = identityLoader.load("A"); CompletableFuture future2 = identityLoader.load("B"); @@ -817,7 +818,7 @@ public void should_work_with_duplicate_keys_when_caching_enabled(TestDataLoaderF @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void should_Accept_objects_with_a_complex_key(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().withCacheKeyFunction(getJsonObjectCacheMapFn()); + DataLoaderOptions options = newOptions().setCacheKeyFunction(getJsonObjectCacheMapFn()).build(); DataLoader identityLoader = factory.idLoader(options, loadCalls); JsonObject key1 = new JsonObject().put("id", 123); @@ -839,7 +840,7 @@ public void should_Accept_objects_with_a_complex_key(TestDataLoaderFactory facto @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void should_Clear_objects_with_complex_key(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().withCacheKeyFunction(getJsonObjectCacheMapFn()); + DataLoaderOptions options = newOptions().setCacheKeyFunction(getJsonObjectCacheMapFn()).build(); DataLoader identityLoader = factory.idLoader(options, loadCalls); JsonObject key1 = new JsonObject().put("id", 123); @@ -864,7 +865,7 @@ public void should_Clear_objects_with_complex_key(TestDataLoaderFactory factory) @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void should_Accept_objects_with_different_order_of_keys(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().withCacheKeyFunction(getJsonObjectCacheMapFn()); + DataLoaderOptions options = newOptions().setCacheKeyFunction(getJsonObjectCacheMapFn()).build(); DataLoader identityLoader = factory.idLoader(options, loadCalls); JsonObject key1 = new JsonObject().put("a", 123).put("b", 321); @@ -887,7 +888,7 @@ public void should_Accept_objects_with_different_order_of_keys(TestDataLoaderFac @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void should_Allow_priming_the_cache_with_an_object_key(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().withCacheKeyFunction(getJsonObjectCacheMapFn()); + DataLoaderOptions options = newOptions().setCacheKeyFunction(getJsonObjectCacheMapFn()).build(); DataLoader identityLoader = factory.idLoader(options, loadCalls); JsonObject key1 = new JsonObject().put("id", 123); @@ -910,7 +911,7 @@ public void should_Allow_priming_the_cache_with_an_object_key(TestDataLoaderFact public void should_Accept_a_custom_cache_map_implementation(TestDataLoaderFactory factory) throws ExecutionException, InterruptedException { CustomCacheMap customMap = new CustomCacheMap(); List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().withCacheMap(customMap); + DataLoaderOptions options = newOptions().setCacheMap(customMap).build(); DataLoader identityLoader = factory.idLoader(options, loadCalls); // Fetches as expected @@ -961,7 +962,7 @@ public void should_Accept_a_custom_cache_map_implementation(TestDataLoaderFactor @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void should_degrade_gracefully_if_cache_get_throws(TestDataLoaderFactory factory) { CacheMap cache = new ThrowingCacheMap(); - DataLoaderOptions options = newOptions().withCachingEnabled(true).withCacheMap(cache); + DataLoaderOptions options = newOptions().setCachingEnabled(true).setCacheMap(cache).build(); List> loadCalls = new ArrayList<>(); DataLoader identityLoader = factory.idLoader(options, loadCalls); @@ -976,7 +977,7 @@ public void should_degrade_gracefully_if_cache_get_throws(TestDataLoaderFactory @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void batching_disabled_should_dispatch_immediately(TestDataLoaderFactory factory) { List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().withBatchingEnabled(false); + DataLoaderOptions options = newOptions().setBatchingEnabled(false).build(); DataLoader identityLoader = factory.idLoader(options, loadCalls); CompletableFuture fa = identityLoader.load("A"); @@ -1005,7 +1006,7 @@ public void batching_disabled_should_dispatch_immediately(TestDataLoaderFactory @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void batching_disabled_and_caching_disabled_should_dispatch_immediately_and_forget(TestDataLoaderFactory factory) { List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().withBatchingEnabled(false).withCachingEnabled(false); + DataLoaderOptions options = newOptions().setBatchingEnabled(false).setCachingEnabled(false).build(); DataLoader identityLoader = factory.idLoader(options, loadCalls); CompletableFuture fa = identityLoader.load("A"); @@ -1037,7 +1038,7 @@ public void batching_disabled_and_caching_disabled_should_dispatch_immediately_a @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void batches_multiple_requests_with_max_batch_size(TestDataLoaderFactory factory) { List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = factory.idLoader(newOptions().withMaxBatchSize(2), loadCalls); + DataLoader identityLoader = factory.idLoader(newOptions().setMaxBatchSize(2).build(), loadCalls); CompletableFuture f1 = identityLoader.load(1); CompletableFuture f2 = identityLoader.load(2); @@ -1059,7 +1060,7 @@ public void batches_multiple_requests_with_max_batch_size(TestDataLoaderFactory @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void can_split_max_batch_sizes_correctly(TestDataLoaderFactory factory) { List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = factory.idLoader(newOptions().withMaxBatchSize(5), loadCalls); + DataLoader identityLoader = factory.idLoader(newOptions().setMaxBatchSize(5).build(), loadCalls); for (int i = 0; i < 21; i++) { identityLoader.load(i); @@ -1082,7 +1083,7 @@ public void can_split_max_batch_sizes_correctly(TestDataLoaderFactory factory) { @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void should_Batch_loads_occurring_within_futures(TestDataLoaderFactory factory) { List> loadCalls = new ArrayList<>(); - DataLoader identityLoader = factory.idLoader(newOptions(), loadCalls); + DataLoader identityLoader = factory.idLoader(newDefaultOptions(), loadCalls); Supplier nullValue = () -> null; diff --git a/src/test/java/org/dataloader/DataLoaderValueCacheTest.java b/src/test/java/org/dataloader/DataLoaderValueCacheTest.java index 8217690f..24d111bd 100644 --- a/src/test/java/org/dataloader/DataLoaderValueCacheTest.java +++ b/src/test/java/org/dataloader/DataLoaderValueCacheTest.java @@ -36,7 +36,7 @@ public class DataLoaderValueCacheTest { @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") public void test_by_default_we_have_no_value_caching(TestDataLoaderFactory factory) { List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions(); + DataLoaderOptions options = newOptions().build(); DataLoader identityLoader = factory.idLoader(options, loadCalls); CompletableFuture fA = identityLoader.load("a"); @@ -72,7 +72,7 @@ public void test_by_default_we_have_no_value_caching(TestDataLoaderFactory facto public void should_accept_a_remote_value_store_for_caching(TestDataLoaderFactory factory) { CustomValueCache customValueCache = new CustomValueCache(); List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().withValueCache(customValueCache); + DataLoaderOptions options = newOptions().setValueCache(customValueCache).build(); DataLoader identityLoader = factory.idLoader(options, loadCalls); // Fetches as expected @@ -127,7 +127,7 @@ public void can_use_caffeine_for_caching(TestDataLoaderFactory factory) { ValueCache caffeineValueCache = new CaffeineValueCache(caffeineCache); List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().withValueCache(caffeineValueCache); + DataLoaderOptions options = newOptions().setValueCache(caffeineValueCache).build(); DataLoader identityLoader = factory.idLoader(options, loadCalls); // Fetches as expected @@ -170,7 +170,7 @@ public CompletableFuture get(String key) { customValueCache.set("b", "From Cache"); List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().withValueCache(customValueCache); + DataLoaderOptions options = newOptions().setValueCache(customValueCache).build(); DataLoader identityLoader = factory.idLoader(options, loadCalls); CompletableFuture fA = identityLoader.load("a"); @@ -198,7 +198,7 @@ public CompletableFuture set(String key, Object value) { }; List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().withValueCache(customValueCache); + DataLoaderOptions options = newOptions().setValueCache(customValueCache).build(); DataLoader identityLoader = factory.idLoader(options, loadCalls); CompletableFuture fA = identityLoader.load("a"); @@ -237,7 +237,7 @@ public CompletableFuture get(String key) { List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().withValueCache(customValueCache); + DataLoaderOptions options = newOptions().setValueCache(customValueCache).build(); DataLoader identityLoader = factory.idLoader(options, loadCalls); CompletableFuture fA = identityLoader.load("a"); @@ -279,7 +279,7 @@ public CompletableFuture>> getValues(List keys) { List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().withValueCache(customValueCache); + DataLoaderOptions options = newOptions().setValueCache(customValueCache).build(); DataLoader identityLoader = factory.idLoader(options, loadCalls); CompletableFuture fA = identityLoader.load("a"); @@ -323,7 +323,7 @@ public CompletableFuture>> getValues(List keys) { }; List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().withValueCache(customValueCache); + DataLoaderOptions options = newOptions().setValueCache(customValueCache).build(); DataLoader identityLoader = factory.idLoader(options, loadCalls); CompletableFuture fA = identityLoader.load("a"); @@ -359,7 +359,7 @@ public CompletableFuture get(String key) { }; List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().withValueCache(customValueCache).withCachingEnabled(false); + DataLoaderOptions options = newOptions().setValueCache(customValueCache).setCachingEnabled(false).build(); DataLoader identityLoader = factory.idLoader(options, loadCalls); CompletableFuture fA = identityLoader.load("a"); @@ -403,7 +403,7 @@ public CompletableFuture> setValues(List keys, List customValueCache.asMap().put("c", "cachedC"); List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().withValueCache(customValueCache).withCachingEnabled(true); + DataLoaderOptions options = newOptions().setValueCache(customValueCache).setCachingEnabled(true).build(); DataLoader identityLoader = factory.idLoader(options, loadCalls); CompletableFuture fA = identityLoader.load("a"); @@ -444,7 +444,7 @@ public CompletableFuture> setValues(List keys, List customValueCache.asMap().put("a", "cachedA"); List> loadCalls = new ArrayList<>(); - DataLoaderOptions options = newOptions().withValueCache(customValueCache).withCachingEnabled(true).withBatchingEnabled(false); + DataLoaderOptions options = newOptions().setValueCache(customValueCache).setCachingEnabled(true).setBatchingEnabled(false).build(); DataLoader identityLoader = factory.idLoader(options, loadCalls); CompletableFuture fA = identityLoader.load("a"); diff --git a/src/test/java/org/dataloader/instrumentation/ChainedDataLoaderInstrumentationTest.java b/src/test/java/org/dataloader/instrumentation/ChainedDataLoaderInstrumentationTest.java index 0d5ddb1f..28cef700 100644 --- a/src/test/java/org/dataloader/instrumentation/ChainedDataLoaderInstrumentationTest.java +++ b/src/test/java/org/dataloader/instrumentation/ChainedDataLoaderInstrumentationTest.java @@ -14,7 +14,7 @@ import java.util.concurrent.CompletableFuture; import static org.awaitility.Awaitility.await; -import static org.dataloader.DataLoaderOptions.newOptionsBuilder; +import static org.dataloader.DataLoaderOptions.newOptions; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; @@ -37,7 +37,7 @@ void canChainTogetherZeroInstrumentation() { // just to prove its useless but harmless ChainedDataLoaderInstrumentation chainedItn = new ChainedDataLoaderInstrumentation(); - DataLoaderOptions options = newOptionsBuilder().setInstrumentation(chainedItn).build(); + DataLoaderOptions options = newOptions().setInstrumentation(chainedItn).build(); DataLoader dl = DataLoaderFactory.newDataLoader(TestKit.keysAsValues(), options); @@ -57,7 +57,7 @@ void canChainTogetherOneInstrumentation() { ChainedDataLoaderInstrumentation chainedItn = new ChainedDataLoaderInstrumentation() .add(capturingA); - DataLoaderOptions options = newOptionsBuilder().setInstrumentation(chainedItn).build(); + DataLoaderOptions options = newOptions().setInstrumentation(chainedItn).build(); DataLoader dl = DataLoaderFactory.newDataLoader(TestKit.keysAsValues(), options); @@ -88,7 +88,7 @@ public void canChainTogetherManyInstrumentationsWithDifferentBatchLoaders(TestDa .add(capturingB) .add(capturingButReturnsNull); - DataLoaderOptions options = newOptionsBuilder().setInstrumentation(chainedItn).build(); + DataLoaderOptions options = newOptions().setInstrumentation(chainedItn).build(); DataLoader dl = factory.idLoader(options); diff --git a/src/test/java/org/dataloader/instrumentation/DataLoaderInstrumentationTest.java b/src/test/java/org/dataloader/instrumentation/DataLoaderInstrumentationTest.java index a0fc1086..3e2b94f1 100644 --- a/src/test/java/org/dataloader/instrumentation/DataLoaderInstrumentationTest.java +++ b/src/test/java/org/dataloader/instrumentation/DataLoaderInstrumentationTest.java @@ -48,9 +48,10 @@ public DataLoaderInstrumentationContext> beginBatchLoader(DataLoader dl = DataLoaderFactory.newDataLoader(snoozingBatchLoader, options); @@ -109,9 +110,10 @@ public DataLoaderInstrumentationContext> beginBatchLoader(DataLoader dl = DataLoaderFactory.newDataLoader(snoozingBatchLoader, options); @@ -155,7 +157,7 @@ public void onCompleted(List result, Throwable t) { } }; - DataLoaderOptions options = new DataLoaderOptions().withInstrumentation(instrumentation); + DataLoaderOptions options = DataLoaderOptions.newOptions().setInstrumentation(instrumentation).build(); DataLoader dl = DataLoaderFactory.newDataLoader(snoozingBatchLoader, options); dl.load("A", "kcA"); diff --git a/src/test/java/org/dataloader/instrumentation/DataLoaderRegistryInstrumentationTest.java b/src/test/java/org/dataloader/instrumentation/DataLoaderRegistryInstrumentationTest.java index 8d919345..37b64094 100644 --- a/src/test/java/org/dataloader/instrumentation/DataLoaderRegistryInstrumentationTest.java +++ b/src/test/java/org/dataloader/instrumentation/DataLoaderRegistryInstrumentationTest.java @@ -120,9 +120,9 @@ void wontDoAnyThingIfThereIsNoRegistryInstrumentation() { @Test void wontDoAnyThingIfThereTheyAreTheSameInstrumentationAlready() { - DataLoader newX = dlX.transform(builder -> builder.options(dlX.getOptions().withInstrumentation(instrA))); - DataLoader newY = dlX.transform(builder -> builder.options(dlY.getOptions().withInstrumentation(instrA))); - DataLoader newZ = dlX.transform(builder -> builder.options(dlZ.getOptions().withInstrumentation(instrA))); + DataLoader newX = dlX.transform(builder -> builder.options(dlX.getOptions().transform(b -> b.setInstrumentation(instrA)))); + DataLoader newY = dlX.transform(builder -> builder.options(dlY.getOptions().transform(b -> b.setInstrumentation(instrA)))); + DataLoader newZ = dlX.transform(builder -> builder.options(dlZ.getOptions().transform(b -> b.setInstrumentation(instrA)))); DataLoaderRegistry registry = DataLoaderRegistry.newRegistry() .instrumentation(instrA) .register("X", newX) @@ -145,7 +145,7 @@ void wontDoAnyThingIfThereTheyAreTheSameInstrumentationAlready() { @Test void ifTheDLHasAInstrumentationThenItsTurnedIntoAChainedOne() { - DataLoaderOptions options = dlX.getOptions().withInstrumentation(instrA); + DataLoaderOptions options = dlX.getOptions().transform(b -> b.setInstrumentation(instrA)); DataLoader newX = dlX.transform(builder -> builder.options(options)); DataLoaderRegistry registry = DataLoaderRegistry.newRegistry() @@ -164,7 +164,7 @@ void ifTheDLHasAInstrumentationThenItsTurnedIntoAChainedOne() { @Test void chainedInstrumentationsWillBeCombined() { - DataLoaderOptions options = dlX.getOptions().withInstrumentation(chainedInstrB); + DataLoaderOptions options = dlX.getOptions().transform(b -> b.setInstrumentation(chainedInstrB)); DataLoader newX = dlX.transform(builder -> builder.options(options)); DataLoaderRegistry registry = DataLoaderRegistry.newRegistry() diff --git a/src/test/java/org/dataloader/scheduler/BatchLoaderSchedulerTest.java b/src/test/java/org/dataloader/scheduler/BatchLoaderSchedulerTest.java index fd590afc..b9a7c01d 100644 --- a/src/test/java/org/dataloader/scheduler/BatchLoaderSchedulerTest.java +++ b/src/test/java/org/dataloader/scheduler/BatchLoaderSchedulerTest.java @@ -83,7 +83,7 @@ private static void commonSetupAndSimpleAsserts(DataLoader ide @Test public void can_allow_a_simple_scheduler() { - DataLoaderOptions options = DataLoaderOptions.newOptions().withBatchLoaderScheduler(immediateScheduling); + DataLoaderOptions options = DataLoaderOptions.newOptions().setBatchLoaderScheduler(immediateScheduling).build(); DataLoader identityLoader = newDataLoader(keysAsValues(), options); @@ -92,7 +92,7 @@ public void can_allow_a_simple_scheduler() { @Test public void can_allow_a_simple_scheduler_with_context() { - DataLoaderOptions options = DataLoaderOptions.newOptions().withBatchLoaderScheduler(immediateScheduling); + DataLoaderOptions options = DataLoaderOptions.newOptions().setBatchLoaderScheduler(immediateScheduling).build(); DataLoader identityLoader = newDataLoader(keysAsValuesWithContext(), options); @@ -101,7 +101,7 @@ public void can_allow_a_simple_scheduler_with_context() { @Test public void can_allow_a_simple_scheduler_with_mapped_batch_load() { - DataLoaderOptions options = DataLoaderOptions.newOptions().withBatchLoaderScheduler(immediateScheduling); + DataLoaderOptions options = DataLoaderOptions.newOptions().setBatchLoaderScheduler(immediateScheduling).build(); DataLoader identityLoader = newMappedDataLoader(keysAsMapOfValues(), options); @@ -110,7 +110,7 @@ public void can_allow_a_simple_scheduler_with_mapped_batch_load() { @Test public void can_allow_a_simple_scheduler_with_mapped_batch_load_with_context() { - DataLoaderOptions options = DataLoaderOptions.newOptions().withBatchLoaderScheduler(immediateScheduling); + DataLoaderOptions options = DataLoaderOptions.newOptions().setBatchLoaderScheduler(immediateScheduling).build(); DataLoader identityLoader = newMappedDataLoader(keysAsMapOfValuesWithContext(), options); @@ -119,7 +119,7 @@ public void can_allow_a_simple_scheduler_with_mapped_batch_load_with_context() { @Test public void can_allow_an_async_scheduler() { - DataLoaderOptions options = DataLoaderOptions.newOptions().withBatchLoaderScheduler(delayedScheduling(50)); + DataLoaderOptions options = DataLoaderOptions.newOptions().setBatchLoaderScheduler(delayedScheduling(50)).build(); DataLoader identityLoader = newDataLoader(keysAsValues(), options); @@ -160,7 +160,7 @@ public void scheduleBatchPublisher(ScheduledBatchPublisherCall scheduledCall }); } }; - DataLoaderOptions options = DataLoaderOptions.newOptions().withBatchLoaderScheduler(funkyScheduler); + DataLoaderOptions options = DataLoaderOptions.newOptions().setBatchLoaderScheduler(funkyScheduler).build(); DataLoader identityLoader = newDataLoader(keysAsValues(), options); From 02a485d49bdff88925f2ebef9254b76ec6b6811a Mon Sep 17 00:00:00 2001 From: bbaker Date: Fri, 2 May 2025 16:48:05 +1000 Subject: [PATCH 090/156] Breaking change - adds a name to a DataLoader and deprecates a bunch of factory methods --- src/main/java/org/dataloader/DataLoader.java | 321 +---------------- .../org/dataloader/DataLoaderFactory.java | 336 ++++++++++++++++-- .../org/dataloader/DelegatingDataLoader.java | 6 +- .../java/org/dataloader/ClockDataLoader.java | 2 +- .../org/dataloader/DataLoaderFactoryTest.java | 51 +++ .../java/org/dataloader/DataLoaderTest.java | 37 +- .../dataloader/DelegatingDataLoaderTest.java | 16 + 7 files changed, 432 insertions(+), 337 deletions(-) create mode 100644 src/test/java/org/dataloader/DataLoaderFactoryTest.java diff --git a/src/main/java/org/dataloader/DataLoader.java b/src/main/java/org/dataloader/DataLoader.java index d03e5acd..94b69b1b 100644 --- a/src/main/java/org/dataloader/DataLoader.java +++ b/src/main/java/org/dataloader/DataLoader.java @@ -58,7 +58,7 @@ *

* A call to the batch loader might result in individual exception failures for item with the returned list. if * you want to capture these specific item failures then use {@link org.dataloader.Try} as a return value and - * create the data loader with {@link #newDataLoaderWithTry(BatchLoader)} form. The Try values will be interpreted + * create the data loader with {@link DataLoaderFactory#newDataLoaderWithTry(BatchLoader)} form. The Try values will be interpreted * as either success values or cause the {@link #load(Object)} promise to complete exceptionally. * * @param type parameter indicating the type of the data load keys @@ -70,6 +70,7 @@ @NullMarked public class DataLoader { + private final @Nullable String name; private final DataLoaderHelper helper; private final StatisticsCollector stats; private final CacheMap futureCache; @@ -77,317 +78,13 @@ public class DataLoader { private final DataLoaderOptions options; private final Object batchLoadFunction; - /** - * Creates new DataLoader with the specified batch loader function and default options - * (batching, caching and unlimited batch size). - * - * @param batchLoadFunction the batch load function to use - * @param the key type - * @param the value type - * @return a new DataLoader - * @deprecated use {@link DataLoaderFactory} instead - */ - @Deprecated - public static DataLoader newDataLoader(BatchLoader batchLoadFunction) { - return newDataLoader(batchLoadFunction, null); - } - - /** - * Creates new DataLoader with the specified batch loader function with the provided options - * - * @param batchLoadFunction the batch load function to use - * @param options the options to use - * @param the key type - * @param the value type - * @return a new DataLoader - * @deprecated use {@link DataLoaderFactory} instead - */ - @Deprecated - public static DataLoader newDataLoader(BatchLoader batchLoadFunction, @Nullable DataLoaderOptions options) { - return DataLoaderFactory.mkDataLoader(batchLoadFunction, options); - } - - /** - * Creates new DataLoader with the specified batch loader function and default options - * (batching, caching and unlimited batch size) where the batch loader function returns a list of - * {@link org.dataloader.Try} objects. - *

- * If it's important you to know the exact status of each item in a batch call and whether it threw exceptions then - * you can use this form to create the data loader. - *

- * Using Try objects allows you to capture a value returned or an exception that might - * have occurred trying to get a value. . - * - * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects - * @param the key type - * @param the value type - * @return a new DataLoader - * @deprecated use {@link DataLoaderFactory} instead - */ - @Deprecated - public static DataLoader newDataLoaderWithTry(BatchLoader> batchLoadFunction) { - return newDataLoaderWithTry(batchLoadFunction, null); - } - - /** - * Creates new DataLoader with the specified batch loader function and with the provided options - * where the batch loader function returns a list of - * {@link org.dataloader.Try} objects. - * - * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects - * @param options the options to use - * @param the key type - * @param the value type - * @return a new DataLoader - * @see DataLoaderFactory#newDataLoaderWithTry(BatchLoader) - * @deprecated use {@link DataLoaderFactory} instead - */ - @Deprecated - public static DataLoader newDataLoaderWithTry(BatchLoader> batchLoadFunction, @Nullable DataLoaderOptions options) { - return DataLoaderFactory.mkDataLoader(batchLoadFunction, options); - } - - /** - * Creates new DataLoader with the specified batch loader function and default options - * (batching, caching and unlimited batch size). - * - * @param batchLoadFunction the batch load function to use - * @param the key type - * @param the value type - * @return a new DataLoader - * @deprecated use {@link DataLoaderFactory} instead - */ - @Deprecated - public static DataLoader newDataLoader(BatchLoaderWithContext batchLoadFunction) { - return newDataLoader(batchLoadFunction, null); - } - - /** - * Creates new DataLoader with the specified batch loader function with the provided options - * - * @param batchLoadFunction the batch load function to use - * @param options the options to use - * @param the key type - * @param the value type - * @return a new DataLoader - * @deprecated use {@link DataLoaderFactory} instead - */ - @Deprecated - public static DataLoader newDataLoader(BatchLoaderWithContext batchLoadFunction, @Nullable DataLoaderOptions options) { - return DataLoaderFactory.mkDataLoader(batchLoadFunction, options); - } - - /** - * Creates new DataLoader with the specified batch loader function and default options - * (batching, caching and unlimited batch size) where the batch loader function returns a list of - * {@link org.dataloader.Try} objects. - *

- * If it's important you to know the exact status of each item in a batch call and whether it threw exceptions then - * you can use this form to create the data loader. - *

- * Using Try objects allows you to capture a value returned or an exception that might - * have occurred trying to get a value. . - * - * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects - * @param the key type - * @param the value type - * @return a new DataLoader - * @deprecated use {@link DataLoaderFactory} instead - */ - @Deprecated - public static DataLoader newDataLoaderWithTry(BatchLoaderWithContext> batchLoadFunction) { - return newDataLoaderWithTry(batchLoadFunction, null); - } - - /** - * Creates new DataLoader with the specified batch loader function and with the provided options - * where the batch loader function returns a list of - * {@link org.dataloader.Try} objects. - * - * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects - * @param options the options to use - * @param the key type - * @param the value type - * @return a new DataLoader - * @see DataLoaderFactory#newDataLoaderWithTry(BatchLoader) - * @deprecated use {@link DataLoaderFactory} instead - */ - @Deprecated - public static DataLoader newDataLoaderWithTry(BatchLoaderWithContext> batchLoadFunction, @Nullable DataLoaderOptions options) { - return DataLoaderFactory.mkDataLoader(batchLoadFunction, options); - } - - /** - * Creates new DataLoader with the specified batch loader function and default options - * (batching, caching and unlimited batch size). - * - * @param batchLoadFunction the batch load function to use - * @param the key type - * @param the value type - * @return a new DataLoader - * @deprecated use {@link DataLoaderFactory} instead - */ - @Deprecated - public static DataLoader newMappedDataLoader(MappedBatchLoader batchLoadFunction) { - return newMappedDataLoader(batchLoadFunction, null); - } - - /** - * Creates new DataLoader with the specified batch loader function with the provided options - * - * @param batchLoadFunction the batch load function to use - * @param options the options to use - * @param the key type - * @param the value type - * @return a new DataLoader - * @deprecated use {@link DataLoaderFactory} instead - */ - @Deprecated - public static DataLoader newMappedDataLoader(MappedBatchLoader batchLoadFunction, @Nullable DataLoaderOptions options) { - return DataLoaderFactory.mkDataLoader(batchLoadFunction, options); - } - - /** - * Creates new DataLoader with the specified batch loader function and default options - * (batching, caching and unlimited batch size) where the batch loader function returns a list of - * {@link org.dataloader.Try} objects. - *

- * If it's important you to know the exact status of each item in a batch call and whether it threw exceptions then - * you can use this form to create the data loader. - *

- * Using Try objects allows you to capture a value returned or an exception that might - * have occurred trying to get a value. . - *

- * - * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects - * @param the key type - * @param the value type - * @return a new DataLoader - * @deprecated use {@link DataLoaderFactory} instead - */ - @Deprecated - public static DataLoader newMappedDataLoaderWithTry(MappedBatchLoader> batchLoadFunction) { - return newMappedDataLoaderWithTry(batchLoadFunction, null); - } - - /** - * Creates new DataLoader with the specified batch loader function and with the provided options - * where the batch loader function returns a list of - * {@link org.dataloader.Try} objects. - * - * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects - * @param options the options to use - * @param the key type - * @param the value type - * @return a new DataLoader - * @see DataLoaderFactory#newDataLoaderWithTry(BatchLoader) - * @deprecated use {@link DataLoaderFactory} instead - */ - @Deprecated - public static DataLoader newMappedDataLoaderWithTry(MappedBatchLoader> batchLoadFunction, @Nullable DataLoaderOptions options) { - return DataLoaderFactory.mkDataLoader(batchLoadFunction, options); - } - - /** - * Creates new DataLoader with the specified mapped batch loader function and default options - * (batching, caching and unlimited batch size). - * - * @param batchLoadFunction the batch load function to use - * @param the key type - * @param the value type - * @return a new DataLoader - * @deprecated use {@link DataLoaderFactory} instead - */ - @Deprecated - public static DataLoader newMappedDataLoader(MappedBatchLoaderWithContext batchLoadFunction) { - return newMappedDataLoader(batchLoadFunction, null); - } - - /** - * Creates new DataLoader with the specified batch loader function with the provided options - * - * @param batchLoadFunction the batch load function to use - * @param options the options to use - * @param the key type - * @param the value type - * @return a new DataLoader - * @deprecated use {@link DataLoaderFactory} instead - */ - @Deprecated - public static DataLoader newMappedDataLoader(MappedBatchLoaderWithContext batchLoadFunction, @Nullable DataLoaderOptions options) { - return DataLoaderFactory.mkDataLoader(batchLoadFunction, options); - } - - /** - * Creates new DataLoader with the specified batch loader function and default options - * (batching, caching and unlimited batch size) where the batch loader function returns a list of - * {@link org.dataloader.Try} objects. - *

- * If it's important you to know the exact status of each item in a batch call and whether it threw exceptions then - * you can use this form to create the data loader. - *

- * Using Try objects allows you to capture a value returned or an exception that might - * have occurred trying to get a value. . - * - * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects - * @param the key type - * @param the value type - * @return a new DataLoader - * @deprecated use {@link DataLoaderFactory} instead - */ - @Deprecated - public static DataLoader newMappedDataLoaderWithTry(MappedBatchLoaderWithContext> batchLoadFunction) { - return newMappedDataLoaderWithTry(batchLoadFunction, null); - } - - /** - * Creates new DataLoader with the specified batch loader function and with the provided options - * where the batch loader function returns a list of - * {@link org.dataloader.Try} objects. - * - * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects - * @param options the options to use - * @param the key type - * @param the value type - * @return a new DataLoader - * @see DataLoaderFactory#newDataLoaderWithTry(BatchLoader) - * @deprecated use {@link DataLoaderFactory} instead - */ - @Deprecated - public static DataLoader newMappedDataLoaderWithTry(MappedBatchLoaderWithContext> batchLoadFunction, @Nullable DataLoaderOptions options) { - return DataLoaderFactory.mkDataLoader(batchLoadFunction, options); - } - - /** - * Creates a new data loader with the provided batch load function, and default options. - * - * @param batchLoadFunction the batch load function to use - * @deprecated use {@link DataLoaderFactory} instead - */ - @Deprecated - public DataLoader(BatchLoader batchLoadFunction) { - this((Object) batchLoadFunction, null); - } - - /** - * Creates a new data loader with the provided batch load function and options. - * - * @param batchLoadFunction the batch load function to use - * @param options the batch load options - * @deprecated use {@link DataLoaderFactory} instead - */ - @Deprecated - public DataLoader(BatchLoader batchLoadFunction, @Nullable DataLoaderOptions options) { - this((Object) batchLoadFunction, options); - } - @VisibleForTesting - DataLoader(Object batchLoadFunction, @Nullable DataLoaderOptions options) { - this(batchLoadFunction, options, Clock.systemUTC()); + DataLoader(@Nullable String name, Object batchLoadFunction, @Nullable DataLoaderOptions options) { + this(name, batchLoadFunction, options, Clock.systemUTC()); } @VisibleForTesting - DataLoader(Object batchLoadFunction, @Nullable DataLoaderOptions options, Clock clock) { + DataLoader(@Nullable String name, Object batchLoadFunction, @Nullable DataLoaderOptions options, Clock clock) { DataLoaderOptions loaderOptions = options == null ? new DataLoaderOptions() : options; this.futureCache = determineFutureCache(loaderOptions); this.valueCache = determineValueCache(loaderOptions); @@ -395,6 +92,7 @@ public DataLoader(BatchLoader batchLoadFunction, @Nullable DataLoaderOptio this.stats = nonNull(loaderOptions.getStatisticsCollector()); this.batchLoadFunction = nonNull(batchLoadFunction); this.options = loaderOptions; + this.name = name; this.helper = new DataLoaderHelper<>(this, batchLoadFunction, loaderOptions, this.futureCache, this.valueCache, this.stats, clock); } @@ -410,6 +108,13 @@ private ValueCache determineValueCache(DataLoaderOptions loaderOptions) { return (ValueCache) loaderOptions.valueCache().orElseGet(ValueCache::defaultValueCache); } + /** + * @return the name of the DataLoader which can be null + */ + public @Nullable String getName() { + return name; + } + /** * @return the options used to build this {@link DataLoader} */ diff --git a/src/main/java/org/dataloader/DataLoaderFactory.java b/src/main/java/org/dataloader/DataLoaderFactory.java index ef1a2870..c85f4677 100644 --- a/src/main/java/org/dataloader/DataLoaderFactory.java +++ b/src/main/java/org/dataloader/DataLoaderFactory.java @@ -33,7 +33,21 @@ public static DataLoader newDataLoader(BatchLoader batchLoadF * @return a new DataLoader */ public static DataLoader newDataLoader(BatchLoader batchLoadFunction, DataLoaderOptions options) { - return mkDataLoader(batchLoadFunction, options); + return mkDataLoader(null, batchLoadFunction, options); + } + + /** + * Creates new DataLoader with the specified batch loader function with the provided options + * + * @param name the name to use + * @param batchLoadFunction the batch load function to use + * @param options the options to use + * @param the key type + * @param the value type + * @return a new DataLoader + */ + public static DataLoader newDataLoader(@Nullable String name, BatchLoader batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(name, batchLoadFunction, options); } /** @@ -69,7 +83,24 @@ public static DataLoader newDataLoaderWithTry(BatchLoader * @see #newDataLoaderWithTry(BatchLoader) */ public static DataLoader newDataLoaderWithTry(BatchLoader> batchLoadFunction, DataLoaderOptions options) { - return mkDataLoader(batchLoadFunction, options); + return mkDataLoader(null, batchLoadFunction, options); + } + + /** + * Creates new DataLoader with the specified batch loader function and with the provided options + * where the batch loader function returns a list of + * {@link org.dataloader.Try} objects. + * + * @param name the name to use + * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects + * @param options the options to use + * @param the key type + * @param the value type + * @return a new DataLoader + * @see #newDataLoaderWithTry(BatchLoader) + */ + public static DataLoader newDataLoaderWithTry(@Nullable String name, BatchLoader> batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(name, batchLoadFunction, options); } /** @@ -95,7 +126,21 @@ public static DataLoader newDataLoader(BatchLoaderWithContext * @return a new DataLoader */ public static DataLoader newDataLoader(BatchLoaderWithContext batchLoadFunction, DataLoaderOptions options) { - return mkDataLoader(batchLoadFunction, options); + return mkDataLoader(null, batchLoadFunction, options); + } + + /** + * Creates new DataLoader with the specified batch loader function with the provided options + * + * @param name the name to use + * @param batchLoadFunction the batch load function to use + * @param options the options to use + * @param the key type + * @param the value type + * @return a new DataLoader + */ + public static DataLoader newDataLoader(@Nullable String name, BatchLoaderWithContext batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(name, batchLoadFunction, options); } /** @@ -131,7 +176,24 @@ public static DataLoader newDataLoaderWithTry(BatchLoaderWithContex * @see #newDataLoaderWithTry(BatchLoader) */ public static DataLoader newDataLoaderWithTry(BatchLoaderWithContext> batchLoadFunction, DataLoaderOptions options) { - return mkDataLoader(batchLoadFunction, options); + return mkDataLoader(null, batchLoadFunction, options); + } + + /** + * Creates new DataLoader with the specified batch loader function and with the provided options + * where the batch loader function returns a list of + * {@link org.dataloader.Try} objects. + * + * @param name the name to use + * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects + * @param options the options to use + * @param the key type + * @param the value type + * @return a new DataLoader + * @see #newDataLoaderWithTry(BatchLoader) + */ + public static DataLoader newDataLoaderWithTry(@Nullable String name, BatchLoaderWithContext> batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(name, batchLoadFunction, options); } /** @@ -157,7 +219,20 @@ public static DataLoader newMappedDataLoader(MappedBatchLoader DataLoader newMappedDataLoader(MappedBatchLoader batchLoadFunction, @Nullable DataLoaderOptions options) { - return mkDataLoader(batchLoadFunction, options); + return mkDataLoader(null, batchLoadFunction, options); + } + + /** + * Creates new DataLoader with the specified batch loader function with the provided options + * + * @param batchLoadFunction the batch load function to use + * @param options the options to use + * @param the key type + * @param the value type + * @return a new DataLoader + */ + public static DataLoader newMappedDataLoader(@Nullable String name, MappedBatchLoader batchLoadFunction, @Nullable DataLoaderOptions options) { + return mkDataLoader(name, batchLoadFunction, options); } /** @@ -194,7 +269,24 @@ public static DataLoader newMappedDataLoaderWithTry(MappedBatchLoad * @see #newDataLoaderWithTry(BatchLoader) */ public static DataLoader newMappedDataLoaderWithTry(MappedBatchLoader> batchLoadFunction, DataLoaderOptions options) { - return mkDataLoader(batchLoadFunction, options); + return mkDataLoader(null, batchLoadFunction, options); + } + + /** + * Creates new DataLoader with the specified batch loader function and with the provided options + * where the batch loader function returns a list of + * {@link org.dataloader.Try} objects. + * + * @param name the name to use + * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects + * @param options the options to use + * @param the key type + * @param the value type + * @return a new DataLoader + * @see #newDataLoaderWithTry(BatchLoader) + */ + public static DataLoader newMappedDataLoaderWithTry(@Nullable String name, MappedBatchLoader> batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(name, batchLoadFunction, options); } /** @@ -220,7 +312,21 @@ public static DataLoader newMappedDataLoader(MappedBatchLoaderWithC * @return a new DataLoader */ public static DataLoader newMappedDataLoader(MappedBatchLoaderWithContext batchLoadFunction, DataLoaderOptions options) { - return mkDataLoader(batchLoadFunction, options); + return mkDataLoader(null, batchLoadFunction, options); + } + + /** + * Creates new DataLoader with the specified batch loader function with the provided options + * + * @param name the name to use + * @param batchLoadFunction the batch load function to use + * @param options the options to use + * @param the key type + * @param the value type + * @return a new DataLoader + */ + public static DataLoader newMappedDataLoader(@Nullable String name, MappedBatchLoaderWithContext batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(name, batchLoadFunction, options); } /** @@ -256,7 +362,24 @@ public static DataLoader newMappedDataLoaderWithTry(MappedBatchLoad * @see #newDataLoaderWithTry(BatchLoader) */ public static DataLoader newMappedDataLoaderWithTry(MappedBatchLoaderWithContext> batchLoadFunction, DataLoaderOptions options) { - return mkDataLoader(batchLoadFunction, options); + return mkDataLoader(null, batchLoadFunction, options); + } + + /** + * Creates new DataLoader with the specified batch loader function and with the provided options + * where the batch loader function returns a list of + * {@link org.dataloader.Try} objects. + * + * @param name the name to use + * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects + * @param options the options to use + * @param the key type + * @param the value type + * @return a new DataLoader + * @see #newDataLoaderWithTry(BatchLoader) + */ + public static DataLoader newMappedDataLoaderWithTry(@Nullable String name, MappedBatchLoaderWithContext> batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(name, batchLoadFunction, options); } /** @@ -282,7 +405,21 @@ public static DataLoader newPublisherDataLoader(BatchPublisher DataLoader newPublisherDataLoader(BatchPublisher batchLoadFunction, DataLoaderOptions options) { - return mkDataLoader(batchLoadFunction, options); + return mkDataLoader(null, batchLoadFunction, options); + } + + /** + * Creates new DataLoader with the specified batch loader function with the provided options + * + * @param name the name to use + * @param batchLoadFunction the batch load function to use + * @param options the options to use + * @param the key type + * @param the value type + * @return a new DataLoader + */ + public static DataLoader newPublisherDataLoader(@Nullable String name, BatchPublisher batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(name, batchLoadFunction, options); } /** @@ -318,7 +455,24 @@ public static DataLoader newPublisherDataLoaderWithTry(BatchPublish * @see #newDataLoaderWithTry(BatchLoader) */ public static DataLoader newPublisherDataLoaderWithTry(BatchPublisher> batchLoadFunction, DataLoaderOptions options) { - return mkDataLoader(batchLoadFunction, options); + return mkDataLoader(null, batchLoadFunction, options); + } + + /** + * Creates new DataLoader with the specified batch loader function and with the provided options + * where the batch loader function returns a list of + * {@link org.dataloader.Try} objects. + * + * @param name the name to use + * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects + * @param options the options to use + * @param the key type + * @param the value type + * @return a new DataLoader + * @see #newDataLoaderWithTry(BatchLoader) + */ + public static DataLoader newPublisherDataLoaderWithTry(@Nullable String name, BatchPublisher> batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(name, batchLoadFunction, options); } /** @@ -344,7 +498,21 @@ public static DataLoader newPublisherDataLoader(BatchPublisherWithC * @return a new DataLoader */ public static DataLoader newPublisherDataLoader(BatchPublisherWithContext batchLoadFunction, DataLoaderOptions options) { - return mkDataLoader(batchLoadFunction, options); + return mkDataLoader(null, batchLoadFunction, options); + } + + /** + * Creates new DataLoader with the specified batch loader function with the provided options + * + * @param name the name to use + * @param batchLoadFunction the batch load function to use + * @param options the options to use + * @param the key type + * @param the value type + * @return a new DataLoader + */ + public static DataLoader newPublisherDataLoader(@Nullable String name, BatchPublisherWithContext batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(name, batchLoadFunction, options); } /** @@ -380,7 +548,24 @@ public static DataLoader newPublisherDataLoaderWithTry(BatchPublish * @see #newPublisherDataLoaderWithTry(BatchPublisher) */ public static DataLoader newPublisherDataLoaderWithTry(BatchPublisherWithContext> batchLoadFunction, DataLoaderOptions options) { - return mkDataLoader(batchLoadFunction, options); + return mkDataLoader(null, batchLoadFunction, options); + } + + /** + * Creates new DataLoader with the specified batch loader function and with the provided options + * where the batch loader function returns a list of + * {@link org.dataloader.Try} objects. + * + * @param name the name to use + * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects + * @param options the options to use + * @param the key type + * @param the value type + * @return a new DataLoader + * @see #newPublisherDataLoaderWithTry(BatchPublisher) + */ + public static DataLoader newPublisherDataLoaderWithTry(@Nullable String name, BatchPublisherWithContext> batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(name, batchLoadFunction, options); } /** @@ -406,7 +591,21 @@ public static DataLoader newMappedPublisherDataLoader(MappedBatchPu * @return a new DataLoader */ public static DataLoader newMappedPublisherDataLoader(MappedBatchPublisher batchLoadFunction, DataLoaderOptions options) { - return mkDataLoader(batchLoadFunction, options); + return mkDataLoader(null, batchLoadFunction, options); + } + + /** + * Creates new DataLoader with the specified batch loader function with the provided options + * + * @param name the name to use + * @param batchLoadFunction the batch load function to use + * @param options the options to use + * @param the key type + * @param the value type + * @return a new DataLoader + */ + public static DataLoader newMappedPublisherDataLoader(@Nullable String name, MappedBatchPublisher batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(name, batchLoadFunction, options); } /** @@ -442,7 +641,24 @@ public static DataLoader newMappedPublisherDataLoaderWithTry(Mapped * @see #newDataLoaderWithTry(BatchLoader) */ public static DataLoader newMappedPublisherDataLoaderWithTry(MappedBatchPublisher> batchLoadFunction, DataLoaderOptions options) { - return mkDataLoader(batchLoadFunction, options); + return mkDataLoader(null, batchLoadFunction, options); + } + + /** + * Creates new DataLoader with the specified batch loader function and with the provided options + * where the batch loader function returns a list of + * {@link org.dataloader.Try} objects. + * + * @param name the name to use + * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects + * @param options the options to use + * @param the key type + * @param the value type + * @return a new DataLoader + * @see #newDataLoaderWithTry(BatchLoader) + */ + public static DataLoader newMappedPublisherDataLoaderWithTry(@Nullable String name, MappedBatchPublisher> batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(name, batchLoadFunction, options); } /** @@ -468,7 +684,21 @@ public static DataLoader newMappedPublisherDataLoader(MappedBatchPu * @return a new DataLoader */ public static DataLoader newMappedPublisherDataLoader(MappedBatchPublisherWithContext batchLoadFunction, DataLoaderOptions options) { - return mkDataLoader(batchLoadFunction, options); + return mkDataLoader(null, batchLoadFunction, options); + } + + /** + * Creates new DataLoader with the specified batch loader function with the provided options + * + * @param name the name to use + * @param batchLoadFunction the batch load function to use + * @param options the options to use + * @param the key type + * @param the value type + * @return a new DataLoader + */ + public static DataLoader newMappedPublisherDataLoader(@Nullable String name, MappedBatchPublisherWithContext batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(name, batchLoadFunction, options); } /** @@ -504,11 +734,28 @@ public static DataLoader newMappedPublisherDataLoaderWithTry(Mapped * @see #newMappedPublisherDataLoaderWithTry(MappedBatchPublisher) */ public static DataLoader newMappedPublisherDataLoaderWithTry(MappedBatchPublisherWithContext> batchLoadFunction, DataLoaderOptions options) { - return mkDataLoader(batchLoadFunction, options); + return mkDataLoader(null, batchLoadFunction, options); + } + + /** + * Creates new DataLoader with the specified batch loader function and with the provided options + * where the batch loader function returns a list of + * {@link org.dataloader.Try} objects. + * + * @param name the name to use + * @param batchLoadFunction the batch load function to use that uses {@link org.dataloader.Try} objects + * @param options the options to use + * @param the key type + * @param the value type + * @return a new DataLoader + * @see #newMappedPublisherDataLoaderWithTry(MappedBatchPublisher) + */ + public static DataLoader newMappedPublisherDataLoaderWithTry(@Nullable String name, MappedBatchPublisherWithContext> batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(name, batchLoadFunction, options); } - static DataLoader mkDataLoader(Object batchLoadFunction, DataLoaderOptions options) { - return new DataLoader<>(batchLoadFunction, options); + static DataLoader mkDataLoader(@Nullable String name, Object batchLoadFunction, DataLoaderOptions options) { + return new DataLoader<>(name, batchLoadFunction, options); } /** @@ -541,6 +788,7 @@ public static Builder builder(DataLoader dataLoader) { * @param the value type */ public static class Builder { + String name; Object batchLoadFunction; DataLoaderOptions options = DataLoaderOptions.newOptions(); @@ -548,12 +796,13 @@ public static class Builder { } Builder(DataLoader dataLoader) { + this.name = dataLoader.getName(); this.batchLoadFunction = dataLoader.getBatchLoadFunction(); this.options = dataLoader.getOptions(); } - public Builder batchLoadFunction(Object batchLoadFunction) { - this.batchLoadFunction = batchLoadFunction; + public Builder name(String name) { + this.name = name; return this; } @@ -562,8 +811,53 @@ public Builder options(DataLoaderOptions options) { return this; } + public Builder batchLoadFunction(Object batchLoadFunction) { + this.batchLoadFunction = batchLoadFunction; + return this; + } + + public Builder batchLoader(BatchLoader batchLoadFunction) { + this.batchLoadFunction = batchLoadFunction; + return this; + } + + public Builder batchLoader(BatchLoaderWithContext batchLoadFunction) { + this.batchLoadFunction = batchLoadFunction; + return this; + } + + public Builder mappedBatchLoader(MappedBatchLoader batchLoadFunction) { + this.batchLoadFunction = batchLoadFunction; + return this; + } + + public Builder mappedBatchLoader(MappedBatchLoaderWithContext batchLoadFunction) { + this.batchLoadFunction = batchLoadFunction; + return this; + } + + public Builder publisherBatchLoader(BatchPublisher batchLoadFunction) { + this.batchLoadFunction = batchLoadFunction; + return this; + } + + public Builder publisherBatchLoader(BatchPublisherWithContext batchLoadFunction) { + this.batchLoadFunction = batchLoadFunction; + return this; + } + + public Builder mappedPublisherBatchLoader(MappedBatchPublisher batchLoadFunction) { + this.batchLoadFunction = batchLoadFunction; + return this; + } + + public Builder mappedPublisherBatchLoader(MappedBatchPublisherWithContext batchLoadFunction) { + this.batchLoadFunction = batchLoadFunction; + return this; + } + public DataLoader build() { - return mkDataLoader(batchLoadFunction, options); + return mkDataLoader(name, batchLoadFunction, options); } } } diff --git a/src/main/java/org/dataloader/DelegatingDataLoader.java b/src/main/java/org/dataloader/DelegatingDataLoader.java index c54a7317..7cffced0 100644 --- a/src/main/java/org/dataloader/DelegatingDataLoader.java +++ b/src/main/java/org/dataloader/DelegatingDataLoader.java @@ -29,8 +29,8 @@ * CompletableFuture cf = super.load(key, keyContext); * return cf.thenApply(v -> "|" + v + "|"); * } - *}; - *} + * }; + * } * * @param type parameter indicating the type of the data load keys * @param type parameter indicating the type of the data that is returned @@ -58,7 +58,7 @@ public static DataLoader unwrap(DataLoader dataLoader) { } public DelegatingDataLoader(DataLoader delegate) { - super(delegate.getBatchLoadFunction(), delegate.getOptions()); + super(delegate.getName(), delegate.getBatchLoadFunction(), delegate.getOptions()); this.delegate = delegate; } diff --git a/src/test/java/org/dataloader/ClockDataLoader.java b/src/test/java/org/dataloader/ClockDataLoader.java index 21faeea4..0c833167 100644 --- a/src/test/java/org/dataloader/ClockDataLoader.java +++ b/src/test/java/org/dataloader/ClockDataLoader.java @@ -9,7 +9,7 @@ public ClockDataLoader(Object batchLoadFunction, Clock clock) { } public ClockDataLoader(Object batchLoadFunction, DataLoaderOptions options, Clock clock) { - super(batchLoadFunction, options, clock); + super(null, batchLoadFunction, options, clock); } } diff --git a/src/test/java/org/dataloader/DataLoaderFactoryTest.java b/src/test/java/org/dataloader/DataLoaderFactoryTest.java new file mode 100644 index 00000000..265aa62c --- /dev/null +++ b/src/test/java/org/dataloader/DataLoaderFactoryTest.java @@ -0,0 +1,51 @@ +package org.dataloader; + +import org.junit.jupiter.api.Test; + +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +class DataLoaderFactoryTest { + + @Test + void can_create_via_builder() { + BatchLoaderWithContext loader = (keys, environment) -> CompletableFuture.completedFuture(keys); + DataLoaderOptions options = DataLoaderOptions.newOptionsBuilder().setBatchingEnabled(true).build(); + + DataLoader dl = DataLoaderFactory.builder() + .name("x").batchLoader(loader).options(options).build(); + + assertNotNull(dl.getName()); + assertThat(dl.getName(), equalTo("x")); + assertThat(dl.getBatchLoadFunction(), equalTo(loader)); + assertThat(dl.getOptions(), equalTo(options)); + + BatchLoaderWithContext> loaderTry = (keys, environment) + -> CompletableFuture.completedFuture(keys.stream().map(Try::succeeded).collect(Collectors.toList())); + + DataLoader> dlTry = DataLoaderFactory.>builder() + .name("try").batchLoader(loaderTry).options(options).build(); + + assertNotNull(dlTry.getName()); + assertThat(dlTry.getName(), equalTo("try")); + assertThat(dlTry.getBatchLoadFunction(), equalTo(loaderTry)); + assertThat(dlTry.getOptions(), equalTo(options)); + + MappedBatchLoader> mappedLoaderTry = (keys) + -> CompletableFuture.completedFuture( + keys.stream().collect(Collectors.toMap(k -> k, Try::succeeded)) + ); + + DataLoader> dlTry2 = DataLoaderFactory.>builder() + .name("try2").mappedBatchLoader(mappedLoaderTry).options(options).build(); + + assertNotNull(dlTry2.getName()); + assertThat(dlTry2.getName(), equalTo("try2")); + assertThat(dlTry2.getBatchLoadFunction(), equalTo(mappedLoaderTry)); + assertThat(dlTry2.getOptions(), equalTo(options)); + } +} \ No newline at end of file diff --git a/src/test/java/org/dataloader/DataLoaderTest.java b/src/test/java/org/dataloader/DataLoaderTest.java index 069d390a..630dd188 100644 --- a/src/test/java/org/dataloader/DataLoaderTest.java +++ b/src/test/java/org/dataloader/DataLoaderTest.java @@ -33,7 +33,13 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; import java.util.concurrent.ExecutionException; @@ -43,8 +49,12 @@ import java.util.stream.Collectors; import static java.util.Arrays.asList; -import static java.util.Collections.*; -import static java.util.concurrent.CompletableFuture.*; +import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; +import static java.util.Collections.singletonList; +import static java.util.concurrent.CompletableFuture.allOf; +import static java.util.concurrent.CompletableFuture.completedFuture; +import static java.util.concurrent.CompletableFuture.supplyAsync; import static org.awaitility.Awaitility.await; import static org.dataloader.DataLoaderFactory.newDataLoader; import static org.dataloader.DataLoaderOptions.newOptions; @@ -52,8 +62,13 @@ import static org.dataloader.fixtures.TestKit.listFrom; import static org.dataloader.impl.CompletableFutureKit.cause; import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.anEmptyMap; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; /** * Tests for {@link DataLoader}. @@ -82,6 +97,20 @@ public void should_Build_a_really_really_simple_data_loader() { await().untilAtomic(success, is(true)); } + @Test + public void should_Build_a_named_data_loader() { + BatchLoader loadFunction = CompletableFuture::completedFuture; + DataLoader dl = newDataLoader("name", loadFunction, DataLoaderOptions.newOptions()); + + assertNotNull(dl.getName()); + assertThat(dl.getName(), equalTo("name")); + + DataLoader dl2 = DataLoaderFactory.builder().name("name2").batchLoader(loadFunction).build(); + + assertNotNull(dl2.getName()); + assertThat(dl2.getName(), equalTo("name2")); + } + @Test public void basic_map_batch_loading() { MappedBatchLoader evensOnlyMappedBatchLoader = (keys) -> { diff --git a/src/test/java/org/dataloader/DelegatingDataLoaderTest.java b/src/test/java/org/dataloader/DelegatingDataLoaderTest.java index 9103ecaf..f1aec2d4 100644 --- a/src/test/java/org/dataloader/DelegatingDataLoaderTest.java +++ b/src/test/java/org/dataloader/DelegatingDataLoaderTest.java @@ -4,6 +4,7 @@ import org.dataloader.fixtures.parameterized.DelegatingDataLoaderFactory; import org.jspecify.annotations.NonNull; import org.jspecify.annotations.Nullable; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import java.util.List; @@ -13,6 +14,7 @@ import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.*; /** * There are WAY more tests via the {@link DelegatingDataLoaderFactory} @@ -61,4 +63,18 @@ public CompletableFuture load(@NonNull String key, @Nullable Object keyC assertThat(delegatingDataLoader.getIfCompleted("A").isEmpty(), equalTo(false)); assertThat(delegatingDataLoader.getIfCompleted("X").isEmpty(), equalTo(true)); } + + @Test + void can_delegate_simple_properties() { + DataLoaderOptions options = DataLoaderOptions.newOptions(); + BatchLoader loadFunction = CompletableFuture::completedFuture; + + DataLoader rawLoader = DataLoaderFactory.newDataLoader("name", loadFunction, options); + DelegatingDataLoader delegate = new DelegatingDataLoader<>(rawLoader); + + assertNotNull(delegate.getName()); + assertThat(delegate.getName(),equalTo("name")); + assertThat(delegate.getOptions(),equalTo(options)); + assertThat(delegate.getBatchLoadFunction(),equalTo(loadFunction)); + } } \ No newline at end of file From bf3edd29c8d7bef31e024732beca6b63239f80d4 Mon Sep 17 00:00:00 2001 From: bbaker Date: Fri, 2 May 2025 23:51:10 +1000 Subject: [PATCH 091/156] Breaking change - adds a name to a DataLoader and deprecates a bunch of factory methods - added support in DLR for namings DLs --- src/main/java/org/dataloader/DataLoader.java | 7 ++ .../org/dataloader/DataLoaderFactory.java | 82 +++++++++++-------- .../org/dataloader/DataLoaderRegistry.java | 48 +++++++++-- .../ScheduledDataLoaderRegistry.java | 20 ++--- .../dataloader/DataLoaderRegistryTest.java | 28 ++++--- .../java/org/dataloader/fixtures/TestKit.java | 12 ++- ...DataLoaderRegistryInstrumentationTest.java | 10 +-- 7 files changed, 133 insertions(+), 74 deletions(-) diff --git a/src/main/java/org/dataloader/DataLoader.java b/src/main/java/org/dataloader/DataLoader.java index 94b69b1b..7a506197 100644 --- a/src/main/java/org/dataloader/DataLoader.java +++ b/src/main/java/org/dataloader/DataLoader.java @@ -492,4 +492,11 @@ public ValueCache getValueCache() { return valueCache; } + @Override + public String toString() { + return "DataLoader{" + + "name='" + name + '\'' + + ", stats=" + stats + + '}'; + } } diff --git a/src/main/java/org/dataloader/DataLoaderFactory.java b/src/main/java/org/dataloader/DataLoaderFactory.java index c85f4677..a95e1582 100644 --- a/src/main/java/org/dataloader/DataLoaderFactory.java +++ b/src/main/java/org/dataloader/DataLoaderFactory.java @@ -3,6 +3,8 @@ import org.dataloader.annotations.PublicApi; import org.jspecify.annotations.Nullable; +import static org.dataloader.impl.Assertions.nonNull; + /** * A factory class to create {@link DataLoader}s */ @@ -23,6 +25,20 @@ public static DataLoader newDataLoader(BatchLoader batchLoadF return newDataLoader(batchLoadFunction, null); } + /** + * Creates new DataLoader with the specified batch loader function and default options + * (batching, caching and unlimited batch size). + * + * @param name the name to use + * @param batchLoadFunction the batch load function to use + * @param the key type + * @param the value type + * @return a new DataLoader + */ + public static DataLoader newDataLoader(String name, BatchLoader batchLoadFunction) { + return newDataLoader(name, batchLoadFunction, null); + } + /** * Creates new DataLoader with the specified batch loader function with the provided options * @@ -46,8 +62,8 @@ public static DataLoader newDataLoader(BatchLoader batchLoadF * @param the value type * @return a new DataLoader */ - public static DataLoader newDataLoader(@Nullable String name, BatchLoader batchLoadFunction, DataLoaderOptions options) { - return mkDataLoader(name, batchLoadFunction, options); + public static DataLoader newDataLoader(String name, BatchLoader batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(nonNull(name), batchLoadFunction, options); } /** @@ -99,8 +115,8 @@ public static DataLoader newDataLoaderWithTry(BatchLoader * @return a new DataLoader * @see #newDataLoaderWithTry(BatchLoader) */ - public static DataLoader newDataLoaderWithTry(@Nullable String name, BatchLoader> batchLoadFunction, DataLoaderOptions options) { - return mkDataLoader(name, batchLoadFunction, options); + public static DataLoader newDataLoaderWithTry(String name, BatchLoader> batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(nonNull(name), batchLoadFunction, options); } /** @@ -139,8 +155,8 @@ public static DataLoader newDataLoader(BatchLoaderWithContext * @param the value type * @return a new DataLoader */ - public static DataLoader newDataLoader(@Nullable String name, BatchLoaderWithContext batchLoadFunction, DataLoaderOptions options) { - return mkDataLoader(name, batchLoadFunction, options); + public static DataLoader newDataLoader(String name, BatchLoaderWithContext batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(nonNull(name), batchLoadFunction, options); } /** @@ -192,8 +208,8 @@ public static DataLoader newDataLoaderWithTry(BatchLoaderWithContex * @return a new DataLoader * @see #newDataLoaderWithTry(BatchLoader) */ - public static DataLoader newDataLoaderWithTry(@Nullable String name, BatchLoaderWithContext> batchLoadFunction, DataLoaderOptions options) { - return mkDataLoader(name, batchLoadFunction, options); + public static DataLoader newDataLoaderWithTry(String name, BatchLoaderWithContext> batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(nonNull(name), batchLoadFunction, options); } /** @@ -231,8 +247,8 @@ public static DataLoader newMappedDataLoader(MappedBatchLoader the value type * @return a new DataLoader */ - public static DataLoader newMappedDataLoader(@Nullable String name, MappedBatchLoader batchLoadFunction, @Nullable DataLoaderOptions options) { - return mkDataLoader(name, batchLoadFunction, options); + public static DataLoader newMappedDataLoader(String name, MappedBatchLoader batchLoadFunction, @Nullable DataLoaderOptions options) { + return mkDataLoader(nonNull(name), batchLoadFunction, options); } /** @@ -285,8 +301,8 @@ public static DataLoader newMappedDataLoaderWithTry(MappedBatchLoad * @return a new DataLoader * @see #newDataLoaderWithTry(BatchLoader) */ - public static DataLoader newMappedDataLoaderWithTry(@Nullable String name, MappedBatchLoader> batchLoadFunction, DataLoaderOptions options) { - return mkDataLoader(name, batchLoadFunction, options); + public static DataLoader newMappedDataLoaderWithTry(String name, MappedBatchLoader> batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(nonNull(name), batchLoadFunction, options); } /** @@ -325,8 +341,8 @@ public static DataLoader newMappedDataLoader(MappedBatchLoaderWithC * @param the value type * @return a new DataLoader */ - public static DataLoader newMappedDataLoader(@Nullable String name, MappedBatchLoaderWithContext batchLoadFunction, DataLoaderOptions options) { - return mkDataLoader(name, batchLoadFunction, options); + public static DataLoader newMappedDataLoader(String name, MappedBatchLoaderWithContext batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(nonNull(name), batchLoadFunction, options); } /** @@ -378,8 +394,8 @@ public static DataLoader newMappedDataLoaderWithTry(MappedBatchLoad * @return a new DataLoader * @see #newDataLoaderWithTry(BatchLoader) */ - public static DataLoader newMappedDataLoaderWithTry(@Nullable String name, MappedBatchLoaderWithContext> batchLoadFunction, DataLoaderOptions options) { - return mkDataLoader(name, batchLoadFunction, options); + public static DataLoader newMappedDataLoaderWithTry(String name, MappedBatchLoaderWithContext> batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(nonNull(name), batchLoadFunction, options); } /** @@ -418,8 +434,8 @@ public static DataLoader newPublisherDataLoader(BatchPublisher the value type * @return a new DataLoader */ - public static DataLoader newPublisherDataLoader(@Nullable String name, BatchPublisher batchLoadFunction, DataLoaderOptions options) { - return mkDataLoader(name, batchLoadFunction, options); + public static DataLoader newPublisherDataLoader(String name, BatchPublisher batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(nonNull(name), batchLoadFunction, options); } /** @@ -471,8 +487,8 @@ public static DataLoader newPublisherDataLoaderWithTry(BatchPublish * @return a new DataLoader * @see #newDataLoaderWithTry(BatchLoader) */ - public static DataLoader newPublisherDataLoaderWithTry(@Nullable String name, BatchPublisher> batchLoadFunction, DataLoaderOptions options) { - return mkDataLoader(name, batchLoadFunction, options); + public static DataLoader newPublisherDataLoaderWithTry(String name, BatchPublisher> batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(nonNull(name), batchLoadFunction, options); } /** @@ -511,8 +527,8 @@ public static DataLoader newPublisherDataLoader(BatchPublisherWithC * @param the value type * @return a new DataLoader */ - public static DataLoader newPublisherDataLoader(@Nullable String name, BatchPublisherWithContext batchLoadFunction, DataLoaderOptions options) { - return mkDataLoader(name, batchLoadFunction, options); + public static DataLoader newPublisherDataLoader(String name, BatchPublisherWithContext batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(nonNull(name), batchLoadFunction, options); } /** @@ -564,8 +580,8 @@ public static DataLoader newPublisherDataLoaderWithTry(BatchPublish * @return a new DataLoader * @see #newPublisherDataLoaderWithTry(BatchPublisher) */ - public static DataLoader newPublisherDataLoaderWithTry(@Nullable String name, BatchPublisherWithContext> batchLoadFunction, DataLoaderOptions options) { - return mkDataLoader(name, batchLoadFunction, options); + public static DataLoader newPublisherDataLoaderWithTry(String name, BatchPublisherWithContext> batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(nonNull(name), batchLoadFunction, options); } /** @@ -604,8 +620,8 @@ public static DataLoader newMappedPublisherDataLoader(MappedBatchPu * @param the value type * @return a new DataLoader */ - public static DataLoader newMappedPublisherDataLoader(@Nullable String name, MappedBatchPublisher batchLoadFunction, DataLoaderOptions options) { - return mkDataLoader(name, batchLoadFunction, options); + public static DataLoader newMappedPublisherDataLoader(String name, MappedBatchPublisher batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(nonNull(name), batchLoadFunction, options); } /** @@ -657,8 +673,8 @@ public static DataLoader newMappedPublisherDataLoaderWithTry(Mapped * @return a new DataLoader * @see #newDataLoaderWithTry(BatchLoader) */ - public static DataLoader newMappedPublisherDataLoaderWithTry(@Nullable String name, MappedBatchPublisher> batchLoadFunction, DataLoaderOptions options) { - return mkDataLoader(name, batchLoadFunction, options); + public static DataLoader newMappedPublisherDataLoaderWithTry(String name, MappedBatchPublisher> batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(nonNull(name), batchLoadFunction, options); } /** @@ -697,8 +713,8 @@ public static DataLoader newMappedPublisherDataLoader(MappedBatchPu * @param the value type * @return a new DataLoader */ - public static DataLoader newMappedPublisherDataLoader(@Nullable String name, MappedBatchPublisherWithContext batchLoadFunction, DataLoaderOptions options) { - return mkDataLoader(name, batchLoadFunction, options); + public static DataLoader newMappedPublisherDataLoader(String name, MappedBatchPublisherWithContext batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(nonNull(name), batchLoadFunction, options); } /** @@ -750,11 +766,11 @@ public static DataLoader newMappedPublisherDataLoaderWithTry(Mapped * @return a new DataLoader * @see #newMappedPublisherDataLoaderWithTry(MappedBatchPublisher) */ - public static DataLoader newMappedPublisherDataLoaderWithTry(@Nullable String name, MappedBatchPublisherWithContext> batchLoadFunction, DataLoaderOptions options) { - return mkDataLoader(name, batchLoadFunction, options); + public static DataLoader newMappedPublisherDataLoaderWithTry(String name, MappedBatchPublisherWithContext> batchLoadFunction, DataLoaderOptions options) { + return mkDataLoader(nonNull(name), batchLoadFunction, options); } - static DataLoader mkDataLoader(@Nullable String name, Object batchLoadFunction, DataLoaderOptions options) { + static DataLoader mkDataLoader(@Nullable String name, Object batchLoadFunction, @Nullable DataLoaderOptions options) { return new DataLoader<>(name, batchLoadFunction, options); } diff --git a/src/main/java/org/dataloader/DataLoaderRegistry.java b/src/main/java/org/dataloader/DataLoaderRegistry.java index 06c93c41..0d7b1f6a 100644 --- a/src/main/java/org/dataloader/DataLoaderRegistry.java +++ b/src/main/java/org/dataloader/DataLoaderRegistry.java @@ -5,6 +5,8 @@ import org.dataloader.instrumentation.DataLoaderInstrumentation; import org.dataloader.instrumentation.DataLoaderInstrumentationHelper; import org.dataloader.stats.Statistics; +import org.jspecify.annotations.NullMarked; +import org.jspecify.annotations.Nullable; import java.util.ArrayList; import java.util.HashMap; @@ -16,6 +18,8 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.function.Function; +import static org.dataloader.impl.Assertions.assertState; + /** * This allows data loaders to be registered together into a single place, so * they can be dispatched as one. It also allows you to retrieve data loaders by @@ -35,9 +39,10 @@ * are the same object, then nothing is changed, since the same instrumentation code is being run. */ @PublicApi +@NullMarked public class DataLoaderRegistry { protected final Map> dataLoaders; - protected final DataLoaderInstrumentation instrumentation; + protected final @Nullable DataLoaderInstrumentation instrumentation; public DataLoaderRegistry() { @@ -48,15 +53,15 @@ private DataLoaderRegistry(Builder builder) { this(builder.dataLoaders, builder.instrumentation); } - protected DataLoaderRegistry(Map> dataLoaders, DataLoaderInstrumentation instrumentation) { + protected DataLoaderRegistry(Map> dataLoaders, @Nullable DataLoaderInstrumentation instrumentation) { this.dataLoaders = instrumentDLs(dataLoaders, instrumentation); this.instrumentation = instrumentation; } - private Map> instrumentDLs(Map> incomingDataLoaders, DataLoaderInstrumentation registryInstrumentation) { + private Map> instrumentDLs(Map> incomingDataLoaders, @Nullable DataLoaderInstrumentation registryInstrumentation) { Map> dataLoaders = new ConcurrentHashMap<>(incomingDataLoaders); if (registryInstrumentation != null) { - dataLoaders.replaceAll((k, existingDL) -> instrumentDL(registryInstrumentation, existingDL)); + dataLoaders.replaceAll((k, existingDL) -> nameAndInstrumentDL(k, registryInstrumentation, existingDL)); } return dataLoaders; } @@ -64,11 +69,14 @@ protected DataLoaderRegistry(Map> dataLoaders, DataLoad /** * Can be called to tweak a {@link DataLoader} so that it has the registry {@link DataLoaderInstrumentation} added as the first one. * + * @param key the key used to register the data loader * @param registryInstrumentation the common registry {@link DataLoaderInstrumentation} * @param existingDL the existing data loader * @return a new {@link DataLoader} or the same one if there is nothing to change */ - private static DataLoader instrumentDL(DataLoaderInstrumentation registryInstrumentation, DataLoader existingDL) { + private static DataLoader nameAndInstrumentDL(String key, @Nullable DataLoaderInstrumentation registryInstrumentation, DataLoader existingDL) { + existingDL = checkAndSetName(key, existingDL); + if (registryInstrumentation == null) { return existingDL; } @@ -97,6 +105,15 @@ protected DataLoaderRegistry(Map> dataLoaders, DataLoad } } + private static DataLoader checkAndSetName(String key, DataLoader dataLoader) { + if (dataLoader.getName() == null) { + return dataLoader.transform(b -> b.name(key)); + } + assertState(key.equals(dataLoader.getName()), + () -> String.format("Data loader name '%s' is not the same as registered key '%s'", dataLoader.getName(), key)); + return dataLoader; + } + private static DataLoader mkInstrumentedDataLoader(DataLoader existingDL, DataLoaderOptions options, DataLoaderInstrumentation newInstrumentation) { return existingDL.transform(builder -> builder.options(setInInstrumentation(options, newInstrumentation))); } @@ -108,7 +125,7 @@ private static DataLoaderOptions setInInstrumentation(DataLoaderOptions options, /** * @return the {@link DataLoaderInstrumentation} associated with this registry which can be null */ - public DataLoaderInstrumentation getInstrumentation() { + public @Nullable DataLoaderInstrumentation getInstrumentation() { return instrumentation; } @@ -120,10 +137,23 @@ public DataLoaderInstrumentation getInstrumentation() { * @return this registry */ public DataLoaderRegistry register(String key, DataLoader dataLoader) { - dataLoaders.put(key, instrumentDL(instrumentation, dataLoader)); + dataLoaders.put(key, nameAndInstrumentDL(key, instrumentation, dataLoader)); return this; } + /** + * This will register a new dataloader and then return it. It might have been wrapped into a new instance + * because of the registry instrumentation for example. + * + * @param key the key to put the data loader under + * @param dataLoader the data loader to register + * @return the data loader instance that was registered + */ + public DataLoader registerAndGet(String key, DataLoader dataLoader) { + dataLoaders.put(key, nameAndInstrumentDL(key, instrumentation, dataLoader)); + return getDataLoader(key); + } + /** * Computes a data loader if absent or return it if it was * already registered at that key. @@ -142,7 +172,7 @@ public DataLoader computeIfAbsent(final String key, final Function> mappingFunction) { return (DataLoader) dataLoaders.computeIfAbsent(key, (k) -> { DataLoader dl = mappingFunction.apply(k); - return instrumentDL(instrumentation, dl); + return nameAndInstrumentDL(key, instrumentation, dl); }); } @@ -262,7 +292,7 @@ public static Builder newRegistry() { public static class Builder { private final Map> dataLoaders = new HashMap<>(); - private DataLoaderInstrumentation instrumentation; + private @Nullable DataLoaderInstrumentation instrumentation; /** * This will register a new dataloader diff --git a/src/main/java/org/dataloader/registries/ScheduledDataLoaderRegistry.java b/src/main/java/org/dataloader/registries/ScheduledDataLoaderRegistry.java index b6bc2573..4f62378d 100644 --- a/src/main/java/org/dataloader/registries/ScheduledDataLoaderRegistry.java +++ b/src/main/java/org/dataloader/registries/ScheduledDataLoaderRegistry.java @@ -3,7 +3,10 @@ import org.dataloader.DataLoader; import org.dataloader.DataLoaderRegistry; import org.dataloader.annotations.ExperimentalApi; +import org.dataloader.impl.Assertions; import org.dataloader.instrumentation.DataLoaderInstrumentation; +import org.jspecify.annotations.NullMarked; +import org.jspecify.annotations.Nullable; import java.time.Duration; import java.util.LinkedHashMap; @@ -54,6 +57,7 @@ * This code is currently marked as {@link ExperimentalApi} */ @ExperimentalApi +@NullMarked public class ScheduledDataLoaderRegistry extends DataLoaderRegistry implements AutoCloseable { private final Map, DispatchPredicate> dataLoaderPredicates = new ConcurrentHashMap<>(); @@ -66,7 +70,7 @@ public class ScheduledDataLoaderRegistry extends DataLoaderRegistry implements A private ScheduledDataLoaderRegistry(Builder builder) { super(builder.dataLoaders, builder.instrumentation); - this.scheduledExecutorService = builder.scheduledExecutorService; + this.scheduledExecutorService = Assertions.nonNull(builder.scheduledExecutorService); this.defaultExecutorUsed = builder.defaultExecutorUsed; this.schedule = builder.schedule; this.tickerMode = builder.tickerMode; @@ -112,7 +116,6 @@ public boolean isTickerMode() { * and return a new combined registry * * @param registry the registry to combine into this registry - * * @return a new combined registry */ public ScheduledDataLoaderRegistry combine(DataLoaderRegistry registry) { @@ -128,7 +131,6 @@ public ScheduledDataLoaderRegistry combine(DataLoaderRegistry registry) { * This will unregister a new dataloader * * @param key the key of the data loader to unregister - * * @return this registry */ public ScheduledDataLoaderRegistry unregister(String key) { @@ -161,7 +163,6 @@ public DispatchPredicate getDispatchPredicate() { * @param key the key to put the data loader under * @param dataLoader the data loader to register * @param dispatchPredicate the dispatch predicate to associate with this data loader - * * @return this registry */ public ScheduledDataLoaderRegistry register(String key, DataLoader dataLoader, DispatchPredicate dispatchPredicate) { @@ -222,7 +223,6 @@ public void rescheduleNow() { * * @param dataLoaderKey the key in the dataloader map * @param dataLoader the dataloader - * * @return true if it should dispatch */ private boolean shouldDispatch(String dataLoaderKey, DataLoader dataLoader) { @@ -267,11 +267,11 @@ public static class Builder { private final Map> dataLoaders = new LinkedHashMap<>(); private final Map, DispatchPredicate> dataLoaderPredicates = new LinkedHashMap<>(); private DispatchPredicate dispatchPredicate = DispatchPredicate.DISPATCH_ALWAYS; - private ScheduledExecutorService scheduledExecutorService; + private @Nullable ScheduledExecutorService scheduledExecutorService; private boolean defaultExecutorUsed = false; private Duration schedule = Duration.ofMillis(10); private boolean tickerMode = false; - private DataLoaderInstrumentation instrumentation; + private @Nullable DataLoaderInstrumentation instrumentation; /** @@ -279,7 +279,6 @@ public static class Builder { * {@link ScheduledDataLoaderRegistry#close()} is called. This is left to the code that made this setup code * * @param executorService the executor service to run the ticker on - * * @return this builder for a fluent pattern */ public Builder scheduledExecutorService(ScheduledExecutorService executorService) { @@ -297,7 +296,6 @@ public Builder schedule(Duration schedule) { * * @param key the key to put the data loader under * @param dataLoader the data loader to register - * * @return this builder for a fluent pattern */ public Builder register(String key, DataLoader dataLoader) { @@ -312,7 +310,6 @@ public Builder register(String key, DataLoader dataLoader) { * @param key the key to put the data loader under * @param dataLoader the data loader to register * @param dispatchPredicate the dispatch predicate - * * @return this builder for a fluent pattern */ public Builder register(String key, DataLoader dataLoader, DispatchPredicate dispatchPredicate) { @@ -326,7 +323,6 @@ public Builder register(String key, DataLoader dataLoader, DispatchPredica * from a previous {@link DataLoaderRegistry} * * @param otherRegistry the previous {@link DataLoaderRegistry} - * * @return this builder for a fluent pattern */ public Builder registerAll(DataLoaderRegistry otherRegistry) { @@ -343,7 +339,6 @@ public Builder registerAll(DataLoaderRegistry otherRegistry) { * whether all {@link DataLoader}s in the {@link DataLoaderRegistry }should be dispatched. * * @param dispatchPredicate the predicate - * * @return this builder for a fluent pattern */ public Builder dispatchPredicate(DispatchPredicate dispatchPredicate) { @@ -357,7 +352,6 @@ public Builder dispatchPredicate(DispatchPredicate dispatchPredicate) { * to dispatchAll. * * @param tickerMode true or false - * * @return this builder for a fluent pattern */ public Builder tickerMode(boolean tickerMode) { diff --git a/src/test/java/org/dataloader/DataLoaderRegistryTest.java b/src/test/java/org/dataloader/DataLoaderRegistryTest.java index bd1534dd..16b0af6b 100644 --- a/src/test/java/org/dataloader/DataLoaderRegistryTest.java +++ b/src/test/java/org/dataloader/DataLoaderRegistryTest.java @@ -18,9 +18,9 @@ public class DataLoaderRegistryTest { @Test public void registration_works() { - DataLoader dlA = newDataLoader(identityBatchLoader); - DataLoader dlB = newDataLoader(identityBatchLoader); - DataLoader dlC = newDataLoader(identityBatchLoader); + DataLoader dlA = newDataLoader("a", identityBatchLoader); + DataLoader dlB = newDataLoader("b", identityBatchLoader); + DataLoader dlC = newDataLoader("c", identityBatchLoader); DataLoaderRegistry registry = new DataLoaderRegistry(); @@ -54,10 +54,10 @@ public void registration_works() { @Test public void registries_can_be_combined() { - DataLoader dlA = newDataLoader(identityBatchLoader); - DataLoader dlB = newDataLoader(identityBatchLoader); - DataLoader dlC = newDataLoader(identityBatchLoader); - DataLoader dlD = newDataLoader(identityBatchLoader); + DataLoader dlA = newDataLoader("a",identityBatchLoader); + DataLoader dlB = newDataLoader("b", identityBatchLoader); + DataLoader dlC = newDataLoader("c", identityBatchLoader); + DataLoader dlD = newDataLoader("d", identityBatchLoader); DataLoaderRegistry registry1 = new DataLoaderRegistry(); @@ -90,6 +90,10 @@ public void stats_can_be_collected() { registry.register("a", dlA).register("b", dlB).register("c", dlC); + dlA = registry.getDataLoader("a"); + dlB = registry.getDataLoader("b"); + dlC = registry.getDataLoader("b"); + dlA.load("X"); dlB.load("Y"); dlC.load("Z"); @@ -116,7 +120,7 @@ public void computeIfAbsent_creates_a_data_loader_if_there_was_no_value_at_key() DataLoaderRegistry registry = new DataLoaderRegistry(); - DataLoader dlA = newDataLoader(identityBatchLoader); + DataLoader dlA = newDataLoader("a",identityBatchLoader); DataLoader registered = registry.computeIfAbsent("a", (key) -> dlA); assertThat(registered, equalTo(dlA)); @@ -129,11 +133,11 @@ public void computeIfAbsent_returns_an_existing_data_loader_if_there_was_a_value DataLoaderRegistry registry = new DataLoaderRegistry(); - DataLoader dlA = newDataLoader(identityBatchLoader); + DataLoader dlA = newDataLoader("a",identityBatchLoader); registry.computeIfAbsent("a", (key) -> dlA); // register again at same key - DataLoader dlA2 = newDataLoader(identityBatchLoader); + DataLoader dlA2 = newDataLoader("a", identityBatchLoader); DataLoader registered = registry.computeIfAbsent("a", (key) -> dlA2); assertThat(registered, equalTo(dlA)); @@ -149,8 +153,8 @@ public void dispatch_counts_are_maintained() { DataLoader dlA = newDataLoader(identityBatchLoader); DataLoader dlB = newDataLoader(identityBatchLoader); - registry.register("a", dlA); - registry.register("b", dlB); + dlA = registry.registerAndGet("a", dlA); + dlB = registry.registerAndGet("b", dlB); dlA.load("av1"); dlA.load("av2"); diff --git a/src/test/java/org/dataloader/fixtures/TestKit.java b/src/test/java/org/dataloader/fixtures/TestKit.java index 04ec5e50..e6ba3197 100644 --- a/src/test/java/org/dataloader/fixtures/TestKit.java +++ b/src/test/java/org/dataloader/fixtures/TestKit.java @@ -11,8 +11,8 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.LinkedHashSet; import java.util.HashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -64,10 +64,18 @@ public static DataLoader idLoader() { return idLoader(null, new ArrayList<>()); } + public static DataLoader idLoader(String name) { + return idLoader(name, null, new ArrayList<>()); + } + public static DataLoader idLoader(DataLoaderOptions options, List> loadCalls) { return DataLoaderFactory.newDataLoader(keysAsValues(loadCalls), options); } + public static DataLoader idLoader(String name, DataLoaderOptions options, List> loadCalls) { + return DataLoaderFactory.newDataLoader(name, keysAsValues(loadCalls), options); + } + public static Collection listFrom(int i, int max) { List ints = new ArrayList<>(); for (int j = i; j < max; j++) { @@ -104,7 +112,7 @@ public static Set asSet(Collection elements) { public static boolean areAllDone(CompletableFuture... cfs) { for (CompletableFuture cf : cfs) { - if (! cf.isDone()) { + if (!cf.isDone()) { return false; } } diff --git a/src/test/java/org/dataloader/instrumentation/DataLoaderRegistryInstrumentationTest.java b/src/test/java/org/dataloader/instrumentation/DataLoaderRegistryInstrumentationTest.java index 49ccf0ee..b091c393 100644 --- a/src/test/java/org/dataloader/instrumentation/DataLoaderRegistryInstrumentationTest.java +++ b/src/test/java/org/dataloader/instrumentation/DataLoaderRegistryInstrumentationTest.java @@ -32,9 +32,9 @@ public class DataLoaderRegistryInstrumentationTest { @BeforeEach void setUp() { - dlX = TestKit.idLoader(); - dlY = TestKit.idLoader(); - dlZ = TestKit.idLoader(); + dlX = TestKit.idLoader("X"); + dlY = TestKit.idLoader("Y"); + dlZ = TestKit.idLoader("Z"); instrA = new CapturingInstrumentation("A"); instrB = new CapturingInstrumentation("B"); chainedInstrA = new ChainedDataLoaderInstrumentation().add(instrA); @@ -121,8 +121,8 @@ void wontDoAnyThingIfThereIsNoRegistryInstrumentation() { @Test void wontDoAnyThingIfThereTheyAreTheSameInstrumentationAlready() { DataLoader newX = dlX.transform(builder -> builder.options(dlX.getOptions().setInstrumentation(instrA))); - DataLoader newY = dlX.transform(builder -> builder.options(dlY.getOptions().setInstrumentation(instrA))); - DataLoader newZ = dlX.transform(builder -> builder.options(dlZ.getOptions().setInstrumentation(instrA))); + DataLoader newY = dlY.transform(builder -> builder.options(dlY.getOptions().setInstrumentation(instrA))); + DataLoader newZ = dlZ.transform(builder -> builder.options(dlZ.getOptions().setInstrumentation(instrA))); DataLoaderRegistry registry = DataLoaderRegistry.newRegistry() .instrumentation(instrA) .register("X", newX) From 496a298f48ad4343c31e84914ac87ab20ed41179 Mon Sep 17 00:00:00 2001 From: bbaker Date: Sat, 3 May 2025 00:04:52 +1000 Subject: [PATCH 092/156] Breaking change - adds a name to a DataLoader and deprecates a bunch of factory methods - added support in DLR for namings DLs - added direct register --- .../org/dataloader/DataLoaderRegistry.java | 35 +++++++++++++++++-- .../dataloader/DataLoaderRegistryTest.java | 24 ++++++++++--- 2 files changed, 52 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/dataloader/DataLoaderRegistry.java b/src/main/java/org/dataloader/DataLoaderRegistry.java index 0d7b1f6a..0988697b 100644 --- a/src/main/java/org/dataloader/DataLoaderRegistry.java +++ b/src/main/java/org/dataloader/DataLoaderRegistry.java @@ -130,7 +130,29 @@ private static DataLoaderOptions setInInstrumentation(DataLoaderOptions options, } /** - * This will register a new dataloader + * This will register a new named dataloader. The {@link DataLoader} must be named something and + * cannot have a null name. + *

+ * Note: Registration can change the data loader instance since it might get an {@link DataLoaderInstrumentation} applied to + * it. So the {@link DataLoader} instance your read via {@link DataLoaderRegistry#getDataLoader(String)} might not be the same + * object that was registered. + * + * @param dataLoader the named data loader to register + * @return this registry + */ + public DataLoaderRegistry register(DataLoader dataLoader) { + String name = dataLoader.getName(); + assertState(name != null, () -> "The DataLoader must have a non null name"); + dataLoaders.put(name, nameAndInstrumentDL(name, instrumentation, dataLoader)); + return this; + } + + /** + * This will register a new {@link DataLoader} + *

+ * Note: Registration can change the data loader instance since it might get an {@link DataLoaderInstrumentation} applied to + * it. So the {@link DataLoader} instance your read via {@link DataLoaderRegistry#getDataLoader(String)} might not be the same + * object that was registered. * * @param key the key to put the data loader under * @param dataLoader the data loader to register @@ -142,8 +164,11 @@ public DataLoaderRegistry register(String key, DataLoader dataLoader) { } /** - * This will register a new dataloader and then return it. It might have been wrapped into a new instance - * because of the registry instrumentation for example. + * This will register a new {@link DataLoader} and then return it. + *

+ * Note: Registration can change the data loader instance since it might get an {@link DataLoaderInstrumentation} applied to + * it. So the {@link DataLoader} instance your read via {@link DataLoaderRegistry#getDataLoader(String)} might not be the same + * object that was registered. * * @param key the key to put the data loader under * @param dataLoader the data loader to register @@ -160,6 +185,10 @@ public DataLoader registerAndGet(String key, DataLoader dataL *

* Note: The entire method invocation is performed atomically, * so the function is applied at most once per key. + *

+ * Note: Registration can change the data loader instance since it might get an {@link DataLoaderInstrumentation} applied to + * it. So the {@link DataLoader} instance your read via {@link DataLoaderRegistry#getDataLoader(String)} might not be the same + * object that was registered. * * @param key the key of the data loader * @param mappingFunction the function to compute a data loader diff --git a/src/test/java/org/dataloader/DataLoaderRegistryTest.java b/src/test/java/org/dataloader/DataLoaderRegistryTest.java index 16b0af6b..c97c0879 100644 --- a/src/test/java/org/dataloader/DataLoaderRegistryTest.java +++ b/src/test/java/org/dataloader/DataLoaderRegistryTest.java @@ -1,9 +1,12 @@ package org.dataloader; +import org.dataloader.impl.DataLoaderAssertionException; import org.dataloader.stats.SimpleStatisticsCollector; import org.dataloader.stats.Statistics; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.List; import java.util.concurrent.CompletableFuture; import static java.util.Arrays.asList; @@ -21,6 +24,7 @@ public void registration_works() { DataLoader dlA = newDataLoader("a", identityBatchLoader); DataLoader dlB = newDataLoader("b", identityBatchLoader); DataLoader dlC = newDataLoader("c", identityBatchLoader); + DataLoader dlUnnamed = newDataLoader(identityBatchLoader); DataLoaderRegistry registry = new DataLoaderRegistry(); @@ -40,7 +44,7 @@ public void registration_works() { // and unregister (fluently) DataLoaderRegistry dlR = registry.unregister("c"); - assertThat(dlR,equalTo(registry)); + assertThat(dlR, equalTo(registry)); assertThat(registry.getDataLoaders(), equalTo(asList(dlA, dlB))); @@ -49,12 +53,24 @@ public void registration_works() { assertThat(readDL, sameInstance(dlA)); assertThat(registry.getKeys(), hasItems("a", "b")); + + + // named registry + registry = new DataLoaderRegistry(); + registry.register(dlA); + assertThat(registry.getDataLoaders(), equalTo(List.of(dlA))); + + try { + registry.register(dlUnnamed); + Assertions.fail("Should have thrown an exception"); + } catch (DataLoaderAssertionException ignored) { + } } @Test public void registries_can_be_combined() { - DataLoader dlA = newDataLoader("a",identityBatchLoader); + DataLoader dlA = newDataLoader("a", identityBatchLoader); DataLoader dlB = newDataLoader("b", identityBatchLoader); DataLoader dlC = newDataLoader("c", identityBatchLoader); DataLoader dlD = newDataLoader("d", identityBatchLoader); @@ -120,7 +136,7 @@ public void computeIfAbsent_creates_a_data_loader_if_there_was_no_value_at_key() DataLoaderRegistry registry = new DataLoaderRegistry(); - DataLoader dlA = newDataLoader("a",identityBatchLoader); + DataLoader dlA = newDataLoader("a", identityBatchLoader); DataLoader registered = registry.computeIfAbsent("a", (key) -> dlA); assertThat(registered, equalTo(dlA)); @@ -133,7 +149,7 @@ public void computeIfAbsent_returns_an_existing_data_loader_if_there_was_a_value DataLoaderRegistry registry = new DataLoaderRegistry(); - DataLoader dlA = newDataLoader("a",identityBatchLoader); + DataLoader dlA = newDataLoader("a", identityBatchLoader); registry.computeIfAbsent("a", (key) -> dlA); // register again at same key From 7259df977ed83c326c7e98b94ad53226d2024927 Mon Sep 17 00:00:00 2001 From: Martin Schulze Date: Mon, 5 May 2025 13:06:48 +0200 Subject: [PATCH 093/156] OSGI - Make org.jspecify.* imports optional --- build.gradle | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build.gradle b/build.gradle index 6072f4f6..eb571583 100644 --- a/build.gradle +++ b/build.gradle @@ -63,6 +63,9 @@ jar { '-exportcontents': 'org.dataloader.*', '-removeheaders': 'Private-Package') } + bnd(''' +Import-Package: org.jspecify.annotations;resolution:=optional,* +''') } dependencies { From 51c21d5954b876c77fe233393683834f5bfa7de9 Mon Sep 17 00:00:00 2001 From: bbaker Date: Mon, 12 May 2025 15:20:04 +1000 Subject: [PATCH 094/156] Fixed tests after merging master --- src/test/java/org/dataloader/DataLoaderFactoryTest.java | 2 +- src/test/java/org/dataloader/DataLoaderTest.java | 2 +- src/test/java/org/dataloader/DelegatingDataLoaderTest.java | 2 +- .../DataLoaderRegistryInstrumentationTest.java | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/test/java/org/dataloader/DataLoaderFactoryTest.java b/src/test/java/org/dataloader/DataLoaderFactoryTest.java index 265aa62c..3b3f368a 100644 --- a/src/test/java/org/dataloader/DataLoaderFactoryTest.java +++ b/src/test/java/org/dataloader/DataLoaderFactoryTest.java @@ -14,7 +14,7 @@ class DataLoaderFactoryTest { @Test void can_create_via_builder() { BatchLoaderWithContext loader = (keys, environment) -> CompletableFuture.completedFuture(keys); - DataLoaderOptions options = DataLoaderOptions.newOptionsBuilder().setBatchingEnabled(true).build(); + DataLoaderOptions options = DataLoaderOptions.newOptions().setBatchingEnabled(true).build(); DataLoader dl = DataLoaderFactory.builder() .name("x").batchLoader(loader).options(options).build(); diff --git a/src/test/java/org/dataloader/DataLoaderTest.java b/src/test/java/org/dataloader/DataLoaderTest.java index e196abab..224b54d3 100644 --- a/src/test/java/org/dataloader/DataLoaderTest.java +++ b/src/test/java/org/dataloader/DataLoaderTest.java @@ -101,7 +101,7 @@ public void should_Build_a_really_really_simple_data_loader() { @Test public void should_Build_a_named_data_loader() { BatchLoader loadFunction = CompletableFuture::completedFuture; - DataLoader dl = newDataLoader("name", loadFunction, DataLoaderOptions.newOptions()); + DataLoader dl = newDataLoader("name", loadFunction, DataLoaderOptions.newDefaultOptions()); assertNotNull(dl.getName()); assertThat(dl.getName(), equalTo("name")); diff --git a/src/test/java/org/dataloader/DelegatingDataLoaderTest.java b/src/test/java/org/dataloader/DelegatingDataLoaderTest.java index f1aec2d4..88497528 100644 --- a/src/test/java/org/dataloader/DelegatingDataLoaderTest.java +++ b/src/test/java/org/dataloader/DelegatingDataLoaderTest.java @@ -66,7 +66,7 @@ public CompletableFuture load(@NonNull String key, @Nullable Object keyC @Test void can_delegate_simple_properties() { - DataLoaderOptions options = DataLoaderOptions.newOptions(); + DataLoaderOptions options = DataLoaderOptions.newOptions().build(); BatchLoader loadFunction = CompletableFuture::completedFuture; DataLoader rawLoader = DataLoaderFactory.newDataLoader("name", loadFunction, options); diff --git a/src/test/java/org/dataloader/instrumentation/DataLoaderRegistryInstrumentationTest.java b/src/test/java/org/dataloader/instrumentation/DataLoaderRegistryInstrumentationTest.java index 68df9c7c..e672d80d 100644 --- a/src/test/java/org/dataloader/instrumentation/DataLoaderRegistryInstrumentationTest.java +++ b/src/test/java/org/dataloader/instrumentation/DataLoaderRegistryInstrumentationTest.java @@ -120,9 +120,9 @@ void wontDoAnyThingIfThereIsNoRegistryInstrumentation() { @Test void wontDoAnyThingIfThereTheyAreTheSameInstrumentationAlready() { - DataLoader newX = dlX.transform(builder -> builder.options(dlX.getOptions().setInstrumentation(instrA))); - DataLoader newY = dlY.transform(builder -> builder.options(dlY.getOptions().setInstrumentation(instrA))); - DataLoader newZ = dlZ.transform(builder -> builder.options(dlZ.getOptions().setInstrumentation(instrA))); + DataLoader newX = dlX.transform(builder -> builder.options(dlX.getOptions().transform(b-> b.setInstrumentation(instrA)))); + DataLoader newY = dlY.transform(builder -> builder.options(dlY.getOptions().transform(b-> b.setInstrumentation(instrA)))); + DataLoader newZ = dlZ.transform(builder -> builder.options(dlZ.getOptions().transform(b-> b.setInstrumentation(instrA)))); DataLoaderRegistry registry = DataLoaderRegistry.newRegistry() .instrumentation(instrA) .register("X", newX) From 3f676acfa7251988fb9169695fce3ebf7d47dec2 Mon Sep 17 00:00:00 2001 From: bbaker Date: Wed, 4 Jun 2025 13:57:51 +1000 Subject: [PATCH 095/156] Error Prone / NullAway support for JSpecify --- build.gradle | 29 ++++++++++++++++++- .../dataloader/BatchLoaderEnvironment.java | 7 ++--- .../org/dataloader/DataLoaderRegistry.java | 10 ++++--- .../dataloader/DataLoaderRegistryTest.java | 2 +- 4 files changed, 38 insertions(+), 10 deletions(-) diff --git a/build.gradle b/build.gradle index eb571583..e07821fd 100644 --- a/build.gradle +++ b/build.gradle @@ -11,11 +11,12 @@ plugins { id 'io.github.gradle-nexus.publish-plugin' version '1.0.0' id 'com.github.ben-manes.versions' version '0.51.0' id "me.champeau.jmh" version "0.7.3" + id "net.ltgt.errorprone" version '4.2.0' } java { toolchain { - languageVersion = JavaLanguageVersion.of(11) + languageVersion = JavaLanguageVersion.of(17) } } @@ -75,8 +76,34 @@ dependencies { // this is needed for the idea jmh plugin to work correctly jmh 'org.openjdk.jmh:jmh-core:1.37' jmh 'org.openjdk.jmh:jmh-generator-annprocess:1.37' + + errorprone 'com.uber.nullaway:nullaway:0.12.6' + errorprone 'com.google.errorprone:error_prone_core:2.37.0' +} + +import net.ltgt.gradle.errorprone.CheckSeverity + +tasks.withType(JavaCompile) { + options.release = 11 + options.errorprone { + disableAllChecks = true + check("NullAway", CheckSeverity.ERROR) + // + // end state has us with this config turned on - eg all classes + // + //option("NullAway:AnnotatedPackages", "org.dataloader") + option("NullAway:OnlyNullMarked", "true") + option("NullAway:JSpecifyMode", "true") + } + // Include to disable NullAway on test code + if (name.toLowerCase().contains("test")) { + options.errorprone { + disable("NullAway") + } + } } + task sourcesJar(type: Jar) { dependsOn classes archiveClassifier.set('sources') diff --git a/src/main/java/org/dataloader/BatchLoaderEnvironment.java b/src/main/java/org/dataloader/BatchLoaderEnvironment.java index 6b84e709..c7a2ed8b 100644 --- a/src/main/java/org/dataloader/BatchLoaderEnvironment.java +++ b/src/main/java/org/dataloader/BatchLoaderEnvironment.java @@ -19,11 +19,11 @@ @NullMarked public class BatchLoaderEnvironment { - private final Object context; + private final @Nullable Object context; private final Map keyContexts; private final List keyContextsList; - private BatchLoaderEnvironment(Object context, List keyContextsList, Map keyContexts) { + private BatchLoaderEnvironment(@Nullable Object context, List keyContextsList, Map keyContexts) { this.context = context; this.keyContexts = keyContexts; this.keyContextsList = keyContextsList; @@ -33,7 +33,6 @@ private BatchLoaderEnvironment(Object context, List keyContextsList, Map * Returns the overall context object provided by {@link org.dataloader.BatchLoaderContextProvider} * * @param the type you would like the object to be - * * @return a context object or null if there isn't one */ @SuppressWarnings("unchecked") @@ -68,7 +67,7 @@ public static Builder newBatchLoaderEnvironment() { } public static class Builder { - private Object context; + private @Nullable Object context; private Map keyContexts = Collections.emptyMap(); private List keyContextsList = Collections.emptyList(); diff --git a/src/main/java/org/dataloader/DataLoaderRegistry.java b/src/main/java/org/dataloader/DataLoaderRegistry.java index 0988697b..3b7409c9 100644 --- a/src/main/java/org/dataloader/DataLoaderRegistry.java +++ b/src/main/java/org/dataloader/DataLoaderRegistry.java @@ -1,6 +1,7 @@ package org.dataloader; import org.dataloader.annotations.PublicApi; +import org.dataloader.impl.Assertions; import org.dataloader.instrumentation.ChainedDataLoaderInstrumentation; import org.dataloader.instrumentation.DataLoaderInstrumentation; import org.dataloader.instrumentation.DataLoaderInstrumentationHelper; @@ -14,6 +15,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.function.Function; @@ -142,7 +144,7 @@ private static DataLoaderOptions setInInstrumentation(DataLoaderOptions options, */ public DataLoaderRegistry register(DataLoader dataLoader) { String name = dataLoader.getName(); - assertState(name != null, () -> "The DataLoader must have a non null name"); + Objects.requireNonNull(name, "The DataLoader must have a non null name"); dataLoaders.put(name, nameAndInstrumentDL(name, instrumentation, dataLoader)); return this; } @@ -176,7 +178,7 @@ public DataLoaderRegistry register(String key, DataLoader dataLoader) { */ public DataLoader registerAndGet(String key, DataLoader dataLoader) { dataLoaders.put(key, nameAndInstrumentDL(key, instrumentation, dataLoader)); - return getDataLoader(key); + return Objects.requireNonNull(getDataLoader(key)); } /** @@ -251,10 +253,10 @@ public DataLoaderRegistry unregister(String key) { * @param key the key of the data loader * @param the type of keys * @param the type of values - * @return a data loader or null if its not present + * @return a data loader or null if it's not present */ @SuppressWarnings("unchecked") - public DataLoader getDataLoader(String key) { + public @Nullable DataLoader getDataLoader(String key) { return (DataLoader) dataLoaders.get(key); } diff --git a/src/test/java/org/dataloader/DataLoaderRegistryTest.java b/src/test/java/org/dataloader/DataLoaderRegistryTest.java index 270bd500..43353755 100644 --- a/src/test/java/org/dataloader/DataLoaderRegistryTest.java +++ b/src/test/java/org/dataloader/DataLoaderRegistryTest.java @@ -63,7 +63,7 @@ public void registration_works() { try { registry.register(dlUnnamed); Assertions.fail("Should have thrown an exception"); - } catch (DataLoaderAssertionException ignored) { + } catch (NullPointerException ignored) { } } From 4fbeccdc96de752a02879bbb74a974973b1d7d5b Mon Sep 17 00:00:00 2001 From: bbaker Date: Wed, 4 Jun 2025 15:17:21 +1000 Subject: [PATCH 096/156] Error Prone / NullAway support for JSpecify - added Kotlin --- build.gradle | 24 +++++++++++- src/main/java/org/dataloader/DataLoader.java | 2 +- .../kotlin/org/dataloader/KotlinExamples.kt | 39 +++++++++++++++++++ 3 files changed, 62 insertions(+), 3 deletions(-) create mode 100644 src/test/kotlin/org/dataloader/KotlinExamples.kt diff --git a/build.gradle b/build.gradle index e07821fd..2d22afd1 100644 --- a/build.gradle +++ b/build.gradle @@ -1,3 +1,6 @@ +import org.jetbrains.kotlin.gradle.dsl.JvmTarget +import org.jetbrains.kotlin.gradle.dsl.KotlinVersion +import net.ltgt.gradle.errorprone.CheckSeverity import java.text.SimpleDateFormat plugins { @@ -12,6 +15,9 @@ plugins { id 'com.github.ben-manes.versions' version '0.51.0' id "me.champeau.jmh" version "0.7.3" id "net.ltgt.errorprone" version '4.2.0' + + // Kotlin just for tests - not + id 'org.jetbrains.kotlin.jvm' version '2.1.21' } java { @@ -20,6 +26,19 @@ java { } } +kotlin { + compilerOptions { + apiVersion = KotlinVersion.KOTLIN_2_0 + languageVersion = KotlinVersion.KOTLIN_2_0 + jvmTarget = JvmTarget.JVM_11 + javaParameters = true + freeCompilerArgs = [ + '-Xemit-jvm-type-annotations', + '-Xjspecify-annotations=strict', + ] + } +} + def getDevelopmentVersion() { def output = new StringBuilder() def error = new StringBuilder() @@ -79,9 +98,10 @@ dependencies { errorprone 'com.uber.nullaway:nullaway:0.12.6' errorprone 'com.google.errorprone:error_prone_core:2.37.0' -} -import net.ltgt.gradle.errorprone.CheckSeverity + // just tests + testCompileOnly 'org.jetbrains.kotlin:kotlin-stdlib-jdk8' +} tasks.withType(JavaCompile) { options.release = 11 diff --git a/src/main/java/org/dataloader/DataLoader.java b/src/main/java/org/dataloader/DataLoader.java index 7a506197..321b58c4 100644 --- a/src/main/java/org/dataloader/DataLoader.java +++ b/src/main/java/org/dataloader/DataLoader.java @@ -68,7 +68,7 @@ */ @PublicApi @NullMarked -public class DataLoader { +public class DataLoader { private final @Nullable String name; private final DataLoaderHelper helper; diff --git a/src/test/kotlin/org/dataloader/KotlinExamples.kt b/src/test/kotlin/org/dataloader/KotlinExamples.kt new file mode 100644 index 00000000..da532e49 --- /dev/null +++ b/src/test/kotlin/org/dataloader/KotlinExamples.kt @@ -0,0 +1,39 @@ +package org.dataloader + +import org.junit.jupiter.api.Test +import java.util.concurrent.CompletableFuture + +/** + * Some Kotlin code to prove that are JSpecify annotations work here + * as expected in Kotlin land. We don't intend to ue Kotlin in our tests + * or to deliver Kotlin code in the java + */ +class KotlinExamples { + + @Test + fun `basic kotlin test of non nullable value types`() { + val dataLoader: DataLoader = DataLoaderFactory.newDataLoader { keys -> CompletableFuture.completedFuture(keys.toList()) } + + val cfA = dataLoader.load("A") + val cfB = dataLoader.load("B") + + dataLoader.dispatch() + + cfA.join().equals("A") + cfB.join().equals("B") + } + + @Test + fun `basic kotlin test of nullable value types`() { + val dataLoader: DataLoader = DataLoaderFactory.newDataLoader { keys -> CompletableFuture.completedFuture(keys.toList()) } + + val cfA = dataLoader.load("A") + val cfB = dataLoader.load("B") + + dataLoader.dispatch() + + cfA.join().equals("A") + cfB.join().equals("B") + } + +} \ No newline at end of file From 4a68a3930aae0f83ade11a2ccf70117a16f9ba38 Mon Sep 17 00:00:00 2001 From: bbaker Date: Wed, 4 Jun 2025 15:23:44 +1000 Subject: [PATCH 097/156] Error Prone / NullAway support for JSpecify - added Kotlin - tweak --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 2d22afd1..551f4ec4 100644 --- a/build.gradle +++ b/build.gradle @@ -16,7 +16,7 @@ plugins { id "me.champeau.jmh" version "0.7.3" id "net.ltgt.errorprone" version '4.2.0' - // Kotlin just for tests - not + // Kotlin just for tests - not production code id 'org.jetbrains.kotlin.jvm' version '2.1.21' } From 7a6749e913e5ef7bcf281e15bc99d03a113b11b4 Mon Sep 17 00:00:00 2001 From: bbaker Date: Wed, 4 Jun 2025 15:27:18 +1000 Subject: [PATCH 098/156] Error Prone / NullAway support for JSpecify - added Kotlin - tweak to assert --- src/test/kotlin/org/dataloader/KotlinExamples.kt | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/test/kotlin/org/dataloader/KotlinExamples.kt b/src/test/kotlin/org/dataloader/KotlinExamples.kt index da532e49..c415b1aa 100644 --- a/src/test/kotlin/org/dataloader/KotlinExamples.kt +++ b/src/test/kotlin/org/dataloader/KotlinExamples.kt @@ -2,6 +2,7 @@ package org.dataloader import org.junit.jupiter.api.Test import java.util.concurrent.CompletableFuture +import java.util.concurrent.CompletableFuture.* /** * Some Kotlin code to prove that are JSpecify annotations work here @@ -12,28 +13,28 @@ class KotlinExamples { @Test fun `basic kotlin test of non nullable value types`() { - val dataLoader: DataLoader = DataLoaderFactory.newDataLoader { keys -> CompletableFuture.completedFuture(keys.toList()) } + val dataLoader: DataLoader = DataLoaderFactory.newDataLoader { keys -> completedFuture(keys.toList()) } val cfA = dataLoader.load("A") val cfB = dataLoader.load("B") dataLoader.dispatch() - cfA.join().equals("A") - cfB.join().equals("B") + assert(cfA.join().equals("A")) + assert(cfA.join().equals("A")) } @Test fun `basic kotlin test of nullable value types`() { - val dataLoader: DataLoader = DataLoaderFactory.newDataLoader { keys -> CompletableFuture.completedFuture(keys.toList()) } + val dataLoader: DataLoader = DataLoaderFactory.newDataLoader { keys -> completedFuture(keys.toList()) } val cfA = dataLoader.load("A") val cfB = dataLoader.load("B") dataLoader.dispatch() - cfA.join().equals("A") - cfB.join().equals("B") + assert(cfA.join().equals("A")) + assert(cfA.join().equals("A")) } } \ No newline at end of file From afd5dc1c1b027242bc39549e7616860535dc9cad Mon Sep 17 00:00:00 2001 From: bbaker Date: Thu, 5 Jun 2025 08:48:59 +1000 Subject: [PATCH 099/156] Error Prone / NullAway support for JSpecify - added Kotlin - reverted to old assertions --- src/main/java/org/dataloader/DataLoaderRegistry.java | 3 +-- src/test/java/org/dataloader/DataLoaderRegistryTest.java | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/java/org/dataloader/DataLoaderRegistry.java b/src/main/java/org/dataloader/DataLoaderRegistry.java index 3b7409c9..6bc79f64 100644 --- a/src/main/java/org/dataloader/DataLoaderRegistry.java +++ b/src/main/java/org/dataloader/DataLoaderRegistry.java @@ -143,8 +143,7 @@ private static DataLoaderOptions setInInstrumentation(DataLoaderOptions options, * @return this registry */ public DataLoaderRegistry register(DataLoader dataLoader) { - String name = dataLoader.getName(); - Objects.requireNonNull(name, "The DataLoader must have a non null name"); + String name = Assertions.nonNull(dataLoader.getName(), () -> "The DataLoader must have a non null name"); dataLoaders.put(name, nameAndInstrumentDL(name, instrumentation, dataLoader)); return this; } diff --git a/src/test/java/org/dataloader/DataLoaderRegistryTest.java b/src/test/java/org/dataloader/DataLoaderRegistryTest.java index 43353755..89624d71 100644 --- a/src/test/java/org/dataloader/DataLoaderRegistryTest.java +++ b/src/test/java/org/dataloader/DataLoaderRegistryTest.java @@ -1,6 +1,5 @@ package org.dataloader; -import org.dataloader.impl.DataLoaderAssertionException; import org.dataloader.stats.SimpleStatisticsCollector; import org.dataloader.stats.Statistics; import org.junit.jupiter.api.Assertions; From c6d87199de304a95306f897b96091eef787c4e3f Mon Sep 17 00:00:00 2001 From: bbaker Date: Wed, 18 Jun 2025 12:10:51 +1000 Subject: [PATCH 100/156] Move to new Sonatype URLs --- .github/workflows/master.yml | 2 ++ .github/workflows/release.yml | 2 ++ build.gradle | 9 ++++++--- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 01b89bc4..8643910b 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -10,6 +10,8 @@ jobs: env: MAVEN_CENTRAL_USER: ${{ secrets.MAVEN_CENTRAL_USER }} MAVEN_CENTRAL_PASSWORD: ${{ secrets.MAVEN_CENTRAL_PASSWORD }} + MAVEN_CENTRAL_USER_NEW: ${{ secrets.MAVEN_CENTRAL_USER_NEW }} + MAVEN_CENTRAL_PASSWORD_NEW: ${{ secrets.MAVEN_CENTRAL_PASSWORD_NEW }} MAVEN_CENTRAL_PGP_KEY: ${{ secrets.MAVEN_CENTRAL_PGP_KEY }} steps: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a574a68f..fd614cca 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,6 +14,8 @@ jobs: MAVEN_CENTRAL_PGP_KEY: ${{ secrets.MAVEN_CENTRAL_PGP_KEY }} MAVEN_CENTRAL_USER: ${{ secrets.MAVEN_CENTRAL_USER }} MAVEN_CENTRAL_PASSWORD: ${{ secrets.MAVEN_CENTRAL_PASSWORD }} + MAVEN_CENTRAL_USER_NEW: ${{ secrets.MAVEN_CENTRAL_USER_NEW }} + MAVEN_CENTRAL_PASSWORD_NEW: ${{ secrets.MAVEN_CENTRAL_PASSWORD_NEW }} RELEASE_VERSION: ${{ github.event.inputs.version }} steps: diff --git a/build.gradle b/build.gradle index 551f4ec4..80de227f 100644 --- a/build.gradle +++ b/build.gradle @@ -224,9 +224,12 @@ publishing { nexusPublishing { repositories { sonatype { - username = System.env.MAVEN_CENTRAL_USER - password = System.env.MAVEN_CENTRAL_PASSWORD - } + username = System.env.MAVEN_CENTRAL_USER_NEW + password = System.env.MAVEN_CENTRAL_PASSWORD_NEW + // https://central.sonatype.org/publish/publish-portal-ossrh-staging-api/#configuration + nexusUrl.set(uri("https://ossrh-staging-api.central.sonatype.com/service/local/")) + // GraphQL Java does not publish snapshots, but adding this URL for completeness + snapshotRepositoryUrl.set(uri("https://central.sonatype.com/repository/maven-snapshots/")) } } } From d7f53ad2654a8b4fbc993b388bd28a820245e259 Mon Sep 17 00:00:00 2001 From: dondonz <13839920+dondonz@users.noreply.github.com> Date: Sun, 22 Jun 2025 09:34:24 +1000 Subject: [PATCH 101/156] Update Gradle Github action --- .github/workflows/master.yml | 2 +- .github/workflows/pull_request.yml | 2 +- .github/workflows/release.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 8643910b..ad3f9f29 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -16,7 +16,7 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: gradle/actions/wrapper-validation@v3 + - uses: gradle/actions/wrapper-validation@v4 - name: Set up JDK 11 uses: actions/setup-java@v4 with: diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index f16bf96b..573e169a 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: gradle/actions/wrapper-validation@v3 + - uses: gradle/actions/wrapper-validation@v4 - name: Set up JDK 11 uses: actions/setup-java@v4 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fd614cca..64ef19c9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,7 +20,7 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: gradle/actions/wrapper-validation@v3 + - uses: gradle/actions/wrapper-validation@v4 - name: Set up JDK 11 uses: actions/setup-java@v4 with: From dc948178890571faccc2a45468271ea667fad338 Mon Sep 17 00:00:00 2001 From: dondonz <13839920+dondonz@users.noreply.github.com> Date: Sun, 22 Jun 2025 10:22:13 +1000 Subject: [PATCH 102/156] Add dependabot --- .github/dependabot.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..10ef8311 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,10 @@ +version: 2 +updates: + - package-ecosystem: "gradle" + directory: "/" + schedule: + interval: "weekly" + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" From a1cd77c130d135dfe2b3252bf7bd9a0e994ea68f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 22 Jun 2025 00:24:37 +0000 Subject: [PATCH 103/156] Bump com.gradle.develocity from 3.19 to 4.0.2 Bumps com.gradle.develocity from 3.19 to 4.0.2. --- updated-dependencies: - dependency-name: com.gradle.develocity dependency-version: 4.0.2 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- settings.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.gradle b/settings.gradle index 47404e7c..d5a82f45 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,5 +1,5 @@ plugins { - id 'com.gradle.develocity' version '3.19' + id 'com.gradle.develocity' version '4.0.2' id 'org.gradle.toolchains.foojay-resolver-convention' version '0.9.0' } From 5764dffcea7cb89d07f4265573efd217f47806a6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 22 Jun 2025 00:24:38 +0000 Subject: [PATCH 104/156] Bump com.github.ben-manes.versions from 0.51.0 to 0.52.0 Bumps com.github.ben-manes.versions from 0.51.0 to 0.52.0. --- updated-dependencies: - dependency-name: com.github.ben-manes.versions dependency-version: 0.52.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 80de227f..296c0224 100644 --- a/build.gradle +++ b/build.gradle @@ -12,7 +12,7 @@ plugins { id 'groovy' id 'biz.aQute.bnd.builder' version '6.2.0' id 'io.github.gradle-nexus.publish-plugin' version '1.0.0' - id 'com.github.ben-manes.versions' version '0.51.0' + id 'com.github.ben-manes.versions' version '0.52.0' id "me.champeau.jmh" version "0.7.3" id "net.ltgt.errorprone" version '4.2.0' From 262739b5b95e75372ba40aaeb7042620badd19c4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 22 Jun 2025 00:24:41 +0000 Subject: [PATCH 105/156] Bump com.uber.nullaway:nullaway from 0.12.6 to 0.12.7 Bumps [com.uber.nullaway:nullaway](https://github.com/uber/NullAway) from 0.12.6 to 0.12.7. - [Release notes](https://github.com/uber/NullAway/releases) - [Changelog](https://github.com/uber/NullAway/blob/master/CHANGELOG.md) - [Commits](https://github.com/uber/NullAway/compare/v0.12.6...v0.12.7) --- updated-dependencies: - dependency-name: com.uber.nullaway:nullaway dependency-version: 0.12.7 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 80de227f..8a361482 100644 --- a/build.gradle +++ b/build.gradle @@ -96,7 +96,7 @@ dependencies { jmh 'org.openjdk.jmh:jmh-core:1.37' jmh 'org.openjdk.jmh:jmh-generator-annprocess:1.37' - errorprone 'com.uber.nullaway:nullaway:0.12.6' + errorprone 'com.uber.nullaway:nullaway:0.12.7' errorprone 'com.google.errorprone:error_prone_core:2.37.0' // just tests From aa7349f7c4a44d16ae85a3c9ef10fb56c44b7c1d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 25 Jun 2025 09:28:38 +1000 Subject: [PATCH 106/156] More explicit nullable annotations on BatchLoader interfaces --- src/main/java/org/dataloader/BatchLoader.java | 9 ++- .../dataloader/BatchLoaderWithContext.java | 3 +- .../java/org/dataloader/DataLoaderHelper.java | 4 +- .../org/dataloader/MappedBatchLoader.java | 3 +- .../MappedBatchLoaderWithContext.java | 3 +- .../org/dataloader/MappedBatchPublisher.java | 3 +- .../MappedBatchPublisherWithContext.java | 3 +- .../kotlin/org/dataloader/KotlinExamples.kt | 67 +++++++++++++++++-- 8 files changed, 77 insertions(+), 18 deletions(-) diff --git a/src/main/java/org/dataloader/BatchLoader.java b/src/main/java/org/dataloader/BatchLoader.java index 2b0c3c53..df11f89b 100644 --- a/src/main/java/org/dataloader/BatchLoader.java +++ b/src/main/java/org/dataloader/BatchLoader.java @@ -17,8 +17,8 @@ package org.dataloader; import org.dataloader.annotations.PublicSpi; -import org.jspecify.annotations.NonNull; import org.jspecify.annotations.NullMarked; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.concurrent.CompletionStage; @@ -40,7 +40,7 @@ * 2, 9, 6, 1 * ] * - * + *

* and loading from a back-end service returned this list of values: * *

@@ -50,7 +50,7 @@
  *      { id: 2, name: 'San Francisco' },
  *  ]
  * 
- * + *

* then the batch loader function contract has been broken. *

* The back-end service returned results in a different order than we requested, likely because it was more efficient for it to @@ -77,7 +77,7 @@ @FunctionalInterface @PublicSpi @NullMarked -public interface BatchLoader { +public interface BatchLoader { /** * Called to batch load the provided keys and return a promise to a list of values. @@ -85,7 +85,6 @@ public interface BatchLoader { * If you need calling context then implement {@link org.dataloader.BatchLoaderWithContext} * * @param keys the collection of keys to load - * * @return a promise of the values for those keys */ CompletionStage> load(List keys); diff --git a/src/main/java/org/dataloader/BatchLoaderWithContext.java b/src/main/java/org/dataloader/BatchLoaderWithContext.java index eba26e4b..fb6ff71d 100644 --- a/src/main/java/org/dataloader/BatchLoaderWithContext.java +++ b/src/main/java/org/dataloader/BatchLoaderWithContext.java @@ -2,6 +2,7 @@ import org.dataloader.annotations.PublicSpi; import org.jspecify.annotations.NullMarked; +import org.jspecify.annotations.Nullable; import java.util.List; import java.util.concurrent.CompletionStage; @@ -16,7 +17,7 @@ */ @PublicSpi @NullMarked -public interface BatchLoaderWithContext { +public interface BatchLoaderWithContext { /** * Called to batch load the provided keys and return a promise to a list of values. This default * version can be given an environment object to that maybe be useful during the call. A typical use case diff --git a/src/main/java/org/dataloader/DataLoaderHelper.java b/src/main/java/org/dataloader/DataLoaderHelper.java index 78587802..f4a8f106 100644 --- a/src/main/java/org/dataloader/DataLoaderHelper.java +++ b/src/main/java/org/dataloader/DataLoaderHelper.java @@ -594,11 +594,11 @@ private boolean isMapLoader() { } private boolean isPublisher() { - return batchLoadFunction instanceof BatchPublisher; + return batchLoadFunction instanceof BatchPublisher || batchLoadFunction instanceof BatchPublisherWithContext; } private boolean isMappedPublisher() { - return batchLoadFunction instanceof MappedBatchPublisher; + return batchLoadFunction instanceof MappedBatchPublisher || batchLoadFunction instanceof MappedBatchPublisherWithContext; } private DataLoaderInstrumentation instrumentation() { diff --git a/src/main/java/org/dataloader/MappedBatchLoader.java b/src/main/java/org/dataloader/MappedBatchLoader.java index 1ad4c79a..179d6a20 100644 --- a/src/main/java/org/dataloader/MappedBatchLoader.java +++ b/src/main/java/org/dataloader/MappedBatchLoader.java @@ -18,6 +18,7 @@ import org.dataloader.annotations.PublicSpi; import org.jspecify.annotations.NullMarked; +import org.jspecify.annotations.Nullable; import java.util.Map; import java.util.Set; @@ -59,7 +60,7 @@ */ @PublicSpi @NullMarked -public interface MappedBatchLoader { +public interface MappedBatchLoader { /** * Called to batch load the provided keys and return a promise to a map of values. diff --git a/src/main/java/org/dataloader/MappedBatchLoaderWithContext.java b/src/main/java/org/dataloader/MappedBatchLoaderWithContext.java index 95592607..9f342d44 100644 --- a/src/main/java/org/dataloader/MappedBatchLoaderWithContext.java +++ b/src/main/java/org/dataloader/MappedBatchLoaderWithContext.java @@ -18,6 +18,7 @@ import org.dataloader.annotations.PublicSpi; import org.jspecify.annotations.NullMarked; +import org.jspecify.annotations.Nullable; import java.util.Map; import java.util.Set; @@ -33,7 +34,7 @@ */ @PublicSpi @NullMarked -public interface MappedBatchLoaderWithContext { +public interface MappedBatchLoaderWithContext { /** * Called to batch load the provided keys and return a promise to a map of values. * diff --git a/src/main/java/org/dataloader/MappedBatchPublisher.java b/src/main/java/org/dataloader/MappedBatchPublisher.java index 493401f9..66371577 100644 --- a/src/main/java/org/dataloader/MappedBatchPublisher.java +++ b/src/main/java/org/dataloader/MappedBatchPublisher.java @@ -2,6 +2,7 @@ import org.dataloader.annotations.PublicSpi; import org.jspecify.annotations.NullMarked; +import org.jspecify.annotations.Nullable; import org.reactivestreams.Subscriber; import java.util.Map; @@ -20,7 +21,7 @@ */ @PublicSpi @NullMarked -public interface MappedBatchPublisher { +public interface MappedBatchPublisher { /** * Called to batch the provided keys into a stream of map entries of keys and values. *

diff --git a/src/main/java/org/dataloader/MappedBatchPublisherWithContext.java b/src/main/java/org/dataloader/MappedBatchPublisherWithContext.java index 7b862caa..dd8b5f9c 100644 --- a/src/main/java/org/dataloader/MappedBatchPublisherWithContext.java +++ b/src/main/java/org/dataloader/MappedBatchPublisherWithContext.java @@ -2,6 +2,7 @@ import org.dataloader.annotations.PublicSpi; import org.jspecify.annotations.NullMarked; +import org.jspecify.annotations.Nullable; import org.reactivestreams.Subscriber; import java.util.List; @@ -17,7 +18,7 @@ */ @PublicSpi @NullMarked -public interface MappedBatchPublisherWithContext { +public interface MappedBatchPublisherWithContext { /** * Called to batch the provided keys into a stream of map entries of keys and values. diff --git a/src/test/kotlin/org/dataloader/KotlinExamples.kt b/src/test/kotlin/org/dataloader/KotlinExamples.kt index c415b1aa..480a965a 100644 --- a/src/test/kotlin/org/dataloader/KotlinExamples.kt +++ b/src/test/kotlin/org/dataloader/KotlinExamples.kt @@ -1,8 +1,8 @@ package org.dataloader import org.junit.jupiter.api.Test -import java.util.concurrent.CompletableFuture -import java.util.concurrent.CompletableFuture.* +import reactor.core.publisher.Flux +import java.util.concurrent.CompletableFuture.completedFuture /** * Some Kotlin code to prove that are JSpecify annotations work here @@ -13,7 +13,10 @@ class KotlinExamples { @Test fun `basic kotlin test of non nullable value types`() { - val dataLoader: DataLoader = DataLoaderFactory.newDataLoader { keys -> completedFuture(keys.toList()) } + val batchLoadFunction = BatchLoader + { keys -> completedFuture(keys.toList()) } + val dataLoader: DataLoader = + DataLoaderFactory.newDataLoader(batchLoadFunction) val cfA = dataLoader.load("A") val cfB = dataLoader.load("B") @@ -21,20 +24,72 @@ class KotlinExamples { dataLoader.dispatch() assert(cfA.join().equals("A")) - assert(cfA.join().equals("A")) + assert(cfB.join().equals("B")) } @Test fun `basic kotlin test of nullable value types`() { - val dataLoader: DataLoader = DataLoaderFactory.newDataLoader { keys -> completedFuture(keys.toList()) } + val batchLoadFunction: BatchLoader = BatchLoader { keys -> completedFuture(keys.toList()) } + val dataLoader: DataLoader = DataLoaderFactory.newDataLoader(batchLoadFunction) + + standardNullableAsserts(dataLoader) + } + + @Test + fun `basic kotlin test of nullable value types in mapped batch loader`() { + val batchLoadFunction = MappedBatchLoader + { keys -> completedFuture(keys.associateBy({ it })) } + + val dataLoader: DataLoader = DataLoaderFactory.newMappedDataLoader(batchLoadFunction) + + standardNullableAsserts(dataLoader) + } + + @Test + fun `basic kotlin test of nullable value types in mapped batch loader with context`() { + val batchLoadFunction = MappedBatchLoaderWithContext + { keys, env -> completedFuture(keys.associateBy({ it })) } + + val dataLoader: DataLoader = DataLoaderFactory.newMappedDataLoader(batchLoadFunction) + + standardNullableAsserts(dataLoader) + } + @Test + fun `basic kotlin test of nullable value types in mapped batch publisher`() { + val batchLoadFunction = MappedBatchPublisher + { keys, subscriber -> + val map: Map = keys.associateBy({ it }) + Flux.fromIterable(map.entries).subscribe(subscriber); + } + + val dataLoader: DataLoader = DataLoaderFactory.newMappedPublisherDataLoader(batchLoadFunction) + + standardNullableAsserts(dataLoader) + } + + @Test + fun `basic kotlin test of nullable value types in mapped batch publisher with context`() { + val batchLoadFunction = MappedBatchPublisherWithContext + { keys, subscriber, env -> + val map: Map = keys.associateBy({ it }) + Flux.fromIterable(map.entries).subscribe(subscriber); + } + + val dataLoader: DataLoader = DataLoaderFactory.newMappedPublisherDataLoader(batchLoadFunction) + + standardNullableAsserts(dataLoader) + } + + private fun standardNullableAsserts(dataLoader: DataLoader) { val cfA = dataLoader.load("A") val cfB = dataLoader.load("B") dataLoader.dispatch() assert(cfA.join().equals("A")) - assert(cfA.join().equals("A")) + assert(cfB.join().equals("B")) } + } \ No newline at end of file From e10797eff8ef6e4fcf75e3fe60e8c4a5104d157a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Jun 2025 13:33:29 +0000 Subject: [PATCH 107/156] Bump org.jetbrains.kotlin.jvm from 2.1.21 to 2.2.0 Bumps [org.jetbrains.kotlin.jvm](https://github.com/JetBrains/kotlin) from 2.1.21 to 2.2.0. - [Release notes](https://github.com/JetBrains/kotlin/releases) - [Changelog](https://github.com/JetBrains/kotlin/blob/master/ChangeLog.md) - [Commits](https://github.com/JetBrains/kotlin/compare/v2.1.21...v2.2.0) --- updated-dependencies: - dependency-name: org.jetbrains.kotlin.jvm dependency-version: 2.2.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 8a361482..5ed119a2 100644 --- a/build.gradle +++ b/build.gradle @@ -17,7 +17,7 @@ plugins { id "net.ltgt.errorprone" version '4.2.0' // Kotlin just for tests - not production code - id 'org.jetbrains.kotlin.jvm' version '2.1.21' + id 'org.jetbrains.kotlin.jvm' version '2.2.0' } java { From e6795fe988487b237cfc924937d4e526652310f4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Jul 2025 22:48:18 +0000 Subject: [PATCH 108/156] Bump io.github.gradle-nexus.publish-plugin from 1.0.0 to 2.0.0 Bumps io.github.gradle-nexus.publish-plugin from 1.0.0 to 2.0.0. --- updated-dependencies: - dependency-name: io.github.gradle-nexus.publish-plugin dependency-version: 2.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 065f7453..3edf4f84 100644 --- a/build.gradle +++ b/build.gradle @@ -11,7 +11,7 @@ plugins { id 'signing' id 'groovy' id 'biz.aQute.bnd.builder' version '6.2.0' - id 'io.github.gradle-nexus.publish-plugin' version '1.0.0' + id 'io.github.gradle-nexus.publish-plugin' version '2.0.0' id 'com.github.ben-manes.versions' version '0.52.0' id "me.champeau.jmh" version "0.7.3" id "net.ltgt.errorprone" version '4.2.0' From 1d34141834b0ec89b663bbb6d76df36c7c227805 Mon Sep 17 00:00:00 2001 From: bbaker Date: Wed, 2 Jul 2025 16:29:20 +1000 Subject: [PATCH 109/156] #208 to ensure tests are in place --- .../java/org/dataloader/DataLoaderTest.java | 26 +++++++++++++++++++ .../DelegatingDataLoaderFactory.java | 7 +++++ .../parameterized/ListDataLoaderFactory.java | 11 ++++++++ .../MappedDataLoaderFactory.java | 14 +++++++++- .../MappedPublisherDataLoaderFactory.java | 18 ++++++++++--- .../PublisherDataLoaderFactory.java | 12 ++++++++- .../parameterized/TestDataLoaderFactory.java | 7 +++++ 7 files changed, 89 insertions(+), 6 deletions(-) diff --git a/src/test/java/org/dataloader/DataLoaderTest.java b/src/test/java/org/dataloader/DataLoaderTest.java index 224b54d3..37ae0307 100644 --- a/src/test/java/org/dataloader/DataLoaderTest.java +++ b/src/test/java/org/dataloader/DataLoaderTest.java @@ -45,6 +45,7 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -1230,6 +1231,31 @@ public void when_values_size_are_more_then_key_size(TestDataLoaderFactory factor } } + @ParameterizedTest + @MethodSource("org.dataloader.fixtures.parameterized.TestDataLoaderFactories#get") + public void should_Support_loading_values_with_context(TestDataLoaderFactory factory) { + AtomicReference environmentREF = new AtomicReference<>(); + DataLoader identityLoader = factory.idLoaderWithContext(new DataLoaderOptions(), new ArrayList<>(), environmentREF); + + identityLoader.load(1, "ctx1"); + identityLoader.load(2, "ctx2"); + identityLoader.loadMany(List.of(3, 4), List.of("ctx3", "ctx4")); + + CompletableFuture> cf = identityLoader.dispatch(); + await().atMost(Duration.FIVE_SECONDS).until(cf::isDone); + + assertThat(cf.toCompletableFuture().join(), equalTo(asList(1, 2, 3, 4))); + + Map keyContexts = environmentREF.get().getKeyContexts(); + assertThat(keyContexts, equalTo(Map.of( + 1, "ctx1", + 2, "ctx2", + 3, "ctx3", + 4, "ctx4" + ))); + } + + @Test public void can_call_a_loader_from_a_loader() throws Exception { List> deepLoadCalls = new ArrayList<>(); diff --git a/src/test/java/org/dataloader/fixtures/parameterized/DelegatingDataLoaderFactory.java b/src/test/java/org/dataloader/fixtures/parameterized/DelegatingDataLoaderFactory.java index 0cbd3f34..8d1f815c 100644 --- a/src/test/java/org/dataloader/fixtures/parameterized/DelegatingDataLoaderFactory.java +++ b/src/test/java/org/dataloader/fixtures/parameterized/DelegatingDataLoaderFactory.java @@ -1,5 +1,6 @@ package org.dataloader.fixtures.parameterized; +import org.dataloader.BatchLoaderEnvironment; import org.dataloader.DataLoader; import org.dataloader.DataLoaderOptions; import org.dataloader.DelegatingDataLoader; @@ -8,6 +9,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.concurrent.atomic.AtomicReference; public class DelegatingDataLoaderFactory implements TestDataLoaderFactory { // its delegates all the way down to the turtles @@ -38,6 +40,11 @@ public DataLoader idLoader(DataLoaderOptions options, List DataLoader idLoaderWithContext(DataLoaderOptions options, List> loadCalls, AtomicReference environmentREF) { + return mkDelegateDataLoader(delegateFactory.idLoaderWithContext(options, loadCalls, environmentREF)); + } + @Override public DataLoader idLoaderDelayed(DataLoaderOptions options, List> loadCalls, Duration delay) { return mkDelegateDataLoader(delegateFactory.idLoaderDelayed(options, loadCalls, delay)); diff --git a/src/test/java/org/dataloader/fixtures/parameterized/ListDataLoaderFactory.java b/src/test/java/org/dataloader/fixtures/parameterized/ListDataLoaderFactory.java index 0644d3ca..8ec69d7c 100644 --- a/src/test/java/org/dataloader/fixtures/parameterized/ListDataLoaderFactory.java +++ b/src/test/java/org/dataloader/fixtures/parameterized/ListDataLoaderFactory.java @@ -1,5 +1,6 @@ package org.dataloader.fixtures.parameterized; +import org.dataloader.BatchLoaderEnvironment; import org.dataloader.DataLoader; import org.dataloader.DataLoaderOptions; import org.dataloader.fixtures.TestKit; @@ -9,6 +10,7 @@ import java.util.Collection; import java.util.List; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import static java.util.concurrent.CompletableFuture.completedFuture; @@ -23,6 +25,15 @@ public DataLoader idLoader(DataLoaderOptions options, List DataLoader idLoaderWithContext(DataLoaderOptions options, List> loadCalls, AtomicReference environmentREF) { + return newDataLoader((keys, env) -> { + environmentREF.set(env); + loadCalls.add(new ArrayList<>(keys)); + return completedFuture(keys); + }, options); + } + @Override public DataLoader idLoaderDelayed(DataLoaderOptions options, List> loadCalls, Duration delay) { return newDataLoader(keys -> CompletableFuture.supplyAsync(() -> { diff --git a/src/test/java/org/dataloader/fixtures/parameterized/MappedDataLoaderFactory.java b/src/test/java/org/dataloader/fixtures/parameterized/MappedDataLoaderFactory.java index e7c47ec4..f1c548e9 100644 --- a/src/test/java/org/dataloader/fixtures/parameterized/MappedDataLoaderFactory.java +++ b/src/test/java/org/dataloader/fixtures/parameterized/MappedDataLoaderFactory.java @@ -1,5 +1,6 @@ package org.dataloader.fixtures.parameterized; +import org.dataloader.BatchLoaderEnvironment; import org.dataloader.DataLoader; import org.dataloader.DataLoaderOptions; import org.dataloader.fixtures.TestKit; @@ -11,10 +12,10 @@ import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import static java.util.concurrent.CompletableFuture.completedFuture; -import static org.dataloader.DataLoaderFactory.newDataLoader; import static org.dataloader.DataLoaderFactory.newMappedDataLoader; import static org.dataloader.fixtures.TestKit.futureError; @@ -31,6 +32,17 @@ public DataLoader idLoader( }, options); } + @Override + public DataLoader idLoaderWithContext(DataLoaderOptions options, List> loadCalls, AtomicReference environmentREF) { + return newMappedDataLoader((keys, environment) -> { + environmentREF.set(environment); + loadCalls.add(new ArrayList<>(keys)); + Map map = new HashMap<>(); + keys.forEach(k -> map.put(k, k)); + return completedFuture(map); + }, options); + } + @Override public DataLoader idLoaderDelayed( DataLoaderOptions options, List> loadCalls, Duration delay) { diff --git a/src/test/java/org/dataloader/fixtures/parameterized/MappedPublisherDataLoaderFactory.java b/src/test/java/org/dataloader/fixtures/parameterized/MappedPublisherDataLoaderFactory.java index fa920cf7..3a0f54ec 100644 --- a/src/test/java/org/dataloader/fixtures/parameterized/MappedPublisherDataLoaderFactory.java +++ b/src/test/java/org/dataloader/fixtures/parameterized/MappedPublisherDataLoaderFactory.java @@ -1,5 +1,6 @@ package org.dataloader.fixtures.parameterized; +import org.dataloader.BatchLoaderEnvironment; import org.dataloader.DataLoader; import org.dataloader.DataLoaderOptions; import org.dataloader.Try; @@ -12,16 +13,13 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.concurrent.CompletableFuture; -import java.util.stream.Collectors; +import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Stream; import static java.util.stream.Collectors.toList; -import static java.util.stream.Collectors.toSet; import static org.dataloader.DataLoaderFactory.newMappedPublisherDataLoader; import static org.dataloader.DataLoaderFactory.newMappedPublisherDataLoaderWithTry; -import static org.dataloader.DataLoaderFactory.newPublisherDataLoader; public class MappedPublisherDataLoaderFactory implements TestDataLoaderFactory, TestReactiveDataLoaderFactory { @@ -36,6 +34,18 @@ public DataLoader idLoader( }, options); } + @Override + public DataLoader idLoaderWithContext(DataLoaderOptions options, List> loadCalls, AtomicReference environmentREF) { + return newMappedPublisherDataLoader((keys, subscriber, environment) -> { + environmentREF.set(environment); + + loadCalls.add(new ArrayList<>(keys)); + Map map = new HashMap<>(); + keys.forEach(k -> map.put(k, k)); + Flux.fromIterable(map.entrySet()).subscribe(subscriber); + }, options); + } + @Override public DataLoader idLoaderDelayed( DataLoaderOptions options, List> loadCalls, Duration delay) { diff --git a/src/test/java/org/dataloader/fixtures/parameterized/PublisherDataLoaderFactory.java b/src/test/java/org/dataloader/fixtures/parameterized/PublisherDataLoaderFactory.java index 20497192..c8e8b671 100644 --- a/src/test/java/org/dataloader/fixtures/parameterized/PublisherDataLoaderFactory.java +++ b/src/test/java/org/dataloader/fixtures/parameterized/PublisherDataLoaderFactory.java @@ -1,5 +1,6 @@ package org.dataloader.fixtures.parameterized; +import org.dataloader.BatchLoaderEnvironment; import org.dataloader.DataLoader; import org.dataloader.DataLoaderOptions; import org.dataloader.Try; @@ -11,9 +12,9 @@ import java.util.Collection; import java.util.List; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Stream; -import static org.dataloader.DataLoaderFactory.newDataLoader; import static org.dataloader.DataLoaderFactory.newPublisherDataLoader; import static org.dataloader.DataLoaderFactory.newPublisherDataLoaderWithTry; @@ -28,6 +29,15 @@ public DataLoader idLoader( }, options); } + @Override + public DataLoader idLoaderWithContext(DataLoaderOptions options, List> loadCalls, AtomicReference environmentREF) { + return newPublisherDataLoader((keys, subscriber, environment) -> { + environmentREF.set(environment); + loadCalls.add(new ArrayList<>(keys)); + Flux.fromIterable(keys).subscribe(subscriber); + }, options); + } + @Override public DataLoader idLoaderDelayed(DataLoaderOptions options, List> loadCalls, Duration delay) { return newPublisherDataLoader((keys, subscriber) -> { diff --git a/src/test/java/org/dataloader/fixtures/parameterized/TestDataLoaderFactory.java b/src/test/java/org/dataloader/fixtures/parameterized/TestDataLoaderFactory.java index 789b136d..3c584fda 100644 --- a/src/test/java/org/dataloader/fixtures/parameterized/TestDataLoaderFactory.java +++ b/src/test/java/org/dataloader/fixtures/parameterized/TestDataLoaderFactory.java @@ -1,5 +1,6 @@ package org.dataloader.fixtures.parameterized; +import org.dataloader.BatchLoaderEnvironment; import org.dataloader.DataLoader; import org.dataloader.DataLoaderOptions; @@ -7,6 +8,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.concurrent.atomic.AtomicReference; public interface TestDataLoaderFactory { DataLoader idLoader(DataLoaderOptions options, List> loadCalls); @@ -23,6 +25,11 @@ public interface TestDataLoaderFactory { DataLoader idLoaderReturnsTooMany(int howManyMore, DataLoaderOptions options, ArrayList loadCalls); + // similar to above but batch loaders with context + + DataLoader idLoaderWithContext(DataLoaderOptions options, List> loadCalls, AtomicReference environmentREF); + + // Convenience methods default DataLoader idLoader(DataLoaderOptions options) { From 62555dbaa1deb480e71e799e57349b6e3fc03945 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Jul 2025 13:02:57 +0000 Subject: [PATCH 110/156] Bump net.ltgt.errorprone from 4.2.0 to 4.3.0 Bumps net.ltgt.errorprone from 4.2.0 to 4.3.0. --- updated-dependencies: - dependency-name: net.ltgt.errorprone dependency-version: 4.3.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 3edf4f84..102a4336 100644 --- a/build.gradle +++ b/build.gradle @@ -14,7 +14,7 @@ plugins { id 'io.github.gradle-nexus.publish-plugin' version '2.0.0' id 'com.github.ben-manes.versions' version '0.52.0' id "me.champeau.jmh" version "0.7.3" - id "net.ltgt.errorprone" version '4.2.0' + id "net.ltgt.errorprone" version '4.3.0' // Kotlin just for tests - not production code id 'org.jetbrains.kotlin.jvm' version '2.2.0' From 00ff1182080d9e107c300102b0c9810e22d2a0f5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Jul 2025 13:03:07 +0000 Subject: [PATCH 111/156] Bump com.google.errorprone:error_prone_core from 2.37.0 to 2.39.0 Bumps [com.google.errorprone:error_prone_core](https://github.com/google/error-prone) from 2.37.0 to 2.39.0. - [Release notes](https://github.com/google/error-prone/releases) - [Commits](https://github.com/google/error-prone/compare/v2.37.0...v2.39.0) --- updated-dependencies: - dependency-name: com.google.errorprone:error_prone_core dependency-version: 2.39.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 3edf4f84..4ef0e0f5 100644 --- a/build.gradle +++ b/build.gradle @@ -97,7 +97,7 @@ dependencies { jmh 'org.openjdk.jmh:jmh-generator-annprocess:1.37' errorprone 'com.uber.nullaway:nullaway:0.12.7' - errorprone 'com.google.errorprone:error_prone_core:2.37.0' + errorprone 'com.google.errorprone:error_prone_core:2.39.0' // just tests testCompileOnly 'org.jetbrains.kotlin:kotlin-stdlib-jdk8' From 6d63714005af65983ca3b30d01db453cda0663c3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Jul 2025 12:45:41 +0000 Subject: [PATCH 112/156] Bump com.google.errorprone:error_prone_core from 2.39.0 to 2.40.0 Bumps [com.google.errorprone:error_prone_core](https://github.com/google/error-prone) from 2.39.0 to 2.40.0. - [Release notes](https://github.com/google/error-prone/releases) - [Commits](https://github.com/google/error-prone/compare/v2.39.0...v2.40.0) --- updated-dependencies: - dependency-name: com.google.errorprone:error_prone_core dependency-version: 2.40.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index fadeff19..41cc28c0 100644 --- a/build.gradle +++ b/build.gradle @@ -97,7 +97,7 @@ dependencies { jmh 'org.openjdk.jmh:jmh-generator-annprocess:1.37' errorprone 'com.uber.nullaway:nullaway:0.12.7' - errorprone 'com.google.errorprone:error_prone_core:2.39.0' + errorprone 'com.google.errorprone:error_prone_core:2.40.0' // just tests testCompileOnly 'org.jetbrains.kotlin:kotlin-stdlib-jdk8' From fb203f54bd7fb3b3181c6b078343d4177a92a228 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Jul 2025 11:47:07 +0000 Subject: [PATCH 113/156] Bump com.gradle.develocity from 4.0.2 to 4.1 Bumps com.gradle.develocity from 4.0.2 to 4.1. --- updated-dependencies: - dependency-name: com.gradle.develocity dependency-version: '4.1' dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- settings.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.gradle b/settings.gradle index d5a82f45..0f127bf1 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,5 +1,5 @@ plugins { - id 'com.gradle.develocity' version '4.0.2' + id 'com.gradle.develocity' version '4.1' id 'org.gradle.toolchains.foojay-resolver-convention' version '0.9.0' } From 174d52f431828dc2608ecca4396b4b4b134e7863 Mon Sep 17 00:00:00 2001 From: Andreas Marek Date: Fri, 25 Jul 2025 15:42:22 +1000 Subject: [PATCH 114/156] update workflows to use 21 and update toolchain plugin --- .github/workflows/master.yml | 6 +++--- .github/workflows/pull_request.yml | 6 +++--- .github/workflows/release.yml | 6 +++--- settings.gradle | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index ad3f9f29..b7dbe7eb 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -17,11 +17,11 @@ jobs: steps: - uses: actions/checkout@v4 - uses: gradle/actions/wrapper-validation@v4 - - name: Set up JDK 11 + - name: Set up JDK 21 uses: actions/setup-java@v4 with: - java-version: '11' - distribution: 'temurin' + java-version: '21' + distribution: 'corretto' check-latest: true # Configure Gradle for optimal use in GiHub Actions, including caching of downloaded dependencies. # See: https://github.com/gradle/actions/blob/main/setup-gradle/README.md diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 573e169a..ce61f96b 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -15,11 +15,11 @@ jobs: steps: - uses: actions/checkout@v4 - uses: gradle/actions/wrapper-validation@v4 - - name: Set up JDK 11 + - name: Set up JDK 21 uses: actions/setup-java@v4 with: - java-version: '11' - distribution: 'temurin' + java-version: '21' + distribution: 'corretto' check-latest: true # Configure Gradle for optimal use in GiHub Actions, including caching of downloaded dependencies. # See: https://github.com/gradle/actions/blob/main/setup-gradle/README.md diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 64ef19c9..3e0c2d7e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -21,11 +21,11 @@ jobs: steps: - uses: actions/checkout@v4 - uses: gradle/actions/wrapper-validation@v4 - - name: Set up JDK 11 + - name: Set up JDK 21 uses: actions/setup-java@v4 with: - java-version: '11' - distribution: 'temurin' + java-version: '21' + distribution: 'corretto' check-latest: true # Configure Gradle for optimal use in GiHub Actions, including caching of downloaded dependencies. # See: https://github.com/gradle/actions/blob/main/setup-gradle/README.md diff --git a/settings.gradle b/settings.gradle index 0f127bf1..d1e33ace 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,6 +1,6 @@ plugins { id 'com.gradle.develocity' version '4.1' - id 'org.gradle.toolchains.foojay-resolver-convention' version '0.9.0' + id 'org.gradle.toolchains.foojay-resolver-convention' version '1.0.0' } develocity { From f6a625584c9b9f153e1375748a3480e523abd2e3 Mon Sep 17 00:00:00 2001 From: Paulius Dambrauskas Date: Fri, 25 Jul 2025 11:25:06 +0300 Subject: [PATCH 115/156] Allow value nullability for ValueCache --- src/main/java/org/dataloader/ValueCache.java | 3 ++- src/test/kotlin/org/dataloader/KotlinExamples.kt | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/dataloader/ValueCache.java b/src/main/java/org/dataloader/ValueCache.java index 80c8402e..b06fdb8b 100644 --- a/src/main/java/org/dataloader/ValueCache.java +++ b/src/main/java/org/dataloader/ValueCache.java @@ -3,6 +3,7 @@ import org.dataloader.annotations.PublicSpi; import org.dataloader.impl.CompletableFutureKit; import org.dataloader.impl.NoOpValueCache; +import org.jspecify.annotations.Nullable; import org.jspecify.annotations.NullMarked; import java.util.ArrayList; @@ -40,7 +41,7 @@ */ @PublicSpi @NullMarked -public interface ValueCache { +public interface ValueCache { /** * Creates a new value cache, using the default no-op implementation. diff --git a/src/test/kotlin/org/dataloader/KotlinExamples.kt b/src/test/kotlin/org/dataloader/KotlinExamples.kt index 480a965a..f53faf4a 100644 --- a/src/test/kotlin/org/dataloader/KotlinExamples.kt +++ b/src/test/kotlin/org/dataloader/KotlinExamples.kt @@ -1,8 +1,10 @@ package org.dataloader +import java.util.concurrent.CompletableFuture import org.junit.jupiter.api.Test import reactor.core.publisher.Flux import java.util.concurrent.CompletableFuture.completedFuture +import org.dataloader.impl.NoOpValueCache /** * Some Kotlin code to prove that are JSpecify annotations work here @@ -81,6 +83,19 @@ class KotlinExamples { standardNullableAsserts(dataLoader) } + @Test + fun `basic kotlin test of nullable value types in value cache`() { + val valueCache = object : ValueCache by NoOpValueCache() { + override fun get(key: String): CompletableFuture = if (key == "null") + completedFuture(null) + else + completedFuture(key) + } + + assert(valueCache["key"].get() == "key") + assert(valueCache["null"].get() == null) + } + private fun standardNullableAsserts(dataLoader: DataLoader) { val cfA = dataLoader.load("A") val cfB = dataLoader.load("B") From 03eddbc40bf7542e7501c7fa5066b89b2c0b04aa Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Jul 2025 15:55:49 +0000 Subject: [PATCH 116/156] Bump com.google.errorprone:error_prone_core from 2.40.0 to 2.41.0 Bumps [com.google.errorprone:error_prone_core](https://github.com/google/error-prone) from 2.40.0 to 2.41.0. - [Release notes](https://github.com/google/error-prone/releases) - [Commits](https://github.com/google/error-prone/compare/v2.40.0...v2.41.0) --- updated-dependencies: - dependency-name: com.google.errorprone:error_prone_core dependency-version: 2.41.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 41cc28c0..1342fb3d 100644 --- a/build.gradle +++ b/build.gradle @@ -97,7 +97,7 @@ dependencies { jmh 'org.openjdk.jmh:jmh-generator-annprocess:1.37' errorprone 'com.uber.nullaway:nullaway:0.12.7' - errorprone 'com.google.errorprone:error_prone_core:2.40.0' + errorprone 'com.google.errorprone:error_prone_core:2.41.0' // just tests testCompileOnly 'org.jetbrains.kotlin:kotlin-stdlib-jdk8' From e379027e19b98db82bab44edbf2f093d680134e2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Aug 2025 15:56:36 +0000 Subject: [PATCH 117/156] Bump com.uber.nullaway:nullaway from 0.12.7 to 0.12.8 Bumps [com.uber.nullaway:nullaway](https://github.com/uber/NullAway) from 0.12.7 to 0.12.8. - [Release notes](https://github.com/uber/NullAway/releases) - [Changelog](https://github.com/uber/NullAway/blob/master/CHANGELOG.md) - [Commits](https://github.com/uber/NullAway/compare/v0.12.7...v0.12.8) --- updated-dependencies: - dependency-name: com.uber.nullaway:nullaway dependency-version: 0.12.8 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 1342fb3d..7a44da10 100644 --- a/build.gradle +++ b/build.gradle @@ -96,7 +96,7 @@ dependencies { jmh 'org.openjdk.jmh:jmh-core:1.37' jmh 'org.openjdk.jmh:jmh-generator-annprocess:1.37' - errorprone 'com.uber.nullaway:nullaway:0.12.7' + errorprone 'com.uber.nullaway:nullaway:0.12.8' errorprone 'com.google.errorprone:error_prone_core:2.41.0' // just tests From a07cb2a28e9453d23fe4deb7d9acb6d1f10b9a0d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Aug 2025 15:55:58 +0000 Subject: [PATCH 118/156] Bump actions/checkout from 4 to 5 Bumps [actions/checkout](https://github.com/actions/checkout) from 4 to 5. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: '5' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/master.yml | 2 +- .github/workflows/pull_request.yml | 2 +- .github/workflows/release.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index b7dbe7eb..ad30fb2b 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -15,7 +15,7 @@ jobs: MAVEN_CENTRAL_PGP_KEY: ${{ secrets.MAVEN_CENTRAL_PGP_KEY }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: gradle/actions/wrapper-validation@v4 - name: Set up JDK 21 uses: actions/setup-java@v4 diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index ce61f96b..4900736d 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -13,7 +13,7 @@ jobs: buildAndTest: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: gradle/actions/wrapper-validation@v4 - name: Set up JDK 21 uses: actions/setup-java@v4 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3e0c2d7e..de556bb1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -19,7 +19,7 @@ jobs: RELEASE_VERSION: ${{ github.event.inputs.version }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: gradle/actions/wrapper-validation@v4 - name: Set up JDK 21 uses: actions/setup-java@v4 From f83c35bcb11bb70e7a67c359fd27294b3dfc4d0a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Aug 2025 16:28:25 +0000 Subject: [PATCH 119/156] Bump org.jetbrains.kotlin.jvm from 2.2.0 to 2.2.10 Bumps [org.jetbrains.kotlin.jvm](https://github.com/JetBrains/kotlin) from 2.2.0 to 2.2.10. - [Release notes](https://github.com/JetBrains/kotlin/releases) - [Changelog](https://github.com/JetBrains/kotlin/blob/master/ChangeLog.md) - [Commits](https://github.com/JetBrains/kotlin/compare/v2.2.0...v2.2.10) --- updated-dependencies: - dependency-name: org.jetbrains.kotlin.jvm dependency-version: 2.2.10 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 7a44da10..e542776e 100644 --- a/build.gradle +++ b/build.gradle @@ -17,7 +17,7 @@ plugins { id "net.ltgt.errorprone" version '4.3.0' // Kotlin just for tests - not production code - id 'org.jetbrains.kotlin.jvm' version '2.2.0' + id 'org.jetbrains.kotlin.jvm' version '2.2.10' } java { From b2be2d2116e5936de6af7cea8fbdd4ede82db25c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Aug 2025 18:09:37 +0000 Subject: [PATCH 120/156] Bump actions/setup-java from 4 to 5 Bumps [actions/setup-java](https://github.com/actions/setup-java) from 4 to 5. - [Release notes](https://github.com/actions/setup-java/releases) - [Commits](https://github.com/actions/setup-java/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/setup-java dependency-version: '5' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/master.yml | 2 +- .github/workflows/pull_request.yml | 2 +- .github/workflows/release.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index ad30fb2b..14bf6f7f 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -18,7 +18,7 @@ jobs: - uses: actions/checkout@v5 - uses: gradle/actions/wrapper-validation@v4 - name: Set up JDK 21 - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: java-version: '21' distribution: 'corretto' diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 4900736d..56afef32 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -16,7 +16,7 @@ jobs: - uses: actions/checkout@v5 - uses: gradle/actions/wrapper-validation@v4 - name: Set up JDK 21 - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: java-version: '21' distribution: 'corretto' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index de556bb1..0d208027 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -22,7 +22,7 @@ jobs: - uses: actions/checkout@v5 - uses: gradle/actions/wrapper-validation@v4 - name: Set up JDK 21 - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: java-version: '21' distribution: 'corretto' From 6c213936f36a26d195c78214292e4d7f2a27ec2f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Aug 2025 19:25:45 +0000 Subject: [PATCH 121/156] Bump com.uber.nullaway:nullaway from 0.12.8 to 0.12.9 Bumps [com.uber.nullaway:nullaway](https://github.com/uber/NullAway) from 0.12.8 to 0.12.9. - [Release notes](https://github.com/uber/NullAway/releases) - [Changelog](https://github.com/uber/NullAway/blob/master/CHANGELOG.md) - [Commits](https://github.com/uber/NullAway/compare/v0.12.8...v0.12.9) --- updated-dependencies: - dependency-name: com.uber.nullaway:nullaway dependency-version: 0.12.9 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index e542776e..8838b9cb 100644 --- a/build.gradle +++ b/build.gradle @@ -96,7 +96,7 @@ dependencies { jmh 'org.openjdk.jmh:jmh-core:1.37' jmh 'org.openjdk.jmh:jmh-generator-annprocess:1.37' - errorprone 'com.uber.nullaway:nullaway:0.12.8' + errorprone 'com.uber.nullaway:nullaway:0.12.9' errorprone 'com.google.errorprone:error_prone_core:2.41.0' // just tests From 4744403f91d8c92c346c19254110bdd400b420fe Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Aug 2025 19:51:57 +0000 Subject: [PATCH 122/156] Bump com.gradle.develocity from 4.1 to 4.1.1 Bumps com.gradle.develocity from 4.1 to 4.1.1. --- updated-dependencies: - dependency-name: com.gradle.develocity dependency-version: 4.1.1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- settings.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.gradle b/settings.gradle index d1e33ace..446724ea 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,5 +1,5 @@ plugins { - id 'com.gradle.develocity' version '4.1' + id 'com.gradle.develocity' version '4.1.1' id 'org.gradle.toolchains.foojay-resolver-convention' version '1.0.0' } From 8520307f7d6bf0ca1431e84944fede277a868b40 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Sep 2025 11:01:37 +0000 Subject: [PATCH 123/156] Bump actions/stale from 9 to 10 Bumps [actions/stale](https://github.com/actions/stale) from 9 to 10. - [Release notes](https://github.com/actions/stale/releases) - [Changelog](https://github.com/actions/stale/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/stale/compare/v9...v10) --- updated-dependencies: - dependency-name: actions/stale dependency-version: '10' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/stale-pr-issue.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stale-pr-issue.yml b/.github/workflows/stale-pr-issue.yml index d9454022..0f385ee2 100644 --- a/.github/workflows/stale-pr-issue.yml +++ b/.github/workflows/stale-pr-issue.yml @@ -16,7 +16,7 @@ jobs: close-pending: runs-on: ubuntu-latest steps: - - uses: actions/stale@v9 + - uses: actions/stale@v10 with: # GLOBAL ------------------------------------------------------------ # Exempt any PRs or issues already added to a milestone From e6218bfa784bee713643501895b041e4840c0b4e Mon Sep 17 00:00:00 2001 From: bbaker Date: Thu, 11 Sep 2025 21:50:09 +1000 Subject: [PATCH 124/156] Dont allow the Kotlin stdlib to be a dep --- gradle.properties | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 428b6e29..06592abf 100644 --- a/gradle.properties +++ b/gradle.properties @@ -23,4 +23,9 @@ hamcrest_version=2.2 awaitility_version=2.0.0 reactor_core_version=3.6.6 caffeine_version=3.1.8 -reactive_streams_version=1.0.3 \ No newline at end of file +reactive_streams_version=1.0.3 +`` +# Prevents the Kotlin stdlib being a POM dependency +# +# https://kotlinlang.org/docs/gradle-configure-project.html#dependency-on-the-standard-library +kotlin.stdlib.default.dependency=false \ No newline at end of file From bed9e01ff4904c12162f06ad14f1d75fdcc6bd53 Mon Sep 17 00:00:00 2001 From: bbaker Date: Thu, 11 Sep 2025 22:00:42 +1000 Subject: [PATCH 125/156] Dont allow the Kotlin stdlib to be a dep --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 8838b9cb..a9683250 100644 --- a/build.gradle +++ b/build.gradle @@ -100,7 +100,7 @@ dependencies { errorprone 'com.google.errorprone:error_prone_core:2.41.0' // just tests - testCompileOnly 'org.jetbrains.kotlin:kotlin-stdlib-jdk8' + testImplementation 'org.jetbrains.kotlin:kotlin-stdlib-jdk8' } tasks.withType(JavaCompile) { From 78133186f4f7c987778ac452e80520474a14dcf3 Mon Sep 17 00:00:00 2001 From: bbaker Date: Fri, 12 Sep 2025 16:01:50 +1000 Subject: [PATCH 126/156] ooops --- gradle.properties | 1 - 1 file changed, 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 06592abf..db4b3329 100644 --- a/gradle.properties +++ b/gradle.properties @@ -24,7 +24,6 @@ awaitility_version=2.0.0 reactor_core_version=3.6.6 caffeine_version=3.1.8 reactive_streams_version=1.0.3 -`` # Prevents the Kotlin stdlib being a POM dependency # # https://kotlinlang.org/docs/gradle-configure-project.html#dependency-on-the-standard-library From ac7dc17f3976f079d21a970fb31e381c6db43e8c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Sep 2025 11:50:12 +0000 Subject: [PATCH 127/156] Bump org.jetbrains.kotlin.jvm from 2.2.10 to 2.2.20 Bumps [org.jetbrains.kotlin.jvm](https://github.com/JetBrains/kotlin) from 2.2.10 to 2.2.20. - [Release notes](https://github.com/JetBrains/kotlin/releases) - [Changelog](https://github.com/JetBrains/kotlin/blob/master/ChangeLog.md) - [Commits](https://github.com/JetBrains/kotlin/compare/v2.2.10...v2.2.20) --- updated-dependencies: - dependency-name: org.jetbrains.kotlin.jvm dependency-version: 2.2.20 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index a9683250..006b18c1 100644 --- a/build.gradle +++ b/build.gradle @@ -17,7 +17,7 @@ plugins { id "net.ltgt.errorprone" version '4.3.0' // Kotlin just for tests - not production code - id 'org.jetbrains.kotlin.jvm' version '2.2.10' + id 'org.jetbrains.kotlin.jvm' version '2.2.20' } java { From ad8ebba583d9a0f3ec7a734e009996f33ea77191 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Sep 2025 10:52:54 +0000 Subject: [PATCH 128/156] Bump com.uber.nullaway:nullaway from 0.12.9 to 0.12.10 Bumps [com.uber.nullaway:nullaway](https://github.com/uber/NullAway) from 0.12.9 to 0.12.10. - [Release notes](https://github.com/uber/NullAway/releases) - [Changelog](https://github.com/uber/NullAway/blob/master/CHANGELOG.md) - [Commits](https://github.com/uber/NullAway/compare/v0.12.9...v0.12.10) --- updated-dependencies: - dependency-name: com.uber.nullaway:nullaway dependency-version: 0.12.10 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index a9683250..106eacdf 100644 --- a/build.gradle +++ b/build.gradle @@ -96,7 +96,7 @@ dependencies { jmh 'org.openjdk.jmh:jmh-core:1.37' jmh 'org.openjdk.jmh:jmh-generator-annprocess:1.37' - errorprone 'com.uber.nullaway:nullaway:0.12.9' + errorprone 'com.uber.nullaway:nullaway:0.12.10' errorprone 'com.google.errorprone:error_prone_core:2.41.0' // just tests From 99e2a3ae05fd0a13b37e7404544214d86a1d1c8b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Sep 2025 10:54:13 +0000 Subject: [PATCH 129/156] Bump com.google.errorprone:error_prone_core from 2.41.0 to 2.42.0 Bumps [com.google.errorprone:error_prone_core](https://github.com/google/error-prone) from 2.41.0 to 2.42.0. - [Release notes](https://github.com/google/error-prone/releases) - [Commits](https://github.com/google/error-prone/compare/v2.41.0...v2.42.0) --- updated-dependencies: - dependency-name: com.google.errorprone:error_prone_core dependency-version: 2.42.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index a9683250..8ece9d9e 100644 --- a/build.gradle +++ b/build.gradle @@ -97,7 +97,7 @@ dependencies { jmh 'org.openjdk.jmh:jmh-generator-annprocess:1.37' errorprone 'com.uber.nullaway:nullaway:0.12.9' - errorprone 'com.google.errorprone:error_prone_core:2.41.0' + errorprone 'com.google.errorprone:error_prone_core:2.42.0' // just tests testImplementation 'org.jetbrains.kotlin:kotlin-stdlib-jdk8' From 03334ba1bf426e41ae3d105c584b87a81741092c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Sep 2025 10:55:39 +0000 Subject: [PATCH 130/156] Bump com.gradle.develocity from 4.1.1 to 4.2 Bumps com.gradle.develocity from 4.1.1 to 4.2. --- updated-dependencies: - dependency-name: com.gradle.develocity dependency-version: '4.2' dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- settings.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.gradle b/settings.gradle index 446724ea..9ec1e250 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,5 +1,5 @@ plugins { - id 'com.gradle.develocity' version '4.1.1' + id 'com.gradle.develocity' version '4.2' id 'org.gradle.toolchains.foojay-resolver-convention' version '1.0.0' } From 24d83ea7e044378a5440d7b5cb39f7eb29af3a37 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Sep 2025 15:07:48 +0000 Subject: [PATCH 131/156] Bump com.github.ben-manes.versions from 0.52.0 to 0.53.0 Bumps com.github.ben-manes.versions from 0.52.0 to 0.53.0. --- updated-dependencies: - dependency-name: com.github.ben-manes.versions dependency-version: 0.53.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index f85bddd3..7b5de5c4 100644 --- a/build.gradle +++ b/build.gradle @@ -12,7 +12,7 @@ plugins { id 'groovy' id 'biz.aQute.bnd.builder' version '6.2.0' id 'io.github.gradle-nexus.publish-plugin' version '2.0.0' - id 'com.github.ben-manes.versions' version '0.52.0' + id 'com.github.ben-manes.versions' version '0.53.0' id "me.champeau.jmh" version "0.7.3" id "net.ltgt.errorprone" version '4.3.0' From 54e1d786d1096b1ee8cf10034923609d86d1aa8a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Oct 2025 11:05:32 +0000 Subject: [PATCH 132/156] Bump com.gradle.develocity from 4.2 to 4.2.1 Bumps com.gradle.develocity from 4.2 to 4.2.1. --- updated-dependencies: - dependency-name: com.gradle.develocity dependency-version: 4.2.1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- settings.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.gradle b/settings.gradle index 9ec1e250..40f2ffd6 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,5 +1,5 @@ plugins { - id 'com.gradle.develocity' version '4.2' + id 'com.gradle.develocity' version '4.2.1' id 'org.gradle.toolchains.foojay-resolver-convention' version '1.0.0' } From 4759ea77ed77b371affa3931ffe9e05194d74adc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Oct 2025 11:17:15 +0000 Subject: [PATCH 133/156] Bump gradle/actions from 4 to 5 Bumps [gradle/actions](https://github.com/gradle/actions) from 4 to 5. - [Release notes](https://github.com/gradle/actions/releases) - [Commits](https://github.com/gradle/actions/compare/v4...v5) --- updated-dependencies: - dependency-name: gradle/actions dependency-version: '5' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/master.yml | 4 ++-- .github/workflows/pull_request.yml | 4 ++-- .github/workflows/release.yml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 14bf6f7f..e6aa3a2e 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -16,7 +16,7 @@ jobs: steps: - uses: actions/checkout@v5 - - uses: gradle/actions/wrapper-validation@v4 + - uses: gradle/actions/wrapper-validation@v5 - name: Set up JDK 21 uses: actions/setup-java@v5 with: @@ -26,7 +26,7 @@ jobs: # Configure Gradle for optimal use in GiHub Actions, including caching of downloaded dependencies. # See: https://github.com/gradle/actions/blob/main/setup-gradle/README.md - name: Setup Gradle - uses: gradle/actions/setup-gradle@v4 + uses: gradle/actions/setup-gradle@v5 - name: build test and publish run: ./gradlew assemble && ./gradlew check --info && ./gradlew publishToSonatype closeAndReleaseSonatypeStagingRepository -x check --info --stacktrace env: diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 56afef32..5258ad28 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v5 - - uses: gradle/actions/wrapper-validation@v4 + - uses: gradle/actions/wrapper-validation@v5 - name: Set up JDK 21 uses: actions/setup-java@v5 with: @@ -24,7 +24,7 @@ jobs: # Configure Gradle for optimal use in GiHub Actions, including caching of downloaded dependencies. # See: https://github.com/gradle/actions/blob/main/setup-gradle/README.md - name: Setup Gradle - uses: gradle/actions/setup-gradle@v4 + uses: gradle/actions/setup-gradle@v5 - name: build and test run: ./gradlew assemble && ./gradlew check --info --stacktrace env: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0d208027..ba437442 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,7 +20,7 @@ jobs: steps: - uses: actions/checkout@v5 - - uses: gradle/actions/wrapper-validation@v4 + - uses: gradle/actions/wrapper-validation@v5 - name: Set up JDK 21 uses: actions/setup-java@v5 with: @@ -30,7 +30,7 @@ jobs: # Configure Gradle for optimal use in GiHub Actions, including caching of downloaded dependencies. # See: https://github.com/gradle/actions/blob/main/setup-gradle/README.md - name: Setup Gradle - uses: gradle/actions/setup-gradle@v4 + uses: gradle/actions/setup-gradle@v5 - name: build test and publish run: ./gradlew assemble && ./gradlew check --info && ./gradlew publishToSonatype closeAndReleaseSonatypeStagingRepository -x check --info --stacktrace env: From 19b5c305308ac6dcf2b51b202580bc633f3cc6dd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Oct 2025 11:27:56 +0000 Subject: [PATCH 134/156] Bump com.gradle.develocity from 4.2.1 to 4.2.2 Bumps com.gradle.develocity from 4.2.1 to 4.2.2. --- updated-dependencies: - dependency-name: com.gradle.develocity dependency-version: 4.2.2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- settings.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.gradle b/settings.gradle index 40f2ffd6..0d61ab8a 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,5 +1,5 @@ plugins { - id 'com.gradle.develocity' version '4.2.1' + id 'com.gradle.develocity' version '4.2.2' id 'org.gradle.toolchains.foojay-resolver-convention' version '1.0.0' } From 1804706fc1f3e6cdc7c91f94648d6f3be1fef001 Mon Sep 17 00:00:00 2001 From: bbaker Date: Tue, 14 Oct 2025 11:54:03 +1100 Subject: [PATCH 135/156] Updating the code to use reentrant locks --- src/main/java/org/dataloader/DataLoader.java | 52 ++++++----- .../java/org/dataloader/DataLoaderHelper.java | 37 ++++++-- .../reactive/AbstractBatchSubscriber.java | 3 + .../reactive/BatchSubscriberImpl.java | 90 ++++++++++-------- .../reactive/MappedBatchSubscriberImpl.java | 92 +++++++++++-------- 5 files changed, 170 insertions(+), 104 deletions(-) diff --git a/src/main/java/org/dataloader/DataLoader.java b/src/main/java/org/dataloader/DataLoader.java index 321b58c4..68e699bc 100644 --- a/src/main/java/org/dataloader/DataLoader.java +++ b/src/main/java/org/dataloader/DataLoader.java @@ -35,6 +35,8 @@ import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; import java.util.function.BiConsumer; import java.util.function.Consumer; @@ -77,6 +79,7 @@ public class DataLoader { private final ValueCache valueCache; private final DataLoaderOptions options; private final Object batchLoadFunction; + final Lock lock; @VisibleForTesting DataLoader(@Nullable String name, Object batchLoadFunction, @Nullable DataLoaderOptions options) { @@ -93,7 +96,7 @@ public class DataLoader { this.batchLoadFunction = nonNull(batchLoadFunction); this.options = loaderOptions; this.name = name; - + this.lock = new ReentrantLock(); this.helper = new DataLoaderHelper<>(this, batchLoadFunction, loaderOptions, this.futureCache, this.valueCache, this.stats, clock); } @@ -261,18 +264,16 @@ public CompletableFuture> loadMany(List keys, List keyContext nonNull(keys); nonNull(keyContexts); - synchronized (this) { - List> collect = new ArrayList<>(keys.size()); - for (int i = 0; i < keys.size(); i++) { - K key = keys.get(i); - Object keyContext = null; - if (i < keyContexts.size()) { - keyContext = keyContexts.get(i); - } - collect.add(load(key, keyContext)); + List> collect = new ArrayList<>(keys.size()); + for (int i = 0; i < keys.size(); i++) { + K key = keys.get(i); + Object keyContext = null; + if (i < keyContexts.size()) { + keyContext = keyContexts.get(i); } - return CompletableFutureKit.allOf(collect); + collect.add(load(key, keyContext)); } + return CompletableFutureKit.allOf(collect); } /** @@ -292,15 +293,13 @@ public CompletableFuture> loadMany(List keys, List keyContext public CompletableFuture> loadMany(Map keysAndContexts) { nonNull(keysAndContexts); - synchronized (this) { - Map> collect = new HashMap<>(keysAndContexts.size()); - for (Map.Entry entry : keysAndContexts.entrySet()) { - K key = entry.getKey(); - Object keyContext = entry.getValue(); - collect.put(key, load(key, keyContext)); - } - return CompletableFutureKit.allOf(collect); + Map> collect = new HashMap<>(keysAndContexts.size()); + for (Map.Entry entry : keysAndContexts.entrySet()) { + K key = entry.getKey(); + Object keyContext = entry.getValue(); + collect.put(key, load(key, keyContext)); } + return CompletableFutureKit.allOf(collect); } /** @@ -376,9 +375,12 @@ public DataLoader clear(K key) { */ public DataLoader clear(K key, BiConsumer handler) { Object cacheKey = getCacheKey(key); - synchronized (this) { + try { + lock.lock(); futureCache.delete(cacheKey); valueCache.delete(key).whenComplete(handler); + } finally { + lock.unlock(); } return this; } @@ -400,9 +402,12 @@ public DataLoader clearAll() { * @return the data loader for fluent coding */ public DataLoader clearAll(BiConsumer handler) { - synchronized (this) { + try { + lock.lock(); futureCache.clear(); valueCache.clear().whenComplete(handler); + } finally { + lock.unlock(); } return this; } @@ -442,10 +447,13 @@ public DataLoader prime(K key, Exception error) { */ public DataLoader prime(K key, CompletableFuture value) { Object cacheKey = getCacheKey(key); - synchronized (this) { + try { + lock.lock(); if (!futureCache.containsKey(cacheKey)) { futureCache.set(cacheKey, value); } + } finally { + lock.unlock(); } return this; } diff --git a/src/main/java/org/dataloader/DataLoaderHelper.java b/src/main/java/org/dataloader/DataLoaderHelper.java index f4a8f106..83389e4f 100644 --- a/src/main/java/org/dataloader/DataLoaderHelper.java +++ b/src/main/java/org/dataloader/DataLoaderHelper.java @@ -28,6 +28,7 @@ import java.util.concurrent.CompletionException; import java.util.concurrent.CompletionStage; import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.Lock; import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; @@ -82,6 +83,7 @@ Object getCallContext() { private final StatisticsCollector stats; private final Clock clock; private final AtomicReference lastDispatchTime; + private final Lock lock; DataLoaderHelper(DataLoader dataLoader, Object batchLoadFunction, @@ -95,6 +97,7 @@ Object getCallContext() { this.loaderOptions = loaderOptions; this.futureCache = futureCache; this.valueCache = valueCache; + this.lock = dataLoader.lock; this.loaderQueue = new ArrayList<>(); this.stats = stats; this.clock = clock; @@ -111,7 +114,8 @@ public Instant getLastDispatchTime() { } Optional> getIfPresent(K key) { - synchronized (dataLoader) { + try { + lock.lock(); boolean cachingEnabled = loaderOptions.cachingEnabled(); if (cachingEnabled) { Object cacheKey = getCacheKey(nonNull(key)); @@ -124,12 +128,16 @@ Optional> getIfPresent(K key) { } catch (Exception ignored) { } } + } finally { + lock.unlock(); } return Optional.empty(); } Optional> getIfCompleted(K key) { - synchronized (dataLoader) { + try { + lock.lock(); + Optional> cachedPromise = getIfPresent(key); if (cachedPromise.isPresent()) { CompletableFuture promise = cachedPromise.get(); @@ -137,13 +145,18 @@ Optional> getIfCompleted(K key) { return cachedPromise; } } + } finally { + lock.unlock(); } return Optional.empty(); } + @GuardedBy("lock") CompletableFuture load(K key, Object loadContext) { - synchronized (dataLoader) { + try { + lock.lock(); + boolean batchingEnabled = loaderOptions.batchingEnabled(); boolean cachingEnabled = loaderOptions.cachingEnabled(); @@ -158,6 +171,8 @@ CompletableFuture load(K key, Object loadContext) { ctx.onDispatched(); cf.whenComplete(ctx::onCompleted); return cf; + } finally { + lock.unlock(); } } @@ -173,6 +188,7 @@ Object getCacheKeyWithContext(K key, Object context) { loaderOptions.cacheKeyFunction().get().getKeyWithContext(key, context) : key; } + @GuardedBy("lock") DispatchResult dispatch() { DataLoaderInstrumentationContext> instrCtx = ctxOrNoopCtx(instrumentation().beginDispatch(dataLoader)); @@ -180,7 +196,9 @@ DispatchResult dispatch() { final List keys; final List callContexts; final List> queuedFutures; - synchronized (dataLoader) { + try { + lock.lock(); + int queueSize = loaderQueue.size(); if (queueSize == 0) { lastDispatchTime.set(now()); @@ -200,6 +218,8 @@ DispatchResult dispatch() { }); loaderQueue.clear(); lastDispatchTime.set(now()); + } finally { + lock.unlock(); } if (!batchingEnabled) { instrCtx.onDispatched(); @@ -334,7 +354,7 @@ private void possiblyClearCacheEntriesOnExceptions(List keys) { } } - @GuardedBy("dataLoader") + @GuardedBy("lock") private CompletableFuture loadFromCache(K key, Object loadContext, boolean batchingEnabled) { final Object cacheKey = loadContext == null ? getCacheKey(key) : getCacheKeyWithContext(key, loadContext); @@ -353,7 +373,7 @@ private CompletableFuture loadFromCache(K key, Object loadContext, boolean ba return loadCallFuture; } - @GuardedBy("dataLoader") + @GuardedBy("lock") private CompletableFuture queueOrInvokeLoader(K key, Object loadContext, boolean batchingEnabled, boolean cachingEnabled) { if (batchingEnabled) { CompletableFuture loadCallFuture = new CompletableFuture<>(); @@ -606,8 +626,11 @@ private DataLoaderInstrumentation instrumentation() { } int dispatchDepth() { - synchronized (dataLoader) { + try { + lock.lock(); return loaderQueue.size(); + } finally { + lock.unlock(); } } diff --git a/src/main/java/org/dataloader/reactive/AbstractBatchSubscriber.java b/src/main/java/org/dataloader/reactive/AbstractBatchSubscriber.java index c2f54380..76c94b38 100644 --- a/src/main/java/org/dataloader/reactive/AbstractBatchSubscriber.java +++ b/src/main/java/org/dataloader/reactive/AbstractBatchSubscriber.java @@ -10,6 +10,8 @@ import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; import static org.dataloader.impl.Assertions.assertState; @@ -25,6 +27,7 @@ abstract class AbstractBatchSubscriber implements Subscriber { final List callContexts; final List> queuedFutures; final ReactiveSupport.HelperIntegration helperIntegration; + final Lock lock = new ReentrantLock(); List clearCacheKeys = new ArrayList<>(); List completedValues = new ArrayList<>(); diff --git a/src/main/java/org/dataloader/reactive/BatchSubscriberImpl.java b/src/main/java/org/dataloader/reactive/BatchSubscriberImpl.java index d0b81100..74a500a5 100644 --- a/src/main/java/org/dataloader/reactive/BatchSubscriberImpl.java +++ b/src/main/java/org/dataloader/reactive/BatchSubscriberImpl.java @@ -29,58 +29,74 @@ class BatchSubscriberImpl extends AbstractBatchSubscriber { super(valuesFuture, keys, callContexts, queuedFutures, helperIntegration); } - // onNext may be called by multiple threads - for the time being, we pass 'synchronized' to guarantee + // onNext may be called by multiple threads - for the time being, we use a ReentrantLock to guarantee // correctness (at the cost of speed). @Override - public synchronized void onNext(V value) { - super.onNext(value); + public void onNext(V value) { + try { + lock.lock(); - if (idx >= keys.size()) { - // hang on they have given us more values than we asked for in keys - // we cant handle this - return; - } - K key = keys.get(idx); - Object callContext = callContexts.get(idx); - CompletableFuture future = queuedFutures.get(idx); - onNextValue(key, value, callContext, List.of(future)); + super.onNext(value); + + if (idx >= keys.size()) { + // hang on they have given us more values than we asked for in keys + // we cant handle this + return; + } + K key = keys.get(idx); + Object callContext = callContexts.get(idx); + CompletableFuture future = queuedFutures.get(idx); + onNextValue(key, value, callContext, List.of(future)); - completedValues.add(value); - idx++; + completedValues.add(value); + idx++; + } finally { + lock.unlock(); + } } @Override - public synchronized void onComplete() { - super.onComplete(); - if (keys.size() != completedValues.size()) { - // we have more or less values than promised - // we will go through all the outstanding promises and mark those that - // have not finished as failed - for (CompletableFuture queuedFuture : queuedFutures) { - if (!queuedFuture.isDone()) { - queuedFuture.completeExceptionally(new DataLoaderAssertionException("The size of the promised values MUST be the same size as the key list")); + public void onComplete() { + try { + lock.lock(); + super.onComplete(); + if (keys.size() != completedValues.size()) { + // we have more or less values than promised + // we will go through all the outstanding promises and mark those that + // have not finished as failed + for (CompletableFuture queuedFuture : queuedFutures) { + if (!queuedFuture.isDone()) { + queuedFuture.completeExceptionally(new DataLoaderAssertionException("The size of the promised values MUST be the same size as the key list")); + } } } + possiblyClearCacheEntriesOnExceptions(); + valuesFuture.complete(completedValues); + } finally { + lock.unlock(); } - possiblyClearCacheEntriesOnExceptions(); - valuesFuture.complete(completedValues); } @Override - public synchronized void onError(Throwable ex) { - super.onError(ex); - ex = unwrapThrowable(ex); - // Set the remaining keys to the exception. - for (int i = idx; i < queuedFutures.size(); i++) { - K key = keys.get(i); - CompletableFuture future = queuedFutures.get(i); - if (!future.isDone()) { - future.completeExceptionally(ex); - // clear any cached view of this key because it failed - helperIntegration.clearCacheView(key); + public void onError(Throwable ex) { + try { + lock.lock(); + super.onError(ex); + ex = unwrapThrowable(ex); + // Set the remaining keys to the exception. + for (int i = idx; i < queuedFutures.size(); i++) { + K key = keys.get(i); + CompletableFuture future = queuedFutures.get(i); + if (!future.isDone()) { + future.completeExceptionally(ex); + // clear any cached view of this key because it failed + helperIntegration.clearCacheView(key); + } } + valuesFuture.completeExceptionally(ex); + } finally { + lock.unlock(); } - valuesFuture.completeExceptionally(ex); } } diff --git a/src/main/java/org/dataloader/reactive/MappedBatchSubscriberImpl.java b/src/main/java/org/dataloader/reactive/MappedBatchSubscriberImpl.java index d56efa0e..3c937b03 100644 --- a/src/main/java/org/dataloader/reactive/MappedBatchSubscriberImpl.java +++ b/src/main/java/org/dataloader/reactive/MappedBatchSubscriberImpl.java @@ -43,61 +43,77 @@ class MappedBatchSubscriberImpl extends AbstractBatchSubscriber entry) { - super.onNext(entry); - K key = entry.getKey(); - V value = entry.getValue(); + public void onNext(Map.Entry entry) { + try { + lock.lock(); + super.onNext(entry); + K key = entry.getKey(); + V value = entry.getValue(); - Object callContext = callContextByKey.get(key); - List> futures = queuedFuturesByKey.getOrDefault(key, List.of()); + Object callContext = callContextByKey.get(key); + List> futures = queuedFuturesByKey.getOrDefault(key, List.of()); - onNextValue(key, value, callContext, futures); + onNextValue(key, value, callContext, futures); - // did we have an actual key for this value - ignore it if they send us one outside the key set - if (!futures.isEmpty()) { - completedValuesByKey.put(key, value); + // did we have an actual key for this value - ignore it if they send us one outside the key set + if (!futures.isEmpty()) { + completedValuesByKey.put(key, value); + } + } finally { + lock.unlock(); } + } @Override - public synchronized void onComplete() { - super.onComplete(); + public void onComplete() { + try { + lock.lock(); + super.onComplete(); - possiblyClearCacheEntriesOnExceptions(); - List values = new ArrayList<>(keys.size()); - for (K key : keys) { - V value = completedValuesByKey.get(key); - values.add(value); + possiblyClearCacheEntriesOnExceptions(); + List values = new ArrayList<>(keys.size()); + for (K key : keys) { + V value = completedValuesByKey.get(key); + values.add(value); - List> futures = queuedFuturesByKey.getOrDefault(key, List.of()); - for (CompletableFuture future : futures) { - if (!future.isDone()) { - // we have a future that never came back for that key - // but the publisher is done sending in data - it must be null - // e.g. for key X when found no value - future.complete(null); + List> futures = queuedFuturesByKey.getOrDefault(key, List.of()); + for (CompletableFuture future : futures) { + if (!future.isDone()) { + // we have a future that never came back for that key + // but the publisher is done sending in data - it must be null + // e.g. for key X when found no value + future.complete(null); + } } } + valuesFuture.complete(values); + } finally { + lock.unlock(); } - valuesFuture.complete(values); } @Override - public synchronized void onError(Throwable ex) { - super.onError(ex); - ex = unwrapThrowable(ex); - // Complete the futures for the remaining keys with the exception. - for (int idx = 0; idx < queuedFutures.size(); idx++) { - K key = keys.get(idx); - List> futures = queuedFuturesByKey.get(key); - if (!completedValuesByKey.containsKey(key)) { - for (CompletableFuture future : futures) { - future.completeExceptionally(ex); + public void onError(Throwable ex) { + try { + lock.lock(); + super.onError(ex); + ex = unwrapThrowable(ex); + // Complete the futures for the remaining keys with the exception. + for (int idx = 0; idx < queuedFutures.size(); idx++) { + K key = keys.get(idx); + List> futures = queuedFuturesByKey.get(key); + if (!completedValuesByKey.containsKey(key)) { + for (CompletableFuture future : futures) { + future.completeExceptionally(ex); + } + // clear any cached view of this key because they all failed + helperIntegration.clearCacheView(key); } - // clear any cached view of this key because they all failed - helperIntegration.clearCacheView(key); } + valuesFuture.completeExceptionally(ex); + } finally { + lock.unlock(); } - valuesFuture.completeExceptionally(ex); } } From 46950a9d1d44dc46f1d9125f343e523bb1028a30 Mon Sep 17 00:00:00 2001 From: bbaker Date: Tue, 14 Oct 2025 13:41:44 +1100 Subject: [PATCH 136/156] Added a loadImpl function to allow delegate overriding better --- src/main/java/org/dataloader/DataLoader.java | 38 +++++----- .../org/dataloader/DelegatingDataLoader.java | 29 ++++++-- .../dataloader/DelegatingDataLoaderTest.java | 73 +++++++++++++++++-- 3 files changed, 110 insertions(+), 30 deletions(-) diff --git a/src/main/java/org/dataloader/DataLoader.java b/src/main/java/org/dataloader/DataLoader.java index 321b58c4..ded7b0dc 100644 --- a/src/main/java/org/dataloader/DataLoader.java +++ b/src/main/java/org/dataloader/DataLoader.java @@ -159,20 +159,6 @@ public Duration getTimeSinceDispatch() { return Duration.between(helper.getLastDispatchTime(), helper.now()); } - /** - * Requests to load the data with the specified key asynchronously, and returns a future of the resulting value. - *

- * If batching is enabled (the default), you'll have to call {@link DataLoader#dispatch()} at a later stage to - * start batch execution. If you forget this call the future will never be completed (unless already completed, - * and returned from cache). - * - * @param key the key to load - * @return the future of the value - */ - public CompletableFuture load(K key) { - return load(key, null); - } - /** * This will return an optional promise to a value previously loaded via a {@link #load(Object)} call or empty if not call has been made for that key. *

@@ -209,6 +195,24 @@ public Optional> getIfCompleted(K key) { } + private CompletableFuture loadImpl(@NonNull K key, @Nullable Object keyContext) { + return helper.load(nonNull(key), keyContext); + } + + /** + * Requests to load the data with the specified key asynchronously, and returns a future of the resulting value. + *

+ * If batching is enabled (the default), you'll have to call {@link DataLoader#dispatch()} at a later stage to + * start batch execution. If you forget this call the future will never be completed (unless already completed, + * and returned from cache). + * + * @param key the key to load + * @return the future of the value + */ + public CompletableFuture load(K key) { + return loadImpl(key, null); + } + /** * Requests to load the data with the specified key asynchronously, and returns a future of the resulting value. *

@@ -224,7 +228,7 @@ public Optional> getIfCompleted(K key) { * @return the future of the value */ public CompletableFuture load(@NonNull K key, @Nullable Object keyContext) { - return helper.load(nonNull(key), keyContext); + return loadImpl(key, keyContext); } /** @@ -269,7 +273,7 @@ public CompletableFuture> loadMany(List keys, List keyContext if (i < keyContexts.size()) { keyContext = keyContexts.get(i); } - collect.add(load(key, keyContext)); + collect.add(loadImpl(key, keyContext)); } return CompletableFutureKit.allOf(collect); } @@ -297,7 +301,7 @@ public CompletableFuture> loadMany(Map keysAndContexts) { for (Map.Entry entry : keysAndContexts.entrySet()) { K key = entry.getKey(); Object keyContext = entry.getValue(); - collect.put(key, load(key, keyContext)); + collect.put(key, loadImpl(key, keyContext)); } return CompletableFutureKit.allOf(collect); } diff --git a/src/main/java/org/dataloader/DelegatingDataLoader.java b/src/main/java/org/dataloader/DelegatingDataLoader.java index 7cffced0..01750592 100644 --- a/src/main/java/org/dataloader/DelegatingDataLoader.java +++ b/src/main/java/org/dataloader/DelegatingDataLoader.java @@ -9,6 +9,7 @@ import java.time.Duration; import java.time.Instant; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.function.BiConsumer; @@ -66,19 +67,31 @@ public DataLoader getDelegate() { return delegate; } - /** - * The {@link DataLoader#load(Object)} and {@link DataLoader#loadMany(List)} type methods all call back - * to the {@link DataLoader#load(Object, Object)} and hence we don't override them. - * - * @param key the key to load - * @param keyContext a context object that is specific to this key - * @return the future of the value - */ + @Override + public CompletableFuture load(K key) { + return delegate.load(key); + } + @Override public CompletableFuture load(@NonNull K key, @Nullable Object keyContext) { return delegate.load(key, keyContext); } + @Override + public CompletableFuture> loadMany(List keys) { + return delegate.loadMany(keys); + } + + @Override + public CompletableFuture> loadMany(List keys, List keyContexts) { + return delegate.loadMany(keys, keyContexts); + } + + @Override + public CompletableFuture> loadMany(Map keysAndContexts) { + return delegate.loadMany(keysAndContexts); + } + @Override public DataLoader transform(Consumer> builderConsumer) { return delegate.transform(builderConsumer); diff --git a/src/test/java/org/dataloader/DelegatingDataLoaderTest.java b/src/test/java/org/dataloader/DelegatingDataLoaderTest.java index 88497528..a76beb99 100644 --- a/src/test/java/org/dataloader/DelegatingDataLoaderTest.java +++ b/src/test/java/org/dataloader/DelegatingDataLoaderTest.java @@ -3,18 +3,20 @@ import org.dataloader.fixtures.TestKit; import org.dataloader.fixtures.parameterized.DelegatingDataLoaderFactory; import org.jspecify.annotations.NonNull; +import org.jspecify.annotations.NullMarked; import org.jspecify.annotations.Nullable; -import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import java.util.List; +import java.util.Map; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicInteger; import static org.awaitility.Awaitility.await; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertNotNull; /** * There are WAY more tests via the {@link DelegatingDataLoaderFactory} @@ -73,8 +75,69 @@ void can_delegate_simple_properties() { DelegatingDataLoader delegate = new DelegatingDataLoader<>(rawLoader); assertNotNull(delegate.getName()); - assertThat(delegate.getName(),equalTo("name")); - assertThat(delegate.getOptions(),equalTo(options)); - assertThat(delegate.getBatchLoadFunction(),equalTo(loadFunction)); + assertThat(delegate.getName(), equalTo("name")); + assertThat(delegate.getOptions(), equalTo(options)); + assertThat(delegate.getBatchLoadFunction(), equalTo(loadFunction)); + } + + @NullMarked + @Test + void can_create_a_delegate_class_that_has_post_side_effects() { + DataLoaderOptions options = DataLoaderOptions.newOptions().build(); + BatchLoader loadFunction = CompletableFuture::completedFuture; + DataLoader rawLoader = DataLoaderFactory.newDataLoader("name", loadFunction, options); + + AtomicInteger loadCalled = new AtomicInteger(0); + AtomicInteger loadManyCalled = new AtomicInteger(0); + AtomicInteger loadManyMapCalled = new AtomicInteger(0); + DelegatingDataLoader delegate = new DelegatingDataLoader<>(rawLoader) { + + @Override + public CompletableFuture load(String key) { + CompletableFuture cf = super.load(key); + loadCalled.incrementAndGet(); + return cf; + } + + @Override + public CompletableFuture load(String key, @Nullable Object keyContext) { + CompletableFuture cf = super.load(key, keyContext); + loadCalled.incrementAndGet(); + return cf; + } + + @Override + public CompletableFuture> loadMany(List keys, List keyContexts) { + CompletableFuture> cf = super.loadMany(keys, keyContexts); + loadManyCalled.incrementAndGet(); + return cf; + } + + @Override + public CompletableFuture> loadMany(List keys) { + CompletableFuture> cf = super.loadMany(keys); + loadManyCalled.incrementAndGet(); + return cf; + } + + @Override + public CompletableFuture> loadMany(Map keysAndContexts) { + CompletableFuture> cf = super.loadMany(keysAndContexts); + loadManyMapCalled.incrementAndGet(); + return cf; + } + }; + + + delegate.load("L1"); + delegate.load("L2", null); + delegate.loadMany(List.of("LM1", "LM2"), List.of()); + delegate.loadMany(List.of("LM3")); + delegate.loadMany(Map.of("LMM1", "kc1", "LMM2", "kc2")); + + assertNotNull(delegate.getDelegate()); + assertThat(loadCalled.get(), equalTo(2)); + assertThat(loadManyCalled.get(), equalTo(2)); + assertThat(loadManyMapCalled.get(), equalTo(1)); } } \ No newline at end of file From 2f3fa1e6d058d37c79c82e9c49b7f3d696ffb4e8 Mon Sep 17 00:00:00 2001 From: bbaker Date: Tue, 14 Oct 2025 14:08:54 +1100 Subject: [PATCH 137/156] Added a loadImpl function to allow delegate overriding better - fixed test --- .../dataloader/DelegatingDataLoaderTest.java | 31 ++++++++++++++++--- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/src/test/java/org/dataloader/DelegatingDataLoaderTest.java b/src/test/java/org/dataloader/DelegatingDataLoaderTest.java index a76beb99..0f51b5f7 100644 --- a/src/test/java/org/dataloader/DelegatingDataLoaderTest.java +++ b/src/test/java/org/dataloader/DelegatingDataLoaderTest.java @@ -2,7 +2,6 @@ import org.dataloader.fixtures.TestKit; import org.dataloader.fixtures.parameterized.DelegatingDataLoaderFactory; -import org.jspecify.annotations.NonNull; import org.jspecify.annotations.NullMarked; import org.jspecify.annotations.Nullable; import org.junit.jupiter.api.Test; @@ -11,6 +10,7 @@ import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; import static org.awaitility.Awaitility.await; import static org.hamcrest.CoreMatchers.equalTo; @@ -34,14 +34,37 @@ void canUnwrapDataLoaders() { } @Test + @NullMarked void canCreateAClassOk() { DataLoader rawLoader = TestKit.idLoader(); DelegatingDataLoader delegatingDataLoader = new DelegatingDataLoader<>(rawLoader) { - @Override - public CompletableFuture load(@NonNull String key, @Nullable Object keyContext) { - CompletableFuture cf = super.load(key, keyContext); + private CompletableFuture enhance(CompletableFuture cf) { return cf.thenApply(v -> "|" + v + "|"); } + + private CompletableFuture> enhanceList(CompletableFuture> cf) { + return cf.thenApply(v -> v.stream().map(s -> "|" + s + "|").collect(Collectors.toList())); + } + + @Override + public CompletableFuture load(String key, @Nullable Object keyContext) { + return enhance(super.load(key, keyContext)); + } + + @Override + public CompletableFuture load(String key) { + return enhance(super.load(key)); + } + + @Override + public CompletableFuture> loadMany(List keys) { + return enhanceList(super.loadMany(keys)); + } + + @Override + public CompletableFuture> loadMany(List keys, List keyContexts) { + return enhanceList(super.loadMany(keys, keyContexts)); + } }; assertThat(delegatingDataLoader.getDelegate(), is(rawLoader)); From 7b5041c5cd7b3fe21d4917bafd8e5af01ce90791 Mon Sep 17 00:00:00 2001 From: Andreas Marek Date: Sat, 18 Oct 2025 15:36:53 +1000 Subject: [PATCH 138/156] remove any Reentrant locks in favor of a CAS linked list --- src/main/java/org/dataloader/CacheMap.java | 2 +- src/main/java/org/dataloader/DataLoader.java | 53 ++-- .../java/org/dataloader/DataLoaderHelper.java | 229 ++++++++++-------- .../org/dataloader/impl/DefaultCacheMap.java | 12 +- src/test/java/ReadmeExamples.java | 2 +- .../java/org/dataloader/DataLoaderTest.java | 4 +- .../dataloader/fixtures/CustomCacheMap.java | 12 +- 7 files changed, 161 insertions(+), 153 deletions(-) diff --git a/src/main/java/org/dataloader/CacheMap.java b/src/main/java/org/dataloader/CacheMap.java index 54b1b497..36913e64 100644 --- a/src/main/java/org/dataloader/CacheMap.java +++ b/src/main/java/org/dataloader/CacheMap.java @@ -90,7 +90,7 @@ static CacheMap simpleMap() { * * @return the cache map for fluent coding */ - CacheMap set(K key, CompletableFuture value); + CompletableFuture setIfAbsent(K key, CompletableFuture value); /** * Deletes the entry with the specified key from the cache map, if it exists. diff --git a/src/main/java/org/dataloader/DataLoader.java b/src/main/java/org/dataloader/DataLoader.java index 68e699bc..4077e552 100644 --- a/src/main/java/org/dataloader/DataLoader.java +++ b/src/main/java/org/dataloader/DataLoader.java @@ -35,8 +35,6 @@ import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; import java.util.function.BiConsumer; import java.util.function.Consumer; @@ -65,6 +63,7 @@ * * @param type parameter indicating the type of the data load keys * @param type parameter indicating the type of the data that is returned + * * @author Arnold Schrijver * @author Brad Baker */ @@ -79,7 +78,6 @@ public class DataLoader { private final ValueCache valueCache; private final DataLoaderOptions options; private final Object batchLoadFunction; - final Lock lock; @VisibleForTesting DataLoader(@Nullable String name, Object batchLoadFunction, @Nullable DataLoaderOptions options) { @@ -96,7 +94,6 @@ public class DataLoader { this.batchLoadFunction = nonNull(batchLoadFunction); this.options = loaderOptions; this.name = name; - this.lock = new ReentrantLock(); this.helper = new DataLoaderHelper<>(this, batchLoadFunction, loaderOptions, this.futureCache, this.valueCache, this.stats, clock); } @@ -136,6 +133,7 @@ public Object getBatchLoadFunction() { * This allows you to change the current {@link DataLoader} and turn it into a new one * * @param builderConsumer the {@link DataLoaderFactory.Builder} consumer for changing the {@link DataLoader} + * * @return a newly built {@link DataLoader} instance */ public DataLoader transform(Consumer> builderConsumer) { @@ -170,6 +168,7 @@ public Duration getTimeSinceDispatch() { * and returned from cache). * * @param key the key to load + * * @return the future of the value */ public CompletableFuture load(K key) { @@ -187,6 +186,7 @@ public CompletableFuture load(K key) { * NOTE : This will NOT cause a data load to happen. You must call {@link #load(Object)} for that to happen. * * @param key the key to check + * * @return an Optional to the future of the value */ public Optional> getIfPresent(K key) { @@ -205,6 +205,7 @@ public Optional> getIfPresent(K key) { * NOTE : This will NOT cause a data load to happen. You must call {@link #load(Object)} for that to happen. * * @param key the key to check + * * @return an Optional to the future of the value */ public Optional> getIfCompleted(K key) { @@ -224,6 +225,7 @@ public Optional> getIfCompleted(K key) { * * @param key the key to load * @param keyContext a context object that is specific to this key + * * @return the future of the value */ public CompletableFuture load(@NonNull K key, @Nullable Object keyContext) { @@ -239,6 +241,7 @@ public CompletableFuture load(@NonNull K key, @Nullable Object keyContext) { * and returned from cache). * * @param keys the list of keys to load + * * @return the composite future of the list of values */ public CompletableFuture> loadMany(List keys) { @@ -258,6 +261,7 @@ public CompletableFuture> loadMany(List keys) { * * @param keys the list of keys to load * @param keyContexts the list of key calling context objects + * * @return the composite future of the list of values */ public CompletableFuture> loadMany(List keys, List keyContexts) { @@ -288,6 +292,7 @@ public CompletableFuture> loadMany(List keys, List keyContext * {@link org.dataloader.MappedBatchLoaderWithContext} to help retrieve data. * * @param keysAndContexts the map of keys to their respective contexts + * * @return the composite future of the map of keys and values */ public CompletableFuture> loadMany(Map keysAndContexts) { @@ -358,6 +363,7 @@ public int dispatchDepth() { * on the next load request. * * @param key the key to remove + * * @return the data loader for fluent coding */ public DataLoader clear(K key) { @@ -371,17 +377,13 @@ public DataLoader clear(K key) { * * @param key the key to remove * @param handler a handler that will be called after the async remote clear completes + * * @return the data loader for fluent coding */ public DataLoader clear(K key, BiConsumer handler) { Object cacheKey = getCacheKey(key); - try { - lock.lock(); - futureCache.delete(cacheKey); - valueCache.delete(key).whenComplete(handler); - } finally { - lock.unlock(); - } + futureCache.delete(cacheKey); + valueCache.delete(key).whenComplete(handler); return this; } @@ -399,16 +401,12 @@ public DataLoader clearAll() { * Clears the entire cache map of the loader, and of the cached value store. * * @param handler a handler that will be called after the async remote clear all completes + * * @return the data loader for fluent coding */ public DataLoader clearAll(BiConsumer handler) { - try { - lock.lock(); - futureCache.clear(); - valueCache.clear().whenComplete(handler); - } finally { - lock.unlock(); - } + futureCache.clear(); + valueCache.clear().whenComplete(handler); return this; } @@ -419,6 +417,7 @@ public DataLoader clearAll(BiConsumer handler) { * * @param key the key * @param value the value + * * @return the data loader for fluent coding */ public DataLoader prime(K key, V value) { @@ -430,6 +429,7 @@ public DataLoader prime(K key, V value) { * * @param key the key * @param error the exception to prime instead of a value + * * @return the data loader for fluent coding */ public DataLoader prime(K key, Exception error) { @@ -443,18 +443,12 @@ public DataLoader prime(K key, Exception error) { * * @param key the key * @param value the value + * * @return the data loader for fluent coding */ public DataLoader prime(K key, CompletableFuture value) { Object cacheKey = getCacheKey(key); - try { - lock.lock(); - if (!futureCache.containsKey(cacheKey)) { - futureCache.set(cacheKey, value); - } - } finally { - lock.unlock(); - } + futureCache.setIfAbsent(cacheKey, value); return this; } @@ -465,6 +459,7 @@ public DataLoader prime(K key, CompletableFuture value) { * If no cache key function is present in {@link DataLoaderOptions}, then the returned value equals the input key. * * @param key the input key + * * @return the cache key after the input is transformed with the cache key function */ public Object getCacheKey(K key) { @@ -503,8 +498,8 @@ public ValueCache getValueCache() { @Override public String toString() { return "DataLoader{" + - "name='" + name + '\'' + - ", stats=" + stats + - '}'; + "name='" + name + '\'' + + ", stats=" + stats + + '}'; } } diff --git a/src/main/java/org/dataloader/DataLoaderHelper.java b/src/main/java/org/dataloader/DataLoaderHelper.java index 83389e4f..eeb68081 100644 --- a/src/main/java/org/dataloader/DataLoaderHelper.java +++ b/src/main/java/org/dataloader/DataLoaderHelper.java @@ -13,12 +13,14 @@ import org.dataloader.stats.context.IncrementCacheHitCountStatisticsContext; import org.dataloader.stats.context.IncrementLoadCountStatisticsContext; import org.dataloader.stats.context.IncrementLoadErrorCountStatisticsContext; +import org.jspecify.annotations.Nullable; import org.reactivestreams.Subscriber; import java.time.Clock; import java.time.Instant; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; @@ -28,7 +30,6 @@ import java.util.concurrent.CompletionException; import java.util.concurrent.CompletionStage; import java.util.concurrent.atomic.AtomicReference; -import java.util.concurrent.locks.Lock; import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; @@ -52,20 +53,22 @@ class DataLoaderHelper { static class LoaderQueueEntry { final K key; - final V value; + final CompletableFuture value; final Object callContext; + final LoaderQueueEntry prev; - public LoaderQueueEntry(K key, V value, Object callContext) { + public LoaderQueueEntry(K key, CompletableFuture value, Object callContext, LoaderQueueEntry prev) { this.key = key; this.value = value; this.callContext = callContext; + this.prev = prev; } K getKey() { return key; } - V getValue() { + CompletableFuture getValue() { return value; } @@ -79,11 +82,11 @@ Object getCallContext() { private final DataLoaderOptions loaderOptions; private final CacheMap futureCache; private final ValueCache valueCache; - private final List>> loaderQueue; + // private final List>> loaderQueue; + private final AtomicReference<@Nullable LoaderQueueEntry> loaderQueue = new AtomicReference<>(); private final StatisticsCollector stats; private final Clock clock; private final AtomicReference lastDispatchTime; - private final Lock lock; DataLoaderHelper(DataLoader dataLoader, Object batchLoadFunction, @@ -97,8 +100,6 @@ Object getCallContext() { this.loaderOptions = loaderOptions; this.futureCache = futureCache; this.valueCache = valueCache; - this.lock = dataLoader.lock; - this.loaderQueue = new ArrayList<>(); this.stats = stats; this.clock = clock; this.lastDispatchTime = new AtomicReference<>(); @@ -114,39 +115,28 @@ public Instant getLastDispatchTime() { } Optional> getIfPresent(K key) { - try { - lock.lock(); - boolean cachingEnabled = loaderOptions.cachingEnabled(); - if (cachingEnabled) { - Object cacheKey = getCacheKey(nonNull(key)); - try { - CompletableFuture cacheValue = futureCache.get(cacheKey); - if (cacheValue != null) { - stats.incrementCacheHitCount(new IncrementCacheHitCountStatisticsContext<>(key)); - return Optional.of(cacheValue); - } - } catch (Exception ignored) { + boolean cachingEnabled = loaderOptions.cachingEnabled(); + if (cachingEnabled) { + Object cacheKey = getCacheKey(nonNull(key)); + try { + CompletableFuture cacheValue = futureCache.get(cacheKey); + if (cacheValue != null) { + stats.incrementCacheHitCount(new IncrementCacheHitCountStatisticsContext<>(key)); + return Optional.of(cacheValue); } + } catch (Exception ignored) { } - } finally { - lock.unlock(); } return Optional.empty(); } Optional> getIfCompleted(K key) { - try { - lock.lock(); - - Optional> cachedPromise = getIfPresent(key); - if (cachedPromise.isPresent()) { - CompletableFuture promise = cachedPromise.get(); - if (promise.isDone()) { - return cachedPromise; - } + Optional> cachedPromise = getIfPresent(key); + if (cachedPromise.isPresent()) { + CompletableFuture promise = cachedPromise.get(); + if (promise.isDone()) { + return cachedPromise; } - } finally { - lock.unlock(); } return Optional.empty(); } @@ -154,25 +144,70 @@ Optional> getIfCompleted(K key) { @GuardedBy("lock") CompletableFuture load(K key, Object loadContext) { - try { - lock.lock(); + boolean batchingEnabled = loaderOptions.batchingEnabled(); + boolean futureCachingEnabled = loaderOptions.cachingEnabled(); + + stats.incrementLoadCount(new IncrementLoadCountStatisticsContext<>(key, loadContext)); + DataLoaderInstrumentationContext ctx = ctxOrNoopCtx(instrumentation().beginLoad(dataLoader, key, loadContext)); + Object cacheKey = null; + if (futureCachingEnabled) { + cacheKey = loadContext == null ? getCacheKey(key) : getCacheKeyWithContext(key, loadContext); + try { + CompletableFuture cachedFuture = futureCache.get(cacheKey); + if (cachedFuture != null) { + // We already have a promise for this key, no need to check value cache or queue up load + stats.incrementCacheHitCount(new IncrementCacheHitCountStatisticsContext<>(key, loadContext)); + ctx.onDispatched(); + cachedFuture.whenComplete(ctx::onCompleted); + return cachedFuture; + } + } catch (Exception ignored) { + } + } + CompletableFuture loadCallFuture; + if (batchingEnabled) { + loadCallFuture = new CompletableFuture<>(); + if (futureCachingEnabled) { + CompletableFuture cachedFuture = futureCache.setIfAbsent(cacheKey, loadCallFuture); + if (cachedFuture != null) { + // another thread was faster and created a matching CF ... hence this is really a cachehit and we are done + stats.incrementCacheHitCount(new IncrementCacheHitCountStatisticsContext<>(key, loadContext)); + ctx.onDispatched(); + cachedFuture.whenComplete(ctx::onCompleted); + return cachedFuture; + } + } + addEntryToLoaderQueue(key, loadCallFuture, loadContext); + } else { + stats.incrementBatchLoadCountBy(1, new IncrementBatchLoadCountByStatisticsContext<>(key, loadContext)); + // immediate execution of batch function + loadCallFuture = invokeLoaderImmediately(key, loadContext, true); + if (futureCachingEnabled) { + CompletableFuture cachedFuture = futureCache.setIfAbsent(cacheKey, loadCallFuture); + if (cachedFuture != null) { + // another thread was faster and the loader was invoked twice with the same key + // we are disregarding the resul of our dispatch call and use the already cached value + // meaning this is a cache hit and we are done + stats.incrementCacheHitCount(new IncrementCacheHitCountStatisticsContext<>(key, loadContext)); + ctx.onDispatched(); + cachedFuture.whenComplete(ctx::onCompleted); + return cachedFuture; + } + } + } - boolean batchingEnabled = loaderOptions.batchingEnabled(); - boolean cachingEnabled = loaderOptions.cachingEnabled(); + ctx.onDispatched(); + loadCallFuture.whenComplete(ctx::onCompleted); + return loadCallFuture; + } - stats.incrementLoadCount(new IncrementLoadCountStatisticsContext<>(key, loadContext)); - DataLoaderInstrumentationContext ctx = ctxOrNoopCtx(instrumentation().beginLoad(dataLoader, key,loadContext)); - CompletableFuture cf; - if (cachingEnabled) { - cf = loadFromCache(key, loadContext, batchingEnabled); - } else { - cf = queueOrInvokeLoader(key, loadContext, batchingEnabled, false); + private void addEntryToLoaderQueue(K key, CompletableFuture future, Object loadContext) { + while (true) { + LoaderQueueEntry prev = loaderQueue.get(); + LoaderQueueEntry curr = new LoaderQueueEntry<>(key, future, loadContext, prev); + if (loaderQueue.compareAndSet(prev, curr)) { + return; } - ctx.onDispatched(); - cf.whenComplete(ctx::onCompleted); - return cf; - } finally { - lock.unlock(); } } @@ -196,35 +231,42 @@ DispatchResult dispatch() { final List keys; final List callContexts; final List> queuedFutures; - try { - lock.lock(); - int queueSize = loaderQueue.size(); - if (queueSize == 0) { - lastDispatchTime.set(now()); - instrCtx.onDispatched(); - return endDispatchCtx(instrCtx, emptyDispatchResult()); + LoaderQueueEntry loaderQueueEntryHead; + while (true) { + loaderQueueEntryHead = loaderQueue.get(); + if (loaderQueue.compareAndSet(loaderQueueEntryHead, null)) { + break; } - - // we copy the pre-loaded set of futures ready for dispatch - keys = new ArrayList<>(queueSize); - callContexts = new ArrayList<>(queueSize); - queuedFutures = new ArrayList<>(queueSize); - - loaderQueue.forEach(entry -> { - keys.add(entry.getKey()); - queuedFutures.add(entry.getValue()); - callContexts.add(entry.getCallContext()); - }); - loaderQueue.clear(); + } + if (loaderQueueEntryHead == null) { lastDispatchTime.set(now()); - } finally { - lock.unlock(); + instrCtx.onDispatched(); + return endDispatchCtx(instrCtx, emptyDispatchResult()); } + int queueSize = calcQueueDepth(loaderQueueEntryHead); + // we copy the pre-loaded set of futures ready for dispatch + keys = new ArrayList<>(queueSize); + callContexts = new ArrayList<>(queueSize); + queuedFutures = new ArrayList<>(queueSize); + + while (loaderQueueEntryHead != null) { + keys.add(loaderQueueEntryHead.getKey()); + queuedFutures.add(loaderQueueEntryHead.getValue()); + callContexts.add(loaderQueueEntryHead.getCallContext()); + loaderQueueEntryHead = loaderQueueEntryHead.prev; + } + //TODO: to many test depend on the previous order, therefore we reverse the lists here + // but this should not matter and we should change the tests + Collections.reverse(keys); + Collections.reverse(callContexts); + Collections.reverse(queuedFutures); + lastDispatchTime.set(now()); if (!batchingEnabled) { instrCtx.onDispatched(); return endDispatchCtx(instrCtx, emptyDispatchResult()); } + final int totalEntriesHandled = keys.size(); // // order of keys -> values matter in data loader hence the use of linked hash map @@ -354,38 +396,6 @@ private void possiblyClearCacheEntriesOnExceptions(List keys) { } } - @GuardedBy("lock") - private CompletableFuture loadFromCache(K key, Object loadContext, boolean batchingEnabled) { - final Object cacheKey = loadContext == null ? getCacheKey(key) : getCacheKeyWithContext(key, loadContext); - - try { - CompletableFuture cacheValue = futureCache.get(cacheKey); - if (cacheValue != null) { - // We already have a promise for this key, no need to check value cache or queue up load - stats.incrementCacheHitCount(new IncrementCacheHitCountStatisticsContext<>(key, loadContext)); - return cacheValue; - } - } catch (Exception ignored) { - } - - CompletableFuture loadCallFuture = queueOrInvokeLoader(key, loadContext, batchingEnabled, true); - futureCache.set(cacheKey, loadCallFuture); - return loadCallFuture; - } - - @GuardedBy("lock") - private CompletableFuture queueOrInvokeLoader(K key, Object loadContext, boolean batchingEnabled, boolean cachingEnabled) { - if (batchingEnabled) { - CompletableFuture loadCallFuture = new CompletableFuture<>(); - loaderQueue.add(new LoaderQueueEntry<>(key, loadCallFuture, loadContext)); - return loadCallFuture; - } else { - stats.incrementBatchLoadCountBy(1, new IncrementBatchLoadCountByStatisticsContext<>(key, loadContext)); - // immediate execution of batch function - return invokeLoaderImmediately(key, loadContext, cachingEnabled); - } - } - CompletableFuture invokeLoaderImmediately(K key, Object keyContext, boolean cachingEnabled) { List keys = singletonList(key); List keyContexts = singletonList(keyContext); @@ -626,14 +636,19 @@ private DataLoaderInstrumentation instrumentation() { } int dispatchDepth() { - try { - lock.lock(); - return loaderQueue.size(); - } finally { - lock.unlock(); + return calcQueueDepth(loaderQueue.get()); + } + + private int calcQueueDepth(LoaderQueueEntry head) { + int count = 0; + while (head != null) { + count++; + head = head.prev; } + return count; } + private final List> NOT_SUPPORTED_LIST = emptyList(); private final CompletableFuture>> NOT_SUPPORTED = CompletableFuture.completedFuture(NOT_SUPPORTED_LIST); private final Try ALWAYS_FAILED = Try.alwaysFailed(); diff --git a/src/main/java/org/dataloader/impl/DefaultCacheMap.java b/src/main/java/org/dataloader/impl/DefaultCacheMap.java index fa89bb06..c8e1ae23 100644 --- a/src/main/java/org/dataloader/impl/DefaultCacheMap.java +++ b/src/main/java/org/dataloader/impl/DefaultCacheMap.java @@ -20,9 +20,8 @@ import org.dataloader.annotations.Internal; import java.util.Collection; -import java.util.HashMap; -import java.util.Map; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; /** * Default implementation of {@link CacheMap} that is based on a regular {@link java.util.HashMap}. @@ -35,13 +34,13 @@ @Internal public class DefaultCacheMap implements CacheMap { - private final Map> cache; + private final ConcurrentHashMap> cache; /** * Default constructor */ public DefaultCacheMap() { - cache = new HashMap<>(); + cache = new ConcurrentHashMap<>(); } /** @@ -73,9 +72,8 @@ public Collection> getAll() { * {@inheritDoc} */ @Override - public CacheMap set(K key, CompletableFuture value) { - cache.put(key, value); - return this; + public CompletableFuture setIfAbsent(K key, CompletableFuture value) { + return cache.putIfAbsent(key, value); } /** diff --git a/src/test/java/ReadmeExamples.java b/src/test/java/ReadmeExamples.java index f391b80d..8cf5c70c 100644 --- a/src/test/java/ReadmeExamples.java +++ b/src/test/java/ReadmeExamples.java @@ -265,7 +265,7 @@ public Collection> getAll() { } @Override - public CacheMap set(Object key, CompletableFuture value) { + public CompletableFuture setIfAbsent(Object key, CompletableFuture value) { return null; } diff --git a/src/test/java/org/dataloader/DataLoaderTest.java b/src/test/java/org/dataloader/DataLoaderTest.java index 37ae0307..6ec548aa 100644 --- a/src/test/java/org/dataloader/DataLoaderTest.java +++ b/src/test/java/org/dataloader/DataLoaderTest.java @@ -61,6 +61,7 @@ import static org.dataloader.DataLoaderOptions.newDefaultOptions; import static org.dataloader.DataLoaderOptions.newOptions; import static org.dataloader.fixtures.TestKit.areAllDone; +import static org.dataloader.fixtures.TestKit.asSet; import static org.dataloader.fixtures.TestKit.listFrom; import static org.dataloader.impl.CompletableFutureKit.cause; import static org.hamcrest.MatcherAssert.assertThat; @@ -70,6 +71,7 @@ import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; /** @@ -980,7 +982,7 @@ public void should_Accept_a_custom_cache_map_implementation(TestDataLoaderFactor assertThat(future2b.get(), equalTo("b")); assertThat(loadCalls, equalTo(asList(asList("a", "b"), singletonList("c"), singletonList("b")))); - assertArrayEquals(customMap.stash.keySet().toArray(), asList("a", "c", "b").toArray()); + assertEquals(customMap.stash.keySet(), asSet("a", "c", "b")); // Supports clear all diff --git a/src/test/java/org/dataloader/fixtures/CustomCacheMap.java b/src/test/java/org/dataloader/fixtures/CustomCacheMap.java index 695da5ea..0a32ba24 100644 --- a/src/test/java/org/dataloader/fixtures/CustomCacheMap.java +++ b/src/test/java/org/dataloader/fixtures/CustomCacheMap.java @@ -3,16 +3,15 @@ import org.dataloader.CacheMap; import java.util.Collection; -import java.util.LinkedHashMap; -import java.util.Map; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; public class CustomCacheMap implements CacheMap { - public Map> stash; + public ConcurrentHashMap> stash; public CustomCacheMap() { - stash = new LinkedHashMap<>(); + stash = new ConcurrentHashMap<>(); } @Override @@ -31,9 +30,8 @@ public Collection> getAll() { } @Override - public CacheMap set(String key, CompletableFuture value) { - stash.put(key, value); - return this; + public CompletableFuture setIfAbsent(String key, CompletableFuture value) { + return stash.putIfAbsent(key, value); } @Override From 65a496ec4738b261ab72faa31e88823a4f8db3e7 Mon Sep 17 00:00:00 2001 From: Andreas Marek Date: Sat, 18 Oct 2025 20:20:44 +1000 Subject: [PATCH 139/156] add jcstress tests --- .github/workflows/master.yml | 2 +- .github/workflows/pull_request.yml | 2 +- build.gradle | 7 +- .../DataLoaderDispatchJCStress.java | 100 ++++++++++++++++++ 4 files changed, 107 insertions(+), 4 deletions(-) create mode 100644 src/jcstress/java/org/dataloader/DataLoaderDispatchJCStress.java diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index e6aa3a2e..d0762966 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -28,6 +28,6 @@ jobs: - name: Setup Gradle uses: gradle/actions/setup-gradle@v5 - name: build test and publish - run: ./gradlew assemble && ./gradlew check --info && ./gradlew publishToSonatype closeAndReleaseSonatypeStagingRepository -x check --info --stacktrace + run: ./gradlew assemble && ./gradlew check --info && && ./gradlew jcstress && ./gradlew publishToSonatype closeAndReleaseSonatypeStagingRepository -x check --info --stacktrace env: CI: true diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 5258ad28..41a1e883 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -26,6 +26,6 @@ jobs: - name: Setup Gradle uses: gradle/actions/setup-gradle@v5 - name: build and test - run: ./gradlew assemble && ./gradlew check --info --stacktrace + run: ./gradlew assemble && ./gradlew check --info --stacktrace && ./gradlew jcstress env: CI: true diff --git a/build.gradle b/build.gradle index 7b5de5c4..f2a0637f 100644 --- a/build.gradle +++ b/build.gradle @@ -1,6 +1,7 @@ +import net.ltgt.gradle.errorprone.CheckSeverity import org.jetbrains.kotlin.gradle.dsl.JvmTarget import org.jetbrains.kotlin.gradle.dsl.KotlinVersion -import net.ltgt.gradle.errorprone.CheckSeverity + import java.text.SimpleDateFormat plugins { @@ -15,6 +16,7 @@ plugins { id 'com.github.ben-manes.versions' version '0.53.0' id "me.champeau.jmh" version "0.7.3" id "net.ltgt.errorprone" version '4.3.0' + id "io.github.reyerizo.gradle.jcstress" version "0.8.15" // Kotlin just for tests - not production code id 'org.jetbrains.kotlin.jvm' version '2.2.20' @@ -229,7 +231,8 @@ nexusPublishing { // https://central.sonatype.org/publish/publish-portal-ossrh-staging-api/#configuration nexusUrl.set(uri("https://ossrh-staging-api.central.sonatype.com/service/local/")) // GraphQL Java does not publish snapshots, but adding this URL for completeness - snapshotRepositoryUrl.set(uri("https://central.sonatype.com/repository/maven-snapshots/")) } + snapshotRepositoryUrl.set(uri("https://central.sonatype.com/repository/maven-snapshots/")) + } } } diff --git a/src/jcstress/java/org/dataloader/DataLoaderDispatchJCStress.java b/src/jcstress/java/org/dataloader/DataLoaderDispatchJCStress.java new file mode 100644 index 00000000..0226c0ae --- /dev/null +++ b/src/jcstress/java/org/dataloader/DataLoaderDispatchJCStress.java @@ -0,0 +1,100 @@ +package org.dataloader; + +import org.openjdk.jcstress.annotations.Actor; +import org.openjdk.jcstress.annotations.Arbiter; +import org.openjdk.jcstress.annotations.JCStressTest; +import org.openjdk.jcstress.annotations.Outcome; +import org.openjdk.jcstress.annotations.State; +import org.openjdk.jcstress.infra.results.II_Result; + +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.openjdk.jcstress.annotations.Expect.ACCEPTABLE; + +@JCStressTest +@State +@Outcome(id = "2000, 2000", expect = ACCEPTABLE, desc = "accepted") +public class DataLoaderDispatchJCStress { + + + AtomicInteger counter = new AtomicInteger(); + AtomicInteger batchLoaderCount = new AtomicInteger(); + volatile boolean finished1; + volatile boolean finished2; + + + BatchLoader batchLoader = keys -> { + return CompletableFuture.supplyAsync(() -> { + batchLoaderCount.getAndAdd(keys.size()); + return keys; + }); + }; + DataLoader dataLoader = DataLoaderFactory.newDataLoader(batchLoader); + + public DataLoaderDispatchJCStress() { + + } + + @Actor + public void load1() { + for (int i = 0; i < 1000; i++) { + dataLoader.load("load-1-" + i); + } + finished1 = true; + } + + @Actor + public void load2() { + for (int i = 0; i < 1000; i++) { + dataLoader.load("load-2-" + i); + } + finished2 = true; + } + + + @Actor + public void dispatch1() { + while (!finished1 || !finished2) { + try { + List dispatchedResult = dataLoader.dispatch().get(); + counter.getAndAdd(dispatchedResult.size()); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + try { + List dispatchedResult = dataLoader.dispatch().get(); + counter.getAndAdd(dispatchedResult.size()); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Actor + public void dispatch2() { + while (!finished1 || !finished2) { + try { + List dispatchedResult = dataLoader.dispatch().get(); + counter.getAndAdd(dispatchedResult.size()); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + try { + List dispatchedResult = dataLoader.dispatch().get(); + counter.getAndAdd(dispatchedResult.size()); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Arbiter + public void arbiter(II_Result r) { + r.r1 = counter.get(); + r.r2 = batchLoaderCount.get(); + } + + +} From ee8d5685fe47b69a865957205590cb8b56ca137e Mon Sep 17 00:00:00 2001 From: Andreas Marek Date: Sun, 19 Oct 2025 06:42:51 +1000 Subject: [PATCH 140/156] add jcstress tests --- build.gradle | 4 ++++ ...ataLoaderBatchingAndCachingDispatchJCStress.java} | 12 ++++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) rename src/jcstress/java/org/dataloader/{DataLoaderDispatchJCStress.java => DataLoaderBatchingAndCachingDispatchJCStress.java} (87%) diff --git a/build.gradle b/build.gradle index f2a0637f..e49a001e 100644 --- a/build.gradle +++ b/build.gradle @@ -261,3 +261,7 @@ tasks.named("dependencyUpdates").configure { isNonStable(it.candidate.version) } } + +jcstress { + verbose = true +} \ No newline at end of file diff --git a/src/jcstress/java/org/dataloader/DataLoaderDispatchJCStress.java b/src/jcstress/java/org/dataloader/DataLoaderBatchingAndCachingDispatchJCStress.java similarity index 87% rename from src/jcstress/java/org/dataloader/DataLoaderDispatchJCStress.java rename to src/jcstress/java/org/dataloader/DataLoaderBatchingAndCachingDispatchJCStress.java index 0226c0ae..c2df70f8 100644 --- a/src/jcstress/java/org/dataloader/DataLoaderDispatchJCStress.java +++ b/src/jcstress/java/org/dataloader/DataLoaderBatchingAndCachingDispatchJCStress.java @@ -16,7 +16,7 @@ @JCStressTest @State @Outcome(id = "2000, 2000", expect = ACCEPTABLE, desc = "accepted") -public class DataLoaderDispatchJCStress { +public class DataLoaderBatchingAndCachingDispatchJCStress { AtomicInteger counter = new AtomicInteger(); @@ -33,12 +33,16 @@ public class DataLoaderDispatchJCStress { }; DataLoader dataLoader = DataLoaderFactory.newDataLoader(batchLoader); - public DataLoaderDispatchJCStress() { + public DataLoaderBatchingAndCachingDispatchJCStress() { } @Actor public void load1() { + for (int i = 0; i < 1000; i++) { + dataLoader.load("load-1-" + i); + } + // we load the same keys again for (int i = 0; i < 1000; i++) { dataLoader.load("load-1-" + i); } @@ -50,6 +54,10 @@ public void load2() { for (int i = 0; i < 1000; i++) { dataLoader.load("load-2-" + i); } + // we load the same keys again + for (int i = 0; i < 1000; i++) { + dataLoader.load("load-1-" + i); + } finished2 = true; } From bcf7ef6e4e486e2261b2026f7a9dc9e32a24e949 Mon Sep 17 00:00:00 2001 From: Andreas Marek Date: Sun, 19 Oct 2025 07:19:11 +1000 Subject: [PATCH 141/156] add another jcstress test --- build.gradle | 2 +- ...DataLoader_Batching_Caching_JCStress.java} | 4 +- ...ataLoader_NoBatching_Caching_JCStress.java | 66 +++++++++++++++++++ src/main/java/org/dataloader/CacheMap.java | 12 +++- .../org/dataloader/impl/DefaultCacheMap.java | 5 ++ src/test/java/ReadmeExamples.java | 5 ++ .../dataloader/fixtures/CustomCacheMap.java | 5 ++ 7 files changed, 95 insertions(+), 4 deletions(-) rename src/jcstress/java/org/dataloader/{DataLoaderBatchingAndCachingDispatchJCStress.java => DataLoader_Batching_Caching_JCStress.java} (96%) create mode 100644 src/jcstress/java/org/dataloader/DataLoader_NoBatching_Caching_JCStress.java diff --git a/build.gradle b/build.gradle index e49a001e..2384afda 100644 --- a/build.gradle +++ b/build.gradle @@ -263,5 +263,5 @@ tasks.named("dependencyUpdates").configure { } jcstress { - verbose = true +// verbose = true } \ No newline at end of file diff --git a/src/jcstress/java/org/dataloader/DataLoaderBatchingAndCachingDispatchJCStress.java b/src/jcstress/java/org/dataloader/DataLoader_Batching_Caching_JCStress.java similarity index 96% rename from src/jcstress/java/org/dataloader/DataLoaderBatchingAndCachingDispatchJCStress.java rename to src/jcstress/java/org/dataloader/DataLoader_Batching_Caching_JCStress.java index c2df70f8..58f89c60 100644 --- a/src/jcstress/java/org/dataloader/DataLoaderBatchingAndCachingDispatchJCStress.java +++ b/src/jcstress/java/org/dataloader/DataLoader_Batching_Caching_JCStress.java @@ -16,7 +16,7 @@ @JCStressTest @State @Outcome(id = "2000, 2000", expect = ACCEPTABLE, desc = "accepted") -public class DataLoaderBatchingAndCachingDispatchJCStress { +public class DataLoader_Batching_Caching_JCStress { AtomicInteger counter = new AtomicInteger(); @@ -33,7 +33,7 @@ public class DataLoaderBatchingAndCachingDispatchJCStress { }; DataLoader dataLoader = DataLoaderFactory.newDataLoader(batchLoader); - public DataLoaderBatchingAndCachingDispatchJCStress() { + public DataLoader_Batching_Caching_JCStress() { } diff --git a/src/jcstress/java/org/dataloader/DataLoader_NoBatching_Caching_JCStress.java b/src/jcstress/java/org/dataloader/DataLoader_NoBatching_Caching_JCStress.java new file mode 100644 index 00000000..c7f9801f --- /dev/null +++ b/src/jcstress/java/org/dataloader/DataLoader_NoBatching_Caching_JCStress.java @@ -0,0 +1,66 @@ +package org.dataloader; + +import org.openjdk.jcstress.annotations.Actor; +import org.openjdk.jcstress.annotations.Arbiter; +import org.openjdk.jcstress.annotations.JCStressTest; +import org.openjdk.jcstress.annotations.Outcome; +import org.openjdk.jcstress.annotations.State; +import org.openjdk.jcstress.infra.results.II_Result; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.openjdk.jcstress.annotations.Expect.ACCEPTABLE; +import static org.openjdk.jcstress.annotations.Expect.ACCEPTABLE_INTERESTING; + +@JCStressTest +@State +@Outcome(id = "2000, 2000", expect = ACCEPTABLE, desc = "No keys loaded twice") +@Outcome(id = "2.*, 2000", expect = ACCEPTABLE_INTERESTING, desc = "Some keys loaded twice") +public class DataLoader_NoBatching_Caching_JCStress { + + + AtomicInteger batchLoaderCount = new AtomicInteger(); + volatile boolean finished1; + volatile boolean finished2; + + + BatchLoader batchLoader = keys -> { + batchLoaderCount.getAndAdd(keys.size()); + return CompletableFuture.completedFuture(keys); + }; + DataLoader dataLoader = DataLoaderFactory.newDataLoader(batchLoader, DataLoaderOptions.newOptions().setBatchingEnabled(false).build()); + + + @Actor + public void load1() { + for (int i = 0; i < 1000; i++) { + dataLoader.load("load-1-" + i); + } + // we load the same keys again + for (int i = 0; i < 1000; i++) { + dataLoader.load("load-1-" + i); + } + finished1 = true; + } + + @Actor + public void load2() { + for (int i = 0; i < 1000; i++) { + dataLoader.load("load-2-" + i); + } + // we load the same keys again + for (int i = 0; i < 1000; i++) { + dataLoader.load("load-1-" + i); + } + finished2 = true; + } + + + @Arbiter + public void arbiter(II_Result r) { + r.r1 = batchLoaderCount.get(); + r.r2 = dataLoader.getCacheMap().size(); + } + +} diff --git a/src/main/java/org/dataloader/CacheMap.java b/src/main/java/org/dataloader/CacheMap.java index 36913e64..dc60d97f 100644 --- a/src/main/java/org/dataloader/CacheMap.java +++ b/src/main/java/org/dataloader/CacheMap.java @@ -74,10 +74,11 @@ static CacheMap simpleMap() { * * @return the cached value, or {@code null} if not found (depends on cache implementation) */ - @Nullable CompletableFuture get(K key); + @Nullable CompletableFuture get(K key); /** * Gets a collection of CompletableFutures from the cache map. + * * @return the collection of cached values */ Collection> getAll(); @@ -107,4 +108,13 @@ static CacheMap simpleMap() { * @return the cache map for fluent coding */ CacheMap clear(); + + /** + * Returns the current size of the cache. This is not used by DataLoader directly + * and intended for testing and debugging. + * If a cache doesn't support it, it can throw an Exception. + * + * @return + */ + int size(); } diff --git a/src/main/java/org/dataloader/impl/DefaultCacheMap.java b/src/main/java/org/dataloader/impl/DefaultCacheMap.java index c8e1ae23..10c1a5ff 100644 --- a/src/main/java/org/dataloader/impl/DefaultCacheMap.java +++ b/src/main/java/org/dataloader/impl/DefaultCacheMap.java @@ -93,4 +93,9 @@ public CacheMap clear() { cache.clear(); return this; } + + @Override + public int size() { + return cache.size(); + } } diff --git a/src/test/java/ReadmeExamples.java b/src/test/java/ReadmeExamples.java index 8cf5c70c..05d1670f 100644 --- a/src/test/java/ReadmeExamples.java +++ b/src/test/java/ReadmeExamples.java @@ -278,6 +278,11 @@ public CacheMap delete(Object key) { public CacheMap clear() { return null; } + + @Override + public int size() { + return 0; + } } private void customCache() { diff --git a/src/test/java/org/dataloader/fixtures/CustomCacheMap.java b/src/test/java/org/dataloader/fixtures/CustomCacheMap.java index 0a32ba24..c22ce1a4 100644 --- a/src/test/java/org/dataloader/fixtures/CustomCacheMap.java +++ b/src/test/java/org/dataloader/fixtures/CustomCacheMap.java @@ -45,4 +45,9 @@ public CacheMap clear() { stash.clear(); return this; } + + @Override + public int size() { + return stash.size(); + } } From 5ba70ddaea86995de199e159435e1a291bf2e722 Mon Sep 17 00:00:00 2001 From: Andreas Marek Date: Sun, 19 Oct 2025 07:25:14 +1000 Subject: [PATCH 142/156] jcstress tests --- .../DataLoader_Batching_Caching_JCStress.java | 2 +- ...ataLoader_NoBatching_Caching_JCStress.java | 20 ++++--------------- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/src/jcstress/java/org/dataloader/DataLoader_Batching_Caching_JCStress.java b/src/jcstress/java/org/dataloader/DataLoader_Batching_Caching_JCStress.java index 58f89c60..f0d8263f 100644 --- a/src/jcstress/java/org/dataloader/DataLoader_Batching_Caching_JCStress.java +++ b/src/jcstress/java/org/dataloader/DataLoader_Batching_Caching_JCStress.java @@ -56,7 +56,7 @@ public void load2() { } // we load the same keys again for (int i = 0; i < 1000; i++) { - dataLoader.load("load-1-" + i); + dataLoader.load("load-2-" + i); } finished2 = true; } diff --git a/src/jcstress/java/org/dataloader/DataLoader_NoBatching_Caching_JCStress.java b/src/jcstress/java/org/dataloader/DataLoader_NoBatching_Caching_JCStress.java index c7f9801f..6b46ed1f 100644 --- a/src/jcstress/java/org/dataloader/DataLoader_NoBatching_Caching_JCStress.java +++ b/src/jcstress/java/org/dataloader/DataLoader_NoBatching_Caching_JCStress.java @@ -15,45 +15,33 @@ @JCStressTest @State -@Outcome(id = "2000, 2000", expect = ACCEPTABLE, desc = "No keys loaded twice") -@Outcome(id = "2.*, 2000", expect = ACCEPTABLE_INTERESTING, desc = "Some keys loaded twice") +@Outcome(id = "1000, 1000", expect = ACCEPTABLE, desc = "No keys loaded twice") +@Outcome(id = "1.*, 1000", expect = ACCEPTABLE_INTERESTING, desc = "Some keys loaded twice") public class DataLoader_NoBatching_Caching_JCStress { AtomicInteger batchLoaderCount = new AtomicInteger(); - volatile boolean finished1; - volatile boolean finished2; - BatchLoader batchLoader = keys -> { batchLoaderCount.getAndAdd(keys.size()); return CompletableFuture.completedFuture(keys); }; - DataLoader dataLoader = DataLoaderFactory.newDataLoader(batchLoader, DataLoaderOptions.newOptions().setBatchingEnabled(false).build()); + DataLoader dataLoader = DataLoaderFactory.newDataLoader(batchLoader, DataLoaderOptions.newOptions().setBatchingEnabled(false).build()); + @Actor public void load1() { for (int i = 0; i < 1000; i++) { dataLoader.load("load-1-" + i); } - // we load the same keys again - for (int i = 0; i < 1000; i++) { - dataLoader.load("load-1-" + i); - } - finished1 = true; } @Actor public void load2() { - for (int i = 0; i < 1000; i++) { - dataLoader.load("load-2-" + i); - } - // we load the same keys again for (int i = 0; i < 1000; i++) { dataLoader.load("load-1-" + i); } - finished2 = true; } From 002f872c7773b52397ca06788ed608df82a87614 Mon Sep 17 00:00:00 2001 From: Andreas Marek Date: Tue, 21 Oct 2025 12:49:13 +1000 Subject: [PATCH 143/156] dispatch performance test --- .../performance/DataLoaderDispatchPerformance.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/jmh/java/performance/DataLoaderDispatchPerformance.java b/src/jmh/java/performance/DataLoaderDispatchPerformance.java index 0b4696d5..ad2060c8 100644 --- a/src/jmh/java/performance/DataLoaderDispatchPerformance.java +++ b/src/jmh/java/performance/DataLoaderDispatchPerformance.java @@ -12,6 +12,7 @@ import org.openjdk.jmh.annotations.Scope; import org.openjdk.jmh.annotations.Setup; import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Threads; import org.openjdk.jmh.annotations.Warmup; import org.openjdk.jmh.infra.Blackhole; @@ -280,15 +281,20 @@ public void setup() { } + DataLoader ownerDL = DataLoaderFactory.newDataLoader(ownerBatchLoader); + DataLoader petDL = DataLoaderFactory.newDataLoader(petBatchLoader); + + } @Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.NANOSECONDS) + @Threads(Threads.MAX) public void loadAndDispatch(MyState myState, Blackhole blackhole) { - DataLoader ownerDL = DataLoaderFactory.newDataLoader(ownerBatchLoader); - DataLoader petDL = DataLoaderFactory.newDataLoader(petBatchLoader); + DataLoader ownerDL = myState.ownerDL; + DataLoader petDL = myState.petDL; for (Owner owner : owners.values()) { ownerDL.load(owner.id); From 16cf34051e4d7081345759f0a698a7978e5a5771 Mon Sep 17 00:00:00 2001 From: Andreas Marek Date: Wed, 22 Oct 2025 08:42:48 +1000 Subject: [PATCH 144/156] keep order of keys without reversing --- .../java/org/dataloader/DataLoaderHelper.java | 29 +++++++++---------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/src/main/java/org/dataloader/DataLoaderHelper.java b/src/main/java/org/dataloader/DataLoaderHelper.java index eeb68081..4c4ced3c 100644 --- a/src/main/java/org/dataloader/DataLoaderHelper.java +++ b/src/main/java/org/dataloader/DataLoaderHelper.java @@ -19,8 +19,8 @@ import java.time.Clock; import java.time.Instant; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; @@ -228,9 +228,6 @@ DispatchResult dispatch() { DataLoaderInstrumentationContext> instrCtx = ctxOrNoopCtx(instrumentation().beginDispatch(dataLoader)); boolean batchingEnabled = loaderOptions.batchingEnabled(); - final List keys; - final List callContexts; - final List> queuedFutures; LoaderQueueEntry loaderQueueEntryHead; while (true) { @@ -246,21 +243,21 @@ DispatchResult dispatch() { } int queueSize = calcQueueDepth(loaderQueueEntryHead); // we copy the pre-loaded set of futures ready for dispatch - keys = new ArrayList<>(queueSize); - callContexts = new ArrayList<>(queueSize); - queuedFutures = new ArrayList<>(queueSize); - + Object[] keysArray = new Object[queueSize]; + CompletableFuture[] queuedFuturesArray = new CompletableFuture[queueSize]; + Object[] callContextsArray = new Object[queueSize]; + int index = queueSize - 1; while (loaderQueueEntryHead != null) { - keys.add(loaderQueueEntryHead.getKey()); - queuedFutures.add(loaderQueueEntryHead.getValue()); - callContexts.add(loaderQueueEntryHead.getCallContext()); + keysArray[index] = loaderQueueEntryHead.getKey(); + queuedFuturesArray[index] = loaderQueueEntryHead.getValue(); + callContextsArray[index] = loaderQueueEntryHead.getCallContext(); loaderQueueEntryHead = loaderQueueEntryHead.prev; + index--; } - //TODO: to many test depend on the previous order, therefore we reverse the lists here - // but this should not matter and we should change the tests - Collections.reverse(keys); - Collections.reverse(callContexts); - Collections.reverse(queuedFutures); + final List keys = (List) Arrays.asList(keysArray); + final List> queuedFutures = Arrays.asList(queuedFuturesArray); + final List callContexts = Arrays.asList(callContextsArray); + lastDispatchTime.set(now()); if (!batchingEnabled) { instrCtx.onDispatched(); From 9a9105ce322f23c0939d67a938f13b80e37e2e56 Mon Sep 17 00:00:00 2001 From: Andreas Marek Date: Wed, 22 Oct 2025 12:55:39 +1000 Subject: [PATCH 145/156] don't calculate queue lenghth --- .../java/org/dataloader/DataLoaderHelper.java | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/dataloader/DataLoaderHelper.java b/src/main/java/org/dataloader/DataLoaderHelper.java index 4c4ced3c..a0807b27 100644 --- a/src/main/java/org/dataloader/DataLoaderHelper.java +++ b/src/main/java/org/dataloader/DataLoaderHelper.java @@ -56,12 +56,14 @@ static class LoaderQueueEntry { final CompletableFuture value; final Object callContext; final LoaderQueueEntry prev; + final int queueSize; - public LoaderQueueEntry(K key, CompletableFuture value, Object callContext, LoaderQueueEntry prev) { + public LoaderQueueEntry(K key, CompletableFuture value, Object callContext, LoaderQueueEntry prev, int queueSize) { this.key = key; this.value = value; this.callContext = callContext; this.prev = prev; + this.queueSize = queueSize; } K getKey() { @@ -204,7 +206,7 @@ CompletableFuture load(K key, Object loadContext) { private void addEntryToLoaderQueue(K key, CompletableFuture future, Object loadContext) { while (true) { LoaderQueueEntry prev = loaderQueue.get(); - LoaderQueueEntry curr = new LoaderQueueEntry<>(key, future, loadContext, prev); + LoaderQueueEntry curr = new LoaderQueueEntry<>(key, future, loadContext, prev, prev != null ? prev.queueSize + 1 : 1); if (loaderQueue.compareAndSet(prev, curr)) { return; } @@ -241,7 +243,7 @@ DispatchResult dispatch() { instrCtx.onDispatched(); return endDispatchCtx(instrCtx, emptyDispatchResult()); } - int queueSize = calcQueueDepth(loaderQueueEntryHead); + int queueSize = loaderQueueEntryHead.queueSize; // we copy the pre-loaded set of futures ready for dispatch Object[] keysArray = new Object[queueSize]; CompletableFuture[] queuedFuturesArray = new CompletableFuture[queueSize]; @@ -633,16 +635,12 @@ private DataLoaderInstrumentation instrumentation() { } int dispatchDepth() { - return calcQueueDepth(loaderQueue.get()); - } - - private int calcQueueDepth(LoaderQueueEntry head) { - int count = 0; - while (head != null) { - count++; - head = head.prev; + LoaderQueueEntry loaderQueueEntry = loaderQueue.get(); + if (loaderQueueEntry != null) { + return loaderQueueEntry.queueSize; + } else { + return 0; } - return count; } From c026da61dca774293272040d8b7a6a9e02dce936 Mon Sep 17 00:00:00 2001 From: Andreas Marek Date: Wed, 22 Oct 2025 12:56:25 +1000 Subject: [PATCH 146/156] remove guarded by --- src/main/java/org/dataloader/DataLoaderHelper.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/main/java/org/dataloader/DataLoaderHelper.java b/src/main/java/org/dataloader/DataLoaderHelper.java index a0807b27..51ac634f 100644 --- a/src/main/java/org/dataloader/DataLoaderHelper.java +++ b/src/main/java/org/dataloader/DataLoaderHelper.java @@ -1,6 +1,5 @@ package org.dataloader; -import org.dataloader.annotations.GuardedBy; import org.dataloader.annotations.Internal; import org.dataloader.impl.CompletableFutureKit; import org.dataloader.instrumentation.DataLoaderInstrumentation; @@ -144,7 +143,6 @@ Optional> getIfCompleted(K key) { } - @GuardedBy("lock") CompletableFuture load(K key, Object loadContext) { boolean batchingEnabled = loaderOptions.batchingEnabled(); boolean futureCachingEnabled = loaderOptions.cachingEnabled(); @@ -225,7 +223,6 @@ Object getCacheKeyWithContext(K key, Object context) { loaderOptions.cacheKeyFunction().get().getKeyWithContext(key, context) : key; } - @GuardedBy("lock") DispatchResult dispatch() { DataLoaderInstrumentationContext> instrCtx = ctxOrNoopCtx(instrumentation().beginDispatch(dataLoader)); From 54197a08cabc80e9f0a95e70c15b5963c56d8a1b Mon Sep 17 00:00:00 2001 From: Andreas Marek Date: Thu, 23 Oct 2025 10:03:18 +1000 Subject: [PATCH 147/156] name changing to putIfAbsentAtomically --- src/main/java/org/dataloader/CacheMap.java | 2 +- src/main/java/org/dataloader/DataLoader.java | 2 +- src/main/java/org/dataloader/DataLoaderHelper.java | 4 ++-- src/main/java/org/dataloader/impl/DefaultCacheMap.java | 2 +- src/test/java/ReadmeExamples.java | 2 +- src/test/java/org/dataloader/fixtures/CustomCacheMap.java | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/dataloader/CacheMap.java b/src/main/java/org/dataloader/CacheMap.java index dc60d97f..b3929af2 100644 --- a/src/main/java/org/dataloader/CacheMap.java +++ b/src/main/java/org/dataloader/CacheMap.java @@ -91,7 +91,7 @@ static CacheMap simpleMap() { * * @return the cache map for fluent coding */ - CompletableFuture setIfAbsent(K key, CompletableFuture value); + CompletableFuture putIfAbsentAtomically(K key, CompletableFuture value); /** * Deletes the entry with the specified key from the cache map, if it exists. diff --git a/src/main/java/org/dataloader/DataLoader.java b/src/main/java/org/dataloader/DataLoader.java index 4077e552..d80823f9 100644 --- a/src/main/java/org/dataloader/DataLoader.java +++ b/src/main/java/org/dataloader/DataLoader.java @@ -448,7 +448,7 @@ public DataLoader prime(K key, Exception error) { */ public DataLoader prime(K key, CompletableFuture value) { Object cacheKey = getCacheKey(key); - futureCache.setIfAbsent(cacheKey, value); + futureCache.putIfAbsentAtomically(cacheKey, value); return this; } diff --git a/src/main/java/org/dataloader/DataLoaderHelper.java b/src/main/java/org/dataloader/DataLoaderHelper.java index 51ac634f..249c1f25 100644 --- a/src/main/java/org/dataloader/DataLoaderHelper.java +++ b/src/main/java/org/dataloader/DataLoaderHelper.java @@ -168,7 +168,7 @@ CompletableFuture load(K key, Object loadContext) { if (batchingEnabled) { loadCallFuture = new CompletableFuture<>(); if (futureCachingEnabled) { - CompletableFuture cachedFuture = futureCache.setIfAbsent(cacheKey, loadCallFuture); + CompletableFuture cachedFuture = futureCache.putIfAbsentAtomically(cacheKey, loadCallFuture); if (cachedFuture != null) { // another thread was faster and created a matching CF ... hence this is really a cachehit and we are done stats.incrementCacheHitCount(new IncrementCacheHitCountStatisticsContext<>(key, loadContext)); @@ -183,7 +183,7 @@ CompletableFuture load(K key, Object loadContext) { // immediate execution of batch function loadCallFuture = invokeLoaderImmediately(key, loadContext, true); if (futureCachingEnabled) { - CompletableFuture cachedFuture = futureCache.setIfAbsent(cacheKey, loadCallFuture); + CompletableFuture cachedFuture = futureCache.putIfAbsentAtomically(cacheKey, loadCallFuture); if (cachedFuture != null) { // another thread was faster and the loader was invoked twice with the same key // we are disregarding the resul of our dispatch call and use the already cached value diff --git a/src/main/java/org/dataloader/impl/DefaultCacheMap.java b/src/main/java/org/dataloader/impl/DefaultCacheMap.java index 10c1a5ff..b6c811ff 100644 --- a/src/main/java/org/dataloader/impl/DefaultCacheMap.java +++ b/src/main/java/org/dataloader/impl/DefaultCacheMap.java @@ -72,7 +72,7 @@ public Collection> getAll() { * {@inheritDoc} */ @Override - public CompletableFuture setIfAbsent(K key, CompletableFuture value) { + public CompletableFuture putIfAbsentAtomically(K key, CompletableFuture value) { return cache.putIfAbsent(key, value); } diff --git a/src/test/java/ReadmeExamples.java b/src/test/java/ReadmeExamples.java index 05d1670f..6705cb88 100644 --- a/src/test/java/ReadmeExamples.java +++ b/src/test/java/ReadmeExamples.java @@ -265,7 +265,7 @@ public Collection> getAll() { } @Override - public CompletableFuture setIfAbsent(Object key, CompletableFuture value) { + public CompletableFuture putIfAbsentAtomically(Object key, CompletableFuture value) { return null; } diff --git a/src/test/java/org/dataloader/fixtures/CustomCacheMap.java b/src/test/java/org/dataloader/fixtures/CustomCacheMap.java index c22ce1a4..6e20f68c 100644 --- a/src/test/java/org/dataloader/fixtures/CustomCacheMap.java +++ b/src/test/java/org/dataloader/fixtures/CustomCacheMap.java @@ -30,7 +30,7 @@ public Collection> getAll() { } @Override - public CompletableFuture setIfAbsent(String key, CompletableFuture value) { + public CompletableFuture putIfAbsentAtomically(String key, CompletableFuture value) { return stash.putIfAbsent(key, value); } From 0aa02352ad1c41f5ded4f5d8b57cd5732dc57fd8 Mon Sep 17 00:00:00 2001 From: Andreas Marek Date: Thu, 23 Oct 2025 10:26:02 +1000 Subject: [PATCH 148/156] fix master build --- .github/workflows/master.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index d0762966..3be85c96 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -28,6 +28,6 @@ jobs: - name: Setup Gradle uses: gradle/actions/setup-gradle@v5 - name: build test and publish - run: ./gradlew assemble && ./gradlew check --info && && ./gradlew jcstress && ./gradlew publishToSonatype closeAndReleaseSonatypeStagingRepository -x check --info --stacktrace + run: ./gradlew assemble && ./gradlew check --info && ./gradlew jcstress && ./gradlew publishToSonatype closeAndReleaseSonatypeStagingRepository -x check --info --stacktrace env: CI: true From 9671d60862e6a6c4b7099ba72949e47844fbeecc Mon Sep 17 00:00:00 2001 From: bbaker Date: Mon, 27 Oct 2025 09:39:59 +1100 Subject: [PATCH 149/156] Updated the documentation of the CacheMap and also some refacoring of common code --- src/main/java/org/dataloader/CacheMap.java | 25 ++++++++++---- .../java/org/dataloader/DataLoaderHelper.java | 33 ++++++++++--------- .../org/dataloader/impl/DefaultCacheMap.java | 7 ++-- .../dataloader/DataLoaderCacheMapTest.java | 32 +++++++++++++++--- 4 files changed, 68 insertions(+), 29 deletions(-) diff --git a/src/main/java/org/dataloader/CacheMap.java b/src/main/java/org/dataloader/CacheMap.java index b3929af2..3e6d895b 100644 --- a/src/main/java/org/dataloader/CacheMap.java +++ b/src/main/java/org/dataloader/CacheMap.java @@ -28,7 +28,13 @@ * CacheMap is used by data loaders that use caching promises to values aka {@link CompletableFuture}<V>. A better name for this * class might have been FutureCache but that is history now. *

- * The default implementation used by the data loader is based on a {@link java.util.LinkedHashMap}. + * The default implementation used by the data loader is based on a {@link java.util.concurrent.ConcurrentHashMap} because + * the data loader code requires the cache to prove atomic writes especially the {@link #putIfAbsentAtomically(Object, CompletableFuture)} + * method. + *

+ * The data loader code using a Compare and Swap (CAS) algorithm to decide if a cache entry is present or not. If you write your + * own {@link CacheMap} implementation then you MUST provide atomic writes in this method to ensure that the same promise for a key is + * returned always. *

* This is really a cache of completed {@link CompletableFuture}<V> values in memory. It is used, when caching is enabled, to * give back the same future to any code that may call it. If you need a cache of the underlying values that is possible external to the JVM @@ -42,7 +48,7 @@ */ @PublicSpi @NullMarked -public interface CacheMap { +public interface CacheMap { /** * Creates a new cache map, using the default implementation that is based on a {@link java.util.LinkedHashMap}. @@ -84,14 +90,21 @@ static CacheMap simpleMap() { Collection> getAll(); /** - * Creates a new cache map entry with the specified key and value, or updates the value if the key already exists. + * Atomically creates a new cache map entry with the specified key and value, or updates the value if the key already exists. + *

+ * The data loader code using a Compare and Swap (CAS) algorithm to decide if a cache entry is present or not. If you write your + * own {@link CacheMap} implementation then you MUST provide atomic writes in this method to ensure that the same promise for a key is + * returned always. + *

+ * The default implementation of this method uses {@link java.util.concurrent.ConcurrentHashMap} has its implementation so these CAS + * writes work as expected. * * @param key the key to cache * @param value the value to cache * - * @return the cache map for fluent coding + * @return atomically the previous value for the key or null if the value is not present. */ - CompletableFuture putIfAbsentAtomically(K key, CompletableFuture value); + @Nullable CompletableFuture putIfAbsentAtomically(K key, CompletableFuture value); /** * Deletes the entry with the specified key from the cache map, if it exists. @@ -114,7 +127,7 @@ static CacheMap simpleMap() { * and intended for testing and debugging. * If a cache doesn't support it, it can throw an Exception. * - * @return + * @return the size of the cache */ int size(); } diff --git a/src/main/java/org/dataloader/DataLoaderHelper.java b/src/main/java/org/dataloader/DataLoaderHelper.java index 249c1f25..feb6184b 100644 --- a/src/main/java/org/dataloader/DataLoaderHelper.java +++ b/src/main/java/org/dataloader/DataLoaderHelper.java @@ -49,7 +49,7 @@ @Internal class DataLoaderHelper { - static class LoaderQueueEntry { + private static class LoaderQueueEntry { final K key; final CompletableFuture value; @@ -155,11 +155,8 @@ CompletableFuture load(K key, Object loadContext) { try { CompletableFuture cachedFuture = futureCache.get(cacheKey); if (cachedFuture != null) { - // We already have a promise for this key, no need to check value cache or queue up load - stats.incrementCacheHitCount(new IncrementCacheHitCountStatisticsContext<>(key, loadContext)); - ctx.onDispatched(); - cachedFuture.whenComplete(ctx::onCompleted); - return cachedFuture; + // We already have a promise for this key, no need to check value cache or queue this load + return incrementCacheHitAndReturnCF(ctx, key, loadContext, cachedFuture); } } catch (Exception ignored) { } @@ -170,11 +167,8 @@ CompletableFuture load(K key, Object loadContext) { if (futureCachingEnabled) { CompletableFuture cachedFuture = futureCache.putIfAbsentAtomically(cacheKey, loadCallFuture); if (cachedFuture != null) { - // another thread was faster and created a matching CF ... hence this is really a cachehit and we are done - stats.incrementCacheHitCount(new IncrementCacheHitCountStatisticsContext<>(key, loadContext)); - ctx.onDispatched(); - cachedFuture.whenComplete(ctx::onCompleted); - return cachedFuture; + // another thread was faster and created a matching CF ... hence this is really a cache hit and we are done + return incrementCacheHitAndReturnCF(ctx, key, loadContext, cachedFuture); } } addEntryToLoaderQueue(key, loadCallFuture, loadContext); @@ -186,12 +180,9 @@ CompletableFuture load(K key, Object loadContext) { CompletableFuture cachedFuture = futureCache.putIfAbsentAtomically(cacheKey, loadCallFuture); if (cachedFuture != null) { // another thread was faster and the loader was invoked twice with the same key - // we are disregarding the resul of our dispatch call and use the already cached value + // we are disregarding the result of our dispatch call and use the already cached value // meaning this is a cache hit and we are done - stats.incrementCacheHitCount(new IncrementCacheHitCountStatisticsContext<>(key, loadContext)); - ctx.onDispatched(); - cachedFuture.whenComplete(ctx::onCompleted); - return cachedFuture; + return incrementCacheHitAndReturnCF(ctx, key, loadContext, cachedFuture); } } } @@ -201,6 +192,13 @@ CompletableFuture load(K key, Object loadContext) { return loadCallFuture; } + private CompletableFuture incrementCacheHitAndReturnCF(DataLoaderInstrumentationContext ctx, K key, Object loadContext, CompletableFuture cachedFuture) { + stats.incrementCacheHitCount(new IncrementCacheHitCountStatisticsContext<>(key, loadContext)); + ctx.onDispatched(); + cachedFuture.whenComplete(ctx::onCompleted); + return cachedFuture; + } + private void addEntryToLoaderQueue(K key, CompletableFuture future, Object loadContext) { while (true) { LoaderQueueEntry prev = loaderQueue.get(); @@ -223,6 +221,7 @@ Object getCacheKeyWithContext(K key, Object context) { loaderOptions.cacheKeyFunction().get().getKeyWithContext(key, context) : key; } + @SuppressWarnings("unchecked") DispatchResult dispatch() { DataLoaderInstrumentationContext> instrCtx = ctxOrNoopCtx(instrumentation().beginDispatch(dataLoader)); @@ -232,6 +231,8 @@ DispatchResult dispatch() { while (true) { loaderQueueEntryHead = loaderQueue.get(); if (loaderQueue.compareAndSet(loaderQueueEntryHead, null)) { + // one or more threads competed and this one won. This thread holds + // the loader queue root in the local variable loaderQueueEntryHead break; } } diff --git a/src/main/java/org/dataloader/impl/DefaultCacheMap.java b/src/main/java/org/dataloader/impl/DefaultCacheMap.java index b6c811ff..e8db6818 100644 --- a/src/main/java/org/dataloader/impl/DefaultCacheMap.java +++ b/src/main/java/org/dataloader/impl/DefaultCacheMap.java @@ -18,6 +18,8 @@ import org.dataloader.CacheMap; import org.dataloader.annotations.Internal; +import org.jspecify.annotations.NullMarked; +import org.jspecify.annotations.Nullable; import java.util.Collection; import java.util.concurrent.CompletableFuture; @@ -32,6 +34,7 @@ * @author Arnold Schrijver */ @Internal +@NullMarked public class DefaultCacheMap implements CacheMap { private final ConcurrentHashMap> cache; @@ -56,7 +59,7 @@ public boolean containsKey(K key) { * {@inheritDoc} */ @Override - public CompletableFuture get(K key) { + public @Nullable CompletableFuture get(K key) { return cache.get(key); } @@ -72,7 +75,7 @@ public Collection> getAll() { * {@inheritDoc} */ @Override - public CompletableFuture putIfAbsentAtomically(K key, CompletableFuture value) { + public @Nullable CompletableFuture putIfAbsentAtomically(K key, CompletableFuture value) { return cache.putIfAbsent(key, value); } diff --git a/src/test/java/org/dataloader/DataLoaderCacheMapTest.java b/src/test/java/org/dataloader/DataLoaderCacheMapTest.java index df364a20..d3de4aad 100644 --- a/src/test/java/org/dataloader/DataLoaderCacheMapTest.java +++ b/src/test/java/org/dataloader/DataLoaderCacheMapTest.java @@ -14,6 +14,7 @@ /** * Tests for cacheMap functionality.. */ +@SuppressWarnings("NullableProblems") public class DataLoaderCacheMapTest { private BatchLoader keysAsValues() { @@ -24,12 +25,33 @@ private BatchLoader keysAsValues() { public void should_provide_all_futures_from_cache() { DataLoader dataLoader = newDataLoader(keysAsValues()); - dataLoader.load(1); - dataLoader.load(2); - dataLoader.load(1); + CompletableFuture cf1 = dataLoader.load(1); + CompletableFuture cf2 = dataLoader.load(2); + CompletableFuture cf3 = dataLoader.load(3); + + CacheMap cacheMap = dataLoader.getCacheMap(); + Collection> futures = cacheMap.getAll(); + assertThat(futures.size(), equalTo(3)); + + + assertThat(cacheMap.get(1), equalTo(cf1)); + assertThat(cacheMap.get(2), equalTo(cf2)); + assertThat(cacheMap.get(3), equalTo(cf3)); + assertThat(cacheMap.containsKey(1), equalTo(true)); + assertThat(cacheMap.containsKey(2), equalTo(true)); + assertThat(cacheMap.containsKey(3), equalTo(true)); + assertThat(cacheMap.containsKey(4), equalTo(false)); + + cacheMap.delete(1); + assertThat(cacheMap.containsKey(1), equalTo(false)); + assertThat(cacheMap.containsKey(2), equalTo(true)); + + cacheMap.clear(); + assertThat(cacheMap.containsKey(1), equalTo(false)); + assertThat(cacheMap.containsKey(2), equalTo(false)); + assertThat(cacheMap.containsKey(3), equalTo(false)); + assertThat(cacheMap.containsKey(4), equalTo(false)); - Collection> futures = dataLoader.getCacheMap().getAll(); - assertThat(futures.size(), equalTo(2)); } @Test From 5a46931786be3a56578a663f51b25b9c4523c7de Mon Sep 17 00:00:00 2001 From: bbaker Date: Mon, 27 Oct 2025 10:18:06 +1100 Subject: [PATCH 150/156] Merged master --- src/main/java/org/dataloader/DataLoader.java | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/src/main/java/org/dataloader/DataLoader.java b/src/main/java/org/dataloader/DataLoader.java index 6995f2fb..125efe1f 100644 --- a/src/main/java/org/dataloader/DataLoader.java +++ b/src/main/java/org/dataloader/DataLoader.java @@ -160,21 +160,6 @@ public Duration getTimeSinceDispatch() { return Duration.between(helper.getLastDispatchTime(), helper.now()); } - /** - * Requests to load the data with the specified key asynchronously, and returns a future of the resulting value. - *

- * If batching is enabled (the default), you'll have to call {@link DataLoader#dispatch()} at a later stage to - * start batch execution. If you forget this call the future will never be completed (unless already completed, - * and returned from cache). - * - * @param key the key to load - * - * @return the future of the value - */ - public CompletableFuture load(K key) { - return load(key, null); - } - /** * This will return an optional promise to a value previously loaded via a {@link #load(Object)} call or empty if not call has been made for that key. *

From b7fd63fd0ef549e18c82dd000974e6d9d2ec7b40 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Oct 2025 11:23:57 +0000 Subject: [PATCH 151/156] Bump com.google.errorprone:error_prone_core from 2.42.0 to 2.43.0 Bumps [com.google.errorprone:error_prone_core](https://github.com/google/error-prone) from 2.42.0 to 2.43.0. - [Release notes](https://github.com/google/error-prone/releases) - [Commits](https://github.com/google/error-prone/compare/v2.42.0...v2.43.0) --- updated-dependencies: - dependency-name: com.google.errorprone:error_prone_core dependency-version: 2.43.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 2384afda..7b285d6b 100644 --- a/build.gradle +++ b/build.gradle @@ -99,7 +99,7 @@ dependencies { jmh 'org.openjdk.jmh:jmh-generator-annprocess:1.37' errorprone 'com.uber.nullaway:nullaway:0.12.10' - errorprone 'com.google.errorprone:error_prone_core:2.42.0' + errorprone 'com.google.errorprone:error_prone_core:2.43.0' // just tests testImplementation 'org.jetbrains.kotlin:kotlin-stdlib-jdk8' From d82e3531efdd5aab83c56c9560d1641bbf4cf539 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Oct 2025 11:26:36 +0000 Subject: [PATCH 152/156] Bump io.github.reyerizo.gradle.jcstress from 0.8.15 to 0.9.0 Bumps io.github.reyerizo.gradle.jcstress from 0.8.15 to 0.9.0. --- updated-dependencies: - dependency-name: io.github.reyerizo.gradle.jcstress dependency-version: 0.9.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 2384afda..46575495 100644 --- a/build.gradle +++ b/build.gradle @@ -16,7 +16,7 @@ plugins { id 'com.github.ben-manes.versions' version '0.53.0' id "me.champeau.jmh" version "0.7.3" id "net.ltgt.errorprone" version '4.3.0' - id "io.github.reyerizo.gradle.jcstress" version "0.8.15" + id "io.github.reyerizo.gradle.jcstress" version "0.9.0" // Kotlin just for tests - not production code id 'org.jetbrains.kotlin.jvm' version '2.2.20' From ae3af9f7b3b483abdecec06ed16f38dfc6f0a92d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Oct 2025 11:45:43 +0000 Subject: [PATCH 153/156] Bump org.jetbrains.kotlin.jvm from 2.2.20 to 2.2.21 Bumps [org.jetbrains.kotlin.jvm](https://github.com/JetBrains/kotlin) from 2.2.20 to 2.2.21. - [Release notes](https://github.com/JetBrains/kotlin/releases) - [Changelog](https://github.com/JetBrains/kotlin/blob/master/ChangeLog.md) - [Commits](https://github.com/JetBrains/kotlin/compare/v2.2.20...v2.2.21) --- updated-dependencies: - dependency-name: org.jetbrains.kotlin.jvm dependency-version: 2.2.21 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 2384afda..0e3be585 100644 --- a/build.gradle +++ b/build.gradle @@ -19,7 +19,7 @@ plugins { id "io.github.reyerizo.gradle.jcstress" version "0.8.15" // Kotlin just for tests - not production code - id 'org.jetbrains.kotlin.jvm' version '2.2.20' + id 'org.jetbrains.kotlin.jvm' version '2.2.21' } java { From 1194f8975337c1a86da1eab945838a3bff011f58 Mon Sep 17 00:00:00 2001 From: dondonz <13839920+dondonz@users.noreply.github.com> Date: Sat, 1 Nov 2025 10:56:50 +1100 Subject: [PATCH 154/156] Error Prone needs Java 21 --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 7b285d6b..3519cbc3 100644 --- a/build.gradle +++ b/build.gradle @@ -24,7 +24,7 @@ plugins { java { toolchain { - languageVersion = JavaLanguageVersion.of(17) + languageVersion = JavaLanguageVersion.of(21) } } From 3eefb9adbddfb30fdcc01d69b397a1150424f9a4 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 16 Nov 2025 04:48:36 +0000 Subject: [PATCH 155/156] Initial plan From 39258835856165ab4cd6264fa4c6730521cbcd56 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 16 Nov 2025 05:07:28 +0000 Subject: [PATCH 156/156] Upgrade Gradle to 9.2.0 and fix deprecation warnings Co-authored-by: andimarek <1706744+andimarek@users.noreply.github.com> --- build.gradle | 25 +++++++------ gradle/wrapper/gradle-wrapper.jar | Bin 59536 -> 43583 bytes gradle/wrapper/gradle-wrapper.properties | 2 +- gradlew | 44 ++++++++++++++++------- gradlew.bat | 37 ++++++++++--------- 5 files changed, 65 insertions(+), 43 deletions(-) diff --git a/build.gradle b/build.gradle index 1976135b..7d2c8060 100644 --- a/build.gradle +++ b/build.gradle @@ -11,7 +11,7 @@ plugins { id 'maven-publish' id 'signing' id 'groovy' - id 'biz.aQute.bnd.builder' version '6.2.0' + id 'biz.aQute.bnd.builder' version '7.1.0' id 'io.github.gradle-nexus.publish-plugin' version '2.0.0' id 'com.github.ben-manes.versions' version '0.53.0' id "me.champeau.jmh" version "0.7.3" @@ -81,13 +81,15 @@ repositories { jar { manifest { - attributes('Automatic-Module-Name': 'org.dataloader', - '-exportcontents': 'org.dataloader.*', - '-removeheaders': 'Private-Package') + attributes('Automatic-Module-Name': 'org.dataloader') } - bnd(''' + bundle { + bnd(''' +-exportcontents: org.dataloader.* +-removeheaders: Private-Package Import-Package: org.jspecify.annotations;resolution:=optional,* ''') + } } dependencies { @@ -141,10 +143,7 @@ task javadocJar(type: Jar, dependsOn: javadoc) { from javadoc.destinationDir } -artifacts { - archives sourcesJar - archives javadocJar -} + testing { suites { @@ -177,9 +176,9 @@ publishing { publications { graphqlJava(MavenPublication) { from components.java - groupId 'com.graphql-java' - artifactId 'java-dataloader' - version project.version + groupId = 'com.graphql-java' + artifactId = 'java-dataloader' + version = project.version artifact sourcesJar artifact javadocJar @@ -237,7 +236,7 @@ nexusPublishing { } signing { - required { !project.hasProperty('publishToMavenLocal') } + required = { !project.hasProperty('publishToMavenLocal') } def signingKey = System.env.MAVEN_CENTRAL_PGP_KEY useInMemoryPgpKeys(signingKey, "") sign publishing.publications diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 7454180f2ae8848c63b8b4dea2cb829da983f2fa..a4b76b9530d66f5e68d973ea569d8e19de379189 100644 GIT binary patch literal 43583 zcma&N1CXTcmMvW9vTb(Rwr$&4wr$(C?dmSu>@vG-+vuvg^_??!{yS%8zW-#zn-LkA z5&1^$^{lnmUON?}LBF8_K|(?T0Ra(xUH{($5eN!MR#ZihR#HxkUPe+_R8Cn`RRs(P z_^*#_XlXmGv7!4;*Y%p4nw?{bNp@UZHv1?Um8r6)Fei3p@ClJn0ECfg1hkeuUU@Or zDaPa;U3fE=3L}DooL;8f;P0ipPt0Z~9P0)lbStMS)ag54=uL9ia-Lm3nh|@(Y?B`; zx_#arJIpXH!U{fbCbI^17}6Ri*H<>OLR%c|^mh8+)*h~K8Z!9)DPf zR2h?lbDZQ`p9P;&DQ4F0sur@TMa!Y}S8irn(%d-gi0*WxxCSk*A?3lGh=gcYN?FGl z7D=Js!i~0=u3rox^eO3i@$0=n{K1lPNU zwmfjRVmLOCRfe=seV&P*1Iq=^i`502keY8Uy-WNPwVNNtJFx?IwAyRPZo2Wo1+S(xF37LJZ~%i)kpFQ3Fw=mXfd@>%+)RpYQLnr}B~~zoof(JVm^^&f zxKV^+3D3$A1G;qh4gPVjhrC8e(VYUHv#dy^)(RoUFM?o%W-EHxufuWf(l*@-l+7vt z=l`qmR56K~F|v<^Pd*p~1_y^P0P^aPC##d8+HqX4IR1gu+7w#~TBFphJxF)T$2WEa zxa?H&6=Qe7d(#tha?_1uQys2KtHQ{)Qco)qwGjrdNL7thd^G5i8Os)CHqc>iOidS} z%nFEDdm=GXBw=yXe1W-ShHHFb?Cc70+$W~z_+}nAoHFYI1MV1wZegw*0y^tC*s%3h zhD3tN8b=Gv&rj}!SUM6|ajSPp*58KR7MPpI{oAJCtY~JECm)*m_x>AZEu>DFgUcby z1Qaw8lU4jZpQ_$;*7RME+gq1KySGG#Wql>aL~k9tLrSO()LWn*q&YxHEuzmwd1?aAtI zBJ>P=&$=l1efe1CDU;`Fd+_;&wI07?V0aAIgc(!{a z0Jg6Y=inXc3^n!U0Atk`iCFIQooHqcWhO(qrieUOW8X(x?(RD}iYDLMjSwffH2~tB z)oDgNBLB^AJBM1M^c5HdRx6fBfka`(LD-qrlh5jqH~);#nw|iyp)()xVYak3;Ybik z0j`(+69aK*B>)e_p%=wu8XC&9e{AO4c~O1U`5X9}?0mrd*m$_EUek{R?DNSh(=br# z#Q61gBzEpmy`$pA*6!87 zSDD+=@fTY7<4A?GLqpA?Pb2z$pbCc4B4zL{BeZ?F-8`s$?>*lXXtn*NC61>|*w7J* z$?!iB{6R-0=KFmyp1nnEmLsA-H0a6l+1uaH^g%c(p{iT&YFrbQ$&PRb8Up#X3@Zsk zD^^&LK~111%cqlP%!_gFNa^dTYT?rhkGl}5=fL{a`UViaXWI$k-UcHJwmaH1s=S$4 z%4)PdWJX;hh5UoK?6aWoyLxX&NhNRqKam7tcOkLh{%j3K^4Mgx1@i|Pi&}<^5>hs5 zm8?uOS>%)NzT(%PjVPGa?X%`N2TQCKbeH2l;cTnHiHppPSJ<7y-yEIiC!P*ikl&!B z%+?>VttCOQM@ShFguHVjxX^?mHX^hSaO_;pnyh^v9EumqSZTi+#f&_Vaija0Q-e*| z7ulQj6Fs*bbmsWp{`auM04gGwsYYdNNZcg|ph0OgD>7O}Asn7^Z=eI>`$2*v78;sj-}oMoEj&@)9+ycEOo92xSyY344^ z11Hb8^kdOvbf^GNAK++bYioknrpdN>+u8R?JxG=!2Kd9r=YWCOJYXYuM0cOq^FhEd zBg2puKy__7VT3-r*dG4c62Wgxi52EMCQ`bKgf*#*ou(D4-ZN$+mg&7$u!! z-^+Z%;-3IDwqZ|K=ah85OLwkO zKxNBh+4QHh)u9D?MFtpbl)us}9+V!D%w9jfAMYEb>%$A;u)rrI zuBudh;5PN}_6J_}l55P3l_)&RMlH{m!)ai-i$g)&*M`eN$XQMw{v^r@-125^RRCF0 z^2>|DxhQw(mtNEI2Kj(;KblC7x=JlK$@78`O~>V!`|1Lm-^JR$-5pUANAnb(5}B}JGjBsliK4& zk6y(;$e&h)lh2)L=bvZKbvh@>vLlreBdH8No2>$#%_Wp1U0N7Ank!6$dFSi#xzh|( zRi{Uw%-4W!{IXZ)fWx@XX6;&(m_F%c6~X8hx=BN1&q}*( zoaNjWabE{oUPb!Bt$eyd#$5j9rItB-h*5JiNi(v^e|XKAj*8(k<5-2$&ZBR5fF|JA z9&m4fbzNQnAU}r8ab>fFV%J0z5awe#UZ|bz?Ur)U9bCIKWEzi2%A+5CLqh?}K4JHi z4vtM;+uPsVz{Lfr;78W78gC;z*yTch~4YkLr&m-7%-xc ztw6Mh2d>_iO*$Rd8(-Cr1_V8EO1f*^@wRoSozS) zy1UoC@pruAaC8Z_7~_w4Q6n*&B0AjOmMWa;sIav&gu z|J5&|{=a@vR!~k-OjKEgPFCzcJ>#A1uL&7xTDn;{XBdeM}V=l3B8fE1--DHjSaxoSjNKEM9|U9#m2<3>n{Iuo`r3UZp;>GkT2YBNAh|b z^jTq-hJp(ebZh#Lk8hVBP%qXwv-@vbvoREX$TqRGTgEi$%_F9tZES@z8Bx}$#5eeG zk^UsLBH{bc2VBW)*EdS({yw=?qmevwi?BL6*=12k9zM5gJv1>y#ML4!)iiPzVaH9% zgSImetD@dam~e>{LvVh!phhzpW+iFvWpGT#CVE5TQ40n%F|p(sP5mXxna+Ev7PDwA zamaV4m*^~*xV+&p;W749xhb_X=$|LD;FHuB&JL5?*Y2-oIT(wYY2;73<^#46S~Gx| z^cez%V7x$81}UWqS13Gz80379Rj;6~WdiXWOSsdmzY39L;Hg3MH43o*y8ibNBBH`(av4|u;YPq%{R;IuYow<+GEsf@R?=@tT@!}?#>zIIn0CoyV!hq3mw zHj>OOjfJM3F{RG#6ujzo?y32m^tgSXf@v=J$ELdJ+=5j|=F-~hP$G&}tDZsZE?5rX ztGj`!S>)CFmdkccxM9eGIcGnS2AfK#gXwj%esuIBNJQP1WV~b~+D7PJTmWGTSDrR` zEAu4B8l>NPuhsk5a`rReSya2nfV1EK01+G!x8aBdTs3Io$u5!6n6KX%uv@DxAp3F@{4UYg4SWJtQ-W~0MDb|j-$lwVn znAm*Pl!?Ps&3wO=R115RWKb*JKoexo*)uhhHBncEDMSVa_PyA>k{Zm2(wMQ(5NM3# z)jkza|GoWEQo4^s*wE(gHz?Xsg4`}HUAcs42cM1-qq_=+=!Gk^y710j=66(cSWqUe zklbm8+zB_syQv5A2rj!Vbw8;|$@C!vfNmNV!yJIWDQ>{+2x zKjuFX`~~HKG~^6h5FntRpnnHt=D&rq0>IJ9#F0eM)Y-)GpRjiN7gkA8wvnG#K=q{q z9dBn8_~wm4J<3J_vl|9H{7q6u2A!cW{bp#r*-f{gOV^e=8S{nc1DxMHFwuM$;aVI^ zz6A*}m8N-&x8;aunp1w7_vtB*pa+OYBw=TMc6QK=mbA-|Cf* zvyh8D4LRJImooUaSb7t*fVfih<97Gf@VE0|z>NcBwBQze);Rh!k3K_sfunToZY;f2 z^HmC4KjHRVg+eKYj;PRN^|E0>Gj_zagfRbrki68I^#~6-HaHg3BUW%+clM1xQEdPYt_g<2K+z!$>*$9nQ>; zf9Bei{?zY^-e{q_*|W#2rJG`2fy@{%6u0i_VEWTq$*(ZN37|8lFFFt)nCG({r!q#9 z5VK_kkSJ3?zOH)OezMT{!YkCuSSn!K#-Rhl$uUM(bq*jY? zi1xbMVthJ`E>d>(f3)~fozjg^@eheMF6<)I`oeJYx4*+M&%c9VArn(OM-wp%M<-`x z7sLP1&3^%Nld9Dhm@$3f2}87!quhI@nwd@3~fZl_3LYW-B?Ia>ui`ELg z&Qfe!7m6ze=mZ`Ia9$z|ARSw|IdMpooY4YiPN8K z4B(ts3p%2i(Td=tgEHX z0UQ_>URBtG+-?0E;E7Ld^dyZ;jjw0}XZ(}-QzC6+NN=40oDb2^v!L1g9xRvE#@IBR zO!b-2N7wVfLV;mhEaXQ9XAU+>=XVA6f&T4Z-@AX!leJ8obP^P^wP0aICND?~w&NykJ#54x3_@r7IDMdRNy4Hh;h*!u(Ol(#0bJdwEo$5437-UBjQ+j=Ic>Q2z` zJNDf0yO6@mr6y1#n3)s(W|$iE_i8r@Gd@!DWDqZ7J&~gAm1#~maIGJ1sls^gxL9LLG_NhU!pTGty!TbhzQnu)I*S^54U6Yu%ZeCg`R>Q zhBv$n5j0v%O_j{QYWG!R9W?5_b&67KB$t}&e2LdMvd(PxN6Ir!H4>PNlerpBL>Zvyy!yw z-SOo8caEpDt(}|gKPBd$qND5#a5nju^O>V&;f890?yEOfkSG^HQVmEbM3Ugzu+UtH zC(INPDdraBN?P%kE;*Ae%Wto&sgw(crfZ#Qy(<4nk;S|hD3j{IQRI6Yq|f^basLY; z-HB&Je%Gg}Jt@={_C{L$!RM;$$|iD6vu#3w?v?*;&()uB|I-XqEKqZPS!reW9JkLewLb!70T7n`i!gNtb1%vN- zySZj{8-1>6E%H&=V}LM#xmt`J3XQoaD|@XygXjdZ1+P77-=;=eYpoEQ01B@L*a(uW zrZeZz?HJsw_4g0vhUgkg@VF8<-X$B8pOqCuWAl28uB|@r`19DTUQQsb^pfqB6QtiT z*`_UZ`fT}vtUY#%sq2{rchyfu*pCg;uec2$-$N_xgjZcoumE5vSI{+s@iLWoz^Mf; zuI8kDP{!XY6OP~q5}%1&L}CtfH^N<3o4L@J@zg1-mt{9L`s^z$Vgb|mr{@WiwAqKg zp#t-lhrU>F8o0s1q_9y`gQNf~Vb!F%70f}$>i7o4ho$`uciNf=xgJ>&!gSt0g;M>*x4-`U)ysFW&Vs^Vk6m%?iuWU+o&m(2Jm26Y(3%TL; zA7T)BP{WS!&xmxNw%J=$MPfn(9*^*TV;$JwRy8Zl*yUZi8jWYF>==j~&S|Xinsb%c z2?B+kpet*muEW7@AzjBA^wAJBY8i|#C{WtO_or&Nj2{=6JTTX05}|H>N2B|Wf!*3_ z7hW*j6p3TvpghEc6-wufFiY!%-GvOx*bZrhZu+7?iSrZL5q9}igiF^*R3%DE4aCHZ zqu>xS8LkW+Auv%z-<1Xs92u23R$nk@Pk}MU5!gT|c7vGlEA%G^2th&Q*zfg%-D^=f z&J_}jskj|Q;73NP4<4k*Y%pXPU2Thoqr+5uH1yEYM|VtBPW6lXaetokD0u z9qVek6Q&wk)tFbQ8(^HGf3Wp16gKmr>G;#G(HRBx?F`9AIRboK+;OfHaLJ(P>IP0w zyTbTkx_THEOs%Q&aPrxbZrJlio+hCC_HK<4%f3ZoSAyG7Dn`=X=&h@m*|UYO-4Hq0 z-Bq&+Ie!S##4A6OGoC~>ZW`Y5J)*ouaFl_e9GA*VSL!O_@xGiBw!AF}1{tB)z(w%c zS1Hmrb9OC8>0a_$BzeiN?rkPLc9%&;1CZW*4}CDDNr2gcl_3z+WC15&H1Zc2{o~i) z)LLW=WQ{?ricmC`G1GfJ0Yp4Dy~Ba;j6ZV4r{8xRs`13{dD!xXmr^Aga|C=iSmor% z8hi|pTXH)5Yf&v~exp3o+sY4B^^b*eYkkCYl*T{*=-0HniSA_1F53eCb{x~1k3*`W zr~};p1A`k{1DV9=UPnLDgz{aJH=-LQo<5%+Em!DNN252xwIf*wF_zS^!(XSm(9eoj z=*dXG&n0>)_)N5oc6v!>-bd(2ragD8O=M|wGW z!xJQS<)u70m&6OmrF0WSsr@I%T*c#Qo#Ha4d3COcX+9}hM5!7JIGF>7<~C(Ear^Sn zm^ZFkV6~Ula6+8S?oOROOA6$C&q&dp`>oR-2Ym3(HT@O7Sd5c~+kjrmM)YmgPH*tL zX+znN>`tv;5eOfX?h{AuX^LK~V#gPCu=)Tigtq9&?7Xh$qN|%A$?V*v=&-2F$zTUv z`C#WyIrChS5|Kgm_GeudCFf;)!WH7FI60j^0o#65o6`w*S7R@)88n$1nrgU(oU0M9 zx+EuMkC>(4j1;m6NoGqEkpJYJ?vc|B zOlwT3t&UgL!pX_P*6g36`ZXQ; z9~Cv}ANFnJGp(;ZhS(@FT;3e)0)Kp;h^x;$*xZn*k0U6-&FwI=uOGaODdrsp-!K$Ac32^c{+FhI-HkYd5v=`PGsg%6I`4d9Jy)uW0y%) zm&j^9WBAp*P8#kGJUhB!L?a%h$hJgQrx!6KCB_TRo%9{t0J7KW8!o1B!NC)VGLM5! zpZy5Jc{`r{1e(jd%jsG7k%I+m#CGS*BPA65ZVW~fLYw0dA-H_}O zrkGFL&P1PG9p2(%QiEWm6x;U-U&I#;Em$nx-_I^wtgw3xUPVVu zqSuKnx&dIT-XT+T10p;yjo1Y)z(x1fb8Dzfn8e yu?e%!_ptzGB|8GrCfu%p?(_ zQccdaaVK$5bz;*rnyK{_SQYM>;aES6Qs^lj9lEs6_J+%nIiuQC*fN;z8md>r_~Mfl zU%p5Dt_YT>gQqfr@`cR!$NWr~+`CZb%dn;WtzrAOI>P_JtsB76PYe*<%H(y>qx-`Kq!X_; z<{RpAqYhE=L1r*M)gNF3B8r(<%8mo*SR2hu zccLRZwGARt)Hlo1euqTyM>^!HK*!Q2P;4UYrysje@;(<|$&%vQekbn|0Ruu_Io(w4#%p6ld2Yp7tlA`Y$cciThP zKzNGIMPXX%&Ud0uQh!uQZz|FB`4KGD?3!ND?wQt6!n*f4EmCoJUh&b?;B{|lxs#F- z31~HQ`SF4x$&v00@(P+j1pAaj5!s`)b2RDBp*PB=2IB>oBF!*6vwr7Dp%zpAx*dPr zb@Zjq^XjN?O4QcZ*O+8>)|HlrR>oD*?WQl5ri3R#2?*W6iJ>>kH%KnnME&TT@ZzrHS$Q%LC?n|e>V+D+8D zYc4)QddFz7I8#}y#Wj6>4P%34dZH~OUDb?uP%-E zwjXM(?Sg~1!|wI(RVuxbu)-rH+O=igSho_pDCw(c6b=P zKk4ATlB?bj9+HHlh<_!&z0rx13K3ZrAR8W)!@Y}o`?a*JJsD+twZIv`W)@Y?Amu_u zz``@-e2X}27$i(2=9rvIu5uTUOVhzwu%mNazS|lZb&PT;XE2|B&W1>=B58#*!~D&) zfVmJGg8UdP*fx(>Cj^?yS^zH#o-$Q-*$SnK(ZVFkw+er=>N^7!)FtP3y~Xxnu^nzY zikgB>Nj0%;WOltWIob|}%lo?_C7<``a5hEkx&1ku$|)i>Rh6@3h*`slY=9U}(Ql_< zaNG*J8vb&@zpdhAvv`?{=zDedJ23TD&Zg__snRAH4eh~^oawdYi6A3w8<Ozh@Kw)#bdktM^GVb zrG08?0bG?|NG+w^&JvD*7LAbjED{_Zkc`3H!My>0u5Q}m!+6VokMLXxl`Mkd=g&Xx z-a>m*#G3SLlhbKB!)tnzfWOBV;u;ftU}S!NdD5+YtOjLg?X}dl>7m^gOpihrf1;PY zvll&>dIuUGs{Qnd- zwIR3oIrct8Va^Tm0t#(bJD7c$Z7DO9*7NnRZorrSm`b`cxz>OIC;jSE3DO8`hX955ui`s%||YQtt2 z5DNA&pG-V+4oI2s*x^>-$6J?p=I>C|9wZF8z;VjR??Icg?1w2v5Me+FgAeGGa8(3S z4vg*$>zC-WIVZtJ7}o9{D-7d>zCe|z#<9>CFve-OPAYsneTb^JH!Enaza#j}^mXy1 z+ULn^10+rWLF6j2>Ya@@Kq?26>AqK{A_| zQKb*~F1>sE*=d?A?W7N2j?L09_7n+HGi{VY;MoTGr_)G9)ot$p!-UY5zZ2Xtbm=t z@dpPSGwgH=QtIcEulQNI>S-#ifbnO5EWkI;$A|pxJd885oM+ zGZ0_0gDvG8q2xebj+fbCHYfAXuZStH2j~|d^sBAzo46(K8n59+T6rzBwK)^rfPT+B zyIFw)9YC-V^rhtK`!3jrhmW-sTmM+tPH+;nwjL#-SjQPUZ53L@A>y*rt(#M(qsiB2 zx6B)dI}6Wlsw%bJ8h|(lhkJVogQZA&n{?Vgs6gNSXzuZpEyu*xySy8ro07QZ7Vk1!3tJphN_5V7qOiyK8p z#@jcDD8nmtYi1^l8ml;AF<#IPK?!pqf9D4moYk>d99Im}Jtwj6c#+A;f)CQ*f-hZ< z=p_T86jog%!p)D&5g9taSwYi&eP z#JuEK%+NULWus;0w32-SYFku#i}d~+{Pkho&^{;RxzP&0!RCm3-9K6`>KZpnzS6?L z^H^V*s!8<>x8bomvD%rh>Zp3>Db%kyin;qtl+jAv8Oo~1g~mqGAC&Qi_wy|xEt2iz zWAJEfTV%cl2Cs<1L&DLRVVH05EDq`pH7Oh7sR`NNkL%wi}8n>IXcO40hp+J+sC!W?!krJf!GJNE8uj zg-y~Ns-<~D?yqbzVRB}G>0A^f0!^N7l=$m0OdZuqAOQqLc zX?AEGr1Ht+inZ-Qiwnl@Z0qukd__a!C*CKuGdy5#nD7VUBM^6OCpxCa2A(X;e0&V4 zM&WR8+wErQ7UIc6LY~Q9x%Sn*Tn>>P`^t&idaOEnOd(Ufw#>NoR^1QdhJ8s`h^|R_ zXX`c5*O~Xdvh%q;7L!_!ohf$NfEBmCde|#uVZvEo>OfEq%+Ns7&_f$OR9xsihRpBb z+cjk8LyDm@U{YN>+r46?nn{7Gh(;WhFw6GAxtcKD+YWV?uge>;+q#Xx4!GpRkVZYu zzsF}1)7$?%s9g9CH=Zs+B%M_)+~*j3L0&Q9u7!|+T`^O{xE6qvAP?XWv9_MrZKdo& z%IyU)$Q95AB4!#hT!_dA>4e@zjOBD*Y=XjtMm)V|+IXzjuM;(l+8aA5#Kaz_$rR6! zj>#&^DidYD$nUY(D$mH`9eb|dtV0b{S>H6FBfq>t5`;OxA4Nn{J(+XihF(stSche7$es&~N$epi&PDM_N`As;*9D^L==2Q7Z2zD+CiU(|+-kL*VG+&9!Yb3LgPy?A zm7Z&^qRG_JIxK7-FBzZI3Q<;{`DIxtc48k> zc|0dmX;Z=W$+)qE)~`yn6MdoJ4co;%!`ddy+FV538Y)j(vg}5*k(WK)KWZ3WaOG!8 z!syGn=s{H$odtpqFrT#JGM*utN7B((abXnpDM6w56nhw}OY}0TiTG1#f*VFZr+^-g zbP10`$LPq_;PvrA1XXlyx2uM^mrjTzX}w{yuLo-cOClE8MMk47T25G8M!9Z5ypOSV zAJUBGEg5L2fY)ZGJb^E34R2zJ?}Vf>{~gB!8=5Z) z9y$>5c)=;o0HeHHSuE4U)#vG&KF|I%-cF6f$~pdYJWk_dD}iOA>iA$O$+4%@>JU08 zS`ep)$XLPJ+n0_i@PkF#ri6T8?ZeAot$6JIYHm&P6EB=BiaNY|aA$W0I+nz*zkz_z zkEru!tj!QUffq%)8y0y`T&`fuus-1p>=^hnBiBqD^hXrPs`PY9tU3m0np~rISY09> z`P3s=-kt_cYcxWd{de@}TwSqg*xVhp;E9zCsnXo6z z?f&Sv^U7n4`xr=mXle94HzOdN!2kB~4=%)u&N!+2;z6UYKUDqi-s6AZ!haB;@&B`? z_TRX0%@suz^TRdCb?!vNJYPY8L_}&07uySH9%W^Tc&1pia6y1q#?*Drf}GjGbPjBS zbOPcUY#*$3sL2x4v_i*Y=N7E$mR}J%|GUI(>WEr+28+V z%v5{#e!UF*6~G&%;l*q*$V?&r$Pp^sE^i-0$+RH3ERUUdQ0>rAq2(2QAbG}$y{de( z>{qD~GGuOk559Y@%$?N^1ApVL_a704>8OD%8Y%8B;FCt%AoPu8*D1 zLB5X>b}Syz81pn;xnB}%0FnwazlWfUV)Z-~rZg6~b z6!9J$EcE&sEbzcy?CI~=boWA&eeIa%z(7SE^qgVLz??1Vbc1*aRvc%Mri)AJaAG!p z$X!_9Ds;Zz)f+;%s&dRcJt2==P{^j3bf0M=nJd&xwUGlUFn?H=2W(*2I2Gdu zv!gYCwM10aeus)`RIZSrCK=&oKaO_Ry~D1B5!y0R=%!i2*KfXGYX&gNv_u+n9wiR5 z*e$Zjju&ODRW3phN925%S(jL+bCHv6rZtc?!*`1TyYXT6%Ju=|X;6D@lq$8T zW{Y|e39ioPez(pBH%k)HzFITXHvnD6hw^lIoUMA;qAJ^CU?top1fo@s7xT13Fvn1H z6JWa-6+FJF#x>~+A;D~;VDs26>^oH0EI`IYT2iagy23?nyJ==i{g4%HrAf1-*v zK1)~@&(KkwR7TL}L(A@C_S0G;-GMDy=MJn2$FP5s<%wC)4jC5PXoxrQBFZ_k0P{{s@sz+gX`-!=T8rcB(=7vW}^K6oLWMmp(rwDh}b zwaGGd>yEy6fHv%jM$yJXo5oMAQ>c9j`**}F?MCry;T@47@r?&sKHgVe$MCqk#Z_3S z1GZI~nOEN*P~+UaFGnj{{Jo@16`(qVNtbU>O0Hf57-P>x8Jikp=`s8xWs^dAJ9lCQ z)GFm+=OV%AMVqVATtN@|vp61VVAHRn87}%PC^RAzJ%JngmZTasWBAWsoAqBU+8L8u z4A&Pe?fmTm0?mK-BL9t+{y7o(7jm+RpOhL9KnY#E&qu^}B6=K_dB}*VlSEiC9fn)+V=J;OnN)Ta5v66ic1rG+dGAJ1 z1%Zb_+!$=tQ~lxQrzv3x#CPb?CekEkA}0MYSgx$Jdd}q8+R=ma$|&1a#)TQ=l$1tQ z=tL9&_^vJ)Pk}EDO-va`UCT1m#Uty1{v^A3P~83_#v^ozH}6*9mIjIr;t3Uv%@VeW zGL6(CwCUp)Jq%G0bIG%?{_*Y#5IHf*5M@wPo6A{$Um++Co$wLC=J1aoG93&T7Ho}P z=mGEPP7GbvoG!uD$k(H3A$Z))+i{Hy?QHdk>3xSBXR0j!11O^mEe9RHmw!pvzv?Ua~2_l2Yh~_!s1qS`|0~0)YsbHSz8!mG)WiJE| z2f($6TQtt6L_f~ApQYQKSb=`053LgrQq7G@98#igV>y#i==-nEjQ!XNu9 z~;mE+gtj4IDDNQJ~JVk5Ux6&LCSFL!y=>79kE9=V}J7tD==Ga+IW zX)r7>VZ9dY=V&}DR))xUoV!u(Z|%3ciQi_2jl}3=$Agc(`RPb z8kEBpvY>1FGQ9W$n>Cq=DIpski};nE)`p3IUw1Oz0|wxll^)4dq3;CCY@RyJgFgc# zKouFh!`?Xuo{IMz^xi-h=StCis_M7yq$u) z?XHvw*HP0VgR+KR6wI)jEMX|ssqYvSf*_3W8zVTQzD?3>H!#>InzpSO)@SC8q*ii- z%%h}_#0{4JG;Jm`4zg};BPTGkYamx$Xo#O~lBirRY)q=5M45n{GCfV7h9qwyu1NxOMoP4)jjZMxmT|IQQh0U7C$EbnMN<3)Kk?fFHYq$d|ICu>KbY_hO zTZM+uKHe(cIZfEqyzyYSUBZa8;Fcut-GN!HSA9ius`ltNebF46ZX_BbZNU}}ZOm{M2&nANL9@0qvih15(|`S~z}m&h!u4x~(%MAO$jHRWNfuxWF#B)E&g3ghSQ9|> z(MFaLQj)NE0lowyjvg8z0#m6FIuKE9lDO~Glg}nSb7`~^&#(Lw{}GVOS>U)m8bF}x zVjbXljBm34Cs-yM6TVusr+3kYFjr28STT3g056y3cH5Tmge~ASxBj z%|yb>$eF;WgrcOZf569sDZOVwoo%8>XO>XQOX1OyN9I-SQgrm;U;+#3OI(zrWyow3 zk==|{lt2xrQ%FIXOTejR>;wv(Pb8u8}BUpx?yd(Abh6? zsoO3VYWkeLnF43&@*#MQ9-i-d0t*xN-UEyNKeyNMHw|A(k(_6QKO=nKMCxD(W(Yop zsRQ)QeL4X3Lxp^L%wzi2-WVSsf61dqliPUM7srDB?Wm6Lzn0&{*}|IsKQW;02(Y&| zaTKv|`U(pSzuvR6Rduu$wzK_W-Y-7>7s?G$)U}&uK;<>vU}^^ns@Z!p+9?St1s)dG zK%y6xkPyyS1$~&6v{kl?Md6gwM|>mt6Upm>oa8RLD^8T{0?HC!Z>;(Bob7el(DV6x zi`I)$&E&ngwFS@bi4^xFLAn`=fzTC;aimE^!cMI2n@Vo%Ae-ne`RF((&5y6xsjjAZ zVguVoQ?Z9uk$2ON;ersE%PU*xGO@T*;j1BO5#TuZKEf(mB7|g7pcEA=nYJ{s3vlbg zd4-DUlD{*6o%Gc^N!Nptgay>j6E5;3psI+C3Q!1ZIbeCubW%w4pq9)MSDyB{HLm|k zxv-{$$A*pS@csolri$Ge<4VZ}e~78JOL-EVyrbxKra^d{?|NnPp86!q>t<&IP07?Z z^>~IK^k#OEKgRH+LjllZXk7iA>2cfH6+(e&9ku5poo~6y{GC5>(bRK7hwjiurqAiZ zg*DmtgY}v83IjE&AbiWgMyFbaRUPZ{lYiz$U^&Zt2YjG<%m((&_JUbZcfJ22(>bi5 z!J?<7AySj0JZ&<-qXX;mcV!f~>G=sB0KnjWca4}vrtunD^1TrpfeS^4dvFr!65knK zZh`d;*VOkPs4*-9kL>$GP0`(M!j~B;#x?Ba~&s6CopvO86oM?-? zOw#dIRc;6A6T?B`Qp%^<U5 z19x(ywSH$_N+Io!6;e?`tWaM$`=Db!gzx|lQ${DG!zb1Zl&|{kX0y6xvO1o z220r<-oaS^^R2pEyY;=Qllqpmue|5yI~D|iI!IGt@iod{Opz@*ml^w2bNs)p`M(Io z|E;;m*Xpjd9l)4G#KaWfV(t8YUn@A;nK^#xgv=LtnArX|vWQVuw3}B${h+frU2>9^ z!l6)!Uo4`5k`<<;E(ido7M6lKTgWezNLq>U*=uz&s=cc$1%>VrAeOoUtA|T6gO4>UNqsdK=NF*8|~*sl&wI=x9-EGiq*aqV!(VVXA57 zw9*o6Ir8Lj1npUXvlevtn(_+^X5rzdR>#(}4YcB9O50q97%rW2me5_L=%ffYPUSRc z!vv?Kv>dH994Qi>U(a<0KF6NH5b16enCp+mw^Hb3Xs1^tThFpz!3QuN#}KBbww`(h z7GO)1olDqy6?T$()R7y%NYx*B0k_2IBiZ14&8|JPFxeMF{vW>HF-Vi3+ZOI=+qP}n zw(+!WcTd~4ZJX1!ZM&y!+uyt=&i!+~d(V%GjH;-NsEEv6nS1TERt|RHh!0>W4+4pp z1-*EzAM~i`+1f(VEHI8So`S`akPfPTfq*`l{Fz`hS%k#JS0cjT2mS0#QLGf=J?1`he3W*;m4)ce8*WFq1sdP=~$5RlH1EdWm|~dCvKOi4*I_96{^95p#B<(n!d?B z=o`0{t+&OMwKcxiBECznJcfH!fL(z3OvmxP#oWd48|mMjpE||zdiTBdWelj8&Qosv zZFp@&UgXuvJw5y=q6*28AtxZzo-UUpkRW%ne+Ylf!V-0+uQXBW=5S1o#6LXNtY5!I z%Rkz#(S8Pjz*P7bqB6L|M#Er{|QLae-Y{KA>`^} z@lPjeX>90X|34S-7}ZVXe{wEei1<{*e8T-Nbj8JmD4iwcE+Hg_zhkPVm#=@b$;)h6 z<<6y`nPa`f3I6`!28d@kdM{uJOgM%`EvlQ5B2bL)Sl=|y@YB3KeOzz=9cUW3clPAU z^sYc}xf9{4Oj?L5MOlYxR{+>w=vJjvbyO5}ptT(o6dR|ygO$)nVCvNGnq(6;bHlBd zl?w-|plD8spjDF03g5ip;W3Z z><0{BCq!Dw;h5~#1BuQilq*TwEu)qy50@+BE4bX28+7erX{BD4H)N+7U`AVEuREE8 z;X?~fyhF-x_sRfHIj~6f(+^@H)D=ngP;mwJjxhQUbUdzk8f94Ab%59-eRIq?ZKrwD z(BFI=)xrUlgu(b|hAysqK<}8bslmNNeD=#JW*}^~Nrswn^xw*nL@Tx!49bfJecV&KC2G4q5a!NSv)06A_5N3Y?veAz;Gv+@U3R% z)~UA8-0LvVE{}8LVDOHzp~2twReqf}ODIyXMM6=W>kL|OHcx9P%+aJGYi_Om)b!xe zF40Vntn0+VP>o<$AtP&JANjXBn7$}C@{+@3I@cqlwR2MdwGhVPxlTIcRVu@Ho-wO` z_~Or~IMG)A_`6-p)KPS@cT9mu9RGA>dVh5wY$NM9-^c@N=hcNaw4ITjm;iWSP^ZX| z)_XpaI61<+La+U&&%2a z0za$)-wZP@mwSELo#3!PGTt$uy0C(nTT@9NX*r3Ctw6J~7A(m#8fE)0RBd`TdKfAT zCf@$MAxjP`O(u9s@c0Fd@|}UQ6qp)O5Q5DPCeE6mSIh|Rj{$cAVIWsA=xPKVKxdhg zLzPZ`3CS+KIO;T}0Ip!fAUaNU>++ZJZRk@I(h<)RsJUhZ&Ru9*!4Ptn;gX^~4E8W^TSR&~3BAZc#HquXn)OW|TJ`CTahk+{qe`5+ixON^zA9IFd8)kc%*!AiLu z>`SFoZ5bW-%7}xZ>gpJcx_hpF$2l+533{gW{a7ce^B9sIdmLrI0)4yivZ^(Vh@-1q zFT!NQK$Iz^xu%|EOK=n>ug;(7J4OnS$;yWmq>A;hsD_0oAbLYhW^1Vdt9>;(JIYjf zdb+&f&D4@4AS?!*XpH>8egQvSVX`36jMd>$+RgI|pEg))^djhGSo&#lhS~9%NuWfX zDDH;3T*GzRT@5=7ibO>N-6_XPBYxno@mD_3I#rDD?iADxX`! zh*v8^i*JEMzyN#bGEBz7;UYXki*Xr(9xXax(_1qVW=Ml)kSuvK$coq2A(5ZGhs_pF z$*w}FbN6+QDseuB9=fdp_MTs)nQf!2SlROQ!gBJBCXD&@-VurqHj0wm@LWX-TDmS= z71M__vAok|@!qgi#H&H%Vg-((ZfxPAL8AI{x|VV!9)ZE}_l>iWk8UPTGHs*?u7RfP z5MC&=c6X;XlUzrz5q?(!eO@~* zoh2I*%J7dF!!_!vXoSIn5o|wj1#_>K*&CIn{qSaRc&iFVxt*^20ngCL;QonIS>I5^ zMw8HXm>W0PGd*}Ko)f|~dDd%;Wu_RWI_d;&2g6R3S63Uzjd7dn%Svu-OKpx*o|N>F zZg=-~qLb~VRLpv`k zWSdfHh@?dp=s_X`{yxOlxE$4iuyS;Z-x!*E6eqmEm*j2bE@=ZI0YZ5%Yj29!5+J$4h{s($nakA`xgbO8w zi=*r}PWz#lTL_DSAu1?f%-2OjD}NHXp4pXOsCW;DS@BC3h-q4_l`<))8WgzkdXg3! zs1WMt32kS2E#L0p_|x+x**TFV=gn`m9BWlzF{b%6j-odf4{7a4y4Uaef@YaeuPhU8 zHBvRqN^;$Jizy+ z=zW{E5<>2gp$pH{M@S*!sJVQU)b*J5*bX4h>5VJve#Q6ga}cQ&iL#=(u+KroWrxa%8&~p{WEUF0il=db;-$=A;&9M{Rq`ouZ5m%BHT6%st%saGsD6)fQgLN}x@d3q>FC;=f%O3Cyg=Ke@Gh`XW za@RajqOE9UB6eE=zhG%|dYS)IW)&y&Id2n7r)6p_)vlRP7NJL(x4UbhlcFXWT8?K=%s7;z?Vjts?y2+r|uk8Wt(DM*73^W%pAkZa1Jd zNoE)8FvQA>Z`eR5Z@Ig6kS5?0h;`Y&OL2D&xnnAUzQz{YSdh0k zB3exx%A2TyI)M*EM6htrxSlep!Kk(P(VP`$p0G~f$smld6W1r_Z+o?=IB@^weq>5VYsYZZR@` z&XJFxd5{|KPZmVOSxc@^%71C@;z}}WhbF9p!%yLj3j%YOlPL5s>7I3vj25 z@xmf=*z%Wb4;Va6SDk9cv|r*lhZ`(y_*M@>q;wrn)oQx%B(2A$9(74>;$zmQ!4fN; z>XurIk-7@wZys<+7XL@0Fhe-f%*=(weaQEdR9Eh6>Kl-EcI({qoZqyzziGwpg-GM#251sK_ z=3|kitS!j%;fpc@oWn65SEL73^N&t>Ix37xgs= zYG%eQDJc|rqHFia0!_sm7`@lvcv)gfy(+KXA@E{3t1DaZ$DijWAcA)E0@X?2ziJ{v z&KOYZ|DdkM{}t+@{@*6ge}m%xfjIxi%qh`=^2Rwz@w0cCvZ&Tc#UmCDbVwABrON^x zEBK43FO@weA8s7zggCOWhMvGGE`baZ62cC)VHyy!5Zbt%ieH+XN|OLbAFPZWyC6)p z4P3%8sq9HdS3=ih^0OOlqTPbKuzQ?lBEI{w^ReUO{V?@`ARsL|S*%yOS=Z%sF)>-y z(LAQdhgAcuF6LQjRYfdbD1g4o%tV4EiK&ElLB&^VZHbrV1K>tHTO{#XTo>)2UMm`2 z^t4s;vnMQgf-njU-RVBRw0P0-m#d-u`(kq7NL&2T)TjI_@iKuPAK-@oH(J8?%(e!0Ir$yG32@CGUPn5w4)+9@8c&pGx z+K3GKESI4*`tYlmMHt@br;jBWTei&(a=iYslc^c#RU3Q&sYp zSG){)V<(g7+8W!Wxeb5zJb4XE{I|&Y4UrFWr%LHkdQ;~XU zgy^dH-Z3lmY+0G~?DrC_S4@=>0oM8Isw%g(id10gWkoz2Q%7W$bFk@mIzTCcIB(K8 zc<5h&ZzCdT=9n-D>&a8vl+=ZF*`uTvQviG_bLde*k>{^)&0o*b05x$MO3gVLUx`xZ z43j+>!u?XV)Yp@MmG%Y`+COH2?nQcMrQ%k~6#O%PeD_WvFO~Kct za4XoCM_X!c5vhRkIdV=xUB3xI2NNStK*8_Zl!cFjOvp-AY=D;5{uXj}GV{LK1~IE2 z|KffUiBaStRr;10R~K2VVtf{TzM7FaPm;Y(zQjILn+tIPSrJh&EMf6evaBKIvi42-WYU9Vhj~3< zZSM-B;E`g_o8_XTM9IzEL=9Lb^SPhe(f(-`Yh=X6O7+6ALXnTcUFpI>ekl6v)ZQeNCg2 z^H|{SKXHU*%nBQ@I3It0m^h+6tvI@FS=MYS$ZpBaG7j#V@P2ZuYySbp@hA# ze(kc;P4i_-_UDP?%<6>%tTRih6VBgScKU^BV6Aoeg6Uh(W^#J^V$Xo^4#Ekp ztqQVK^g9gKMTHvV7nb64UU7p~!B?>Y0oFH5T7#BSW#YfSB@5PtE~#SCCg3p^o=NkMk$<8- z6PT*yIKGrvne7+y3}_!AC8NNeI?iTY(&nakN>>U-zT0wzZf-RuyZk^X9H-DT_*wk= z;&0}6LsGtfVa1q)CEUPlx#(ED@-?H<1_FrHU#z5^P3lEB|qsxEyn%FOpjx z3S?~gvoXy~L(Q{Jh6*i~=f%9kM1>RGjBzQh_SaIDfSU_9!<>*Pm>l)cJD@wlyxpBV z4Fmhc2q=R_wHCEK69<*wG%}mgD1=FHi4h!98B-*vMu4ZGW~%IrYSLGU{^TuseqVgV zLP<%wirIL`VLyJv9XG_p8w@Q4HzNt-o;U@Au{7%Ji;53!7V8Rv0^Lu^Vf*sL>R(;c zQG_ZuFl)Mh-xEIkGu}?_(HwkB2jS;HdPLSxVU&Jxy9*XRG~^HY(f0g8Q}iqnVmgjI zfd=``2&8GsycjR?M%(zMjn;tn9agcq;&rR!Hp z$B*gzHsQ~aXw8c|a(L^LW(|`yGc!qOnV(ZjU_Q-4z1&0;jG&vAKuNG=F|H?@m5^N@ zq{E!1n;)kNTJ>|Hb2ODt-7U~-MOIFo%9I)_@7fnX+eMMNh>)V$IXesJpBn|uo8f~#aOFytCT zf9&%MCLf8mp4kwHTcojWmM3LU=#|{3L>E}SKwOd?%{HogCZ_Z1BSA}P#O(%H$;z7XyJ^sjGX;j5 zrzp>|Ud;*&VAU3x#f{CKwY7Vc{%TKKqmB@oTHA9;>?!nvMA;8+Jh=cambHz#J18x~ zs!dF>$*AnsQ{{82r5Aw&^7eRCdvcgyxH?*DV5(I$qXh^zS>us*I66_MbL8y4d3ULj z{S(ipo+T3Ag!+5`NU2sc+@*m{_X|&p#O-SAqF&g_n7ObB82~$p%fXA5GLHMC+#qqL zdt`sJC&6C2)=juQ_!NeD>U8lDVpAOkW*khf7MCcs$A(wiIl#B9HM%~GtQ^}yBPjT@ z+E=|A!Z?A(rwzZ;T}o6pOVqHzTr*i;Wrc%&36kc@jXq~+w8kVrs;%=IFdACoLAcCAmhFNpbP8;s`zG|HC2Gv?I~w4ITy=g$`0qMQdkijLSOtX6xW%Z9Nw<;M- zMN`c7=$QxN00DiSjbVt9Mi6-pjv*j(_8PyV-il8Q-&TwBwH1gz1uoxs6~uU}PrgWB zIAE_I-a1EqlIaGQNbcp@iI8W1sm9fBBNOk(k&iLBe%MCo#?xI$%ZmGA?=)M9D=0t7 zc)Q0LnI)kCy{`jCGy9lYX%mUsDWwsY`;jE(;Us@gmWPqjmXL+Hu#^;k%eT>{nMtzj zsV`Iy6leTA8-PndszF;N^X@CJrTw5IIm!GPeu)H2#FQitR{1p;MasQVAG3*+=9FYK zw*k!HT(YQorfQj+1*mCV458(T5=fH`um$gS38hw(OqVMyunQ;rW5aPbF##A3fGH6h z@W)i9Uff?qz`YbK4c}JzQpuxuE3pcQO)%xBRZp{zJ^-*|oryTxJ-rR+MXJ)!f=+pp z10H|DdGd2exhi+hftcYbM0_}C0ZI-2vh+$fU1acsB-YXid7O|=9L!3e@$H*6?G*Zp z%qFB(sgl=FcC=E4CYGp4CN>=M8#5r!RU!u+FJVlH6=gI5xHVD&k;Ta*M28BsxfMV~ zLz+@6TxnfLhF@5=yQo^1&S}cmTN@m!7*c6z;}~*!hNBjuE>NLVl2EwN!F+)0$R1S! zR|lF%n!9fkZ@gPW|x|B={V6x3`=jS*$Pu0+5OWf?wnIy>Y1MbbGSncpKO0qE(qO=ts z!~@&!N`10S593pVQu4FzpOh!tvg}p%zCU(aV5=~K#bKi zHdJ1>tQSrhW%KOky;iW+O_n;`l9~omqM%sdxdLtI`TrJzN6BQz+7xOl*rM>xVI2~# z)7FJ^Dc{DC<%~VS?@WXzuOG$YPLC;>#vUJ^MmtbSL`_yXtNKa$Hk+l-c!aC7gn(Cg ze?YPYZ(2Jw{SF6MiO5(%_pTo7j@&DHNW`|lD`~{iH+_eSTS&OC*2WTT*a`?|9w1dh zh1nh@$a}T#WE5$7Od~NvSEU)T(W$p$s5fe^GpG+7fdJ9=enRT9$wEk+ZaB>G3$KQO zgq?-rZZnIv!p#>Ty~}c*Lb_jxJg$eGM*XwHUwuQ|o^}b3^T6Bxx{!?va8aC@-xK*H ztJBFvFfsSWu89%@b^l3-B~O!CXs)I6Y}y#0C0U0R0WG zybjroj$io0j}3%P7zADXOwHwafT#uu*zfM!oD$6aJx7+WL%t-@6^rD_a_M?S^>c;z zMK580bZXo1f*L$CuMeM4Mp!;P@}b~$cd(s5*q~FP+NHSq;nw3fbWyH)i2)-;gQl{S zZO!T}A}fC}vUdskGSq&{`oxt~0i?0xhr6I47_tBc`fqaSrMOzR4>0H^;A zF)hX1nfHs)%Zb-(YGX;=#2R6C{BG;k=?FfP?9{_uFLri~-~AJ;jw({4MU7e*d)?P@ zXX*GkNY9ItFjhwgAIWq7Y!ksbMzfqpG)IrqKx9q{zu%Mdl+{Dis#p9q`02pr1LG8R z@As?eG!>IoROgS!@J*to<27coFc1zpkh?w=)h9CbYe%^Q!Ui46Y*HO0mr% zEff-*$ndMNw}H2a5@BsGj5oFfd!T(F&0$<{GO!Qdd?McKkorh=5{EIjDTHU`So>8V zBA-fqVLb2;u7UhDV1xMI?y>fe3~4urv3%PX)lDw+HYa;HFkaLqi4c~VtCm&Ca+9C~ zge+67hp#R9`+Euq59WhHX&7~RlXn=--m8$iZ~~1C8cv^2(qO#X0?vl91gzUKBeR1J z^p4!!&7)3#@@X&2aF2-)1Ffcc^F8r|RtdL2X%HgN&XU-KH2SLCbpw?J5xJ*!F-ypZ zMG%AJ!Pr&}`LW?E!K~=(NJxuSVTRCGJ$2a*Ao=uUDSys!OFYu!Vs2IT;xQ6EubLIl z+?+nMGeQQhh~??0!s4iQ#gm3!BpMpnY?04kK375e((Uc7B3RMj;wE?BCoQGu=UlZt!EZ1Q*auI)dj3Jj{Ujgt zW5hd~-HWBLI_3HuO) zNrb^XzPsTIb=*a69wAAA3J6AAZZ1VsYbIG}a`=d6?PjM)3EPaDpW2YP$|GrBX{q*! z$KBHNif)OKMBCFP5>!1d=DK>8u+Upm-{hj5o|Wn$vh1&K!lVfDB&47lw$tJ?d5|=B z^(_9=(1T3Fte)z^>|3**n}mIX;mMN5v2F#l(q*CvU{Ga`@VMp#%rQkDBy7kYbmb-q z<5!4iuB#Q_lLZ8}h|hPODI^U6`gzLJre9u3k3c#%86IKI*^H-@I48Bi*@avYm4v!n0+v zWu{M{&F8#p9cx+gF0yTB_<2QUrjMPo9*7^-uP#~gGW~y3nfPAoV%amgr>PSyVAd@l)}8#X zR5zV6t*uKJZL}?NYvPVK6J0v4iVpwiN|>+t3aYiZSp;m0!(1`bHO}TEtWR1tY%BPB z(W!0DmXbZAsT$iC13p4f>u*ZAy@JoLAkJhzFf1#4;#1deO8#8d&89}en&z!W&A3++^1(;>0SB1*54d@y&9Pn;^IAf3GiXbfT`_>{R+Xv; zQvgL>+0#8-laO!j#-WB~(I>l0NCMt_;@Gp_f0#^c)t?&#Xh1-7RR0@zPyBz!U#0Av zT?}n({(p?p7!4S2ZBw)#KdCG)uPnZe+U|0{BW!m)9 zi_9$F?m<`2!`JNFv+w8MK_K)qJ^aO@7-Ig>cM4-r0bi=>?B_2mFNJ}aE3<+QCzRr*NA!QjHw# z`1OsvcoD0?%jq{*7b!l|L1+Tw0TTAM4XMq7*ntc-Ived>Sj_ZtS|uVdpfg1_I9knY z2{GM_j5sDC7(W&}#s{jqbybqJWyn?{PW*&cQIU|*v8YGOKKlGl@?c#TCnmnAkAzV- zmK={|1G90zz=YUvC}+fMqts0d4vgA%t6Jhjv?d;(Z}(Ep8fTZfHA9``fdUHkA+z3+ zhh{ohP%Bj?T~{i0sYCQ}uC#5BwN`skI7`|c%kqkyWIQ;!ysvA8H`b-t()n6>GJj6xlYDu~8qX{AFo$Cm3d|XFL=4uvc?Keb zzb0ZmMoXca6Mob>JqkNuoP>B2Z>D`Q(TvrG6m`j}-1rGP!g|qoL=$FVQYxJQjFn33lODt3Wb1j8VR zlR++vIT6^DtYxAv_hxupbLLN3e0%A%a+hWTKDV3!Fjr^cWJ{scsAdfhpI)`Bms^M6 zQG$waKgFr=c|p9Piug=fcJvZ1ThMnNhQvBAg-8~b1?6wL*WyqXhtj^g(Ke}mEfZVM zJuLNTUVh#WsE*a6uqiz`b#9ZYg3+2%=C(6AvZGc=u&<6??!slB1a9K)=VL zY9EL^mfyKnD zSJyYBc_>G;5RRnrNgzJz#Rkn3S1`mZgO`(r5;Hw6MveN(URf_XS-r58Cn80K)ArH4 z#Rrd~LG1W&@ttw85cjp8xV&>$b%nSXH_*W}7Ch2pg$$c0BdEo-HWRTZcxngIBJad> z;C>b{jIXjb_9Jis?NZJsdm^EG}e*pR&DAy0EaSGi3XWTa(>C%tz1n$u?5Fb z1qtl?;_yjYo)(gB^iQq?=jusF%kywm?CJP~zEHi0NbZ);$(H$w(Hy@{i>$wcVRD_X|w-~(0Z9BJyh zhNh;+eQ9BEIs;tPz%jSVnfCP!3L&9YtEP;svoj_bNzeGSQIAjd zBss@A;)R^WAu-37RQrM%{DfBNRx>v!G31Z}8-El9IOJlb_MSoMu2}GDYycNaf>uny z+8xykD-7ONCM!APry_Lw6-yT>5!tR}W;W`C)1>pxSs5o1z#j7%m=&=7O4hz+Lsqm` z*>{+xsabZPr&X=}G@obTb{nPTkccJX8w3CG7X+1+t{JcMabv~UNv+G?txRqXib~c^Mo}`q{$`;EBNJ;#F*{gvS12kV?AZ%O0SFB$^ zn+}!HbmEj}w{Vq(G)OGAzH}R~kS^;(-s&=ectz8vN!_)Yl$$U@HNTI-pV`LSj7Opu zTZ5zZ)-S_{GcEQPIQXLQ#oMS`HPu{`SQiAZ)m1at*Hy%3xma|>o`h%E%8BEbi9p0r zVjcsh<{NBKQ4eKlXU|}@XJ#@uQw*$4BxKn6#W~I4T<^f99~(=}a`&3(ur8R9t+|AQ zWkQx7l}wa48-jO@ft2h+7qn%SJtL%~890FG0s5g*kNbL3I&@brh&f6)TlM`K^(bhr zJWM6N6x3flOw$@|C@kPi7yP&SP?bzP-E|HSXQXG>7gk|R9BTj`e=4de9C6+H7H7n# z#GJeVs1mtHhLDmVO?LkYRQc`DVOJ_vdl8VUihO-j#t=0T3%Fc1f9F73ufJz*adn*p zc%&vi(4NqHu^R>sAT_0EDjVR8bc%wTz#$;%NU-kbDyL_dg0%TFafZwZ?5KZpcuaO54Z9hX zD$u>q!-9`U6-D`E#`W~fIfiIF5_m6{fvM)b1NG3xf4Auw;Go~Fu7cth#DlUn{@~yu z=B;RT*dp?bO}o%4x7k9v{r=Y@^YQ^UUm(Qmliw8brO^=NP+UOohLYiaEB3^DB56&V zK?4jV61B|1Uj_5fBKW;8LdwOFZKWp)g{B%7g1~DgO&N& z#lisxf?R~Z@?3E$Mms$$JK8oe@X`5m98V*aV6Ua}8Xs2#A!{x?IP|N(%nxsH?^c{& z@vY&R1QmQs83BW28qAmJfS7MYi=h(YK??@EhjL-t*5W!p z^gYX!Q6-vBqcv~ruw@oMaU&qp0Fb(dbVzm5xJN%0o_^@fWq$oa3X?9s%+b)x4w-q5Koe(@j6Ez7V@~NRFvd zfBH~)U5!ix3isg`6be__wBJp=1@yfsCMw1C@y+9WYD9_C%{Q~7^0AF2KFryfLlUP# zwrtJEcH)jm48!6tUcxiurAMaiD04C&tPe6DI0#aoqz#Bt0_7_*X*TsF7u*zv(iEfA z;$@?XVu~oX#1YXtceQL{dSneL&*nDug^OW$DSLF0M1Im|sSX8R26&)<0Fbh^*l6!5wfSu8MpMoh=2l z^^0Sr$UpZp*9oqa23fcCfm7`ya2<4wzJ`Axt7e4jJrRFVf?nY~2&tRL* zd;6_njcz01c>$IvN=?K}9ie%Z(BO@JG2J}fT#BJQ+f5LFSgup7i!xWRKw6)iITjZU z%l6hPZia>R!`aZjwCp}I zg)%20;}f+&@t;(%5;RHL>K_&7MH^S+7<|(SZH!u zznW|jz$uA`P9@ZWtJgv$EFp>)K&Gt+4C6#*khZQXS*S~6N%JDT$r`aJDs9|uXWdbg zBwho$phWx}x!qy8&}6y5Vr$G{yGSE*r$^r{}pw zVTZKvikRZ`J_IJrjc=X1uw?estdwm&bEahku&D04HD+0Bm~q#YGS6gp!KLf$A{%Qd z&&yX@Hp>~(wU{|(#U&Bf92+1i&Q*-S+=y=3pSZy$#8Uc$#7oiJUuO{cE6=tsPhwPe| zxQpK>`Dbka`V)$}e6_OXKLB%i76~4N*zA?X+PrhH<&)}prET;kel24kW%+9))G^JI zsq7L{P}^#QsZViX%KgxBvEugr>ZmFqe^oAg?{EI=&_O#e)F3V#rc z8$4}0Zr19qd3tE4#$3_f=Bbx9oV6VO!d3(R===i-7p=Vj`520w0D3W6lQfY48}!D* z&)lZMG;~er2qBoI2gsX+Ts-hnpS~NYRDtPd^FPzn!^&yxRy#CSz(b&E*tL|jIkq|l zf%>)7Dtu>jCf`-7R#*GhGn4FkYf;B$+9IxmqH|lf6$4irg{0ept__%)V*R_OK=T06 zyT_m-o@Kp6U{l5h>W1hGq*X#8*y@<;vsOFqEjTQXFEotR+{3}ODDnj;o0@!bB5x=N z394FojuGOtVKBlVRLtHp%EJv_G5q=AgF)SKyRN5=cGBjDWv4LDn$IL`*=~J7u&Dy5 zrMc83y+w^F&{?X(KOOAl-sWZDb{9X9#jrQtmrEXD?;h-}SYT7yM(X_6qksM=K_a;Z z3u0qT0TtaNvDER_8x*rxXw&C^|h{P1qxK|@pS7vdlZ#P z7PdB7MmC2}%sdzAxt>;WM1s0??`1983O4nFK|hVAbHcZ3x{PzytQLkCVk7hA!Lo` zEJH?4qw|}WH{dc4z%aB=0XqsFW?^p=X}4xnCJXK%c#ItOSjdSO`UXJyuc8bh^Cf}8 z@Ht|vXd^6{Fgai8*tmyRGmD_s_nv~r^Fy7j`Bu`6=G)5H$i7Q7lvQnmea&TGvJp9a|qOrUymZ$6G|Ly z#zOCg++$3iB$!6!>215A4!iryregKuUT344X)jQb3|9qY>c0LO{6Vby05n~VFzd?q zgGZv&FGlkiH*`fTurp>B8v&nSxNz)=5IF$=@rgND4d`!AaaX;_lK~)-U8la_Wa8i?NJC@BURO*sUW)E9oyv3RG^YGfN%BmxzjlT)bp*$<| zX3tt?EAy<&K+bhIuMs-g#=d1}N_?isY)6Ay$mDOKRh z4v1asEGWoAp=srraLW^h&_Uw|6O+r;wns=uwYm=JN4Q!quD8SQRSeEcGh|Eb5Jg8m zOT}u;N|x@aq)=&;wufCc^#)5U^VcZw;d_wwaoh9$p@Xrc{DD6GZUqZ ziC6OT^zSq@-lhbgR8B+e;7_Giv;DK5gn^$bs<6~SUadiosfewWDJu`XsBfOd1|p=q zE>m=zF}!lObA%ePey~gqU8S6h-^J2Y?>7)L2+%8kV}Gp=h`Xm_}rlm)SyUS=`=S7msKu zC|T!gPiI1rWGb1z$Md?0YJQ;%>uPLOXf1Z>N~`~JHJ!^@D5kSXQ4ugnFZ>^`zH8CAiZmp z6Ms|#2gcGsQ{{u7+Nb9sA?U>(0e$5V1|WVwY`Kn)rsnnZ4=1u=7u!4WexZD^IQ1Jk zfF#NLe>W$3m&C^ULjdw+5|)-BSHwpegdyt9NYC{3@QtMfd8GrIWDu`gd0nv-3LpGCh@wgBaG z176tikL!_NXM+Bv#7q^cyn9$XSeZR6#!B4JE@GVH zoobHZN_*RF#@_SVYKkQ_igme-Y5U}cV(hkR#k1c{bQNMji zU7aE`?dHyx=1`kOYZo_8U7?3-7vHOp`Qe%Z*i+FX!s?6huNp0iCEW-Z7E&jRWmUW_ z67j>)Ew!yq)hhG4o?^z}HWH-e=es#xJUhDRc4B51M4~E-l5VZ!&zQq`gWe`?}#b~7w1LH4Xa-UCT5LXkXQWheBa2YJYbyQ zl1pXR%b(KCXMO0OsXgl0P0Og<{(@&z1aokU-Pq`eQq*JYgt8xdFQ6S z6Z3IFSua8W&M#`~*L#r>Jfd6*BzJ?JFdBR#bDv$_0N!_5vnmo@!>vULcDm`MFU823 zpG9pqjqz^FE5zMDoGqhs5OMmC{Y3iVcl>F}5Rs24Y5B^mYQ;1T&ks@pIApHOdrzXF z-SdX}Hf{X;TaSxG_T$0~#RhqKISGKNK47}0*x&nRIPtmdwxc&QT3$8&!3fWu1eZ_P zJveQj^hJL#Sn!*4k`3}(d(aasl&7G0j0-*_2xtAnoX1@9+h zO#c>YQg60Z;o{Bi=3i7S`Ic+ZE>K{(u|#)9y}q*j8uKQ1^>+(BI}m%1v3$=4ojGBc zm+o1*!T&b}-lVvZqIUBc8V}QyFEgm#oyIuC{8WqUNV{Toz`oxhYpP!_p2oHHh5P@iB*NVo~2=GQm+8Yrkm2Xjc_VyHg1c0>+o~@>*Qzo zHVBJS>$$}$_4EniTI;b1WShX<5-p#TPB&!;lP!lBVBbLOOxh6FuYloD%m;n{r|;MU3!q4AVkua~fieeWu2 zQAQ$ue(IklX6+V;F1vCu-&V?I3d42FgWgsb_e^29ol}HYft?{SLf>DrmOp9o!t>I^ zY7fBCk+E8n_|apgM|-;^=#B?6RnFKlN`oR)`e$+;D=yO-(U^jV;rft^G_zl`n7qnM zL z*-Y4Phq+ZI1$j$F-f;`CD#|`-T~OM5Q>x}a>B~Gb3-+9i>Lfr|Ca6S^8g*{*?_5!x zH_N!SoRP=gX1?)q%>QTY!r77e2j9W(I!uAz{T`NdNmPBBUzi2{`XMB^zJGGwFWeA9 z{fk33#*9SO0)DjROug+(M)I-pKA!CX;IY(#gE!UxXVsa)X!UftIN98{pt#4MJHOhY zM$_l}-TJlxY?LS6Nuz1T<44m<4i^8k@D$zuCPrkmz@sdv+{ciyFJG2Zwy&%c7;atIeTdh!a(R^QXnu1Oq1b42*OQFWnyQ zWeQrdvP|w_idy53Wa<{QH^lFmEd+VlJkyiC>6B#s)F;w-{c;aKIm;Kp50HnA-o3lY z9B~F$gJ@yYE#g#X&3ADx&tO+P_@mnQTz9gv30_sTsaGXkfNYXY{$(>*PEN3QL>I!k zp)KibPhrfX3%Z$H6SY`rXGYS~143wZrG2;=FLj50+VM6soI~up_>fU(2Wl@{BRsMi zO%sL3x?2l1cXTF)k&moNsHfQrQ+wu(gBt{sk#CU=UhrvJIncy@tJX5klLjgMn>~h= zg|FR&;@eh|C7`>s_9c~0-{IAPV){l|Ts`i=)AW;d9&KPc3fMeoTS%8@V~D8*h;&(^>yjT84MM}=%#LS7shLAuuj(0VAYoozhWjq z4LEr?wUe2^WGwdTIgWBkDUJa>YP@5d9^Rs$kCXmMRxuF*YMVrn?0NFyPl}>`&dqZb z<5eqR=ZG3>n2{6v6BvJ`YBZeeTtB88TAY(x0a58EWyuf>+^|x8Qa6wA|1Nb_p|nA zWWa}|z8a)--Wj`LqyFk_a3gN2>5{Rl_wbW?#by7&i*^hRknK%jwIH6=dQ8*-_{*x0j^DUfMX0`|K@6C<|1cgZ~D(e5vBFFm;HTZF(!vT8=T$K+|F)x3kqzBV4-=p1V(lzi(s7jdu0>LD#N=$Lk#3HkG!a zIF<7>%B7sRNzJ66KrFV76J<2bdYhxll0y2^_rdG=I%AgW4~)1Nvz=$1UkE^J%BxLo z+lUci`UcU062os*=`-j4IfSQA{w@y|3}Vk?i;&SSdh8n+$iHA#%ERL{;EpXl6u&8@ zzg}?hkEOUOJt?ZL=pWZFJ19mI1@P=$U5*Im1e_8Z${JsM>Ov?nh8Z zP5QvI!{Jy@&BP48%P2{Jr_VgzW;P@7)M9n|lDT|Ep#}7C$&ud&6>C^5ZiwKIg2McPU(4jhM!BD@@L(Gd*Nu$ji(ljZ<{FIeW_1Mmf;76{LU z-ywN~=uNN)Xi6$<12A9y)K%X|(W0p|&>>4OXB?IiYr||WKDOJPxiSe01NSV-h24^L z_>m$;|C+q!Mj**-qQ$L-*++en(g|hw;M!^%_h-iDjFHLo-n3JpB;p?+o2;`*jpvJU zLY^lt)Un4joij^^)O(CKs@7E%*!w>!HA4Q?0}oBJ7Nr8NQ7QmY^4~jvf0-`%waOLn zdNjAPaC0_7c|RVhw)+71NWjRi!y>C+Bl;Z`NiL^zn2*0kmj5gyhCLCxts*cWCdRI| zjsd=sT5BVJc^$GxP~YF$-U{-?kW6r@^vHXB%{CqYzU@1>dzf#3SYedJG-Rm6^RB7s zGM5PR(yKPKR)>?~vpUIeTP7A1sc8-knnJk*9)3t^e%izbdm>Y=W{$wm(cy1RB-19i za#828DMBY+ps#7Y8^6t)=Ea@%Nkt)O6JCx|ybC;Ap}Z@Zw~*}3P>MZLPb4Enxz9Wf zssobT^(R@KuShj8>@!1M7tm|2%-pYYDxz-5`rCbaTCG5{;Uxm z*g=+H1X8{NUvFGzz~wXa%Eo};I;~`37*WrRU&K0dPSB$yk(Z*@K&+mFal^?c zurbqB-+|Kb5|sznT;?Pj!+kgFY1#Dr;_%A(GIQC{3ct|{*Bji%FNa6c-thbpBkA;U zURV!Dr&X{0J}iht#-Qp2=xzuh(fM>zRoiGrYl5ttw2#r34gC41CCOC31m~^UPTK@s z6;A@)7O7_%C)>bnAXerYuAHdE93>j2N}H${zEc6&SbZ|-fiG*-qtGuy-qDelH(|u$ zorf8_T6Zqe#Ub!+e3oSyrskt_HyW_^5lrWt#30l)tHk|j$@YyEkXUOV;6B51L;M@=NIWZXU;GrAa(LGxO%|im%7F<-6N;en0Cr zLH>l*y?pMwt`1*cH~LdBPFY_l;~`N!Clyfr;7w<^X;&(ZiVdF1S5e(+Q%60zgh)s4 zn2yj$+mE=miVERP(g8}G4<85^-5f@qxh2ec?n+$A_`?qN=iyT1?U@t?V6DM~BIlBB z>u~eXm-aE>R0sQy!-I4xtCNi!!qh?R1!kKf6BoH2GG{L4%PAz0{Sh6xpuyI%*~u)s z%rLuFl)uQUCBQAtMyN;%)zFMx4loh7uTfKeB2Xif`lN?2gq6NhWhfz0u5WP9J>=V2 zo{mLtSy&BA!mSzs&CrKWq^y40JF5a&GSXIi2= z{EYb59J4}VwikL4P=>+mc6{($FNE@e=VUwG+KV21;<@lrN`mnz5jYGASyvz7BOG_6(p^eTxD-4O#lROgon;R35=|nj#eHIfJBYPWG>H>`dHKCDZ3`R{-?HO0mE~(5_WYcFmp8sU?wr*UkAQiNDGc6T zA%}GOLXlOWqL?WwfHO8MB#8M8*~Y*gz;1rWWoVSXP&IbKxbQ8+s%4Jnt?kDsq7btI zCDr0PZ)b;B%!lu&CT#RJzm{l{2fq|BcY85`w~3LSK<><@(2EdzFLt9Y_`;WXL6x`0 zDoQ?=?I@Hbr;*VVll1Gmd8*%tiXggMK81a+T(5Gx6;eNb8=uYn z5BG-0g>pP21NPn>$ntBh>`*})Fl|38oC^9Qz>~MAazH%3Q~Qb!ALMf$srexgPZ2@&c~+hxRi1;}+)-06)!#Mq<6GhP z-Q?qmgo${aFBApb5p}$1OJKTClfi8%PpnczyVKkoHw7Ml9e7ikrF0d~UB}i3vizos zXW4DN$SiEV9{faLt5bHy2a>33K%7Td-n5C*N;f&ZqAg#2hIqEb(y<&f4u5BWJ>2^4 z414GosL=Aom#m&=x_v<0-fp1r%oVJ{T-(xnomNJ(Dryv zh?vj+%=II_nV+@NR+(!fZZVM&(W6{6%9cm+o+Z6}KqzLw{(>E86uA1`_K$HqINlb1 zKelh3-jr2I9V?ych`{hta9wQ2c9=MM`2cC{m6^MhlL2{DLv7C^j z$xXBCnDl_;l|bPGMX@*tV)B!c|4oZyftUlP*?$YU9C_eAsuVHJ58?)zpbr30P*C`T z7y#ao`uE-SOG(Pi+`$=e^mle~)pRrdwL5)N;o{gpW21of(QE#U6w%*C~`v-z0QqBML!!5EeYA5IQB0 z^l01c;L6E(iytN!LhL}wfwP7W9PNAkb+)Cst?qg#$n;z41O4&v+8-zPs+XNb-q zIeeBCh#ivnFLUCwfS;p{LC0O7tm+Sf9Jn)~b%uwP{%69;QC)Ok0t%*a5M+=;y8j=v z#!*pp$9@!x;UMIs4~hP#pnfVc!%-D<+wsG@R2+J&%73lK|2G!EQC)O05TCV=&3g)C!lT=czLpZ@Sa%TYuoE?v8T8`V;e$#Zf2_Nj6nvBgh1)2 GZ~q4|mN%#X literal 59536 zcma&NbC71ylI~qywr$(CZQJHswz}-9F59+k+g;UV+cs{`J?GrGXYR~=-ydruB3JCa zB64N^cILAcWk5iofq)<(fq;O7{th4@;QxID0)qN`mJ?GIqLY#rX8-|G{5M0pdVW5^ zzXk$-2kQTAC?_N@B`&6-N-rmVFE=$QD?>*=4<|!MJu@}isLc4AW#{m2if&A5T5g&~ ziuMQeS*U5sL6J698wOd)K@oK@1{peP5&Esut<#VH^u)gp`9H4)`uE!2$>RTctN+^u z=ASkePDZA-X8)rp%D;p*~P?*a_=*Kwc<^>QSH|^<0>o37lt^+Mj1;4YvJ(JR-Y+?%Nu}JAYj5 z_Qc5%Ao#F?q32i?ZaN2OSNhWL;2oDEw_({7ZbgUjna!Fqn3NzLM@-EWFPZVmc>(fZ z0&bF-Ch#p9C{YJT9Rcr3+Y_uR^At1^BxZ#eo>$PLJF3=;t_$2|t+_6gg5(j{TmjYU zK12c&lE?Eh+2u2&6Gf*IdKS&6?rYbSEKBN!rv{YCm|Rt=UlPcW9j`0o6{66#y5t9C zruFA2iKd=H%jHf%ypOkxLnO8#H}#Zt{8p!oi6)7#NqoF({t6|J^?1e*oxqng9Q2Cc zg%5Vu!em)}Yuj?kaP!D?b?(C*w!1;>R=j90+RTkyEXz+9CufZ$C^umX^+4|JYaO<5 zmIM3#dv`DGM;@F6;(t!WngZSYzHx?9&$xEF70D1BvfVj<%+b#)vz)2iLCrTeYzUcL z(OBnNoG6Le%M+@2oo)&jdOg=iCszzv59e zDRCeaX8l1hC=8LbBt|k5?CXgep=3r9BXx1uR8!p%Z|0+4Xro=xi0G!e{c4U~1j6!) zH6adq0}#l{%*1U(Cb%4AJ}VLWKBPi0MoKFaQH6x?^hQ!6em@993xdtS%_dmevzeNl z(o?YlOI=jl(`L9^ z0O+H9k$_@`6L13eTT8ci-V0ljDMD|0ifUw|Q-Hep$xYj0hTO@0%IS^TD4b4n6EKDG z??uM;MEx`s98KYN(K0>c!C3HZdZ{+_53DO%9k5W%pr6yJusQAv_;IA}925Y%;+!tY z%2k!YQmLLOr{rF~!s<3-WEUs)`ix_mSU|cNRBIWxOox_Yb7Z=~Q45ZNe*u|m^|)d* zog=i>`=bTe!|;8F+#H>EjIMcgWcG2ORD`w0WD;YZAy5#s{65~qfI6o$+Ty&-hyMyJ z3Ra~t>R!p=5ZpxA;QkDAoPi4sYOP6>LT+}{xp}tk+<0k^CKCFdNYG(Es>p0gqD)jP zWOeX5G;9(m@?GOG7g;e74i_|SmE?`B2i;sLYwRWKLy0RLW!Hx`=!LH3&k=FuCsM=9M4|GqzA)anEHfxkB z?2iK-u(DC_T1};KaUT@3nP~LEcENT^UgPvp!QC@Dw&PVAhaEYrPey{nkcn(ro|r7XUz z%#(=$7D8uP_uU-oPHhd>>^adbCSQetgSG`e$U|7mr!`|bU0aHl_cmL)na-5x1#OsVE#m*+k84Y^+UMeSAa zbrVZHU=mFwXEaGHtXQq`2ZtjfS!B2H{5A<3(nb-6ARVV8kEmOkx6D2x7~-6hl;*-*}2Xz;J#a8Wn;_B5=m zl3dY;%krf?i-Ok^Pal-}4F`{F@TYPTwTEhxpZK5WCpfD^UmM_iYPe}wpE!Djai6_{ z*pGO=WB47#Xjb7!n2Ma)s^yeR*1rTxp`Mt4sfA+`HwZf%!7ZqGosPkw69`Ix5Ku6G z@Pa;pjzV&dn{M=QDx89t?p?d9gna*}jBly*#1!6}5K<*xDPJ{wv4& zM$17DFd~L*Te3A%yD;Dp9UGWTjRxAvMu!j^Tbc}2v~q^59d4bz zvu#!IJCy(BcWTc`;v$9tH;J%oiSJ_i7s;2`JXZF+qd4C)vY!hyCtl)sJIC{ebI*0> z@x>;EzyBv>AI-~{D6l6{ST=em*U( z(r$nuXY-#CCi^8Z2#v#UXOt`dbYN1z5jzNF2 z411?w)whZrfA20;nl&C1Gi+gk<`JSm+{|*2o<< zqM#@z_D`Cn|0H^9$|Tah)0M_X4c37|KQ*PmoT@%xHc3L1ZY6(p(sNXHa&49Frzto& zR`c~ClHpE~4Z=uKa5S(-?M8EJ$zt0&fJk~p$M#fGN1-y$7!37hld`Uw>Urri(DxLa;=#rK0g4J)pXMC zxzraOVw1+kNWpi#P=6(qxf`zSdUC?D$i`8ZI@F>k6k zz21?d+dw7b&i*>Kv5L(LH-?J%@WnqT7j#qZ9B>|Zl+=> z^U-pV@1y_ptHo4hl^cPRWewbLQ#g6XYQ@EkiP z;(=SU!yhjHp%1&MsU`FV1Z_#K1&(|5n(7IHbx&gG28HNT)*~-BQi372@|->2Aw5It z0CBpUcMA*QvsPy)#lr!lIdCi@1k4V2m!NH)%Px(vu-r(Q)HYc!p zJ^$|)j^E#q#QOgcb^pd74^JUi7fUmMiNP_o*lvx*q%_odv49Dsv$NV;6J z9GOXKomA{2Pb{w}&+yHtH?IkJJu~}Z?{Uk++2mB8zyvh*xhHKE``99>y#TdD z&(MH^^JHf;g(Tbb^&8P*;_i*2&fS$7${3WJtV7K&&(MBV2~)2KB3%cWg#1!VE~k#C z!;A;?p$s{ihyojEZz+$I1)L}&G~ml=udD9qh>Tu(ylv)?YcJT3ihapi!zgPtWb*CP zlLLJSRCj-^w?@;RU9aL2zDZY1`I3d<&OMuW=c3$o0#STpv_p3b9Wtbql>w^bBi~u4 z3D8KyF?YE?=HcKk!xcp@Cigvzy=lnFgc^9c%(^F22BWYNAYRSho@~*~S)4%AhEttv zvq>7X!!EWKG?mOd9&n>vvH1p4VzE?HCuxT-u+F&mnsfDI^}*-d00-KAauEaXqg3k@ zy#)MGX!X;&3&0s}F3q40ZmVM$(H3CLfpdL?hB6nVqMxX)q=1b}o_PG%r~hZ4gUfSp zOH4qlEOW4OMUc)_m)fMR_rl^pCfXc{$fQbI*E&mV77}kRF z&{<06AJyJ!e863o-V>FA1a9Eemx6>^F$~9ppt()ZbPGfg_NdRXBWoZnDy2;#ODgf! zgl?iOcF7Meo|{AF>KDwTgYrJLb$L2%%BEtO>T$C?|9bAB&}s;gI?lY#^tttY&hfr# zKhC+&b-rpg_?~uVK%S@mQleU#_xCsvIPK*<`E0fHE1&!J7!xD#IB|SSPW6-PyuqGn3^M^Rz%WT{e?OI^svARX&SAdU77V(C~ zM$H{Kg59op{<|8ry9ecfP%=kFm(-!W&?U0@<%z*+!*<e0XesMxRFu9QnGqun6R_%T+B%&9Dtk?*d$Q zb~>84jEAPi@&F@3wAa^Lzc(AJz5gsfZ7J53;@D<;Klpl?sK&u@gie`~vTsbOE~Cd4 z%kr56mI|#b(Jk&;p6plVwmNB0H@0SmgdmjIn5Ne@)}7Vty(yb2t3ev@22AE^s!KaN zyQ>j+F3w=wnx7w@FVCRe+`vUH)3gW%_72fxzqX!S&!dchdkRiHbXW1FMrIIBwjsai8`CB2r4mAbwp%rrO>3B$Zw;9=%fXI9B{d(UzVap7u z6piC-FQ)>}VOEuPpuqznpY`hN4dGa_1Xz9rVg(;H$5Te^F0dDv*gz9JS<|>>U0J^# z6)(4ICh+N_Q`Ft0hF|3fSHs*?a=XC;e`sJaU9&d>X4l?1W=|fr!5ShD|nv$GK;j46@BV6+{oRbWfqOBRb!ir88XD*SbC(LF}I1h#6@dvK%Toe%@ zhDyG$93H8Eu&gCYddP58iF3oQH*zLbNI;rN@E{T9%A8!=v#JLxKyUe}e}BJpB{~uN zqgxRgo0*-@-iaHPV8bTOH(rS(huwK1Xg0u+e!`(Irzu@Bld&s5&bWgVc@m7;JgELd zimVs`>vQ}B_1(2#rv#N9O`fJpVfPc7V2nv34PC);Dzbb;p!6pqHzvy?2pD&1NE)?A zt(t-ucqy@wn9`^MN5apa7K|L=9>ISC>xoc#>{@e}m#YAAa1*8-RUMKwbm|;5p>T`Z zNf*ph@tnF{gmDa3uwwN(g=`Rh)4!&)^oOy@VJaK4lMT&5#YbXkl`q?<*XtsqD z9PRK6bqb)fJw0g-^a@nu`^?71k|m3RPRjt;pIkCo1{*pdqbVs-Yl>4E>3fZx3Sv44grW=*qdSoiZ9?X0wWyO4`yDHh2E!9I!ZFi zVL8|VtW38}BOJHW(Ax#KL_KQzarbuE{(%TA)AY)@tY4%A%P%SqIU~8~-Lp3qY;U-} z`h_Gel7;K1h}7$_5ZZT0&%$Lxxr-<89V&&TCsu}LL#!xpQ1O31jaa{U34~^le*Y%L za?7$>Jk^k^pS^_M&cDs}NgXlR>16AHkSK-4TRaJSh#h&p!-!vQY%f+bmn6x`4fwTp z$727L^y`~!exvmE^W&#@uY!NxJi`g!i#(++!)?iJ(1)2Wk;RN zFK&O4eTkP$Xn~4bB|q8y(btx$R#D`O@epi4ofcETrx!IM(kWNEe42Qh(8*KqfP(c0 zouBl6>Fc_zM+V;F3znbo{x#%!?mH3`_ANJ?y7ppxS@glg#S9^MXu|FM&ynpz3o&Qh z2ujAHLF3($pH}0jXQsa#?t--TnF1P73b?4`KeJ9^qK-USHE)4!IYgMn-7z|=ALF5SNGkrtPG@Y~niUQV2?g$vzJN3nZ{7;HZHzWAeQ;5P|@Tl3YHpyznGG4-f4=XflwSJY+58-+wf?~Fg@1p1wkzuu-RF3j2JX37SQUc? zQ4v%`V8z9ZVZVqS8h|@@RpD?n0W<=hk=3Cf8R?d^9YK&e9ZybFY%jdnA)PeHvtBe- zhMLD+SSteHBq*q)d6x{)s1UrsO!byyLS$58WK;sqip$Mk{l)Y(_6hEIBsIjCr5t>( z7CdKUrJTrW%qZ#1z^n*Lb8#VdfzPw~OIL76aC+Rhr<~;4Tl!sw?Rj6hXj4XWa#6Tp z@)kJ~qOV)^Rh*-?aG>ic2*NlC2M7&LUzc9RT6WM%Cpe78`iAowe!>(T0jo&ivn8-7 zs{Qa@cGy$rE-3AY0V(l8wjI^uB8Lchj@?L}fYal^>T9z;8juH@?rG&g-t+R2dVDBe zq!K%{e-rT5jX19`(bP23LUN4+_zh2KD~EAYzhpEO3MUG8@}uBHH@4J zd`>_(K4q&>*k82(dDuC)X6JuPrBBubOg7qZ{?x!r@{%0);*`h*^F|%o?&1wX?Wr4b z1~&cy#PUuES{C#xJ84!z<1tp9sfrR(i%Tu^jnXy;4`Xk;AQCdFC@?V%|; zySdC7qS|uQRcH}EFZH%mMB~7gi}a0utE}ZE_}8PQH8f;H%PN41Cb9R%w5Oi5el^fd z$n{3SqLCnrF##x?4sa^r!O$7NX!}&}V;0ZGQ&K&i%6$3C_dR%I7%gdQ;KT6YZiQrW zk%q<74oVBV>@}CvJ4Wj!d^?#Zwq(b$E1ze4$99DuNg?6t9H}k_|D7KWD7i0-g*EO7 z;5{hSIYE4DMOK3H%|f5Edx+S0VI0Yw!tsaRS2&Il2)ea^8R5TG72BrJue|f_{2UHa z@w;^c|K3da#$TB0P3;MPlF7RuQeXT$ zS<<|C0OF(k)>fr&wOB=gP8!Qm>F41u;3esv7_0l%QHt(~+n; zf!G6%hp;Gfa9L9=AceiZs~tK+Tf*Wof=4!u{nIO90jH@iS0l+#%8=~%ASzFv7zqSB^?!@N7)kp0t&tCGLmzXSRMRyxCmCYUD2!B`? zhs$4%KO~m=VFk3Buv9osha{v+mAEq=ik3RdK@;WWTV_g&-$U4IM{1IhGX{pAu%Z&H zFfwCpUsX%RKg);B@7OUzZ{Hn{q6Vv!3#8fAg!P$IEx<0vAx;GU%}0{VIsmFBPq_mb zpe^BChDK>sc-WLKl<6 zwbW|e&d&dv9Wu0goueyu>(JyPx1mz0v4E?cJjFuKF71Q1)AL8jHO$!fYT3(;U3Re* zPPOe%*O+@JYt1bW`!W_1!mN&=w3G9ru1XsmwfS~BJ))PhD(+_J_^N6j)sx5VwbWK| zwRyC?W<`pOCY)b#AS?rluxuuGf-AJ=D!M36l{ua?@SJ5>e!IBr3CXIxWw5xUZ@Xrw z_R@%?{>d%Ld4p}nEsiA@v*nc6Ah!MUs?GA7e5Q5lPpp0@`%5xY$C;{%rz24$;vR#* zBP=a{)K#CwIY%p} zXVdxTQ^HS@O&~eIftU+Qt^~(DGxrdi3k}DdT^I7Iy5SMOp$QuD8s;+93YQ!OY{eB24%xY7ml@|M7I(Nb@K_-?F;2?et|CKkuZK_>+>Lvg!>JE~wN`BI|_h6$qi!P)+K-1Hh(1;a`os z55)4Q{oJiA(lQM#;w#Ta%T0jDNXIPM_bgESMCDEg6rM33anEr}=|Fn6)|jBP6Y}u{ zv9@%7*#RI9;fv;Yii5CI+KrRdr0DKh=L>)eO4q$1zmcSmglsV`*N(x=&Wx`*v!!hn6X-l0 zP_m;X??O(skcj+oS$cIdKhfT%ABAzz3w^la-Ucw?yBPEC+=Pe_vU8nd-HV5YX6X8r zZih&j^eLU=%*;VzhUyoLF;#8QsEfmByk+Y~caBqSvQaaWf2a{JKB9B>V&r?l^rXaC z8)6AdR@Qy_BxQrE2Fk?ewD!SwLuMj@&d_n5RZFf7=>O>hzVE*seW3U?_p|R^CfoY`?|#x9)-*yjv#lo&zP=uI`M?J zbzC<^3x7GfXA4{FZ72{PE*-mNHyy59Q;kYG@BB~NhTd6pm2Oj=_ zizmD?MKVRkT^KmXuhsk?eRQllPo2Ubk=uCKiZ&u3Xjj~<(!M94c)Tez@9M1Gfs5JV z->@II)CDJOXTtPrQudNjE}Eltbjq>6KiwAwqvAKd^|g!exgLG3;wP+#mZYr`cy3#39e653d=jrR-ulW|h#ddHu(m9mFoW~2yE zz5?dB%6vF}+`-&-W8vy^OCxm3_{02royjvmwjlp+eQDzFVEUiyO#gLv%QdDSI#3W* z?3!lL8clTaNo-DVJw@ynq?q!%6hTQi35&^>P85G$TqNt78%9_sSJt2RThO|JzM$iL zg|wjxdMC2|Icc5rX*qPL(coL!u>-xxz-rFiC!6hD1IR%|HSRsV3>Kq~&vJ=s3M5y8SG%YBQ|{^l#LGlg!D?E>2yR*eV%9m$_J6VGQ~AIh&P$_aFbh zULr0Z$QE!QpkP=aAeR4ny<#3Fwyw@rZf4?Ewq`;mCVv}xaz+3ni+}a=k~P+yaWt^L z@w67!DqVf7D%7XtXX5xBW;Co|HvQ8WR1k?r2cZD%U;2$bsM%u8{JUJ5Z0k= zZJARv^vFkmWx15CB=rb=D4${+#DVqy5$C%bf`!T0+epLJLnh1jwCdb*zuCL}eEFvE z{rO1%gxg>1!W(I!owu*mJZ0@6FM(?C+d*CeceZRW_4id*D9p5nzMY&{mWqrJomjIZ z97ZNnZ3_%Hx8dn;H>p8m7F#^2;T%yZ3H;a&N7tm=Lvs&lgJLW{V1@h&6Vy~!+Ffbb zv(n3+v)_D$}dqd!2>Y2B)#<+o}LH#%ogGi2-?xRIH)1!SD)u-L65B&bsJTC=LiaF+YOCif2dUX6uAA|#+vNR z>U+KQekVGon)Yi<93(d!(yw1h3&X0N(PxN2{%vn}cnV?rYw z$N^}_o!XUB!mckL`yO1rnUaI4wrOeQ(+&k?2mi47hzxSD`N#-byqd1IhEoh!PGq>t z_MRy{5B0eKY>;Ao3z$RUU7U+i?iX^&r739F)itdrTpAi-NN0=?^m%?{A9Ly2pVv>Lqs6moTP?T2-AHqFD-o_ znVr|7OAS#AEH}h8SRPQ@NGG47dO}l=t07__+iK8nHw^(AHx&Wb<%jPc$$jl6_p(b$ z)!pi(0fQodCHfM)KMEMUR&UID>}m^(!{C^U7sBDOA)$VThRCI0_+2=( zV8mMq0R(#z;C|7$m>$>`tX+T|xGt(+Y48@ZYu#z;0pCgYgmMVbFb!$?%yhZqP_nhn zy4<#3P1oQ#2b51NU1mGnHP$cf0j-YOgAA}A$QoL6JVLcmExs(kU{4z;PBHJD%_=0F z>+sQV`mzijSIT7xn%PiDKHOujX;n|M&qr1T@rOxTdxtZ!&u&3HHFLYD5$RLQ=heur zb>+AFokUVQeJy-#LP*^)spt{mb@Mqe=A~-4p0b+Bt|pZ+@CY+%x}9f}izU5;4&QFE zO1bhg&A4uC1)Zb67kuowWY4xbo&J=%yoXlFB)&$d*-}kjBu|w!^zbD1YPc0-#XTJr z)pm2RDy%J3jlqSMq|o%xGS$bPwn4AqitC6&e?pqWcjWPt{3I{>CBy;hg0Umh#c;hU3RhCUX=8aR>rmd` z7Orw(5tcM{|-^J?ZAA9KP|)X6n9$-kvr#j5YDecTM6n z&07(nD^qb8hpF0B^z^pQ*%5ePYkv&FabrlI61ntiVp!!C8y^}|<2xgAd#FY=8b*y( zuQOuvy2`Ii^`VBNJB&R!0{hABYX55ooCAJSSevl4RPqEGb)iy_0H}v@vFwFzD%>#I>)3PsouQ+_Kkbqy*kKdHdfkN7NBcq%V{x^fSxgXpg7$bF& zj!6AQbDY(1u#1_A#1UO9AxiZaCVN2F0wGXdY*g@x$ByvUA?ePdide0dmr#}udE%K| z3*k}Vv2Ew2u1FXBaVA6aerI36R&rzEZeDDCl5!t0J=ug6kuNZzH>3i_VN`%BsaVB3 zQYw|Xub_SGf{)F{$ZX5`Jc!X!;eybjP+o$I{Z^Hsj@D=E{MnnL+TbC@HEU2DjG{3-LDGIbq()U87x4eS;JXnSh;lRlJ z>EL3D>wHt-+wTjQF$fGyDO$>d+(fq@bPpLBS~xA~R=3JPbS{tzN(u~m#Po!?H;IYv zE;?8%^vle|%#oux(Lj!YzBKv+Fd}*Ur-dCBoX*t{KeNM*n~ZPYJ4NNKkI^MFbz9!v z4(Bvm*Kc!-$%VFEewYJKz-CQN{`2}KX4*CeJEs+Q(!kI%hN1!1P6iOq?ovz}X0IOi z)YfWpwW@pK08^69#wSyCZkX9?uZD?C^@rw^Y?gLS_xmFKkooyx$*^5#cPqntNTtSG zlP>XLMj2!VF^0k#ole7`-c~*~+_T5ls?x4)ah(j8vo_ zwb%S8qoaZqY0-$ZI+ViIA_1~~rAH7K_+yFS{0rT@eQtTAdz#8E5VpwnW!zJ_^{Utv zlW5Iar3V5t&H4D6A=>?mq;G92;1cg9a2sf;gY9pJDVKn$DYdQlvfXq}zz8#LyPGq@ z+`YUMD;^-6w&r-82JL7mA8&M~Pj@aK!m{0+^v<|t%APYf7`}jGEhdYLqsHW-Le9TL z_hZZ1gbrz7$f9^fAzVIP30^KIz!!#+DRLL+qMszvI_BpOSmjtl$hh;&UeM{ER@INV zcI}VbiVTPoN|iSna@=7XkP&-4#06C};8ajbxJ4Gcq8(vWv4*&X8bM^T$mBk75Q92j z1v&%a;OSKc8EIrodmIiw$lOES2hzGDcjjB`kEDfJe{r}yE6`eZL zEB`9u>Cl0IsQ+t}`-cx}{6jqcANucqIB>Qmga_&<+80E2Q|VHHQ$YlAt{6`Qu`HA3 z03s0-sSlwbvgi&_R8s={6<~M^pGvBNjKOa>tWenzS8s zR>L7R5aZ=mSU{f?ib4Grx$AeFvtO5N|D>9#)ChH#Fny2maHWHOf2G=#<9Myot#+4u zWVa6d^Vseq_0=#AYS(-m$Lp;*8nC_6jXIjEM`omUmtH@QDs3|G)i4j*#_?#UYVZvJ z?YjT-?!4Q{BNun;dKBWLEw2C-VeAz`%?A>p;)PL}TAZn5j~HK>v1W&anteARlE+~+ zj>c(F;?qO3pXBb|#OZdQnm<4xWmn~;DR5SDMxt0UK_F^&eD|KZ=O;tO3vy4@4h^;2 zUL~-z`-P1aOe?|ZC1BgVsL)2^J-&vIFI%q@40w0{jjEfeVl)i9(~bt2z#2Vm)p`V_ z1;6$Ae7=YXk#=Qkd24Y23t&GvRxaOoad~NbJ+6pxqzJ>FY#Td7@`N5xp!n(c!=RE& z&<<@^a$_Ys8jqz4|5Nk#FY$~|FPC0`*a5HH!|Gssa9=~66&xG9)|=pOOJ2KE5|YrR zw!w6K2aC=J$t?L-;}5hn6mHd%hC;p8P|Dgh6D>hGnXPgi;6r+eA=?f72y9(Cf_ho{ zH6#)uD&R=73^$$NE;5piWX2bzR67fQ)`b=85o0eOLGI4c-Tb@-KNi2pz=Ke@SDcPn za$AxXib84`!Sf;Z3B@TSo`Dz7GM5Kf(@PR>Ghzi=BBxK8wRp>YQoXm+iL>H*Jo9M3 z6w&E?BC8AFTFT&Tv8zf+m9<&S&%dIaZ)Aoqkak_$r-2{$d~0g2oLETx9Y`eOAf14QXEQw3tJne;fdzl@wV#TFXSLXM2428F-Q}t+n2g%vPRMUzYPvzQ9f# zu(liiJem9P*?0%V@RwA7F53r~|I!Ty)<*AsMX3J{_4&}{6pT%Tpw>)^|DJ)>gpS~1rNEh z0$D?uO8mG?H;2BwM5a*26^7YO$XjUm40XmBsb63MoR;bJh63J;OngS5sSI+o2HA;W zdZV#8pDpC9Oez&L8loZO)MClRz!_!WD&QRtQxnazhT%Vj6Wl4G11nUk8*vSeVab@N#oJ}`KyJv+8Mo@T1-pqZ1t|?cnaVOd;1(h9 z!$DrN=jcGsVYE-0-n?oCJ^4x)F}E;UaD-LZUIzcD?W^ficqJWM%QLy6QikrM1aKZC zi{?;oKwq^Vsr|&`i{jIphA8S6G4)$KGvpULjH%9u(Dq247;R#l&I0{IhcC|oBF*Al zvLo7Xte=C{aIt*otJD}BUq)|_pdR>{zBMT< z(^1RpZv*l*m*OV^8>9&asGBo8h*_4q*)-eCv*|Pq=XNGrZE)^(SF7^{QE_~4VDB(o zVcPA_!G+2CAtLbl+`=Q~9iW`4ZRLku!uB?;tWqVjB0lEOf}2RD7dJ=BExy=<9wkb- z9&7{XFA%n#JsHYN8t5d~=T~5DcW4$B%3M+nNvC2`0!#@sckqlzo5;hhGi(D9=*A4` z5ynobawSPRtWn&CDLEs3Xf`(8^zDP=NdF~F^s&={l7(aw&EG}KWpMjtmz7j_VLO;@ zM2NVLDxZ@GIv7*gzl1 zjq78tv*8#WSY`}Su0&C;2F$Ze(q>F(@Wm^Gw!)(j;dk9Ad{STaxn)IV9FZhm*n+U} zi;4y*3v%A`_c7a__DJ8D1b@dl0Std3F||4Wtvi)fCcBRh!X9$1x!_VzUh>*S5s!oq z;qd{J_r79EL2wIeiGAqFstWtkfIJpjVh%zFo*=55B9Zq~y0=^iqHWfQl@O!Ak;(o*m!pZqe9 z%U2oDOhR)BvW8&F70L;2TpkzIutIvNQaTjjs5V#8mV4!NQ}zN=i`i@WI1z0eN-iCS z;vL-Wxc^Vc_qK<5RPh(}*8dLT{~GzE{w2o$2kMFaEl&q zP{V=>&3kW7tWaK-Exy{~`v4J0U#OZBk{a9{&)&QG18L@6=bsZ1zC_d{{pKZ-Ey>I> z;8H0t4bwyQqgu4hmO`3|4K{R*5>qnQ&gOfdy?z`XD%e5+pTDzUt3`k^u~SaL&XMe= z9*h#kT(*Q9jO#w2Hd|Mr-%DV8i_1{J1MU~XJ3!WUplhXDYBpJH><0OU`**nIvPIof z|N8@I=wA)sf45SAvx||f?Z5uB$kz1qL3Ky_{%RPdP5iN-D2!p5scq}buuC00C@jom zhfGKm3|f?Z0iQ|K$Z~!`8{nmAS1r+fp6r#YDOS8V*;K&Gs7Lc&f^$RC66O|)28oh`NHy&vq zJh+hAw8+ybTB0@VhWN^0iiTnLsCWbS_y`^gs!LX!Lw{yE``!UVzrV24tP8o;I6-65 z1MUiHw^{bB15tmrVT*7-#sj6cs~z`wk52YQJ*TG{SE;KTm#Hf#a~|<(|ImHH17nNM z`Ub{+J3dMD!)mzC8b(2tZtokKW5pAwHa?NFiso~# z1*iaNh4lQ4TS)|@G)H4dZV@l*Vd;Rw;-;odDhW2&lJ%m@jz+Panv7LQm~2Js6rOW3 z0_&2cW^b^MYW3)@o;neZ<{B4c#m48dAl$GCc=$>ErDe|?y@z`$uq3xd(%aAsX)D%l z>y*SQ%My`yDP*zof|3@_w#cjaW_YW4BdA;#Glg1RQcJGY*CJ9`H{@|D+*e~*457kd z73p<%fB^PV!Ybw@)Dr%(ZJbX}xmCStCYv#K3O32ej{$9IzM^I{6FJ8!(=azt7RWf4 z7ib0UOPqN40X!wOnFOoddd8`!_IN~9O)#HRTyjfc#&MCZ zZAMzOVB=;qwt8gV?{Y2?b=iSZG~RF~uyx18K)IDFLl})G1v@$(s{O4@RJ%OTJyF+Cpcx4jmy|F3euCnMK!P2WTDu5j z{{gD$=M*pH!GGzL%P)V2*ROm>!$Y=z|D`!_yY6e7SU$~a5q8?hZGgaYqaiLnkK%?0 zs#oI%;zOxF@g*@(V4p!$7dS1rOr6GVs6uYCTt2h)eB4?(&w8{#o)s#%gN@BBosRUe z)@P@8_Zm89pr~)b>e{tbPC~&_MR--iB{=)y;INU5#)@Gix-YpgP<-c2Ms{9zuCX|3 z!p(?VaXww&(w&uBHzoT%!A2=3HAP>SDxcljrego7rY|%hxy3XlODWffO_%g|l+7Y_ zqV(xbu)s4lV=l7M;f>vJl{`6qBm>#ZeMA}kXb97Z)?R97EkoI?x6Lp0yu1Z>PS?2{ z0QQ(8D)|lc9CO3B~e(pQM&5(1y&y=e>C^X$`)_&XuaI!IgDTVqt31wX#n+@!a_A0ZQkA zCJ2@M_4Gb5MfCrm5UPggeyh)8 zO9?`B0J#rkoCx(R0I!ko_2?iO@|oRf1;3r+i)w-2&j?=;NVIdPFsB)`|IC0zk6r9c zRrkfxWsiJ(#8QndNJj@{@WP2Ackr|r1VxV{7S&rSU(^)-M8gV>@UzOLXu9K<{6e{T zXJ6b92r$!|lwjhmgqkdswY&}c)KW4A)-ac%sU;2^fvq7gfUW4Bw$b!i@duy1CAxSn z(pyh$^Z=&O-q<{bZUP+$U}=*#M9uVc>CQVgDs4swy5&8RAHZ~$)hrTF4W zPsSa~qYv_0mJnF89RnnJTH`3}w4?~epFl=D(35$ zWa07ON$`OMBOHgCmfO(9RFc<)?$x)N}Jd2A(<*Ll7+4jrRt9w zwGxExUXd9VB#I|DwfxvJ;HZ8Q{37^wDhaZ%O!oO(HpcqfLH%#a#!~;Jl7F5>EX_=8 z{()l2NqPz>La3qJR;_v+wlK>GsHl;uRA8%j`A|yH@k5r%55S9{*Cp%uw6t`qc1!*T za2OeqtQj7sAp#Q~=5Fs&aCR9v>5V+s&RdNvo&H~6FJOjvaj--2sYYBvMq;55%z8^o z|BJDA4vzfow#DO#ZQHh;Oq_{r+qP{R9ox2TOgwQiv7Ow!zjN+A@BN;0tA2lUb#+zO z(^b89eV)D7UVE+h{mcNc6&GtpOqDn_?VAQ)Vob$hlFwW%xh>D#wml{t&Ofmm_d_+; zKDxzdr}`n2Rw`DtyIjrG)eD0vut$}dJAZ0AohZ+ZQdWXn_Z@dI_y=7t3q8x#pDI-K z2VVc&EGq445Rq-j0=U=Zx`oBaBjsefY;%)Co>J3v4l8V(T8H?49_@;K6q#r~Wwppc z4XW0(4k}cP=5ex>-Xt3oATZ~bBWKv)aw|I|Lx=9C1s~&b77idz({&q3T(Y(KbWO?+ zmcZ6?WeUsGk6>km*~234YC+2e6Zxdl~<_g2J|IE`GH%n<%PRv-50; zH{tnVts*S5*_RxFT9eM0z-pksIb^drUq4>QSww=u;UFCv2AhOuXE*V4z?MM`|ABOC4P;OfhS(M{1|c%QZ=!%rQTDFx`+}?Kdx$&FU?Y<$x;j7z=(;Lyz+?EE>ov!8vvMtSzG!nMie zsBa9t8as#2nH}n8xzN%W%U$#MHNXmDUVr@GX{?(=yI=4vks|V)!-W5jHsU|h_&+kY zS_8^kd3jlYqOoiI`ZqBVY!(UfnAGny!FowZWY_@YR0z!nG7m{{)4OS$q&YDyw6vC$ zm4!$h>*|!2LbMbxS+VM6&DIrL*X4DeMO!@#EzMVfr)e4Tagn~AQHIU8?e61TuhcKD zr!F4(kEebk(Wdk-?4oXM(rJwanS>Jc%<>R(siF+>+5*CqJLecP_we33iTFTXr6W^G z7M?LPC-qFHK;E!fxCP)`8rkxZyFk{EV;G-|kwf4b$c1k0atD?85+|4V%YATWMG|?K zLyLrws36p%Qz6{}>7b>)$pe>mR+=IWuGrX{3ZPZXF3plvuv5Huax86}KX*lbPVr}L z{C#lDjdDeHr~?l|)Vp_}T|%$qF&q#U;ClHEPVuS+Jg~NjC1RP=17=aQKGOcJ6B3mp z8?4*-fAD~}sX*=E6!}^u8)+m2j<&FSW%pYr_d|p_{28DZ#Cz0@NF=gC-o$MY?8Ca8 zr5Y8DSR^*urS~rhpX^05r30Ik#2>*dIOGxRm0#0YX@YQ%Mg5b6dXlS!4{7O_kdaW8PFSdj1=ryI-=5$fiieGK{LZ+SX(1b=MNL!q#lN zv98?fqqTUH8r8C7v(cx#BQ5P9W>- zmW93;eH6T`vuJ~rqtIBg%A6>q>gnWb3X!r0wh_q;211+Om&?nvYzL1hhtjB zK_7G3!n7PL>d!kj){HQE zE8(%J%dWLh1_k%gVXTZt zEdT09XSKAx27Ncaq|(vzL3gm83q>6CAw<$fTnMU05*xAe&rDfCiu`u^1)CD<>sx0i z*hr^N_TeN89G(nunZoLBf^81#pmM}>JgD@Nn1l*lN#a=B=9pN%tmvYFjFIoKe_(GF z-26x{(KXdfsQL7Uv6UtDuYwV`;8V3w>oT_I<`Ccz3QqK9tYT5ZQzbop{=I=!pMOCb zCU68`n?^DT%^&m>A%+-~#lvF!7`L7a{z<3JqIlk1$<||_J}vW1U9Y&eX<}l8##6i( zZcTT@2`9(Mecptm@{3A_Y(X`w9K0EwtPq~O!16bq{7c0f7#(3wn-^)h zxV&M~iiF!{-6A@>o;$RzQ5A50kxXYj!tcgme=Qjrbje~;5X2xryU;vH|6bE(8z^<7 zQ>BG7_c*JG8~K7Oe68i#0~C$v?-t@~@r3t2inUnLT(c=URpA9kA8uq9PKU(Ps(LVH zqgcqW>Gm?6oV#AldDPKVRcEyQIdTT`Qa1j~vS{<;SwyTdr&3*t?J)y=M7q*CzucZ&B0M=joT zBbj@*SY;o2^_h*>R0e({!QHF0=)0hOj^B^d*m>SnRrwq>MolNSgl^~r8GR#mDWGYEIJA8B<|{{j?-7p zVnV$zancW3&JVDtVpIlI|5djKq0(w$KxEFzEiiL=h5Jw~4Le23@s(mYyXWL9SX6Ot zmb)sZaly_P%BeX_9 zw&{yBef8tFm+%=--m*J|o~+Xg3N+$IH)t)=fqD+|fEk4AAZ&!wcN5=mi~Vvo^i`}> z#_3ahR}Ju)(Px7kev#JGcSwPXJ2id9%Qd2A#Uc@t8~egZ8;iC{e! z%=CGJOD1}j!HW_sgbi_8suYnn4#Ou}%9u)dXd3huFIb!ytlX>Denx@pCS-Nj$`VO&j@(z!kKSP0hE4;YIP#w9ta=3DO$7f*x zc9M4&NK%IrVmZAe=r@skWD`AEWH=g+r|*13Ss$+{c_R!b?>?UaGXlw*8qDmY#xlR= z<0XFbs2t?8i^G~m?b|!Hal^ZjRjt<@a? z%({Gn14b4-a|#uY^=@iiKH+k?~~wTj5K1A&hU z2^9-HTC)7zpoWK|$JXaBL6C z#qSNYtY>65T@Zs&-0cHeu|RX(Pxz6vTITdzJdYippF zC-EB+n4}#lM7`2Ry~SO>FxhKboIAF#Z{1wqxaCb{#yEFhLuX;Rx(Lz%T`Xo1+a2M}7D+@wol2)OJs$TwtRNJ={( zD@#zTUEE}#Fz#&(EoD|SV#bayvr&E0vzmb%H?o~46|FAcx?r4$N z&67W3mdip-T1RIxwSm_&(%U|+WvtGBj*}t69XVd&ebn>KOuL(7Y8cV?THd-(+9>G7*Nt%T zcH;`p={`SOjaf7hNd(=37Lz3-51;58JffzIPgGs_7xIOsB5p2t&@v1mKS$2D$*GQ6 zM(IR*j4{nri7NMK9xlDy-hJW6sW|ZiDRaFiayj%;(%51DN!ZCCCXz+0Vm#};70nOx zJ#yA0P3p^1DED;jGdPbQWo0WATN=&2(QybbVdhd=Vq*liDk`c7iZ?*AKEYC#SY&2g z&Q(Ci)MJ{mEat$ZdSwTjf6h~roanYh2?9j$CF@4hjj_f35kTKuGHvIs9}Re@iKMxS-OI*`0S z6s)fOtz}O$T?PLFVSeOjSO26$@u`e<>k(OSP!&YstH3ANh>)mzmKGNOwOawq-MPXe zy4xbeUAl6tamnx))-`Gi2uV5>9n(73yS)Ukma4*7fI8PaEwa)dWHs6QA6>$}7?(L8 ztN8M}?{Tf!Zu22J5?2@95&rQ|F7=FK-hihT-vDp!5JCcWrVogEnp;CHenAZ)+E+K5 z$Cffk5sNwD_?4+ymgcHR(5xgt20Z8M`2*;MzOM#>yhk{r3x=EyM226wb&!+j`W<%* zSc&|`8!>dn9D@!pYow~(DsY_naSx7(Z4i>cu#hA5=;IuI88}7f%)bRkuY2B;+9Uep zpXcvFWkJ!mQai63BgNXG26$5kyhZ2&*3Q_tk)Ii4M>@p~_~q_cE!|^A;_MHB;7s#9 zKzMzK{lIxotjc};k67^Xsl-gS!^*m*m6kn|sbdun`O?dUkJ{0cmI0-_2y=lTAfn*Y zKg*A-2sJq)CCJgY0LF-VQvl&6HIXZyxo2#!O&6fOhbHXC?%1cMc6y^*dOS{f$=137Ds1m01qs`>iUQ49JijsaQ( zksqV9@&?il$|4Ua%4!O15>Zy&%gBY&wgqB>XA3!EldQ%1CRSM(pp#k~-pkcCg4LAT zXE=puHbgsw)!xtc@P4r~Z}nTF=D2~j(6D%gTBw$(`Fc=OOQ0kiW$_RDd=hcO0t97h zb86S5r=>(@VGy1&#S$Kg_H@7G^;8Ue)X5Y+IWUi`o;mpvoV)`fcVk4FpcT|;EG!;? zHG^zrVVZOm>1KFaHlaogcWj(v!S)O(Aa|Vo?S|P z5|6b{qkH(USa*Z7-y_Uvty_Z1|B{rTS^qmEMLEYUSk03_Fg&!O3BMo{b^*`3SHvl0 zhnLTe^_vVIdcSHe)SQE}r~2dq)VZJ!aSKR?RS<(9lzkYo&dQ?mubnWmgMM37Nudwo z3Vz@R{=m2gENUE3V4NbIzAA$H1z0pagz94-PTJyX{b$yndsdKptmlKQKaaHj@3=ED zc7L?p@%ui|RegVYutK$64q4pe9+5sv34QUpo)u{1ci?)_7gXQd{PL>b0l(LI#rJmN zGuO+%GO`xneFOOr4EU(Wg}_%bhzUf;d@TU+V*2#}!2OLwg~%D;1FAu=Un>OgjPb3S z7l(riiCwgghC=Lm5hWGf5NdGp#01xQ59`HJcLXbUR3&n%P(+W2q$h2Qd z*6+-QXJ*&Kvk9ht0f0*rO_|FMBALen{j7T1l%=Q>gf#kma zQlg#I9+HB+z*5BMxdesMND`_W;q5|FaEURFk|~&{@qY32N$G$2B=&Po{=!)x5b!#n zxLzblkq{yj05#O7(GRuT39(06FJlalyv<#K4m}+vs>9@q-&31@1(QBv82{}Zkns~K ze{eHC_RDX0#^A*JQTwF`a=IkE6Ze@j#-8Q`tTT?k9`^ZhA~3eCZJ-Jr{~7Cx;H4A3 zcZ+Zj{mzFZbVvQ6U~n>$U2ZotGsERZ@}VKrgGh0xM;Jzt29%TX6_&CWzg+YYMozrM z`nutuS)_0dCM8UVaKRj804J4i%z2BA_8A4OJRQ$N(P9Mfn-gF;4#q788C@9XR0O3< zsoS4wIoyt046d+LnSCJOy@B@Uz*#GGd#+Ln1ek5Dv>(ZtD@tgZlPnZZJGBLr^JK+!$$?A_fA3LOrkoDRH&l7 zcMcD$Hsjko3`-{bn)jPL6E9Ds{WskMrivsUu5apD z?grQO@W7i5+%X&E&p|RBaEZ(sGLR@~(y^BI@lDMot^Ll?!`90KT!JXUhYS`ZgX3jnu@Ja^seA*M5R@f`=`ynQV4rc$uT1mvE?@tz)TN<=&H1%Z?5yjxcpO+6y_R z6EPuPKM5uxKpmZfT(WKjRRNHs@ib)F5WAP7QCADvmCSD#hPz$V10wiD&{NXyEwx5S z6NE`3z!IS^$s7m}PCwQutVQ#~w+V z=+~->DI*bR2j0^@dMr9`p>q^Ny~NrAVxrJtX2DUveic5vM%#N*XO|?YAWwNI$Q)_) zvE|L(L1jP@F%gOGtnlXtIv2&1i8q<)Xfz8O3G^Ea~e*HJsQgBxWL(yuLY+jqUK zRE~`-zklrGog(X}$9@ZVUw!8*=l`6mzYLtsg`AvBYz(cxmAhr^j0~(rzXdiOEeu_p zE$sf2(w(BPAvO5DlaN&uQ$4@p-b?fRs}d7&2UQ4Fh?1Hzu*YVjcndqJLw0#q@fR4u zJCJ}>_7-|QbvOfylj+e^_L`5Ep9gqd>XI3-O?Wp z-gt*P29f$Tx(mtS`0d05nHH=gm~Po_^OxxUwV294BDKT>PHVlC5bndncxGR!n(OOm znsNt@Q&N{TLrmsoKFw0&_M9$&+C24`sIXGWgQaz=kY;S{?w`z^Q0JXXBKFLj0w0U6P*+jPKyZHX9F#b0D1$&(- zrm8PJd?+SrVf^JlfTM^qGDK&-p2Kdfg?f>^%>1n8bu&byH(huaocL>l@f%c*QkX2i znl}VZ4R1en4S&Bcqw?$=Zi7ohqB$Jw9x`aM#>pHc0x z0$!q7iFu zZ`tryM70qBI6JWWTF9EjgG@>6SRzsd}3h+4D8d~@CR07P$LJ}MFsYi-*O%XVvD@yT|rJ+Mk zDllJ7$n0V&A!0flbOf)HE6P_afPWZmbhpliqJuw=-h+r;WGk|ntkWN(8tKlYpq5Ow z(@%s>IN8nHRaYb*^d;M(D$zGCv5C|uqmsDjwy4g=Lz>*OhO3z=)VD}C<65;`89Ye} zSCxrv#ILzIpEx1KdLPlM&%Cctf@FqTKvNPXC&`*H9=l=D3r!GLM?UV zOxa(8ZsB`&+76S-_xuj?G#wXBfDY@Z_tMpXJS7^mp z@YX&u0jYw2A+Z+bD#6sgVK5ZgdPSJV3>{K^4~%HV?rn~4D)*2H!67Y>0aOmzup`{D zzDp3c9yEbGCY$U<8biJ_gB*`jluz1ShUd!QUIQJ$*1;MXCMApJ^m*Fiv88RZ zFopLViw}{$Tyhh_{MLGIE2~sZ)t0VvoW%=8qKZ>h=adTe3QM$&$PO2lfqH@brt!9j ziePM8$!CgE9iz6B<6_wyTQj?qYa;eC^{x_0wuwV~W+^fZmFco-o%wsKSnjXFEx02V zF5C2t)T6Gw$Kf^_c;Ei3G~uC8SM-xyycmXyC2hAVi-IfXqhu$$-C=*|X?R0~hu z8`J6TdgflslhrmDZq1f?GXF7*ALeMmOEpRDg(s*H`4>_NAr`2uqF;k;JQ+8>A|_6ZNsNLECC%NNEb1Y1dP zbIEmNpK)#XagtL4R6BC{C5T(+=yA-(Z|Ap}U-AfZM#gwVpus3(gPn}Q$CExObJ5AC z)ff9Yk?wZ}dZ-^)?cbb9Fw#EjqQ8jxF4G3=L?Ra zg_)0QDMV1y^A^>HRI$x?Op@t;oj&H@1xt4SZ9(kifQ zb59B*`M99Td7@aZ3UWvj1rD0sE)d=BsBuW*KwkCds7ay(7*01_+L}b~7)VHI>F_!{ zyxg-&nCO?v#KOUec0{OOKy+sjWA;8rTE|Lv6I9H?CI?H(mUm8VXGwU$49LGpz&{nQp2}dinE1@lZ1iox6{ghN&v^GZv9J${7WaXj)<0S4g_uiJ&JCZ zr8-hsu`U%N;+9N^@&Q0^kVPB3)wY(rr}p7{p0qFHb3NUUHJb672+wRZs`gd1UjKPX z4o6zljKKA+Kkj?H>Ew63o%QjyBk&1!P22;MkD>sM0=z_s-G{mTixJCT9@_|*(p^bz zJ8?ZZ&;pzV+7#6Mn`_U-)k8Pjg?a;|Oe^us^PoPY$Va~yi8|?+&=y$f+lABT<*pZr zP}D{~Pq1Qyni+@|aP;ixO~mbEW9#c0OU#YbDZIaw=_&$K%Ep2f%hO^&P67hApZe`x zv8b`Mz@?M_7-)b!lkQKk)JXXUuT|B8kJlvqRmRpxtQDgvrHMXC1B$M@Y%Me!BSx3P z#2Eawl$HleZhhTS6Txm>lN_+I`>eV$&v9fOg)%zVn3O5mI*lAl>QcHuW6!Kixmq`X zBCZ*Ck6OYtDiK!N47>jxI&O2a9x7M|i^IagRr-fmrmikEQGgw%J7bO|)*$2FW95O4 zeBs>KR)izRG1gRVL;F*sr8A}aRHO0gc$$j&ds8CIO1=Gwq1%_~E)CWNn9pCtBE}+`Jelk4{>S)M)`Ll=!~gnn1yq^EX(+y*ik@3Ou0qU`IgYi3*doM+5&dU!cho$pZ zn%lhKeZkS72P?Cf68<#kll_6OAO26bIbueZx**j6o;I0cS^XiL`y+>{cD}gd%lux} z)3N>MaE24WBZ}s0ApfdM;5J_Ny}rfUyxfkC``Awo2#sgLnGPewK};dORuT?@I6(5~ z?kE)Qh$L&fwJXzK){iYx!l5$Tt|^D~MkGZPA}(o6f7w~O2G6Vvzdo*a;iXzk$B66$ zwF#;wM7A+(;uFG4+UAY(2`*3XXx|V$K8AYu#ECJYSl@S=uZW$ksfC$~qrrbQj4??z-)uz0QL}>k^?fPnJTPw% zGz)~?B4}u0CzOf@l^um}HZzbaIwPmb<)< zi_3@E9lc)Qe2_`*Z^HH;1CXOceL=CHpHS{HySy3T%<^NrWQ}G0i4e1xm_K3(+~oi$ zoHl9wzb?Z4j#90DtURtjtgvi7uw8DzHYmtPb;?%8vb9n@bszT=1qr)V_>R%s!92_` zfnHQPANx z<#hIjIMm#*(v*!OXtF+w8kLu`o?VZ5k7{`vw{Yc^qYclpUGIM_PBN1+c{#Vxv&E*@ zxg=W2W~JuV{IuRYw3>LSI1)a!thID@R=bU+cU@DbR^_SXY`MC7HOsCN z!dO4OKV7(E_Z8T#8MA1H`99?Z!r0)qKW_#|29X3#Jb+5+>qUidbeP1NJ@)(qi2S-X zao|f0_tl(O+$R|Qwd$H{_ig|~I1fbp_$NkI!0E;Y z6JrnU{1Ra6^on{9gUUB0mwzP3S%B#h0fjo>JvV~#+X0P~JV=IG=yHG$O+p5O3NUgG zEQ}z6BTp^Fie)Sg<){Z&I8NwPR(=mO4joTLHkJ>|Tnk23E(Bo`FSbPc05lF2-+)X? z6vV3*m~IBHTy*^E!<0nA(tCOJW2G4DsH7)BxLV8kICn5lu6@U*R`w)o9;Ro$i8=Q^V%uH8n3q=+Yf;SFRZu z!+F&PKcH#8cG?aSK_Tl@K9P#8o+jry@gdexz&d(Q=47<7nw@e@FFfIRNL9^)1i@;A z28+$Z#rjv-wj#heI|<&J_DiJ*s}xd-f!{J8jfqOHE`TiHHZVIA8CjkNQ_u;Ery^^t zl1I75&u^`1_q)crO+JT4rx|z2ToSC>)Or@-D zy3S>jW*sNIZR-EBsfyaJ+Jq4BQE4?SePtD2+jY8*%FsSLZ9MY>+wk?}}}AFAw)vr{ml)8LUG-y9>^t!{~|sgpxYc0Gnkg`&~R z-pilJZjr@y5$>B=VMdZ73svct%##v%wdX~9fz6i3Q-zOKJ9wso+h?VME7}SjL=!NUG{J?M&i!>ma`eoEa@IX`5G>B1(7;%}M*%-# zfhJ(W{y;>MRz!Ic8=S}VaBKqh;~7KdnGEHxcL$kA-6E~=!hrN*zw9N+_=odt<$_H_8dbo;0=42wcAETPCVGUr~v(`Uai zb{=D!Qc!dOEU6v)2eHSZq%5iqK?B(JlCq%T6av$Cb4Rko6onlG&?CqaX7Y_C_cOC3 zYZ;_oI(}=>_07}Oep&Ws7x7-R)cc8zfe!SYxJYP``pi$FDS)4Fvw5HH=FiU6xfVqIM!hJ;Rx8c0cB7~aPtNH(Nmm5Vh{ibAoU#J6 zImRCr?(iyu_4W_6AWo3*vxTPUw@vPwy@E0`(>1Qi=%>5eSIrp^`` zK*Y?fK_6F1W>-7UsB)RPC4>>Ps9)f+^MqM}8AUm@tZ->j%&h1M8s*s!LX5&WxQcAh z8mciQej@RPm?660%>{_D+7er>%zX_{s|$Z+;G7_sfNfBgY(zLB4Ey}J9F>zX#K0f6 z?dVNIeEh?EIShmP6>M+d|0wMM85Sa4diw1hrg|ITJ}JDg@o8y>(rF9mXk5M z2@D|NA)-7>wD&wF;S_$KS=eE84`BGw3g0?6wGxu8ys4rwI?9U=*^VF22t3%mbGeOh z`!O-OpF7#Vceu~F`${bW0nYVU9ecmk31V{tF%iv&5hWofC>I~cqAt@u6|R+|HLMMX zVxuSlMFOK_EQ86#E8&KwxIr8S9tj_goWtLv4f@!&h8;Ov41{J~496vp9vX=(LK#j! zAwi*21RAV-LD>9Cw3bV_9X(X3)Kr0-UaB*7Y>t82EQ%!)(&(XuAYtTsYy-dz+w=$ir)VJpe!_$ z6SGpX^i(af3{o=VlFPC);|J8#(=_8#vdxDe|Cok+ANhYwbE*FO`Su2m1~w+&9<_9~ z-|tTU_ACGN`~CNW5WYYBn^B#SwZ(t4%3aPp z;o)|L6Rk569KGxFLUPx@!6OOa+5OjQLK5w&nAmwxkC5rZ|m&HT8G%GVZxB_@ME z>>{rnXUqyiJrT(8GMj_ap#yN_!9-lO5e8mR3cJiK3NE{_UM&=*vIU`YkiL$1%kf+1 z4=jk@7EEj`u(jy$HnzE33ZVW_J4bj}K;vT?T91YlO(|Y0FU4r+VdbmQ97%(J5 zkK*Bed8+C}FcZ@HIgdCMioV%A<*4pw_n}l*{Cr4}a(lq|injK#O?$tyvyE`S%(1`H z_wwRvk#13ElkZvij2MFGOj`fhy?nC^8`Zyo%yVcUAfEr8x&J#A{|moUBAV_^f$hpaUuyQeY3da^ zS9iRgf87YBwfe}>BO+T&Fl%rfpZh#+AM?Dq-k$Bq`vG6G_b4z%Kbd&v>qFjow*mBl z-OylnqOpLg}or7_VNwRg2za3VBK6FUfFX{|TD z`Wt0Vm2H$vdlRWYQJqDmM?JUbVqL*ZQY|5&sY*?!&%P8qhA~5+Af<{MaGo(dl&C5t zE%t!J0 zh6jqANt4ABdPxSTrVV}fLsRQal*)l&_*rFq(Ez}ClEH6LHv{J#v?+H-BZ2)Wy{K@9 z+ovXHq~DiDvm>O~r$LJo!cOuwL+Oa--6;UFE2q@g3N8Qkw5E>ytz^(&($!O47+i~$ zKM+tkAd-RbmP{s_rh+ugTD;lriL~`Xwkad#;_aM?nQ7L_muEFI}U_4$phjvYgleK~`Fo`;GiC07&Hq1F<%p;9Q;tv5b?*QnR%8DYJH3P>Svmv47Y>*LPZJy8_{9H`g6kQpyZU{oJ`m%&p~D=K#KpfoJ@ zn-3cqmHsdtN!f?~w+(t+I`*7GQA#EQC^lUA9(i6=i1PqSAc|ha91I%X&nXzjYaM{8$s&wEx@aVkQ6M{E2 zfzId#&r(XwUNtPcq4Ngze^+XaJA1EK-%&C9j>^9(secqe{}z>hR5CFNveMsVA)m#S zk)_%SidkY-XmMWlVnQ(mNJ>)ooszQ#vaK;!rPmGKXV7am^_F!Lz>;~{VrIO$;!#30XRhE1QqO_~#+Ux;B_D{Nk=grn z8Y0oR^4RqtcYM)7a%@B(XdbZCOqnX#fD{BQTeLvRHd(irHKq=4*jq34`6@VAQR8WG z^%)@5CXnD_T#f%@-l${>y$tfb>2LPmc{~5A82|16mH)R?&r#KKLs7xpN-D`=&Cm^R zvMA6#Ahr<3X>Q7|-qfTY)}32HkAz$_mibYV!I)u>bmjK`qwBe(>za^0Kt*HnFbSdO z1>+ryKCNxmm^)*$XfiDOF2|{-v3KKB?&!(S_Y=Ht@|ir^hLd978xuI&N{k>?(*f8H z=ClxVJK_%_z1TH0eUwm2J+2To7FK4o+n_na)&#VLn1m;!+CX+~WC+qg1?PA~KdOlC zW)C@pw75_xoe=w7i|r9KGIvQ$+3K?L{7TGHwrQM{dCp=Z*D}3kX7E-@sZnup!BImw z*T#a=+WcTwL78exTgBn|iNE3#EsOorO z*kt)gDzHiPt07fmisA2LWN?AymkdqTgr?=loT7z@d`wnlr6oN}@o|&JX!yPzC*Y8d zu6kWlTzE1)ckyBn+0Y^HMN+GA$wUO_LN6W>mxCo!0?oiQvT`z$jbSEu&{UHRU0E8# z%B^wOc@S!yhMT49Y)ww(Xta^8pmPCe@eI5C*ed96)AX9<>))nKx0(sci8gwob_1}4 z0DIL&vsJ1_s%<@y%U*-eX z5rN&(zef-5G~?@r79oZGW1d!WaTqQn0F6RIOa9tJ=0(kdd{d1{<*tHT#cCvl*i>YY zH+L7jq8xZNcTUBqj(S)ztTU!TM!RQ}In*n&Gn<>(60G7}4%WQL!o>hbJqNDSGwl#H z`4k+twp0cj%PsS+NKaxslAEu9!#U3xT1|_KB6`h=PI0SW`P9GTa7caD1}vKEglV8# zjKZR`pluCW19c2fM&ZG)c3T3Um;ir3y(tSCJ7Agl6|b524dy5El{^EQBG?E61H0XY z`bqg!;zhGhyMFl&(o=JWEJ8n~z)xI}A@C0d2hQGvw7nGv)?POU@(kS1m=%`|+^ika zXl8zjS?xqW$WlO?Ewa;vF~XbybHBor$f<%I&*t$F5fynwZlTGj|IjZtVfGa7l&tK} zW>I<69w(cZLu)QIVG|M2xzW@S+70NinQzk&Y0+3WT*cC)rx~04O-^<{JohU_&HL5XdUKW!uFy|i$FB|EMu0eUyW;gsf`XfIc!Z0V zeK&*hPL}f_cX=@iv>K%S5kL;cl_$v?n(Q9f_cChk8Lq$glT|=e+T*8O4H2n<=NGmn z+2*h+v;kBvF>}&0RDS>)B{1!_*XuE8A$Y=G8w^qGMtfudDBsD5>T5SB;Qo}fSkkiV ze^K^M(UthkwrD!&*tTsu>Dacdj_q`~V%r_twr$(Ct&_dKeeXE?fA&4&yASJWJ*}~- zel=@W)tusynfC_YqH4ll>4Eg`Xjs5F7Tj>tTLz<0N3)X<1px_d2yUY>X~y>>93*$) z5PuNMQLf9Bu?AAGO~a_|J2akO1M*@VYN^VxvP0F$2>;Zb9;d5Yfd8P%oFCCoZE$ z4#N$^J8rxYjUE_6{T%Y>MmWfHgScpuGv59#4u6fpTF%~KB^Ae`t1TD_^Ud#DhL+Dm zbY^VAM#MrAmFj{3-BpVSWph2b_Y6gCnCAombVa|1S@DU)2r9W<> zT5L8BB^er3zxKt1v(y&OYk!^aoQisqU zH(g@_o)D~BufUXcPt!Ydom)e|aW{XiMnes2z&rE?og>7|G+tp7&^;q?Qz5S5^yd$i z8lWr4g5nctBHtigX%0%XzIAB8U|T6&JsC4&^hZBw^*aIcuNO47de?|pGXJ4t}BB`L^d8tD`H`i zqrP8?#J@8T#;{^B!KO6J=@OWKhAerih(phML`(Rg7N1XWf1TN>=Z3Do{l_!d~DND&)O)D>ta20}@Lt77qSnVsA7>)uZAaT9bsB>u&aUQl+7GiY2|dAEg@%Al3i316y;&IhQL^8fw_nwS>f60M_-m+!5)S_6EPM7Y)(Nq^8gL7(3 zOiot`6Wy6%vw~a_H?1hLVzIT^i1;HedHgW9-P#)}Y6vF%C=P70X0Tk^z9Te@kPILI z_(gk!k+0%CG)%!WnBjjw*kAKs_lf#=5HXC00s-}oM-Q1aXYLj)(1d!_a7 z*Gg4Fe6F$*ujVjI|79Z5+Pr`us%zW@ln++2l+0hsngv<{mJ%?OfSo_3HJXOCys{Ug z00*YR-(fv<=&%Q!j%b-_ppA$JsTm^_L4x`$k{VpfLI(FMCap%LFAyq;#ns5bR7V+x zO!o;c5y~DyBPqdVQX)8G^G&jWkBy2|oWTw>)?5u}SAsI$RjT#)lTV&Rf8;>u*qXnb z8F%Xb=7#$m)83z%`E;49)t3fHInhtc#kx4wSLLms!*~Z$V?bTyUGiS&m>1P(952(H zuHdv=;o*{;5#X-uAyon`hP}d#U{uDlV?W?_5UjJvf%11hKwe&(&9_~{W)*y1nR5f_ z!N(R74nNK`y8>B!0Bt_Vr!;nc3W>~RiKtGSBkNlsR#-t^&;$W#)f9tTlZz>n*+Fjz z3zXZ;jf(sTM(oDzJt4FJS*8c&;PLTW(IQDFs_5QPy+7yhi1syPCarvqrHFcf&yTy)^O<1EBx;Ir`5W{TIM>{8w&PB>ro4;YD<5LF^TjTb0!zAP|QijA+1Vg>{Afv^% zmrkc4o6rvBI;Q8rj4*=AZacy*n8B{&G3VJc)so4$XUoie0)vr;qzPZVbb<#Fc=j+8CGBWe$n|3K& z_@%?{l|TzKSlUEO{U{{%Fz_pVDxs7i9H#bnbCw7@4DR=}r_qV!Zo~CvD4ZI*+j3kO zW6_=|S`)(*gM0Z;;}nj`73OigF4p6_NPZQ-Od~e$c_);;4-7sR>+2u$6m$Gf%T{aq zle>e3(*Rt(TPD}03n5)!Ca8Pu!V}m6v0o1;5<1h$*|7z|^(3$Y&;KHKTT}hV056wuF0Xo@mK-52~r=6^SI1NC%c~CC?n>yX6wPTgiWYVz!Sx^atLby9YNn1Rk{g?|pJaxD4|9cUf|V1_I*w zzxK)hRh9%zOl=*$?XUjly5z8?jPMy%vEN)f%T*|WO|bp5NWv@B(K3D6LMl!-6dQg0 zXNE&O>Oyf%K@`ngCvbGPR>HRg5!1IV$_}m@3dWB7x3t&KFyOJn9pxRXCAzFr&%37wXG;z^xaO$ekR=LJG ztIHpY8F5xBP{mtQidqNRoz= z@){+N3(VO5bD+VrmS^YjG@+JO{EOIW)9=F4v_$Ed8rZtHvjpiEp{r^c4F6Ic#ChlC zJX^DtSK+v(YdCW)^EFcs=XP7S>Y!4=xgmv>{S$~@h=xW-G4FF9?I@zYN$e5oF9g$# zb!eVU#J+NjLyX;yb)%SY)xJdvGhsnE*JEkuOVo^k5PyS=o#vq!KD46UTW_%R=Y&0G zFj6bV{`Y6)YoKgqnir2&+sl+i6foAn-**Zd1{_;Zb7Ki=u394C5J{l^H@XN`_6XTKY%X1AgQM6KycJ+= zYO=&t#5oSKB^pYhNdzPgH~aEGW2=ec1O#s-KG z71}LOg@4UEFtp3GY1PBemXpNs6UK-ax*)#$J^pC_me;Z$Je(OqLoh|ZrW*mAMBFn< zHttjwC&fkVfMnQeen8`Rvy^$pNRFVaiEN4Pih*Y3@jo!T0nsClN)pdrr9AYLcZxZ| zJ5Wlj+4q~($hbtuY zVQ7hl>4-+@6g1i`1a)rvtp-;b0>^`Dloy(#{z~ytgv=j4q^Kl}wD>K_Y!l~ zp(_&7sh`vfO(1*MO!B%<6E_bx1)&s+Ae`O)a|X=J9y~XDa@UB`m)`tSG4AUhoM=5& znWoHlA-(z@3n0=l{E)R-p8sB9XkV zZ#D8wietfHL?J5X0%&fGg@MH~(rNS2`GHS4xTo7L$>TPme+Is~!|79=^}QbPF>m%J zFMkGzSndiPO|E~hrhCeo@&Ea{M(ieIgRWMf)E}qeTxT8Q#g-!Lu*x$v8W^M^>?-g= zwMJ$dThI|~M06rG$Sv@C@tWR>_YgaG&!BAbkGggVQa#KdtDB)lMLNVLN|51C@F^y8 zCRvMB^{GO@j=cHfmy}_pCGbP%xb{pNN>? z?7tBz$1^zVaP|uaatYaIN+#xEN4jBzwZ|YI_)p(4CUAz1ZEbDk>J~Y|63SZaak~#0 zoYKruYsWHoOlC1(MhTnsdUOwQfz5p6-D0}4;DO$B;7#M{3lSE^jnTT;ns`>!G%i*F?@pR1JO{QTuD0U+~SlZxcc8~>IB{)@8p`P&+nDxNj`*gh|u?yrv$phpQcW)Us)bi`kT%qLj(fi{dWRZ%Es2!=3mI~UxiW0$-v3vUl?#g{p6eF zMEUAqo5-L0Ar(s{VlR9g=j7+lt!gP!UN2ICMokAZ5(Agd>})#gkA2w|5+<%-CuEP# zqgcM}u@3(QIC^Gx<2dbLj?cFSws_f3e%f4jeR?4M^M3cx1f+Qr6ydQ>n)kz1s##2w zk}UyQc+Z5G-d-1}{WzjkLXgS-2P7auWSJ%pSnD|Uivj5u!xk0 z_^-N9r9o;(rFDt~q1PvE#iJZ_f>J3gcP$)SOqhE~pD2|$=GvpL^d!r z6u=sp-CrMoF7;)}Zd7XO4XihC4ji?>V&(t^?@3Q&t9Mx=qex6C9d%{FE6dvU6%d94 zIE;hJ1J)cCqjv?F``7I*6bc#X)JW2b4f$L^>j{*$R`%5VHFi*+Q$2;nyieduE}qdS{L8y8F08yLs?w}{>8>$3236T-VMh@B zq-nujsb_1aUv_7g#)*rf9h%sFj*^mIcImRV*k~Vmw;%;YH(&ylYpy!&UjUVqqtfG` zox3esju?`unJJA_zKXRJP)rA3nXc$m^{S&-p|v|-0x9LHJm;XIww7C#R$?00l&Yyj z=e}gKUOpsImwW?N)+E(awoF@HyP^EhL+GlNB#k?R<2>95hz!h9sF@U20DHSB3~WMa zk90+858r@-+vWwkawJ)8ougd(i#1m3GLN{iSTylYz$brAsP%=&m$mQQrH$g%3-^VR zE%B`Vi&m8f3T~&myTEK28BDWCVzfWir1I?03;pX))|kY5ClO^+bae z*7E?g=3g7EiisYOrE+lA)2?Ln6q2*HLNpZEWMB|O-JI_oaHZB%CvYB(%=tU= zE*OY%QY58fW#RG5=gm0NR#iMB=EuNF@)%oZJ}nmm=tsJ?eGjia{e{yuU0l3{d^D@)kVDt=1PE)&tf_hHC%0MB znL|CRCPC}SeuVTdf>-QV70`0(EHizc21s^sU>y%hW0t!0&y<7}Wi-wGy>m%(-jsDj zP?mF|>p_K>liZ6ZP(w5(|9Ga%>tLgb$|doDDfkdW>Z z`)>V2XC?NJT26mL^@ zf+IKr27TfM!UbZ@?zRddC7#6ss1sw%CXJ4FWC+t3lHZupzM77m^=9 z&(a?-LxIq}*nvv)y?27lZ{j zifdl9hyJudyP2LpU$-kXctshbJDKS{WfulP5Dk~xU4Le4c#h^(YjJit4#R8_khheS z|8(>2ibaHES4+J|DBM7I#QF5u-*EdN{n=Kt@4Zt?@Tv{JZA{`4 zU#kYOv{#A&gGPwT+$Ud}AXlK3K7hYzo$(fBSFjrP{QQ zeaKg--L&jh$9N}`pu{Bs>?eDFPaWY4|9|foN%}i;3%;@4{dc+iw>m}{3rELqH21G! z`8@;w-zsJ1H(N3%|1B@#ioLOjib)j`EiJqPQVSbPSPVHCj6t5J&(NcWzBrzCiDt{4 zdlPAUKldz%6x5II1H_+jv)(xVL+a;P+-1hv_pM>gMRr%04@k;DTokASSKKhU1Qms| zrWh3a!b(J3n0>-tipg{a?UaKsP7?+|@A+1WPDiQIW1Sf@qDU~M_P65_s}7(gjTn0X zucyEm)o;f8UyshMy&>^SC3I|C6jR*R_GFwGranWZe*I>K+0k}pBuET&M~ z;Odo*ZcT?ZpduHyrf8E%IBFtv;JQ!N_m>!sV6ly$_1D{(&nO~w)G~Y`7sD3#hQk%^ zp}ucDF_$!6DAz*PM8yE(&~;%|=+h(Rn-=1Wykas_-@d&z#=S}rDf`4w(rVlcF&lF! z=1)M3YVz7orwk^BXhslJ8jR);sh^knJW(Qmm(QdSgIAIdlN4Te5KJisifjr?eB{FjAX1a0AB>d?qY4Wx>BZ8&}5K0fA+d{l8 z?^s&l8#j7pR&ijD?0b%;lL9l$P_mi2^*_OL+b}4kuLR$GAf85sOo02?Y#90}CCDiS zZ%rbCw>=H~CBO=C_JVV=xgDe%b4FaEFtuS7Q1##y686r%F6I)s-~2(}PWK|Z8M+Gu zl$y~5@#0Ka%$M<&Cv%L`a8X^@tY&T7<0|(6dNT=EsRe0%kp1Qyq!^43VAKYnr*A5~ zsI%lK1ewqO;0TpLrT9v}!@vJK{QoVa_+N4FYT#h?Y8rS1S&-G+m$FNMP?(8N`MZP zels(*?kK{{^g9DOzkuZXJ2;SrOQsp9T$hwRB1(phw1c7`!Q!by?Q#YsSM#I12RhU{$Q+{xj83axHcftEc$mNJ8_T7A-BQc*k(sZ+~NsO~xAA zxnbb%dam_fZlHvW7fKXrB~F&jS<4FD2FqY?VG?ix*r~MDXCE^WQ|W|WM;gsIA4lQP zJ2hAK@CF*3*VqPr2eeg6GzWFlICi8S>nO>5HvWzyZTE)hlkdC_>pBej*>o0EOHR|) z$?};&I4+_?wvL*g#PJ9)!bc#9BJu1(*RdNEn>#Oxta(VWeM40ola<0aOe2kSS~{^P zDJBd}0L-P#O-CzX*%+$#v;(x%<*SPgAje=F{Zh-@ucd2DA(yC|N_|ocs*|-!H%wEw z@Q!>siv2W;C^^j^59OAX03&}&D*W4EjCvfi(ygcL#~t8XGa#|NPO+*M@Y-)ctFA@I z-p7npT1#5zOLo>7q?aZpCZ=iecn3QYklP;gF0bq@>oyBq94f6C=;Csw3PkZ|5q=(c zfs`aw?II0e(h=|7o&T+hq&m$; zBrE09Twxd9BJ2P+QPN}*OdZ-JZV7%av@OM7v!!NL8R;%WFq*?{9T3{ct@2EKgc8h) zMxoM$SaF#p<`65BwIDfmXG6+OiK0e)`I=!A3E`+K@61f}0e z!2a*FOaDrOe>U`q%K!QN`&=&0C~)CaL3R4VY(NDt{Xz(Xpqru5=r#uQN1L$Je1*dkdqQ*=lofQaN%lO!<5z9ZlHgxt|`THd>2 zsWfU$9=p;yLyJyM^t zS2w9w?Bpto`@H^xJpZDKR1@~^30Il6oFGfk5%g6w*C+VM)+%R@gfIwNprOV5{F^M2 zO?n3DEzpT+EoSV-%OdvZvNF+pDd-ZVZ&d8 zKeIyrrfPN=EcFRCPEDCVflX#3-)Ik_HCkL(ejmY8vzcf-MTA{oHk!R2*36`O68$7J zf}zJC+bbQk--9Xm!u#lgLvx8TXx2J258E5^*IZ(FXMpq$2LUUvhWQPs((z1+2{Op% z?J}9k5^N=z;7ja~zi8a_-exIqWUBJwohe#4QJ`|FF*$C{lM18z^#hX6!5B8KAkLUX ziP=oti-gpV(BsLD{0(3*dw}4JxK23Y7M{BeFPucw!sHpY&l%Ws4pSm`+~V7;bZ%Dx zeI)MK=4vC&5#;2MT7fS?^ch9?2;%<8Jlu-IB&N~gg8t;6S-#C@!NU{`p7M8@2iGc& zg|JPg%@gCoCQ&s6JvDU&`X2S<57f(k8nJ1wvBu{8r?;q3_kpZZ${?|( z+^)UvR33sjSd)aT!UPkA;ylO6{aE3MQa{g%Mcf$1KONcjO@&g5zPHWtzM1rYC{_K> zgQNcs<{&X{OA=cEWw5JGqpr0O>x*Tfak2PE9?FuWtz^DDNI}rwAaT0(bdo-<+SJ6A z&}S%boGMWIS0L}=S>|-#kRX;e^sUsotry(MjE|3_9duvfc|nwF#NHuM-w7ZU!5ei8 z6Mkf>2)WunY2eU@C-Uj-A zG(z0Tz2YoBk>zCz_9-)4a>T46$(~kF+Y{#sA9MWH%5z#zNoz)sdXq7ZR_+`RZ%0(q zC7&GyS_|BGHNFl8Xa%@>iWh%Gr?=J5<(!OEjauj5jyrA-QXBjn0OAhJJ9+v=!LK`` z@g(`^*84Q4jcDL`OA&ZV60djgwG`|bcD*i50O}Q{9_noRg|~?dj%VtKOnyRs$Uzqg z191aWoR^rDX#@iSq0n z?9Sg$WSRPqSeI<}&n1T3!6%Wj@5iw5`*`Btni~G=&;J+4`7g#OQTa>u`{4ZZ(c@s$ zK0y;ySOGD-UTjREKbru{QaS>HjN<2)R%Nn-TZiQ(Twe4p@-saNa3~p{?^V9Nixz@a zykPv~<@lu6-Ng9i$Lrk(xi2Tri3q=RW`BJYOPC;S0Yly%77c727Yj-d1vF!Fuk{Xh z)lMbA69y7*5ufET>P*gXQrxsW+ zz)*MbHZv*eJPEXYE<6g6_M7N%#%mR{#awV3i^PafNv(zyI)&bH?F}2s8_rR(6%!V4SOWlup`TKAb@ee>!9JKPM=&8g#BeYRH9FpFybxBXQI2|g}FGJfJ+ zY-*2hB?o{TVL;Wt_ek;AP5PBqfDR4@Z->_182W z{P@Mc27j6jE*9xG{R$>6_;i=y{qf(c`5w9fa*`rEzX6t!KJ(p1H|>J1pC-2zqWENF zmm=Z5B4u{cY2XYl(PfrInB*~WGWik3@1oRhiMOS|D;acnf-Bs(QCm#wR;@Vf!hOPJ zgjhDCfDj$HcyVLJ=AaTbQ{@vIv14LWWF$=i-BDoC11}V;2V8A`S>_x)vIq44-VB-v z*w-d}$G+Ql?En8j!~ZkCpQ$|cA0|+rrY>tiCeWxkRGPoarxlGU2?7%k#F693RHT24 z-?JsiXlT2PTqZqNb&sSc>$d;O4V@|b6VKSWQb~bUaWn1Cf0+K%`Q&Wc<>mQ>*iEGB zbZ;aYOotBZ{vH3y<0A*L0QVM|#rf*LIsGx(O*-7)r@yyBIzJnBFSKBUSl1e|8lxU* zzFL+YDVVkIuzFWeJ8AbgN&w(4-7zbiaMn{5!JQXu)SELk*CNL+Fro|2v|YO)1l15t zs(0^&EB6DPMyaqvY>=KL>)tEpsn;N5Q#yJj<9}ImL((SqErWN3Q=;tBO~ExTCs9hB z2E$7eN#5wX4<3m^5pdjm#5o>s#eS_Q^P)tm$@SawTqF*1dj_i#)3};JslbLKHXl_N z)Fxzf>FN)EK&Rz&*|6&%Hs-^f{V|+_vL1S;-1K-l$5xiC@}%uDuwHYhmsV?YcOUlk zOYkG5v2+`+UWqpn0aaaqrD3lYdh0*!L`3FAsNKu=Q!vJu?Yc8n|CoYyDo_`r0mPoo z8>XCo$W4>l(==h?2~PoRR*kEe)&IH{1sM41mO#-36`02m#nTX{r*r`Q5rZ2-sE|nA zhnn5T#s#v`52T5|?GNS`%HgS2;R(*|^egNPDzzH_z^W)-Q98~$#YAe)cEZ%vge965AS_am#DK#pjPRr-!^za8>`kksCAUj(Xr*1NW5~e zpypt_eJpD&4_bl_y?G%>^L}=>xAaV>KR6;^aBytqpiHe%!j;&MzI_>Sx7O%F%D*8s zSN}cS^<{iiK)=Ji`FpO#^zY!_|D)qeRNAtgmH)m;qC|mq^j(|hL`7uBz+ULUj37gj zksdbnU+LSVo35riSX_4z{UX=%n&}7s0{WuZYoSfwAP`8aKN9P@%e=~1`~1ASL-z%# zw>DO&ixr}c9%4InGc*_y42bdEk)ZdG7-mTu0bD@_vGAr*NcFoMW;@r?@LUhRI zCUJgHb`O?M3!w)|CPu~ej%fddw20lod?Ufp8Dmt0PbnA0J%KE^2~AIcnKP()025V> zG>noSM3$5Btmc$GZoyP^v1@Poz0FD(6YSTH@aD0}BXva?LphAiSz9f&Y(aDAzBnUh z?d2m``~{z;{}kZJ>a^wYI?ry(V9hIoh;|EFc0*-#*`$T0DRQ1;WsqInG;YPS+I4{g zJGpKk%%Sdc5xBa$Q^_I~(F97eqDO7AN3EN0u)PNBAb+n+ zWBTxQx^;O9o0`=g+Zrt_{lP!sgWZHW?8bLYS$;1a@&7w9rD9|Ge;Gb?sEjFoF9-6v z#!2)t{DMHZ2@0W*fCx;62d#;jouz`R5Y(t{BT=$N4yr^^o$ON8d{PQ=!O zX17^CrdM~7D-;ZrC!||<+FEOxI_WI3CA<35va%4v>gc zEX-@h8esj=a4szW7x{0g$hwoWRQG$yK{@3mqd-jYiVofJE!Wok1* znV7Gm&Ssq#hFuvj1sRyHg(6PFA5U*Q8Rx>-blOs=lb`qa{zFy&n4xY;sd$fE+<3EI z##W$P9M{B3c3Si9gw^jlPU-JqD~Cye;wr=XkV7BSv#6}DrsXWFJ3eUNrc%7{=^sP> zrp)BWKA9<}^R9g!0q7yWlh;gr_TEOD|#BmGq<@IV;ueg+D2}cjpp+dPf&Q(36sFU&K8}hA85U61faW&{ zlB`9HUl-WWCG|<1XANN3JVAkRYvr5U4q6;!G*MTdSUt*Mi=z_y3B1A9j-@aK{lNvx zK%p23>M&=KTCgR!Ee8c?DAO2_R?B zkaqr6^BSP!8dHXxj%N1l+V$_%vzHjqvu7p@%Nl6;>y*S}M!B=pz=aqUV#`;h%M0rU zHfcog>kv3UZAEB*g7Er@t6CF8kHDmKTjO@rejA^ULqn!`LwrEwOVmHx^;g|5PHm#B zZ+jjWgjJ!043F+&#_;D*mz%Q60=L9Ove|$gU&~As5^uz@2-BfQ!bW)Khn}G+Wyjw- z19qI#oB(RSNydn0t~;tAmK!P-d{b-@@E5|cdgOS#!>%#Rj6ynkMvaW@37E>@hJP^8 z2zk8VXx|>#R^JCcWdBCy{0nPmYFOxN55#^-rlqobe0#L6)bi?E?SPymF*a5oDDeSd zO0gx?#KMoOd&G(2O@*W)HgX6y_aa6iMCl^~`{@UR`nMQE`>n_{_aY5nA}vqU8mt8H z`oa=g0SyiLd~BxAj2~l$zRSDHxvDs;I4>+M$W`HbJ|g&P+$!U7-PHX4RAcR0szJ*( ze-417=bO2q{492SWrqDK+L3#ChUHtz*@MP)e^%@>_&#Yk^1|tv@j4%3T)diEX zATx4K*hcO`sY$jk#jN5WD<=C3nvuVsRh||qDHnc~;Kf59zr0;c7VkVSUPD%NnnJC_ zl3F^#f_rDu8l}l8qcAz0FFa)EAt32IUy_JLIhU_J^l~FRH&6-ivSpG2PRqzDdMWft>Zc(c)#tb%wgmWN%>IOPm zZi-noqS!^Ftb81pRcQi`X#UhWK70hy4tGW1mz|+vI8c*h@ zfFGJtW3r>qV>1Z0r|L>7I3un^gcep$AAWfZHRvB|E*kktY$qQP_$YG60C@X~tTQjB3%@`uz!qxtxF+LE!+=nrS^07hn` zEgAp!h|r03h7B!$#OZW#ACD+M;-5J!W+{h|6I;5cNnE(Y863%1(oH}_FTW})8zYb$7czP zg~Szk1+_NTm6SJ0MS_|oSz%e(S~P-&SFp;!k?uFayytV$8HPwuyELSXOs^27XvK-D zOx-Dl!P|28DK6iX>p#Yb%3`A&CG0X2S43FjN%IB}q(!hC$fG}yl1y9W&W&I@KTg6@ zK^kpH8=yFuP+vI^+59|3%Zqnb5lTDAykf z9S#X`3N(X^SpdMyWQGOQRjhiwlj!0W-yD<3aEj^&X%=?`6lCy~?`&WSWt z?U~EKFcCG_RJ(Qp7j=$I%H8t)Z@6VjA#>1f@EYiS8MRHZphp zMA_5`znM=pzUpBPO)pXGYpQ6gkine{6u_o!P@Q+NKJ}k!_X7u|qfpAyIJb$_#3@wJ z<1SE2Edkfk9C!0t%}8Yio09^F`YGzpaJHGk*-ffsn85@)%4@`;Fv^8q(-Wk7r=Q8p zT&hD`5(f?M{gfzGbbwh8(}G#|#fDuk7v1W)5H9wkorE0ZZjL0Q1=NRGY>zwgfm81DdoaVwNH;or{{eSyybt)m<=zXoA^RALYG-2t zouH|L*BLvmm9cdMmn+KGopyR@4*=&0&4g|FLoreZOhRmh=)R0bg~ zT2(8V_q7~42-zvb)+y959OAv!V$u(O3)%Es0M@CRFmG{5sovIq4%8Ahjk#*5w{+)+ zMWQoJI_r$HxL5km1#6(e@{lK3Udc~n0@g`g$s?VrnQJ$!oPnb?IHh-1qA`Rz$)Ai< z6w$-MJW-gKNvOhL+XMbE7&mFt`x1KY>k4(!KbbpZ`>`K@1J<(#vVbjx@Z@(6Q}MF# zMnbr-f55(cTa^q4+#)=s+ThMaV~E`B8V=|W_fZWDwiso8tNMTNse)RNBGi=gVwgg% zbOg8>mbRN%7^Um-7oj4=6`$|(K7!+t^90a{$18Z>}<#!bm%ZEFQ{X(yBZMc>lCz0f1I2w9Sq zuGh<9<=AO&g6BZte6hn>Qmvv;Rt)*cJfTr2=~EnGD8P$v3R|&1RCl&7)b+`=QGapi zPbLg_pxm`+HZurtFZ;wZ=`Vk*do~$wB zxoW&=j0OTbQ=Q%S8XJ%~qoa3Ea|au5o}_(P;=!y-AjFrERh%8la!z6Fn@lR?^E~H12D?8#ht=1F;7@o4$Q8GDj;sSC%Jfn01xgL&%F2 zwG1|5ikb^qHv&9hT8w83+yv&BQXOQyMVJSBL(Ky~p)gU3#%|blG?IR9rP^zUbs7rOA0X52Ao=GRt@C&zlyjNLv-} z9?*x{y(`509qhCV*B47f2hLrGl^<@SuRGR!KwHei?!CM10Tq*YDIoBNyRuO*>3FU? zHjipIE#B~y3FSfOsMfj~F9PNr*H?0oHyYB^G(YyNh{SxcE(Y-`x5jFMKb~HO*m+R% zrq|ic4fzJ#USpTm;X7K+E%xsT_3VHKe?*uc4-FsILUH;kL>_okY(w`VU*8+l>o>Jm ziU#?2^`>arnsl#)*R&nf_%>A+qwl%o{l(u)M?DK1^mf260_oteV3#E_>6Y4!_hhVD zM8AI6MM2V*^_M^sQ0dmHu11fy^kOqXqzpr?K$`}BKWG`=Es(9&S@K@)ZjA{lj3ea7_MBP zk(|hBFRjHVMN!sNUkrB;(cTP)T97M$0Dtc&UXSec<+q?y>5=)}S~{Z@ua;1xt@=T5 zI7{`Z=z_X*no8s>mY;>BvEXK%b`a6(DTS6t&b!vf_z#HM{Uoy_5fiB(zpkF{})ruka$iX*~pq1ZxD?q68dIo zIZSVls9kFGsTwvr4{T_LidcWtt$u{kJlW7moRaH6+A5hW&;;2O#$oKyEN8kx`LmG)Wfq4ykh+q{I3|RfVpkR&QH_x;t41Uw z`P+tft^E2B$domKT@|nNW`EHwyj>&}K;eDpe z1bNOh=fvIfk`&B61+S8ND<(KC%>y&?>opCnY*r5M+!UrWKxv0_QvTlJc>X#AaI^xo zaRXL}t5Ej_Z$y*|w*$6D+A?Lw-CO-$itm^{2Ct82-<0IW)0KMNvJHgBrdsIR0v~=H z?n6^}l{D``Me90`^o|q!olsF?UX3YSq^6Vu>Ijm>>PaZI8G@<^NGw{Cx&%|PwYrfw zR!gX_%AR=L3BFsf8LxI|K^J}deh0ZdV?$3r--FEX`#INxsOG6_=!v)DI>0q|BxT)z z-G6kzA01M?rba+G_mwNMQD1mbVbNTWmBi*{s_v_Ft9m2Avg!^78(QFu&n6mbRJ2bA zv!b;%yo{g*9l2)>tsZJOOp}U~8VUH`}$ z8p_}t*XIOehezolNa-a2x0BS})Y9}&*TPgua{Ewn-=wVrmJUeU39EKx+%w%=ixQWK zDLpwaNJs65#6o7Ln7~~X+p_o2BR1g~VCfxLzxA{HlWAI6^H;`juI=&r1jQrUv_q0Z z1Ja-tjdktrrP>GOC*#p?*xfQU5MqjMsBe!9lh(u8)w$e@Z|>aUHI5o;MGw*|Myiz3 z-f0;pHg~Q#%*Kx8MxH%AluVXjG2C$)WL-K63@Q`#y9_k_+}eR(x4~dp7oV-ek0H>I zgy8p#i4GN{>#v=pFYUQT(g&b$OeTy-X_#FDgNF8XyfGY6R!>inYn8IR2RDa&O!(6< znXs{W!bkP|s_YI*Yx%4stI`=ZO45IK6rBs`g7sP40ic}GZ58s?Mc$&i`kq_tfci>N zIHrC0H+Qpam1bNa=(`SRKjixBTtm&e`j9porEci!zdlg1RI0Jw#b(_Tb@RQK1Zxr_ z%7SUeH6=TrXt3J@js`4iDD0=IoHhK~I7^W8^Rcp~Yaf>2wVe|Hh1bUpX9ATD#moByY57-f2Ef1TP^lBi&p5_s7WGG9|0T}dlfxOx zXvScJO1Cnq`c`~{Dp;{;l<-KkCDE+pmexJkd}zCgE{eF=)K``-qC~IT6GcRog_)!X z?fK^F8UDz$(zFUrwuR$qro5>qqn>+Z%<5>;_*3pZ8QM|yv9CAtrAx;($>4l^_$_-L z*&?(77!-=zvnCVW&kUcZMb6;2!83si518Y%R*A3JZ8Is|kUCMu`!vxDgaWjs7^0j( ziTaS4HhQ)ldR=r)_7vYFUr%THE}cPF{0H45FJ5MQW^+W>P+eEX2kLp3zzFe*-pFVA zdDZRybv?H|>`9f$AKVjFWJ=wegO7hOOIYCtd?Vj{EYLT*^gl35|HQ`R=ti+ADm{jyQE7K@kdjuqJhWVSks>b^ zxha88-h3s;%3_5b1TqFCPTxVjvuB5U>v=HyZ$?JSk+&I%)M7KE*wOg<)1-Iy)8-K! z^XpIt|0ibmk9RtMmlUd7#Ap3Q!q9N4atQy)TmrhrFhfx1DAN`^vq@Q_SRl|V z#lU<~n67$mT)NvHh`%als+G-)x1`Y%4Bp*6Un5Ri9h=_Db zA-AdP!f>f0m@~>7X#uBM?diI@)Egjuz@jXKvm zJo+==juc9_<;CqeRaU9_Mz@;3e=E4=6TK+c`|uu#pIqhSyNm`G(X)&)B`8q0RBv#> z`gGlw(Q=1Xmf55VHj%C#^1lpc>LY8kfA@|rlC1EA<1#`iuyNO z(=;irt{_&K=i4)^x%;U(Xv<)+o=dczC5H3W~+e|f~{*ucxj@{Yi-cw^MqYr3fN zF5D+~!wd$#al?UfMnz(@K#wn`_5na@rRr8XqN@&M&FGEC@`+OEv}sI1hw>Up0qAWf zL#e4~&oM;TVfjRE+10B_gFlLEP9?Q-dARr3xi6nQqnw>k-S;~b z;!0s2VS4}W8b&pGuK=7im+t(`nz@FnT#VD|!)eQNp-W6)@>aA+j~K*H{$G`y2|QHY z|Hmy+CR@#jWY4~)lr1qBJB_RfHJFfP<}pK5(#ZZGSqcpyS&}01LnTWk5fzmXMGHkJ zTP6L^B+uj;lmB_W<~4=${+v0>z31M!-_O@o-O9GyW)j_mjx}!0@br_LE-7SIuPP84 z;5=O(U*g_um0tyG|61N@d9lEuOeiRd+#NY^{nd5;-CVlw&Ap7J?qwM^?E29wvS}2d zbzar4Fz&RSR(-|s!Z6+za&Z zY#D<5q_JUktIzvL0)yq_kLWG6DO{ri=?c!y!f(Dk%G{8)k`Gym%j#!OgXVDD3;$&v@qy#ISJfp=Vm>pls@9-mapVQChAHHd-x+OGx)(*Yr zC1qDUTZ6mM(b_hi!TuFF2k#8uI2;kD70AQ&di$L*4P*Y-@p`jdm%_c3f)XhYD^6M8&#Y$ZpzQMcR|6nsH>b=*R_Von!$BTRj7yGCXokoAQ z&ANvx0-Epw`QIEPgI(^cS2f(Y85yV@ygI{ewyv5Frng)e}KCZF7JbR(&W618_dcEh(#+^zZFY;o<815<5sOHQdeax9_!PyM&;{P zkBa5xymca0#)c#tke@3KNEM8a_mT&1gm;p&&JlMGH(cL(b)BckgMQ^9&vRwj!~3@l zY?L5}=Jzr080OGKb|y`ee(+`flQg|!lo6>=H)X4`$Gz~hLmu2a%kYW_Uu8x09Pa0J zKZ`E$BKJ=2GPj_3l*TEcZ*uYRr<*J^#5pILTT;k_cgto1ZL-%slyc16J~OH-(RgDA z%;EjEnoUkZ&acS{Q8`{i6T5^nywgqQI5bDIymoa7CSZG|WWVk>GM9)zy*bNih|QIm z%0+(Nnc*a_xo;$=!HQYaapLms>J1ToyjtFByY`C2H1wT#178#4+|{H0BBqtCdd$L% z_3Hc60j@{t9~MjM@LBalR&6@>B;9?r<7J~F+WXyYu*y3?px*=8MAK@EA+jRX8{CG?GI-< z54?Dc9CAh>QTAvyOEm0^+x;r2BWX|{3$Y7)L5l*qVE*y0`7J>l2wCmW zL1?|a`pJ-l{fb_N;R(Z9UMiSj6pQjOvQ^%DvhIJF!+Th7jO2~1f1N+(-TyCFYQZYw z4)>7caf^Ki_KJ^Zx2JUb z&$3zJy!*+rCV4%jqwyuNY3j1ZEiltS0xTzd+=itTb;IPYpaf?8Y+RSdVdpacB(bVQ zC(JupLfFp8y43%PMj2}T|VS@%LVp>hv4Y!RPMF?pp8U_$xCJ)S zQx!69>bphNTIb9yn*_yfj{N%bY)t{L1cs8<8|!f$;UQ*}IN=2<6lA;x^(`8t?;+ST zh)z4qeYYgZkIy{$4x28O-pugO&gauRh3;lti9)9Pvw+^)0!h~%m&8Q!AKX%urEMnl z?yEz?g#ODn$UM`+Q#$Q!6|zsq_`dLO5YK-6bJM6ya>}H+vnW^h?o$z;V&wvuM$dR& zeEq;uUUh$XR`TWeC$$c&Jjau2it3#%J-y}Qm>nW*s?En?R&6w@sDXMEr#8~$=b(gk zwDC3)NtAP;M2BW_lL^5ShpK$D%@|BnD{=!Tq)o(5@z3i7Z){} zGr}Exom_qDO{kAVkZ*MbLNHE666Kina#D{&>Jy%~w7yX$oj;cYCd^p9zy z8*+wgSEcj$4{WxKmCF(5o7U4jqwEvO&dm1H#7z}%VXAbW&W24v-tS6N3}qrm1OnE)fUkoE8yMMn9S$?IswS88tQWm4#Oid#ckgr6 zRtHm!mfNl-`d>O*1~d7%;~n+{Rph6BBy^95zqI{K((E!iFQ+h*C3EsbxNo_aRm5gj zKYug($r*Q#W9`p%Bf{bi6;IY0v`pB^^qu)gbg9QHQ7 zWBj(a1YSu)~2RK8Pi#C>{DMlrqFb9e_RehEHyI{n?e3vL_}L>kYJC z_ly$$)zFi*SFyNrnOt(B*7E$??s67EO%DgoZL2XNk8iVx~X_)o++4oaK1M|ou73vA0K^503j@uuVmLcHH4ya-kOIDfM%5%(E z+Xpt~#7y2!KB&)PoyCA+$~DXqxPxxALy!g-O?<9+9KTk4Pgq4AIdUkl`1<1#j^cJg zgU3`0hkHj_jxV>`Y~%LAZl^3o0}`Sm@iw7kwff{M%VwtN)|~!p{AsfA6vB5UolF~d zHWS%*uBDt<9y!9v2Xe|au&1j&iR1HXCdyCjxSgG*L{wmTD4(NQ=mFjpa~xooc6kju z`~+d{j7$h-;HAB04H!Zscu^hZffL#9!p$)9>sRI|Yovm)g@F>ZnosF2EgkU3ln0bR zTA}|+E(tt)!SG)-bEJi_0m{l+(cAz^pi}`9=~n?y&;2eG;d9{M6nj>BHGn(KA2n|O zt}$=FPq!j`p&kQ8>cirSzkU0c08%8{^Qyqi-w2LoO8)^E7;;I1;HQ6B$u0nNaX2CY zSmfi)F`m94zL8>#zu;8|{aBui@RzRKBlP1&mfFxEC@%cjl?NBs`cr^nm){>;$g?rhKr$AO&6qV_Wbn^}5tfFBry^e1`%du2~o zs$~dN;S_#%iwwA_QvmMjh%Qo?0?rR~6liyN5Xmej8(*V9ym*T`xAhHih-v$7U}8=dfXi2i*aAB!xM(Xekg*ix@r|ymDw*{*s0?dlVys2e)z62u1 z+k3esbJE=-P5S$&KdFp+2H7_2e=}OKDrf( z9-207?6$@f4m4B+9E*e((Y89!q?zH|mz_vM>kp*HGXldO0Hg#!EtFhRuOm$u8e~a9 z5(roy7m$Kh+zjW6@zw{&20u?1f2uP&boD}$#Zy)4o&T;vyBoqFiF2t;*g=|1=)PxB z8eM3Mp=l_obbc?I^xyLz?4Y1YDWPa+nm;O<$Cn;@ane616`J9OO2r=rZr{I_Kizyc zP#^^WCdIEp*()rRT+*YZK>V@^Zs=ht32x>Kwe zab)@ZEffz;VM4{XA6e421^h~`ji5r%)B{wZu#hD}f3$y@L0JV9f3g{-RK!A?vBUA}${YF(vO4)@`6f1 z-A|}e#LN{)(eXloDnX4Vs7eH|<@{r#LodP@Nz--$Dg_Par%DCpu2>2jUnqy~|J?eZ zBG4FVsz_A+ibdwv>mLp>P!(t}E>$JGaK$R~;fb{O3($y1ssQQo|5M;^JqC?7qe|hg zu0ZOqeFcp?qVn&Qu7FQJ4hcFi&|nR!*j)MF#b}QO^lN%5)4p*D^H+B){n8%VPUzi! zDihoGcP71a6!ab`l^hK&*dYrVYzJ0)#}xVrp!e;lI!+x+bfCN0KXwUAPU9@#l7@0& QuEJmfE|#`Dqx|px0L@K;Y5)KL diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index e2847c82..bad7c246 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-9.2.0-bin.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/gradlew b/gradlew index 1b6c7873..f5feea6d 100755 --- a/gradlew +++ b/gradlew @@ -15,6 +15,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# SPDX-License-Identifier: Apache-2.0 +# ############################################################################## # @@ -55,7 +57,7 @@ # Darwin, MinGW, and NonStop. # # (3) This script is generated from the Groovy template -# https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt # within the Gradle project. # # You can find Gradle at https://github.com/gradle/gradle/. @@ -80,13 +82,12 @@ do esac done -APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit - -APP_NAME="Gradle" +# This is normally unused +# shellcheck disable=SC2034 APP_BASE_NAME=${0##*/} - -# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s +' "$PWD" ) || exit # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD=maximum @@ -133,22 +134,29 @@ location of your Java installation." fi else JAVACMD=java - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the location of your Java installation." + fi fi # Increase the maximum file descriptors if we can. if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then case $MAX_FD in #( max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 MAX_FD=$( ulimit -H -n ) || warn "Could not query maximum file descriptor limit" esac case $MAX_FD in #( '' | soft) :;; #( *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 ulimit -n "$MAX_FD" || warn "Could not set maximum file descriptor limit to $MAX_FD" esac @@ -193,11 +201,15 @@ if "$cygwin" || "$msys" ; then done fi -# Collect all arguments for the java command; -# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of -# shell script including quotes and variable substitutions, so put them in -# double quotes to make sure that they get re-expanded; and -# * put everything else in single quotes, so that it's not re-expanded. + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. set -- \ "-Dorg.gradle.appname=$APP_BASE_NAME" \ @@ -205,6 +217,12 @@ set -- \ org.gradle.wrapper.GradleWrapperMain \ "$@" +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + # Use "xargs" to parse quoted args. # # With -n1 it outputs one arg per line, with the quotes and backslashes removed. diff --git a/gradlew.bat b/gradlew.bat index ac1b06f9..9b42019c 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -13,8 +13,10 @@ @rem See the License for the specific language governing permissions and @rem limitations under the License. @rem +@rem SPDX-License-Identifier: Apache-2.0 +@rem -@if "%DEBUG%" == "" @echo off +@if "%DEBUG%"=="" @echo off @rem ########################################################################## @rem @rem Gradle startup script for Windows @@ -25,7 +27,8 @@ if "%OS%"=="Windows_NT" setlocal set DIRNAME=%~dp0 -if "%DIRNAME%" == "" set DIRNAME=. +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused set APP_BASE_NAME=%~n0 set APP_HOME=%DIRNAME% @@ -40,13 +43,13 @@ if defined JAVA_HOME goto findJavaFromJavaHome set JAVA_EXE=java.exe %JAVA_EXE% -version >NUL 2>&1 -if "%ERRORLEVEL%" == "0" goto execute +if %ERRORLEVEL% equ 0 goto execute -echo. -echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. +echo. 1>&2 +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 goto fail @@ -56,11 +59,11 @@ set JAVA_EXE=%JAVA_HOME%/bin/java.exe if exist "%JAVA_EXE%" goto execute -echo. -echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. +echo. 1>&2 +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 goto fail @@ -75,13 +78,15 @@ set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar :end @rem End local scope for the variables with windows NT shell -if "%ERRORLEVEL%"=="0" goto mainEnd +if %ERRORLEVEL% equ 0 goto mainEnd :fail rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of rem the _cmd.exe /c_ return code! -if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 -exit /b 1 +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% :mainEnd if "%OS%"=="Windows_NT" endlocal