Initial implementation of retry logic using Spring Retry

spring-cloud-netflix-core/src/main/java/org/springframework/cloud/netflix/ribbon/support/RetryableLoadBalancingClient.java

+/*
+ *
+ * Copyright 2013-2016 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.springframework.cloud.netflix.ribbon.support;
+
+import java.io.IOException;
+import org.apache.commons.lang.BooleanUtils;
+import org.springframework.cloud.client.ServiceInstance;
+import org.springframework.cloud.client.loadbalancer.InterceptorRetryPolicy;
+import org.springframework.cloud.client.loadbalancer.LoadBalancedRetryPolicy;
+import org.springframework.cloud.client.loadbalancer.LoadBalancedRetryPolicyFactory;
+import org.springframework.cloud.client.loadbalancer.ServiceInstanceChooser;
+import org.springframework.cloud.netflix.feign.ribbon.FeignRetryPolicy;
+import org.springframework.cloud.netflix.ribbon.RibbonLoadBalancerClient;
+import org.springframework.cloud.netflix.ribbon.ServerIntrospector;
+import org.springframework.http.HttpRequest;
+import org.springframework.retry.RetryCallback;
+import org.springframework.retry.policy.NeverRetryPolicy;
+import org.springframework.retry.support.RetryTemplate;
+import com.netflix.client.IResponse;
+import com.netflix.client.RequestSpecificRetryHandler;
+import com.netflix.client.RetryHandler;
+import com.netflix.client.config.IClientConfig;
+import com.netflix.loadbalancer.ILoadBalancer;
+import com.netflix.loadbalancer.Server;
+
+/**
+ * A load balancing client which uses Spring Retry to retry failed requests.
+ * @author Ryan Baxter
+ */
+public abstract class RetryableLoadBalancingClient<S extends ContextAwareRequest, T extends IResponse, D>
+		extends AbstractLoadBalancingClient<S, T, D> implements ServiceInstanceChooser {
+
+	protected LoadBalancedRetryPolicyFactory loadBalancedRetryPolicyFactory =
+			new LoadBalancedRetryPolicyFactory.NeverRetryFactory();
+
+	@Deprecated
+	public RetryableLoadBalancingClient() {
+		super();
+	}
+
+	@Deprecated
+	public RetryableLoadBalancingClient(final ILoadBalancer lb) {
+		super(lb);
+	}
+
+	public RetryableLoadBalancingClient(IClientConfig config, ServerIntrospector serverIntrospector) {
+		super(config, serverIntrospector);
+	}
+
+	public RetryableLoadBalancingClient(D delegate, IClientConfig config, ServerIntrospector serverIntrospector) {
+		super(delegate, config, serverIntrospector);
+	}
+
+	public RetryableLoadBalancingClient(IClientConfig iClientConfig, ServerIntrospector serverIntrospector,
+										 LoadBalancedRetryPolicyFactory loadBalancedRetryPolicyFactory) {
+		this(iClientConfig, serverIntrospector);
+		this.loadBalancedRetryPolicyFactory = loadBalancedRetryPolicyFactory;
+	}
+
+	/**
+	 * Executes a {@link S} using Spring Retry.
+	 * @param request The request to execute.
+	 * @param callback The retry callback to use.
+	 * @return The response.
+	 * @throws Exception Thrown if there is an error making the request and a retry cannot be completed successfully.
+	 */
+	protected T executeWithRetry(S request, RetryCallback<T, IOException> callback) throws Exception {
+		LoadBalancedRetryPolicy retryPolicy = loadBalancedRetryPolicyFactory.create(this.getClientName(), this);
+		RetryTemplate retryTemplate = new RetryTemplate();
+		boolean retryable = request.getContext() == null ? true :
+				BooleanUtils.toBooleanDefaultIfNull(request.getContext().getRetryable(), true);
+		retryTemplate.setRetryPolicy(retryPolicy == null || !retryable ? new NeverRetryPolicy()
+				: new RetryPolicy(request, retryPolicy, this, this.getClientName()));
+		return retryTemplate.execute(callback);
+	}
+
+	@Override
+	public ServiceInstance choose(String serviceId) {
+		Server server = this.getLoadBalancer().chooseServer(serviceId);
+		return new RibbonLoadBalancerClient.RibbonServer(serviceId,
+				server);
+	}
+
+	@Override
+	public RequestSpecificRetryHandler getRequestSpecificRetryHandler(S request, IClientConfig requestConfig) {
+		return new RequestSpecificRetryHandler(false, false, RetryHandler.DEFAULT, null);
+	}
+
+	static class RetryPolicy extends FeignRetryPolicy {
+		public RetryPolicy(HttpRequest request, LoadBalancedRetryPolicy policy, ServiceInstanceChooser serviceInstanceChooser, String serviceName) {
+			super(request, policy, serviceInstanceChooser, serviceName);
+		}
+	}
+
+}

spring-cloud-netflix-core/src/test/java/org/springframework/cloud/netflix/ribbon/support/ContextAwareRequestTest.java

+/*
+ *
+ *  * Copyright 2013-2016 the original author or authors.
+ *  *
+ *  * Licensed under the Apache License, Version 2.0 (the "License");
+ *  * you may not use this file except in compliance with the License.
+ *  * You may obtain a copy of the License at
+ *  *
+ *  *      http://www.apache.org/licenses/LICENSE-2.0
+ *  *
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS,
+ *  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  * See the License for the specific language governing permissions and
+ *  * limitations under the License.
+ *
+ */
+
+package org.springframework.cloud.netflix.ribbon.support;
+
+
+import java.net.URI;
+import java.util.Arrays;
+import java.util.Collections;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.springframework.cloud.netflix.zuul.filters.route.RibbonCommandContext;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.HttpMethod;
+import org.springframework.util.LinkedMultiValueMap;
+import org.springframework.util.MultiValueMap;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+
+/**
+ * @author Ryan Baxter
+ */
+public class ContextAwareRequestTest {
+
+	private RibbonCommandContext context;
+	private ContextAwareRequest request;
+
+	@Before
+	public void setUp() throws Exception {
+		context = mock(RibbonCommandContext.class);
+		doReturn("GET").when(context).getMethod();
+		MultiValueMap<String, String> headers = new LinkedMultiValueMap<>();
+		headers.put("header1", Collections.<String>emptyList());
+		headers.put("header2", Arrays.asList("value1", "value2"));
+		headers.put("header3", Arrays.asList("value1"));
+		doReturn(headers).when(context).getHeaders();
+		doReturn(new URI("http://foo")).when(context).uri();
+		doReturn("foo").when(context).getServiceId();
+		doReturn(new LinkedMultiValueMap<>()).when(context).getParams();
+		request = new TestContextAwareRequest(context);
+	}
+
+	@After
+	public void tearDown() throws Exception {
+		context = null;
+		request = null;
+	}
+
+	@Test
+	public void getContext() throws Exception {
+		assertEquals(context, request.getContext());
+	}
+
+	@Test
+	public void getMethod() throws Exception {
+		assertEquals(HttpMethod.GET, request.getMethod());
+	}
+
+	@Test
+	public void getURI() throws Exception {
+		assertEquals(new URI("http://foo"), request.getURI());
+
+		RibbonCommandContext badUriContext = mock(RibbonCommandContext.class);
+		doReturn(new LinkedMultiValueMap()).when(badUriContext).getHeaders();
+		doReturn("foobar").when(badUriContext).getUri();
+		ContextAwareRequest badUriRequest = new TestContextAwareRequest(badUriContext);
+
+		assertNull(badUriRequest.getURI());
+
+	}
+
+	@Test
+	public void getHeaders() throws Exception {
+		HttpHeaders headers = new HttpHeaders();
+		headers.put("header1", Collections.<String>emptyList());
+		headers.put("header2", Arrays.asList("value1", "value2"));
+		headers.put("header3", Arrays.asList("value1"));
+		assertEquals(headers, request.getHeaders());
+	}
+
+	static class TestContextAwareRequest extends ContextAwareRequest {
+
+		public TestContextAwareRequest(RibbonCommandContext context) {
+			super(context);
+		}
+	}
+
+}
\ No newline at end of file

spring-cloud-netflix-core/src/test/java/org/springframework/cloud/netflix/zuul/filters/route/apache/HttpClientRibbonRetryIntegrationTests.java

+/*
+ *
+ *  * Copyright 2013-2016 the original author or authors.
+ *  *
+ *  * Licensed under the Apache License, Version 2.0 (the "License");
+ *  * you may not use this file except in compliance with the License.
+ *  * You may obtain a copy of the License at
+ *  *
+ *  *      http://www.apache.org/licenses/LICENSE-2.0
+ *  *
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS,
+ *  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  * See the License for the specific language governing permissions and
+ *  * limitations under the License.
+ *
+ */
+
+package org.springframework.cloud.netflix.zuul.filters.route.apache;
+
+import org.junit.runner.RunWith;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.cloud.netflix.zuul.filters.route.support.RibbonRetryIntegrationTestBase;
+import org.springframework.test.annotation.DirtiesContext;
+import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
+
+/**
+ * @author Ryan Baxter
+ */
+@RunWith(SpringJUnit4ClassRunner.class)
+@SpringBootTest(classes = RibbonRetryIntegrationTestBase.RetryableTestConfig.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, value = {
+		"zuul.retryable: false", /* Disable retry by default, have each route enable it */
+		"hystrix.command.default.execution.timeout.enabled: false", /* Disable hystrix so its timeout doesnt get in the way */
+		"ribbon.ReadTimeout: 1000", /* Make sure ribbon will timeout before the thread is done sleeping */
+		"zuul.routes.retryable: /retryable/another/twolevel/**", /* Use /another because there is no fallback configured */
+		"zuul.routes.retryable.retryable: true",
+		"retryable.ribbon.OkToRetryOnAllOperations: true",
+		"retryable.ribbon.MaxAutoRetries: 1",
+		"retryable.ribbon.MaxAutoRetriesNextServer: 1",
+		"zuul.routes.getretryable: /getretryable/another/twolevel/**", /* Use /another because there is no fallback configured */
+		"zuul.routes.getretryable.retryable: true",
+		"getretryable.ribbon.MaxAutoRetries: 1",
+		"getretryable.ribbon.MaxAutoRetriesNextServer: 1",
+		"zuul.routes.disableretry: /disableretry/another/twolevel/**",
+		"zuul.routes.disableretry.retryable: false", /* This will override the global */
+		"disableretry.ribbon.MaxAutoRetries: 1",
+		"disableretry.ribbon.MaxAutoRetriesNextServer: 1",
+		"zuul.routes.globalretrydisabled: /globalretrydisabled/another/twolevel/**",
+		"globalretrydisabled.ribbon.MaxAutoRetries: 1",
+		"globalretrydisabled.ribbon.MaxAutoRetriesNextServer: 1"
+})
+@DirtiesContext
+public class HttpClientRibbonRetryIntegrationTests extends RibbonRetryIntegrationTestBase {
+}

spring-cloud-netflix-core/src/test/java/org/springframework/cloud/netflix/zuul/filters/route/okhttp/OkHttpRibbonRetryIntegrationTests.java

+/*
+ *
+ *  * Copyright 2013-2016 the original author or authors.
+ *  *
+ *  * Licensed under the Apache License, Version 2.0 (the "License");
+ *  * you may not use this file except in compliance with the License.
+ *  * You may obtain a copy of the License at
+ *  *
+ *  *      http://www.apache.org/licenses/LICENSE-2.0
+ *  *
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS,
+ *  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  * See the License for the specific language governing permissions and
+ *  * limitations under the License.
+ *
+ */
+
+package org.springframework.cloud.netflix.zuul.filters.route.okhttp;
+
+import org.junit.runner.RunWith;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.cloud.netflix.zuul.filters.route.support.RibbonRetryIntegrationTestBase;
+import org.springframework.test.annotation.DirtiesContext;
+import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
+
+/**
+ * @author Ryan Baxter
+ */
+@RunWith(SpringJUnit4ClassRunner.class)
+@SpringBootTest(classes = RibbonRetryIntegrationTestBase.RetryableTestConfig.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, value = {
+		"zuul.retryable: false", /* Disable retry by default, have each route enable it */
+		"ribbon.okhttp.enabled: true",
+		"hystrix.command.default.execution.timeout.enabled: false", /* Disable hystrix so its timeout doesnt get in the way */
+		"ribbon.ReadTimeout: 1000", /* Make sure ribbon will timeout before the thread is done sleeping */
+		"zuul.routes.retryable: /retryable/another/twolevel/**", /* Use /another because there is no fallback configured */
+		"zuul.routes.retryable.retryable: true",
+		"retryable.ribbon.OkToRetryOnAllOperations: true",
+		"retryable.ribbon.MaxAutoRetries: 1",
+		"retryable.ribbon.MaxAutoRetriesNextServer: 1",
+		"zuul.routes.getretryable: /getretryable/another/twolevel/**", /* Use /another because there is no fallback configured */
+		"zuul.routes.getretryable.retryable: true",
+		"getretryable.ribbon.MaxAutoRetries: 1",
+		"getretryable.ribbon.MaxAutoRetriesNextServer: 1",
+		"zuul.routes.disableretry: /disableretry/another/twolevel/**",
+		"zuul.routes.disableretry.retryable: false", /* This will override the global */
+		"disableretry.ribbon.MaxAutoRetries: 1",
+		"disableretry.ribbon.MaxAutoRetriesNextServer: 1",
+		"zuul.routes.globalretrydisabled: /globalretrydisabled/another/twolevel/**",
+		"globalretrydisabled.ribbon.MaxAutoRetries: 1",
+		"globalretrydisabled.ribbon.MaxAutoRetriesNextServer: 1"
+})
+@DirtiesContext
+public class OkHttpRibbonRetryIntegrationTests extends RibbonRetryIntegrationTestBase {
+}

spring-cloud-netflix-core/src/test/java/org/springframework/cloud/netflix/zuul/filters/route/support/RibbonRetryIntegrationTestBase.java

+/*
+ *
+ *  * Copyright 2013-2016 the original author or authors.
+ *  *
+ *  * Licensed under the Apache License, Version 2.0 (the "License");
+ *  * you may not use this file except in compliance with the License.
+ *  * You may obtain a copy of the License at
+ *  *
+ *  *      http://www.apache.org/licenses/LICENSE-2.0
+ *  *
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS,
+ *  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  * See the License for the specific language governing permissions and
+ *  * limitations under the License.
+ *
+ */
+
+package org.springframework.cloud.netflix.zuul.filters.route.support;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.test.web.client.TestRestTemplate;
+import org.springframework.cloud.netflix.ribbon.RibbonClient;
+import org.springframework.cloud.netflix.ribbon.RibbonClients;
+import org.springframework.cloud.netflix.ribbon.StaticServerList;
+import org.springframework.cloud.netflix.zuul.EnableZuulProxy;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.http.HttpEntity;
+import org.springframework.http.HttpMethod;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.RestController;
+
+import com.netflix.loadbalancer.Server;
+import com.netflix.loadbalancer.ServerList;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author Ryan Baxter
+ */
+public abstract class RibbonRetryIntegrationTestBase {
+
+	@Value("${local.server.port}")
+	protected int port;
+
+	@Before
+	public void setup() {
+		String uri = "/resetError";
+		new TestRestTemplate().exchange(
+				"http://localhost:" + this.port + uri, HttpMethod.GET,
+				new HttpEntity<>((Void) null), String.class);
+	}
+
+
+	@Test
+	public void retryable() {
+		String uri = "/retryable/another/twolevel/everyothererror";
+		ResponseEntity<String> result = new TestRestTemplate().exchange(
+				"http://localhost:" + this.port + uri, HttpMethod.GET,
+				new HttpEntity<>((Void) null), String.class);
+		assertEquals(HttpStatus.OK, result.getStatusCode());
+	}
+
+	@Test
+	public void postRetryOK() {
+		String uri = "/retryable/another/twolevel/posteveryothererror";
+		ResponseEntity<String> result = new TestRestTemplate().exchange(
+				"http://localhost:" + this.port + uri, HttpMethod.POST,
+				new HttpEntity<>((Void) null), String.class);
+		assertEquals(HttpStatus.OK, result.getStatusCode());
+	}
+
+	@Test
+	public void getRetryable() {
+		String uri = "/getretryable/another/twolevel/everyothererror";
+		ResponseEntity<String> result = new TestRestTemplate().exchange(
+				"http://localhost:" + this.port + uri, HttpMethod.GET,
+				new HttpEntity<>((Void) null), String.class);
+		assertEquals(HttpStatus.OK, result.getStatusCode());
+	}
+
+	@Test
+	public void postNotRetryable() {
+		String uri = "/getretryable/another/twolevel/posteveryothererror";
+		ResponseEntity<String> result = new TestRestTemplate().exchange(
+				"http://localhost:" + this.port + uri, HttpMethod.POST,
+				new HttpEntity<>((Void) null), String.class);
+		assertEquals(HttpStatus.INTERNAL_SERVER_ERROR, result.getStatusCode());
+	}
+
+	@Test
+	public void disbaleRetry() {
+		String uri = "/disableretry/another/twolevel/everyothererror";
+		ResponseEntity<String> result = new TestRestTemplate().exchange(
+				"http://localhost:" + this.port + uri, HttpMethod.GET,
+				new HttpEntity<>((Void) null), String.class);
+		assertEquals(HttpStatus.INTERNAL_SERVER_ERROR, result.getStatusCode());
+	}
+
+	@Test
+	public void globalRetryDisabled() {
+		String uri = "/globalretrydisabled/another/twolevel/everyothererror";
+		ResponseEntity<String> result = new TestRestTemplate().exchange(
+				"http://localhost:" + this.port + uri, HttpMethod.GET,
+				new HttpEntity<>((Void) null), String.class);
+		assertEquals(HttpStatus.INTERNAL_SERVER_ERROR, result.getStatusCode());
+	}
+
+	// Don't use @SpringBootApplication because we don't want to component scan
+	@Configuration
+	@EnableAutoConfiguration
+	@RestController
+	@EnableZuulProxy
+	@RibbonClients({
+			@RibbonClient(name = "retryable", configuration = RibbonClientConfiguration.class),
+			@RibbonClient(name = "disableretry", configuration = RibbonClientConfiguration.class),
+			@RibbonClient(name = "globalretrydisabled", configuration = RibbonClientConfiguration.class),
+			@RibbonClient(name = "getretryable", configuration = RibbonClientConfiguration.class)})
+	public static class RetryableTestConfig {
+
+		private boolean error = true;
+
+		@RequestMapping("/resetError")
+		public void resetError() {
+			error = true;
+		}
+
+		@RequestMapping("/everyothererror")
+		public ResponseEntity<String> timeout() {
+			boolean shouldError = error;
+			error = !error;
+			try {
+				if(shouldError) {
+					Thread.sleep(80000);
+				}
+			} catch (InterruptedException e) {
+				e.printStackTrace();
+			}
+
+
+			return new ResponseEntity<String>("no error", HttpStatus.OK);
+		}
+
+		@RequestMapping(path = "/posteveryothererror", method = RequestMethod.POST)
+		public ResponseEntity<String> postTimeout() {
+			return timeout();
+		}
+
+	}
+
+	@Configuration
+	public static class RibbonClientConfiguration {
+
+		@Value("${local.server.port}")
+		private int port;
+
+		@Bean
+		public ServerList<Server> ribbonServerList() {
+			return new StaticServerList<>(new Server("localhost", this.port));
+		}
+
+	}
+}