Merge pull request #374 from nobodyiam/consumer-audit

apollo-portal/src/main/java/com/ctrip/framework/apollo/openapi/entity/ConsumerAudit.java

+package com.ctrip.framework.apollo.openapi.entity;
+
+import com.google.common.base.MoreObjects;
+
+import java.util.Date;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.GeneratedValue;
+import javax.persistence.Id;
+import javax.persistence.PrePersist;
+import javax.persistence.Table;
+
+/**
+ * @author Jason Song(song_s@ctrip.com)
+ */
+@Entity
+@Table(name = "ConsumerAudit")
+public class ConsumerAudit {
+  @Id
+  @GeneratedValue
+  @Column(name = "Id")
+  private long id;
+
+  @Column(name = "ConsumerId", nullable = false)
+  private long consumerId;
+
+  @Column(name = "Uri", nullable = false)
+  private String uri;
+
+  @Column(name = "Method", nullable = false)
+  private String method;
+
+  @Column(name = "DataChange_CreatedTime")
+  private Date dataChangeCreatedTime;
+
+  @Column(name = "DataChange_LastTime")
+  private Date dataChangeLastModifiedTime;
+
+  @PrePersist
+  protected void prePersist() {
+    if (this.dataChangeCreatedTime == null) {
+      this.dataChangeCreatedTime = new Date();
+    }
+    if (this.dataChangeLastModifiedTime == null) {
+      dataChangeLastModifiedTime = this.dataChangeCreatedTime;
+    }
+  }
+
+  public long getId() {
+    return id;
+  }
+
+  public void setId(long id) {
+    this.id = id;
+  }
+
+  public long getConsumerId() {
+    return consumerId;
+  }
+
+  public void setConsumerId(long consumerId) {
+    this.consumerId = consumerId;
+  }
+
+  public String getUri() {
+    return uri;
+  }
+
+  public void setUri(String uri) {
+    this.uri = uri;
+  }
+
+  public String getMethod() {
+    return method;
+  }
+
+  public void setMethod(String method) {
+    this.method = method;
+  }
+
+  public Date getDataChangeCreatedTime() {
+    return dataChangeCreatedTime;
+  }
+
+  public void setDataChangeCreatedTime(Date dataChangeCreatedTime) {
+    this.dataChangeCreatedTime = dataChangeCreatedTime;
+  }
+
+  public Date getDataChangeLastModifiedTime() {
+    return dataChangeLastModifiedTime;
+  }
+
+  public void setDataChangeLastModifiedTime(Date dataChangeLastModifiedTime) {
+    this.dataChangeLastModifiedTime = dataChangeLastModifiedTime;
+  }
+
+  @Override
+  public String toString() {
+    return MoreObjects.toStringHelper(this)
+        .omitNullValues()
+        .add("id", id)
+        .add("consumerId", consumerId)
+        .add("uri", uri)
+        .add("method", method)
+        .add("dataChangeCreatedTime", dataChangeCreatedTime)
+        .add("dataChangeLastModifiedTime", dataChangeLastModifiedTime)
+        .toString();
+  }
+}

apollo-portal/src/main/java/com/ctrip/framework/apollo/openapi/filter/ConsumerAuthenticationFilter.java

 package com.ctrip.framework.apollo.openapi.filter;
 
+import com.ctrip.framework.apollo.openapi.util.ConsumerAuditUtil;
 import com.ctrip.framework.apollo.openapi.util.ConsumerAuthUtil;
 
 import java.io.IOException;
@@ -18,9 +19,11 @@ import javax.servlet.http.HttpServletResponse;
  */
 public class ConsumerAuthenticationFilter implements Filter {
   private ConsumerAuthUtil consumerAuthUtil;
+  private ConsumerAuditUtil consumerAuditUtil;
 
-  public ConsumerAuthenticationFilter(ConsumerAuthUtil consumerAuthUtil) {
+  public ConsumerAuthenticationFilter(ConsumerAuthUtil consumerAuthUtil, ConsumerAuditUtil consumerAuditUtil) {
     this.consumerAuthUtil = consumerAuthUtil;
+    this.consumerAuditUtil = consumerAuditUtil;
   }
 
   @Override
@@ -44,6 +47,7 @@ public class ConsumerAuthenticationFilter implements Filter {
     }
 
     consumerAuthUtil.storeConsumerId(request, consumerId);
+    consumerAuditUtil.audit(request, consumerId);
 
     chain.doFilter(req, resp);
   }

apollo-portal/src/main/java/com/ctrip/framework/apollo/openapi/repository/ConsumerAuditRepository.java

+package com.ctrip.framework.apollo.openapi.repository;
+
+import com.ctrip.framework.apollo.openapi.entity.ConsumerAudit;
+
+import org.springframework.data.repository.PagingAndSortingRepository;
+
+/**
+ * @author Jason Song(song_s@ctrip.com)
+ */
+public interface ConsumerAuditRepository extends PagingAndSortingRepository<ConsumerAudit, Long> {
+}

apollo-portal/src/main/java/com/ctrip/framework/apollo/openapi/service/ConsumerService.java

@@ -7,7 +7,9 @@ import com.google.common.base.Strings;
 import com.google.common.hash.Hashing;
 
 import com.ctrip.framework.apollo.openapi.entity.Consumer;
+import com.ctrip.framework.apollo.openapi.entity.ConsumerAudit;
 import com.ctrip.framework.apollo.openapi.entity.ConsumerToken;
+import com.ctrip.framework.apollo.openapi.repository.ConsumerAuditRepository;
 import com.ctrip.framework.apollo.openapi.repository.ConsumerRepository;
 import com.ctrip.framework.apollo.openapi.repository.ConsumerTokenRepository;
 import com.ctrip.framework.apollo.portal.service.ServerConfigService;
@@ -34,6 +36,8 @@ public class ConsumerService implements InitializingBean {
   @Autowired
   private ConsumerRepository consumerRepository;
   @Autowired
+  private ConsumerAuditRepository consumerAuditRepository;
+  @Autowired
   private ServerConfigService serverConfigService;
 
   private String consumerTokenSalt;
@@ -71,6 +75,11 @@ public class ConsumerService implements InitializingBean {
     return consumerTokenRepository.save(entity);
   }
 
+  @Transactional
+  public void createConsumerAudits(Iterable<ConsumerAudit> consumerAudits) {
+    consumerAuditRepository.save(consumerAudits);
+  }
+
   String generateConsumerToken(String consumerAppId, Date generationTime, String
       consumerTokenSalt) {
     return Hashing.sha1().hashString(KEY_JOINER.join(consumerAppId, TIMESTAMP_FORMAT.format

apollo-portal/src/main/java/com/ctrip/framework/apollo/openapi/util/ConsumerAuditUtil.java

+package com.ctrip.framework.apollo.openapi.util;
+
+import com.google.common.base.Strings;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Queues;
+
+import com.ctrip.framework.apollo.core.utils.ApolloThreadFactory;
+import com.ctrip.framework.apollo.openapi.entity.ConsumerAudit;
+import com.ctrip.framework.apollo.openapi.service.ConsumerService;
+import com.dianping.cat.Cat;
+
+import org.springframework.beans.factory.InitializingBean;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import javax.servlet.http.HttpServletRequest;
+
+/**
+ * @author Jason Song(song_s@ctrip.com)
+ */
+@Service
+public class ConsumerAuditUtil implements InitializingBean {
+  private static final int CONSUMER_AUDIT_MAX_SIZE = 10000;
+  private BlockingQueue<ConsumerAudit> audits = Queues.newLinkedBlockingQueue(CONSUMER_AUDIT_MAX_SIZE);
+  private final ExecutorService auditExecutorService;
+  private final AtomicBoolean auditStopped;
+  private int BATCH_SIZE = 100;
+  private long BATCH_TIMEOUT = 5;
+  private TimeUnit BATCH_TIMEUNIT = TimeUnit.SECONDS;
+
+  @Autowired
+  private ConsumerService consumerService;
+
+  public ConsumerAuditUtil() {
+    auditExecutorService = Executors.newSingleThreadExecutor(
+        ApolloThreadFactory.create("ConsumerAuditUtil", true));
+    auditStopped = new AtomicBoolean(false);
+  }
+
+  public boolean audit(HttpServletRequest request, long consumerId) {
+    String uri = request.getRequestURI();
+    if (!Strings.isNullOrEmpty(request.getQueryString())) {
+      uri += "?" + request.getQueryString();
+    }
+
+    ConsumerAudit consumerAudit = new ConsumerAudit();
+    Date now = new Date();
+    consumerAudit.setConsumerId(consumerId);
+    consumerAudit.setUri(uri);
+    consumerAudit.setMethod(request.getMethod());
+    consumerAudit.setDataChangeCreatedTime(now);
+    consumerAudit.setDataChangeLastModifiedTime(now);
+
+    //throw away audits if exceeds the max size
+    return this.audits.offer(consumerAudit);
+  }
+
+  @Override
+  public void afterPropertiesSet() throws Exception {
+    auditExecutorService.submit(() -> {
+      while (!auditStopped.get() && !Thread.currentThread().isInterrupted()) {
+        List<ConsumerAudit> toAudit = Lists.newArrayList();
+        try {
+          Queues.drain(audits, toAudit, BATCH_SIZE, BATCH_TIMEOUT, BATCH_TIMEUNIT);
+          if (!toAudit.isEmpty()) {
+            consumerService.createConsumerAudits(toAudit);
+          }
+        } catch (Throwable ex) {
+          Cat.logError(ex);
+        }
+      }
+    });
+  }
+
+  public void stopAudit() {
+    auditStopped.set(true);
+  }
+}

apollo-portal/src/main/java/com/ctrip/framework/apollo/portal/configuration/AuthConfiguration.java

@@ -3,6 +3,7 @@ package com.ctrip.framework.apollo.portal.configuration;
 import com.google.common.collect.Maps;
 
 import com.ctrip.framework.apollo.openapi.filter.ConsumerAuthenticationFilter;
+import com.ctrip.framework.apollo.openapi.util.ConsumerAuditUtil;
 import com.ctrip.framework.apollo.openapi.util.ConsumerAuthUtil;
 import com.ctrip.framework.apollo.portal.auth.LogoutHandler;
 import com.ctrip.framework.apollo.portal.auth.SsoHeartbeatHandler;
@@ -177,10 +178,11 @@ public class AuthConfiguration {
   static class DefaultAuthAutoConfiguration {
 
     @Bean
-    public FilterRegistrationBean openApiAuthenticationFilter(ConsumerAuthUtil consumerAuthUtil) {
+    public FilterRegistrationBean openApiAuthenticationFilter(ConsumerAuthUtil consumerAuthUtil,
+                                                              ConsumerAuditUtil consumerAuditUtil) {
       FilterRegistrationBean openApiFilter = new FilterRegistrationBean();
 
-      openApiFilter.setFilter(new ConsumerAuthenticationFilter(consumerAuthUtil));
+      openApiFilter.setFilter(new ConsumerAuthenticationFilter(consumerAuthUtil, consumerAuditUtil));
       openApiFilter.addUrlPatterns("/openapi/*");
 
       return openApiFilter;

apollo-portal/src/test/java/com/ctrip/framework/apollo/openapi/filter/ConsumerAuthenticationFilterTest.java

 package com.ctrip.framework.apollo.openapi.filter;
 
+import com.ctrip.framework.apollo.openapi.util.ConsumerAuditUtil;
 import com.ctrip.framework.apollo.openapi.util.ConsumerAuthUtil;
 
 import org.junit.Before;
@@ -12,7 +13,6 @@ import javax.servlet.FilterChain;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
-import static org.junit.Assert.*;
 import static org.mockito.Matchers.anyLong;
 import static org.mockito.Matchers.anyString;
 import static org.mockito.Matchers.eq;
@@ -30,6 +30,8 @@ public class ConsumerAuthenticationFilterTest {
   @Mock
   private ConsumerAuthUtil consumerAuthUtil;
   @Mock
+  private ConsumerAuditUtil consumerAuditUtil;
+  @Mock
   private HttpServletRequest request;
   @Mock
   private HttpServletResponse response;
@@ -38,7 +40,7 @@ public class ConsumerAuthenticationFilterTest {
 
   @Before
   public void setUp() throws Exception {
-    authenticationFilter = new ConsumerAuthenticationFilter(consumerAuthUtil);
+    authenticationFilter = new ConsumerAuthenticationFilter(consumerAuthUtil, consumerAuditUtil);
   }
 
   @Test
@@ -52,6 +54,7 @@ public class ConsumerAuthenticationFilterTest {
     authenticationFilter.doFilter(request, response, filterChain);
 
     verify(consumerAuthUtil, times(1)).storeConsumerId(request, someConsumerId);
+    verify(consumerAuditUtil, times(1)).audit(request, someConsumerId);
     verify(filterChain, times(1)).doFilter(request, response);
   }
 
@@ -66,6 +69,7 @@ public class ConsumerAuthenticationFilterTest {
 
     verify(response, times(1)).sendError(eq(HttpServletResponse.SC_UNAUTHORIZED), anyString());
     verify(consumerAuthUtil, never()).storeConsumerId(eq(request), anyLong());
+    verify(consumerAuditUtil, never()).audit(eq(request), anyLong());
     verify(filterChain, never()).doFilter(request, response);
   }
 }
\ No newline at end of file

apollo-portal/src/test/java/com/ctrip/framework/apollo/openapi/util/ConsumerAuditUtilTest.java

+package com.ctrip.framework.apollo.openapi.util;
+
+import com.google.common.util.concurrent.SettableFuture;
+
+import com.ctrip.framework.apollo.openapi.entity.ConsumerAudit;
+import com.ctrip.framework.apollo.openapi.service.ConsumerService;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mock;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.runners.MockitoJUnitRunner;
+import org.mockito.stubbing.Answer;
+import org.springframework.test.util.ReflectionTestUtils;
+
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.TimeUnit;
+
+import javax.servlet.http.HttpServletRequest;
+
+import static org.junit.Assert.*;
+import static org.mockito.Matchers.anyCollectionOf;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.when;
+
+/**
+ * @author Jason Song(song_s@ctrip.com)
+ */
+@RunWith(MockitoJUnitRunner.class)
+public class ConsumerAuditUtilTest {
+  private ConsumerAuditUtil consumerAuditUtil;
+  @Mock
+  private ConsumerService consumerService;
+  @Mock
+  private HttpServletRequest request;
+  private long batchTimeout = 50;
+  private TimeUnit batchTimeUnit = TimeUnit.MILLISECONDS;
+
+  @Before
+  public void setUp() throws Exception {
+    consumerAuditUtil = new ConsumerAuditUtil();
+    ReflectionTestUtils.setField(consumerAuditUtil, "consumerService", consumerService);
+    ReflectionTestUtils.setField(consumerAuditUtil, "BATCH_TIMEOUT", batchTimeout);
+    ReflectionTestUtils.setField(consumerAuditUtil, "BATCH_TIMEUNIT", batchTimeUnit);
+    consumerAuditUtil.afterPropertiesSet();
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    consumerAuditUtil.stopAudit();
+  }
+
+  @Test
+  public void audit() throws Exception {
+    long someConsumerId = 1;
+    String someUri = "someUri";
+    String someQuery = "someQuery";
+    String someMethod = "someMethod";
+
+    when(request.getRequestURI()).thenReturn(someUri);
+    when(request.getQueryString()).thenReturn(someQuery);
+    when(request.getMethod()).thenReturn(someMethod);
+
+    SettableFuture<List<ConsumerAudit>> result = SettableFuture.create();
+
+    doAnswer(new Answer<Void>() {
+      @Override
+      public Void answer(InvocationOnMock invocation) throws Throwable {
+        Object[] args = invocation.getArguments();
+        result.set((List<ConsumerAudit>) args[0]);
+
+        return null;
+      }
+    }).when(consumerService).createConsumerAudits(anyCollectionOf(ConsumerAudit.class));
+
+    consumerAuditUtil.audit(request, someConsumerId);
+
+    List<ConsumerAudit> audits = result.get(batchTimeout * 5, batchTimeUnit);
+
+    assertEquals(1, audits.size());
+
+    ConsumerAudit audit = audits.get(0);
+
+    assertEquals(String.format("%s?%s", someUri, someQuery), audit.getUri());
+    assertEquals(someMethod, audit.getMethod());
+    assertEquals(someConsumerId, audit.getConsumerId());
+  }
+
+}
\ No newline at end of file

apollo-portal/src/test/java/com/ctrip/framework/apollo/portal/AllTests.java

@@ -4,6 +4,7 @@ package com.ctrip.framework.apollo.portal;
 import com.ctrip.framework.apollo.openapi.filter.ConsumerAuthenticationFilterTest;
 import com.ctrip.framework.apollo.openapi.service.ConsumerRolePermissionServiceTest;
 import com.ctrip.framework.apollo.openapi.service.ConsumerServiceTest;
+import com.ctrip.framework.apollo.openapi.util.ConsumerAuditUtilTest;
 import com.ctrip.framework.apollo.openapi.util.ConsumerAuthUtilTest;
 import com.ctrip.framework.apollo.portal.controller.ConsumerControllerTest;
 import com.ctrip.framework.apollo.portal.service.AppNamespaceServiceTest;
@@ -25,7 +26,7 @@ import org.junit.runners.Suite.SuiteClasses;
     AppNamespaceServiceTest.class, RoleInitializationServiceTest.class, FileTextResolverTest.class,
     RetryableRestTemplateTest.class, ConsumerRolePermissionServiceTest.class,
     ConsumerAuthenticationFilterTest.class, ConsumerAuthUtilTest.class, ConsumerServiceTest.class,
-    ConsumerControllerTest.class
+    ConsumerControllerTest.class, ConsumerAuditUtilTest.class
 })
 public class AllTests {