仿牛客網第七章

Spring Security簡介

簡介

Spring Security是一個專注於為Java應用程序提供身份認證，它的強大之處在於可以輕松拓展以滿足其他自定義的需求。

特征

對身份的認證和授權提供全面的、可拓展的支持。

防止各種攻擊，如會話固定攻擊、點擊劫持、csrf攻擊。

支持與Servlet API、Spring MVC等Web技術集成。

在spring4all上可以學習關於Spring Security的信息。

首先是引入依賴，spring-boot-starter-security。

在User里

// true: 賬號未過期.
    @Override
    public boolean isAccountNonExpired() {
        return true;
    }
    // true: 賬號未鎖定.
    @Override
    public boolean isAccountNonLocked() {
        return true;
    }
    // true: 憑證未過期.
    @Override
    public boolean isCredentialsNonExpired() {
        return true;
    }
    // true: 賬號可用.
    @Override
    public boolean isEnabled() {
        return true;
    }
    @Override
    public Collection<? extends GrantedAuthority> getAuthorities() {
        List<GrantedAuthority> list = new ArrayList<>();
        list.add(new GrantedAuthority() {
            @Override
            public String getAuthority() {
                switch (type) {
                    case 1:
                        return "ADMIN";
                    default:
                        return "USER";
                }
            }
        });
        return list;
    }

在UserService上

@Service
public class UserService implements UserDetailsService {
    @Autowired
    private UserMapper userMapper;
    public User findUserByName(String username) {
        return userMapper.selectByName(username);
    }
    @Override
    public UserDetails loadUserByUsername(String username) throws UsernameNotFoundException {
        return this.findUserByName(username);
    }
}

在Config里寫SercurityConfig

@Configuration
public class SecurityConfig extends WebSecurityConfigurerAdapter {
    @Autowired
    private UserService userService;
    @Override
    public void configure(WebSecurity web) throws Exception {
        // 忽略靜態資源的訪問
        web.ignoring().antMatchers("/resources/**");
    }
    // AuthenticationManager: 認證的核心接口.
    // AuthenticationManagerBuilder: 用於構建AuthenticationManager對象的工具.
    // ProviderManager: AuthenticationManager接口的默認實現類.
    @Override
    protected void configure(AuthenticationManagerBuilder auth) throws Exception {
        // 內置的認證規則
        // auth.userDetailsService(userService).passwordEncoder(new Pbkdf2PasswordEncoder("12345"));
        // 自定義認證規則
        // AuthenticationProvider: ProviderManager持有一組AuthenticationProvider,每個AuthenticationProvider負責一種認證.
        // 委托模式: ProviderManager將認證委托給AuthenticationProvider.
        auth.authenticationProvider(new AuthenticationProvider() {
            // Authentication: 用於封裝認證信息的接口,不同的實現類代表不同類型的認證信息.
            @Override
            public Authentication authenticate(Authentication authentication) throws AuthenticationException {
                String username = authentication.getName();
                String password = (String) authentication.getCredentials();
                User user = userService.findUserByName(username);
                if (user == null) {
                    throw new UsernameNotFoundException("賬號不存在!");
                }
                password = CommunityUtil.md5(password + user.getSalt());
                if (!user.getPassword().equals(password)) {
                    throw new BadCredentialsException("密碼不正確!");
                }
                // principal: 主要信息; credentials: 證書; authorities: 權限;
                return new UsernamePasswordAuthenticationToken(user, user.getPassword(), user.getAuthorities());
            }
            // 當前的AuthenticationProvider支持哪種類型的認證.
            @Override
            public boolean supports(Class<?> aClass) {
                // UsernamePasswordAuthenticationToken: Authentication接口的常用的實現類.
                return UsernamePasswordAuthenticationToken.class.equals(aClass);
            }
        });
    }
    @Override
    protected void configure(HttpSecurity http) throws Exception {
        // 登錄相關配置
        http.formLogin()
                .loginPage("/loginpage")
                .loginProcessingUrl("/login")
                .successHandler(new AuthenticationSuccessHandler() {
                    @Override
                    public void onAuthenticationSuccess(HttpServletRequest request, HttpServletResponse response, Authentication authentication) throws IOException, ServletException {
                        response.sendRedirect(request.getContextPath() + "/index");
                    }
                })
                .failureHandler(new AuthenticationFailureHandler() {
                    @Override
                    public void onAuthenticationFailure(HttpServletRequest request, HttpServletResponse response, AuthenticationException e) throws IOException, ServletException {
                        request.setAttribute("error", e.getMessage());
                        request.getRequestDispatcher("/loginpage").forward(request, response);
                    }
                });
        // 退出相關配置
        http.logout()
                .logoutUrl("/logout")
                .logoutSuccessHandler(new LogoutSuccessHandler() {
                    @Override
                    public void onLogoutSuccess(HttpServletRequest request, HttpServletResponse response, Authentication authentication) throws IOException, ServletException {
                        response.sendRedirect(request.getContextPath() + "/index");
                    }
                });
        // 授權配置
        http.authorizeRequests()
                .antMatchers("/letter").hasAnyAuthority("USER", "ADMIN")
                .antMatchers("/admin").hasAnyAuthority("ADMIN")
                .and().exceptionHandling().accessDeniedPage("/denied");
        // 增加Filter,處理驗證碼
        http.addFilterBefore(new Filter() {
            @Override
            public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) throws IOException, ServletException {
                HttpServletRequest request = (HttpServletRequest) servletRequest;
                HttpServletResponse response = (HttpServletResponse) servletResponse;
                if (request.getServletPath().equals("/login")) {
                    String verifyCode = request.getParameter("verifyCode");
                    if (verifyCode == null || !verifyCode.equalsIgnoreCase("1234")) {
                        request.setAttribute("error", "驗證碼錯誤!");
                        request.getRequestDispatcher("/loginpage").forward(request, response);
                        return;
                    }
                }
                // 讓請求繼續向下執行.
                filterChain.doFilter(request, response);
            }
        }, UsernamePasswordAuthenticationFilter.class);
        // 記住我
        http.rememberMe()
                .tokenRepository(new InMemoryTokenRepositoryImpl())
                .tokenValiditySeconds(3600 * 24)
                .userDetailsService(userService);
    }
}

修改HomeController

public class HomeController {
    @RequestMapping(path = "/index", method = RequestMethod.GET)
    public String getIndexPage(Model model) {
        // 認證成功后,結果會通過SecurityContextHolder存入SecurityContext中.
        Object obj = SecurityContextHolder.getContext().getAuthentication().getPrincipal();
        if (obj instanceof User) {
            model.addAttribute("loginUser", obj);
        }
        return "/index";
    }
    @RequestMapping(path = "/discuss", method = RequestMethod.GET)
    public String getDiscussPage() {
        return "/site/discuss";
    }
    @RequestMapping(path = "/letter", method = RequestMethod.GET)
    public String getLetterPage() {
        return "/site/letter";
    }
    @RequestMapping(path = "/admin", method = RequestMethod.GET)
    public String getAdminPage() {
        return "/site/admin";
    }
    @RequestMapping(path = "/loginpage", method = {RequestMethod.GET, RequestMethod.POST})
    public String getLoginPage() {
        return "/site/login";
    }
    // 拒絕訪問時的提示頁面
    @RequestMapping(path = "/denied", method = RequestMethod.GET)
    public String getDeniedPage() {
        return "/error/404";
    }
}

修改之前的登錄

登錄檢查

之前采用攔截器實習登錄檢查，這是簡單的權限管理方案，現在將其廢棄。

廢棄原來的WebMvcConfig的攔截器設置。

授權配置

對當前系統內包含的所有的請求，分配訪問權限(普通用戶、版主、管理員)。

在常量接口上新增

/**
     * 權限：普通用戶
     */
    String AUTHORITY_USER = "user";
    /**
     * 權限：管理員
     */
    String AUTHORITY_ADMIN = "admin";
    /**
     * 權限：版主
     */
    String AUTHORITY_MODERATOR = "moderator";

SecurityConfig

@Configuration
public class SecurityConfig extends WebSecurityConfigurerAdapter implements CommunityConstant {
    @Override
    public void configure(WebSecurity web) throws Exception {
        web.ignoring().antMatchers("/resources/**");
    }
    @Override
    protected void configure(HttpSecurity http) throws Exception {
        //授權
        http.authorizeRequests()
                .antMatchers(
                        "/user/setting",
                        "/user/upload",
                        "/discuss/add",
                        "/comment/add/**",
                        "/letter/**",
                        "/notice/**",
                        "/like",
                        "/follow",
                        "/unfollow"
                ).hasAnyAuthority(
                        AUTHORITY_USER,
                        AUTHORITY_ADMIN,
                        AUTHORITY_MODERATOR
                )
                .anyRequest().permitAll()
                .and().csrf().disable();
        //權限不夠時的處理
        http.exceptionHandling()
                .authenticationEntryPoint(new AuthenticationEntryPoint() {
                    //沒有登錄
                    @Override
                    public void commence(HttpServletRequest request, HttpServletResponse response, AuthenticationException e) throws IOException, ServletException {
                        String xRequestedWith = request.getHeader("x-requested-with");
                        if("XMELHttpRequest".equals(xRequestedWith)){
                            response.setContentType("application/plain;charset=utf-8");
                            PrintWriter writer = response.getWriter();
                            writer.write(CommunityUtil.getJSONString(403, "你還沒有登錄哦!"));
                        }else {
                            response.sendRedirect(request.getContextPath() + "/login");
                        }
                    }
                })
                .accessDeniedHandler(new AccessDeniedHandler() {
                    //權限不足
                    @Override
                    public void handle(HttpServletRequest request, HttpServletResponse response, AccessDeniedException e) throws IOException, ServletException {
                        String xRequestedWith = request.getHeader("x-requested-with");
                        if("XMELHttpRequest".equals(xRequestedWith)){
                            response.setContentType("application/plain;charset=utf-8");
                            PrintWriter writer = response.getWriter();
                            writer.write(CommunityUtil.getJSONString(403, "你沒有訪問此功能的權限!"));
                        }else {
                            response.sendRedirect(request.getContextPath() + "/denied");
                        }
                    }
                });
        //Security底層默認攔截/logout請求，進行退出處理
        //覆蓋它默認的邏輯並能執行我們自己的退出的代碼
        http.logout().logoutUrl("/securitylogout");
    }
}

在UserService

public Collection<? extends GrantedAuthority> getAuthorities(int userId){
        User user = this.findUserById(userId);
        List<GrantedAuthority> list = new ArrayList<>();
        list.add(new GrantedAuthority(){
            @Override
            public String getAuthority() {
                switch (user.getType()){
                    case 1:
                        return AUTHORITY_ADMIN;
                    case 2:
                        return AUTHORITY_MODERATOR;
                    default:
                        return AUTHORITY_USER;
                }
            }
        });
        return list;
    }

認證方案

繞過Security認證系統，采用原來的認證方案。

修改LoginTicketInterceptor

@Component
public class LoginTicketInterceptor implements HandlerInterceptor {
    @Autowired
    private UserService userService;
    @Autowired
    private HostHolder hostHolder;
    @Override
    public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) throws Exception {
        // 從cookie中獲取憑證
        String ticket = CookieUtil.getValue(request, "ticket");
        if (ticket != null) {
            // 查詢憑證
            LoginTicket loginTicket = userService.findLoginTicket(ticket);
            // 檢查憑證是否有效
            if (loginTicket != null && loginTicket.getStatus() == 0 && loginTicket.getExpired().after(new Date())) {
                // 根據憑證查詢用戶
                User user = userService.findUserById(loginTicket.getUserId());
                // 在本次請求中持有用戶
                hostHolder.setUser(user);
                //構建用戶認證的結果,並存入SecurityContent,以便於Security進行授權
                Authentication authentication = new UsernamePasswordAuthenticationToken(
                        user, user.getPassword(), userService.getAuthorities(user.getId()));
                SecurityContextHolder.setContext(new SecurityContextImpl(authentication));
            }
        }
        return true;
    }
    @Override
    public void postHandle(HttpServletRequest request, HttpServletResponse response, Object handler, ModelAndView modelAndView) throws Exception {
        User user = hostHolder.getUser();
        if (user != null && modelAndView != null) {
            modelAndView.addObject("loginUser", user);
        }
    }
    @Override
    public void afterCompletion(HttpServletRequest request, HttpServletResponse response, Object handler, Exception ex) throws Exception {
        hostHolder.clear();
        SecurityContextHolder.clearContext();
    }
}

csrf配置

csrf是其他用戶獲得客戶端的cookie和ticket從而訪問了服務器，security可以生成TOKEN數據，是隱藏的，防止csrf攻擊。

security自帶有這個功能的實現，對於表單的提交，但是對於異步請求就沒有實現。需要自己去實現。

在需要提交異步請求的位置

<meta name="_csrf" th:content="${_csrf.token}">
    <meta name="_csrf_header" th:content="${_csrf.headerName}">

在對應的js文件

//發送AJAX請求之前,將SCRF令牌設置到請求的消息頭中。
    var token = $("meta[name='_csrf']").attr("content");
    var header = $("meta[name='_csrf_header']").attr("content");
    $(document).ajaxSend(function (e, xhr, options) {
        xhr.setRequestHeader(header, token);
    });

防止csrf攻擊，表單，AJAX的配置。

一般需要對所有的異步請求都要配，不然就是不安全的，無法通過，也可以不做配置，那么在授權的時候需要.and().csrf().disable();

置定，加精，刪除

功能實現

點擊置頂，修改帖子的類型。

點擊，“加精”、“刪除”，修改帖子的狀態。

在數據層，在DiscussPostMapper上增加方法

int updateType(int id, int type);
int updateStatus(int id, int status);

在discuss-mapper.xml下

<update id="updateType">
        update discuss_post set type = #{type} where id = #{id}
    </update>
    <update id="updateStatus">
        update discuss_post set status = #{status} where id = #{id}
    </update>

在DiscussPostService下增加方法

    public int updateType(int id, int type){
        return discussPostMapper.updateType(id, type);
    }
    public int updateStatus(int id, int status){
        return discussPostMapper.updateStatus(id, status);
    }

在DiscusspostController下

// 置頂
    @RequestMapping(path = "/top", method = RequestMethod.POST)
    @ResponseBody
    public String setTop(int id) {
        discussPostService.updateType(id, 1);
        // 觸發發帖事件
        Event event = new Event()
                .setTopic(TOPIC_PUBLISH)
                .setUserId(hostHolder.getUser().getId())
                .setEntityType(ENTITY_TYPE_POST)
                .setEntityId(id);
        eventProducer.fireEvent(event);
        return CommunityUtil.getJSONString(0);
    }
    // 加精
    @RequestMapping(path = "/wonderful", method = RequestMethod.POST)
    @ResponseBody
    public String setWonderful(int id) {
        discussPostService.updateStatus(id, 1);
        // 觸發發帖事件
        Event event = new Event()
                .setTopic(TOPIC_PUBLISH)
                .setUserId(hostHolder.getUser().getId())
                .setEntityType(ENTITY_TYPE_POST)
                .setEntityId(id);
        eventProducer.fireEvent(event);
        return CommunityUtil.getJSONString(0);
    }
    // 刪除
    @RequestMapping(path = "/delete", method = RequestMethod.POST)
    @ResponseBody
    public String setDelete(int id) {
        discussPostService.updateStatus(id, 2);
        // 觸發刪帖事件
        Event event = new Event()
                .setTopic(TOPIC_DELETE)
                .setUserId(hostHolder.getUser().getId())
                .setEntityType(ENTITY_TYPE_POST)
                .setEntityId(id);
        eventProducer.fireEvent(event);
        return CommunityUtil.getJSONString(0);
    }

在EventConsumer上消費刪帖事件

// 消費刪帖事件
    @KafkaListener(topics = {TOPIC_DELETE})
    public void handleDeleteMessage(ConsumerRecord record) {
        if (record == null || record.value() == null) {
            logger.error("消息的內容為空!");
            return;
        }
        Event event = JSONObject.parseObject(record.value().toString(), Event.class);
        if (event == null) {
            logger.error("消息格式錯誤!");
            return;
        }
        elasticsearchService.deleteDiscussPost(event.getEntityId());
    }

為置頂、加精、刪除綁定3個js單擊事件。

$(function(){
    $("#topBtn").click(setTop);
    $("#wonderfulBtn").click(setWonderful);
    $("#deleteBtn").click(setDelete);
});
// 置頂
function setTop() {
    $.post(
        CONTEXT_PATH + "/discuss/top",
        {"id":$("#postId").val()},
        function(data) {
            data = $.parseJSON(data);
            if(data.code == 0) {
                $("#topBtn").attr("disabled", "disabled");
            } else {
                alert(data.msg);
            }
        }
    );
}
// 加精
function setWonderful() {
    $.post(
        CONTEXT_PATH + "/discuss/wonderful",
        {"id":$("#postId").val()},
        function(data) {
            data = $.parseJSON(data);
            if(data.code == 0) {
                $("#wonderfulBtn").attr("disabled", "disabled");
            } else {
                alert(data.msg);
            }
        }
    );
}
// 刪除
function setDelete() {
    $.post(
        CONTEXT_PATH + "/discuss/delete",
        {"id":$("#postId").val()},
        function(data) {
            data = $.parseJSON(data);
            if(data.code == 0) {
                location.href = CONTEXT_PATH + "/index";
            } else {
                alert(data.msg);
            }
        }
    );
}

權限管理

版主可以執行“置頂”、“加精”操作。

在SecurityConfig下配置權限。

.antMatchers(
                        "/discuss/top",
                        "/discuss/wonderful"
                )
                .hasAnyAuthority(
                        AUTHORITY_MODERATOR
                )
                .antMatchers(
                        "/discuss/delete"
                )
                .hasAnyAuthority(
                        AUTHORITY_ADMIN
                )

管理員可以看到“刪除”按鈕

在html頁面上，增加xmlns:sec=“http://www.thymeleaf.org/extras/spring-security”，在每個按鈕處，引入權限控制

<button type="button" class="btn btn-danger btn-sm" id="topBtn"
                            th:disabled="${post.type==1}" sec:authorize="hasAnyAuthority('moderator')">置頂</button>
                        <button type="button" class="btn btn-danger btn-sm" id="wonderfulBtn"
                            th:disabled="${post.status==1}" sec:authorize="hasAnyAuthority('moderator')">加精</button>
                        <button type="button" class="btn btn-danger btn-sm" id="deleteBtn"
                            th:disabled="${post.status==2}" sec:authorize="hasAnyAuthority('admin')">刪除</button>

Themleaf做了和Spring security相關的包，可以在github下載，可以在Mavenreposity下載導包。

Redis高級數據類型

HyperLogLog

采用一種基數算法，用於完成獨立數據的統計，特點是占用空間小，無論統計多少數據，只占用12K的內存空間，不足的是統計不精確，誤差在0.81%。

//統計20萬個重復數據的獨立總數。
    @Test
    public void testHyperLogLog(){
        String redisKey = "test:hll:01";
        for (int i = 1; i <= 100000; i++) {
            redisTemplate.opsForHyperLogLog().add(redisKey, i);
        }
        for (int i = 1; i <= 100000; i++) {
            int r = (int)(Math.random() * 100000 + 1);
            redisTemplate.opsForHyperLogLog().add(redisKey, r);
        }
        Long size = redisTemplate.opsForHyperLogLog().size(redisKey);
        System.out.println(size);
    }
    //將3組數據合並，再統計合並后的重復數據的獨立總數
    @Test
    public void testHyperLogLogUnion(){
        String redisKey2 = "test:hll:02";
        for (int i = 1; i <= 10000; i++) {
            redisTemplate.opsForHyperLogLog().add(redisKey2, i);
        }
        String redisKey3 = "test:hll:03";
        for (int i = 5001; i <= 15000; i++) {
            redisTemplate.opsForHyperLogLog().add(redisKey3, i);
        }
        String redisKey4 = "test:hll:04";
        for (int i = 10001; i <= 20000; i++) {
            redisTemplate.opsForHyperLogLog().add(redisKey4, i);
        }
        String unionKey = "test:hll:union";
        redisTemplate.opsForHyperLogLog().union(unionKey, redisKey2, redisKey3, redisKey4);
        long size = redisTemplate.opsForHyperLogLog().size(unionKey);
        System.out.println(size);
    }

Bitmap

不是一種獨立的數據結構，實際上就是字符串。支持按位存取數據，可以將其看成是byte數組。適合儲存大量的連續的布爾值，比如記錄簽到。

//統計一組數據的布爾值
    @Test
    public void testBitMap(){
        String redisKey = "test:bm:01";
        //記錄
        redisTemplate.opsForValue().setBit(redisKey, 1, true);
        redisTemplate.opsForValue().setBit(redisKey, 4, true);
        redisTemplate.opsForValue().setBit(redisKey, 7, true);
        //查詢
        System.out.println(redisTemplate.opsForValue().getBit(redisKey, 0));
        System.out.println(redisTemplate.opsForValue().getBit(redisKey, 1));
        System.out.println(redisTemplate.opsForValue().getBit(redisKey, 2));
        //統計
         Object obj = redisTemplate.execute(new RedisCallback() {
            @Override
            public Object doInRedis(RedisConnection connection) throws DataAccessException {
                return connection.bitCount(redisKey.getBytes());
            }
        });
        System.out.println(obj);
    }
    //統計3組數據的布爾值，並對這3組數據做OR運算
    @Test
    public void testBitMapOperation(){
        String redisKey2 = "test:bm:02";
        redisTemplate.opsForValue().setBit(redisKey2, 0, true);
        redisTemplate.opsForValue().setBit(redisKey2, 1, true);
        redisTemplate.opsForValue().setBit(redisKey2, 2, true);
        String redisKey3 = "test:bm:03";
        redisTemplate.opsForValue().setBit(redisKey3, 2, true);
        redisTemplate.opsForValue().setBit(redisKey3, 3, true);
        redisTemplate.opsForValue().setBit(redisKey3, 4, true);
        String redisKey4 = "test:bm:04";
        redisTemplate.opsForValue().setBit(redisKey3, 4, true);
        redisTemplate.opsForValue().setBit(redisKey3, 5, true);
        redisTemplate.opsForValue().setBit(redisKey3, 6, true);
        String redisKey = "test:bm:or";
        Object obj = redisTemplate.execute(new RedisCallback() {
            @Override
            public Object doInRedis(RedisConnection connection) throws DataAccessException {
                connection.bitOp(RedisStringCommands.BitOperation.OR,
                        redisKey.getBytes(), redisKey2.getBytes(), redisKey3.getBytes(), redisKey4.getBytes());
                return connection.bitCount(redisKey.getBytes());
            }
        });
        System.out.println(obj);
        System.out.println(redisTemplate.opsForValue().getBit(redisKey,0));
        System.out.println(redisTemplate.opsForValue().getBit(redisKey,1));
        System.out.println(redisTemplate.opsForValue().getBit(redisKey,2));
        System.out.println(redisTemplate.opsForValue().getBit(redisKey,3));
        System.out.println(redisTemplate.opsForValue().getBit(redisKey,4));
        System.out.println(redisTemplate.opsForValue().getBit(redisKey,5));
        System.out.println(redisTemplate.opsForValue().getBit(redisKey,6));
    }

網站數據統計

UV(Unique Vistor)

獨立訪客，需要對用戶IP排重統計數據

每次訪問都要進行統計，采用HyperLogLog，性能好，且儲存空間小。

DAU(Daily Active User)

日活躍用戶，需通過用戶ID重排統計數據。

訪問過1次，則認為其活躍。

Bitmap，性能好、且可以統計准確數據。

這些功能是通過redis實現的，在RedisKeyUtil下

private static final String SPLIT = ":";
    private static final String PREFIX_UV = "uv";
    private static final String PREFIX_DAU = "dau";
    //單日UV
    public static String getUVKey(String date){
        return PREFIX_UV + SPLIT + date;
    }
    //區間UV
    public static String getUVKey(String startDate, String endDate){
        return PREFIX_UV + SPLIT + startDate + SPLIT + endDate;
    }
    //單日活躍用戶
    public static String getDAUKey(String date){
        return PREFIX_DAU + SPLIT + date;
    }
    //區間活躍用戶
    public static String getDAUKey(String startDate, String endDate){
        return PREFIX_DAU + SPLIT + startDate + SPLIT + endDate;
    }

DataService

@Service
public class DataService {
    @Autowired
    private RedisTemplate redisTemplate;
    private SimpleDateFormat df = new SimpleDateFormat("yyyyMMdd");
    //將指定的IP計入UV
    public void recordUV(String ip){
        String redisKey = RedisKeyUtil.getUVKey(df.format(new Date()));
        redisTemplate.opsForHyperLogLog().add(redisKey, ip);
    }
    //統計指定日期范圍內的日期
    public long calculateUV(Date start, Date end){
        if(start == null || end == null){
            throw new IllegalArgumentException("參數不能為空!");
        }
        //整理改日期范圍內的key
        List<String> keyList = new ArrayList<>();
        Calendar calendar = Calendar.getInstance();
        calendar.setTime(start);
        while (!calendar.getTime().after(end)){
            String key = RedisKeyUtil.getUVKey(df.format(calendar.getTime()));
            keyList.add(key);
            calendar.add(Calendar.DATE, 1);
        }
        //合並這些數據
        String redisKey = RedisKeyUtil.getUVKey(df.format(start), df.format(end));
        redisTemplate.opsForHyperLogLog().union(redisKey, keyList.toArray());
        //返回這個統計的結果
        return redisTemplate.opsForHyperLogLog().size(redisKey);
    }
    //將指定用戶記錄DAU
    public void recordDAU(int userId){
        String redisKey = RedisKeyUtil.getDAUKey(df.format(new Date()));
        redisTemplate.opsForValue().setBit(redisKey, userId, true);
    }
    //統計指定日期范圍內的DAU
    public long calculateDAU(Date start, Date end){
        if(start == null || end == null){
            throw new IllegalArgumentException("參數不能為空!");
        }
        //整理改日期范圍內的key
        List<byte[]> keyList = new ArrayList<>();
        Calendar calendar = Calendar.getInstance();
        calendar.setTime(start);
        while (!calendar.getTime().after(end)){
            String key = RedisKeyUtil.getDAUKey(df.format(calendar.getTime()));
            keyList.add(key.getBytes());
            calendar.add(Calendar.DATE, 1);
        }
        //進行OR運算
        return (long)redisTemplate.execute(new RedisCallback() {
            @Override
            public Object doInRedis(RedisConnection connection) throws DataAccessException {
                String redisKey = RedisKeyUtil.getDAUKey(df.format(start), df.format(end));
                connection.bitOp(RedisStringCommands.BitOperation.OR,
                        redisKey.getBytes(), keyList.toArray(new byte[0][0]));
                return connection.bitCount(redisKey.getBytes());
            }
        });
    }
}

編寫攔截器DataInterceptor

@Component
public class DataInterceptor implements HandlerInterceptor {
    @Autowired
    private DataService dataService;
    @Autowired
    private HostHolder hostHolder;
    @Override
    public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) throws Exception {
        //統計UV
        String ip = request.getRemoteHost();
        dataService.recordUV(ip);
        //統計DAU
        User user = hostHolder.getUser();
        if(user != null){
            dataService.recordDAU(user.getId());
        }
        return true;
    }
}

在WebConfig配置攔截器。

寫Datacontroller

@Controller
public class DataController {
    @Autowired
    private DataService dataService;
    //統計頁面
    @RequestMapping(path = "data", method = {RequestMethod.GET, RequestMethod.POST})
    public String getDataPage(){
        return "/site/admin/data";
    }
    //處理統計網址UV
    @RequestMapping(path = "/data/uv", method = RequestMethod.POST)
    public String getUV(@DateTimeFormat(pattern = "yyyy-MM-dd") Date start,
                        @DateTimeFormat(pattern = "yyyy-MM-dd")Date end, Model model){
        long uv = dataService.calculateUV(start, end);
        model.addAttribute("uvResult", uv);
        model.addAttribute("uvStartDate", start);
        model.addAttribute("uvEndDate", end);
        return "forward:/data";
    }
    //統計活躍用戶
    @RequestMapping(path = "/data/dau", method = RequestMethod.POST)
    public String getDAU(@DateTimeFormat(pattern = "yyyy-MM-dd") Date start,
                        @DateTimeFormat(pattern = "yyyy-MM-dd")Date end, Model model){
        long dau = dataService.calculateDAU(start, end);
        model.addAttribute("dauResult", dau);
        model.addAttribute("dauStartDate", start);
        model.addAttribute("dauEndDate", end);
        return "forward:/data";
    }
}

我們這個數據查看頁面也是需要一定的權限才能打開，所以我們需要對權限進行一個管理，如果權限不到位，無法訪問

.antMatchers(
                        "/discuss/delete",
                        "/data/**"
                )
                .hasAnyAuthority(
                        AUTHORITY_ADMIN
                )

任務調度與執行

JDK線程池

ExecutorService

private static final Logger logger = LoggerFactory.getLogger(ThreadPoolTests.class);
    //JDK普通線程池
    private ExecutorService executorService = Executors.newFixedThreadPool(5);
    private void sleep(long m){
        try {
            Thread.sleep(m);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
    }
    //1.JDK普通線程池
    @Test
    public void testExecutorService(){
        Runnable task = new Runnable() {
            @Override
            public void run() {
                logger.debug("Hello ExecutorService");
            }
        };
        for (int i = 1; i <= 10; i++) {
            executorService.submit(task);
        }
        sleep(10000);
    }

ScheduledExecutorService

//JDK可執行定時任務的線程池
    private ScheduledExecutorService scheduledExecutorService = Executors.newScheduledThreadPool(5);
    //2.JDK定時任務線程池
    @Test
    public void testScheduledExecutorService(){
        Runnable task = new Runnable() {
            @Override
            public void run() {
                logger.debug("Hello ScheduledExecutorService");
            }
        };
        scheduledExecutorService.scheduleAtFixedRate(task, 10000, 1000, TimeUnit.MILLISECONDS);
        sleep(30000);
    }

Spring線程池

ThreadPoolTaskExecutor

//Spring普通線程池
    @Autowired
    private ThreadPoolTaskExecutor taskExecutor;
    //3.Spring普通線程池
    @Test
    public void testThreadPoolTaskExecutor(){
        Runnable task = new Runnable() {
            @Override
            public void run() {
                logger.debug("Hello ThreadPoolTaskExecutor");
            }
        };
        for (int i = 0; i < 10 ; i++) {
            taskExecutor.submit(task);
        }
        sleep(10000);
    }

ThreadPoolTaskScheduler

//Spring可執行定時任務的線程池
    @Autowired
    private ThreadPoolTaskScheduler taskScheduler;
    //4.Spring定時任務線程池
    @Test
    public void testThreadPoolTaskScheduler(){
        Runnable task = new Runnable() {
            @Override
            public void run() {
                logger.debug("Hello ThreadPoolTaskScheduler");
            }
        };
        Date startTime = new Date(System.currentTimeMillis() + 10000);
        taskScheduler.scheduleAtFixedRate(task, startTime, 1000);
        sleep(30000);
    }

注意Spring的線程池可以自己在Application里做配置

# TaskExecutionProperties
spring.task.execution.pool.core-size=5
spring.task.execution.pool.max-size=15
spring.task.execution.pool.queue-capacity=100

Spring的這種調用是可以簡化的，在AlphaService里

//可以讓該方法在多線程的環境下，被異步的調用。
    @Async
    public void execute1(){
        logger.debug("execute1");
    }
    @Scheduled(initialDelay = 10000, fixedRate = 1000)
    public void execute2(){
        logger.debug("execute2");
    }

添加ThreadPoolConfig

@Configuration
@EnableScheduling
@EnableAsync
public class ThreadPoolConfig {
}

可以進行簡化的調用

//5.Spring普通線程池(簡化)
    @Test
    public void testThreadPoolTaskExecutorSimple(){
        for (int i = 0; i <10 ; i++) {
            alphaService.execute1();
        }
        sleep(10000);
    }
    //6.Spring定時任務線程池(簡化)
    @Test
    public void testThreadPoolTaskSchedulerSimple(){
        sleep(30000);
    }

分布式定時任務

Spring Quartz

Quartz是在數據庫調用的，所以需要導入表在SQL里，還需要引入spring-boot-quart依賴。

在AlphaJob下

public class AlphaJob implements Job {
    @Override
    public void execute(JobExecutionContext context) throws JobExecutionException {
        System.out.println(Thread.currentThread().getName() + ": execute a quartz job.");
    }
}

配置QuartzConfig文件

//配置-> 數據庫 -> 調用
@Configuration
public class QuartzConfig {
    //FactoryBean可簡化Bean的實例化過程:
    //1.通過FactoryBean封裝了Bean的實例化過程.
    //2.將FactoryBean裝配到Spring容器里.
    //3.將FactoryBean注入給其他的Bean
    //4.該Bean得到的是FactoryBean所管理的對象實例.
    //配置JobDetail
    @Bean
    public JobDetailFactoryBean alphaJobDetail(){
        JobDetailFactoryBean factoryBean = new JobDetailFactoryBean();
        factoryBean.setJobClass(AlphaJob.class);
        factoryBean.setName("alphaJob");
        factoryBean.setGroup("alphaJobGroup");
        factoryBean.setDurability(true);
        factoryBean.setRequestsRecovery(true);
        return factoryBean;
    }
    //配置Trigger(SimpleTriggerFactoryBean, CronTriggerFactoryBean)
    @Bean
    public SimpleTriggerFactoryBean alphaTrigger(JobDetail alphaJobDetail){
        SimpleTriggerFactoryBean factoryBean = new SimpleTriggerFactoryBean();
        factoryBean.setJobDetail(alphaJobDetail);
        factoryBean.setName("alphaTrigger");
        factoryBean.setGroup("alphaTriggerGroup");
        factoryBean.setRepeatInterval(3000);
        factoryBean.setJobDataMap(new JobDataMap());
        return factoryBean;
    }
}

在application.properties做配置

# QuartzProperties
spring.quartz.job-store-type=jdbc
spring.quartz.scheduler-name=communityScheduler
spring.quartz.properties.org.quartz.scheduler.instanceId=AUTO
spring.quartz.properties.org.quartz.jobStore.class=org.quartz.impl.jdbcjobstore.JobStoreTX
spring.quartz.properties.org.quartz.jobStore.driverDelegateClass=org.quartz.impl.jdbcjobstore.StdJDBCDelegate
spring.quartz.properties.org.quartz.jobStore.isClustered=true
spring.quartz.properties.org.quartz.threadPool.class=org.quartz.simpl.SimpleThreadPool
spring.quartz.properties.org.quartz.threadPool.threadCount=5
• 1
• 2
• 3
• 4
• 5
• 6
• 7
• 8
• 9

編寫test代碼

@RunWith(SpringRunner.class)
@SpringBootTest
@ContextConfiguration(classes = CommunityApplication.class)
public class QuartzTests {
    @Autowired
    private Scheduler scheduler;
    @Test
    public void testDeleteJob(){
        try {
            boolean result = scheduler.deleteJob(new JobKey("alphaJob", "alphaJobGroup"));
            System.out.println(result);
        } catch (SchedulerException e) {
            e.printStackTrace();
        }
    }
}

如何對帖子進行熱度的計算

一般的帖子，時間越久，量化的分數越低，而點贊和回復的數量越多，量化的分數越高。一般對點贊回復增加的分數做一個log取對數，增加剛剛發布的時候回復點贊的影響，隨着時間的推移，時間的負面作用體現，分數下降，這點和實際的情況相似。

用Redis來實現，每隔5分鍾統計一次分數，但是也不是都統計，把被點贊，回復，或者剛剛發布的帖子放入需要計算的redis的set里，隔一段時間計算分數重排。

現在RedisKeyUtil下

private static final String SPLIT = ":";
    private static final String PREFIX_POST = "post";
    //帖子分數
    public static String getPostScoreKey(){
        return PREFIX_POST + SPLIT + "score";
    }

在DiscussPostController里，對加精，發帖。加入set

//計算帖子分數
        String redisKey = RedisKeyUtil.getPostScoreKey();
        redisTemplate.opsForSet().add(redisKey, id);

在CommentController和LikeController判斷點贊的是帖子，再加入set里。

編寫PostScoreRefreshjob

public class PostScoreRefreshJob implements Job, CommunityConstant {
    private static final Logger logger = LoggerFactory.getLogger(PostScoreRefreshJob.class);
    @Autowired
    private RedisTemplate redisTemplate;
    @Autowired
    private DiscussPostService discussPostService;
    @Autowired
    private LikeService likeService;
    @Autowired
    private ElasticsearchService elasticsearchService;
    //牛客紀元
    private static final Date epoch;
    static {
        try {
            epoch = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").parse("2014-08-01 00:00:00");
        } catch (ParseException e) {
            throw new RuntimeException("初始化牛客紀元失敗!", e);
        }
    }
    @Override
    public void execute(JobExecutionContext context) throws JobExecutionException {
        String redisKey = RedisKeyUtil.getPostScoreKey();
        BoundSetOperations operations = redisTemplate.boundSetOps(redisKey);
        if(operations.size() == 0){
            logger.info("[任務取消] 沒有需要刷新的帖子!");
            return;
        }
        logger.info("[任務開始] 正在刷新帖子分數:" + operations.size());
        while (operations.size()>0){
            this.refresh((Integer)operations.pop());
        }
        logger.info("[任務結束] 帖子分數刷新完畢!");
    }
    private void refresh(int postId){
        DiscussPost post = discussPostService.findDiscussPostById(postId);
        if(post == null){
            logger.error("該帖子不存在: id = " + postId);
            return;
        }
        //是否精華
        boolean wonderful = post.getStatus() == 1;
        //評論數量
        int commentCount = post.getCommentCount();
        //點贊數量
        long likeCount = likeService.findEntityLikeCount(ENTITY_TYPE_POST, postId);
        //計算權重
        double w = (wonderful ? 75 : 0) + commentCount * 10 + likeCount * 2;
        //分數 = 帖子權重 + 距離天數
        double score = Math.log10(Math.max(w, 1))
                + (post.getCreateTime().getTime() - epoch.getTime())/(1000 * 3600 * 24);
        //更新帖子分數
        discussPostService.updateScore(postId, score);
        //同步搜素的數據
        post.setScore(score);
        elasticsearchService.saveDiscussPost(post);
    }
}

寫QuartzConfig

//配置-> 數據庫 -> 調用
@Configuration
public class QuartzConfig {
    //FactoryBean可簡化Bean的實例化過程:
    //1.通過FactoryBean封裝了Bean的實例化過程.
    //2.將FactoryBean裝配到Spring容器里.
    //3.將FactoryBean注入給其他的Bean
    //4.該Bean得到的是FactoryBean所管理的對象實例.
    //刷新帖子分數任務
    @Bean
    public JobDetailFactoryBean postScoreRefreshJobDetail(){
        JobDetailFactoryBean factoryBean = new JobDetailFactoryBean();
        factoryBean.setJobClass(PostScoreRefreshJob.class);
        factoryBean.setName("postScoreRefreshJob");
        factoryBean.setGroup("communityJobGroup");
        factoryBean.setDurability(true);
        factoryBean.setRequestsRecovery(true);
        return factoryBean;
    }
    
    @Bean
    public SimpleTriggerFactoryBean postScoreRefreshTrigger(JobDetail postScoreRefreshJobDetail){
        SimpleTriggerFactoryBean factoryBean = new SimpleTriggerFactoryBean();
        factoryBean.setJobDetail(postScoreRefreshJobDetail);
        factoryBean.setName("postScoreRefreshTrigger");
        factoryBean.setGroup("postScoreRefreshTriggerGroup");
        factoryBean.setRepeatInterval(1000 * 60 * 5);
        factoryBean.setJobDataMap(new JobDataMap());
        return factoryBean;
    }
}

在DiscussPostMapper接口上增加更新分數的方法

int updateScore(int id, double score);

在discusspost-mapper上增加更新分數，並對排序方式更新，增加判斷排序方式的判斷參數。對由此產生變化，調用該方法的進行更新。

<update id="updateScore">
        update discuss_post set score = #{score} where id = #{id}
    </update>
    <select id="selectDiscussPosts" resultType="DiscussPost">
        select <include refid="selectFields"></include>
        from discuss_post
        where status != 2
        <if test="userId!=0">
            and user_id = #{userId}
        </if>
        <if test="orderMode==0">
            order by type desc, create_time desc
        </if>
        <if test="orderMode==1">
            order by type desc, score desc, create_time desc
        </if>
        limit #{offset}, #{limit}
    </select>