Jsoup实现下载layui离线文档

 在银行开发项目的时候，不允许连接外网，每次都得一顿很复杂的操作才能去官方网站看文档，想找个离线本地版的没有找到，还是自己动手爬到本地靠谱点；

package com.bcc.customer.utils;

import java.io.File;
import java.io.IOException;

import org.easitline.common.utils.kit.FileKit;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

public class Download {

    public static void loadPage(String url){
        Connection connection=Jsoup.connect("https://www.layui.com"+url);
        try {
            String[] u=url.split("/");
            String name=u[u.length-1];
            String f=u[u.length-2];
            connection.userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36");
            connection.header("referer","https://www.layui.com/doc/base/infrastructure.html");
            connection.cookie("cookie", "Hm_lvt_d214947968792b839fd669a4decaaffc=1567350006,1567437784,1567518319,1567518344; Hm_lpvt_d214947968792b839fd669a4decaaffc=1567518950");
            Document document = connection.get();
            String title=document.selectFirst("title").text();
            String html=document.html();
            html=html.replaceAll("<script async src=\"https://pagead2.googlesyndication.com/pagead/js/adsbygoogle.js\"></script>","");
            html=html.replaceAll("s.parentNode.insertBefore(hm, s);","");
            html=html.replaceAll("//res.layui.com","/layui-doc");
            html=html.replaceAll("//res.layui.com/staitc","/layui-doc/static");
            html=html.replaceAll("/doc","/layui-doc/html");
            
            String p="D:/develop/layui-doc/html/"+f;
            File file=new File(p);
            if(!file.exists()){
                file.mkdirs();
            }
            FileKit.saveToFile(html, p+"/"+name);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
    public static void main(String[] args) {
        
        Connection connection=Jsoup.connect("https://www.layui.com/doc/base/element.html");
        try {
            Document document=connection.get();
            Elements urls=document.select(".site-tree a");
            for(Element ele:urls){
                String url=ele.attr("href");
                System.out.println(url);
                loadPage(url);
            }
            
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

}

 

public static void saveToFile(InputStream in, String fileName) throws IOException {
        FileOutputStream out=null;
        try {
            fileName=sysPath(fileName);
            File outFile = new File(fileName);
            if (!outFile.exists()) {
                outFile.createNewFile();
            }
            out = new FileOutputStream(fileName);
            byte buffer[] = new byte[1024];
            int read = -1;
            while ((read = in.read(buffer, 0, 1024)) != -1) {
                out.write(buffer, 0, read);
            }
            out.flush();
            
        } catch (IOException e) {
            throw e;
        } catch (Throwable e) {
            throw new RuntimeException("文件写入错误:fileName="+fileName, e);
        } finally {
            if (out != null) {
                try {
                    out.close();
                } catch (Exception e1) {
                }}
            if (in != null) {
                    try {
                        in.close();
                    } catch (Exception e1) {
                    }
            }
        }
    }

 

https://repo1.maven.org/maven2/org/jsoup/jsoup/1.12.1/jsoup-1.12.1.jar