功能
運用到的知識點
- 日志處理
- 向服務端發送get請求
- 遞歸獲取文件夾和創建文件夾
package main
import (
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"os"
"regexp"
"strings"
"time"
)
var (
baseurl = "基礎url"
basePath = "D:\\staticFile" //本地地址
logger *log.Logger
)
func init(){
fmt.Println("創建日記錄日志文件")
f,err:=os.OpenFile("Log.log", os.O_APPEND|os.O_CREATE|os.O_WRONLY, 666)
if err!=nil{
log.Fatal("os.OpenFile err",err)
}
writers := []io.Writer{
f,
os.Stdout}
//defer f.Close() 因為這里，就已經將文件關閉了，看來不能夠隨便使用defer
fileAndStdoutWriter := io.MultiWriter(writers...)
logger = log.New(fileAndStdoutWriter, "", log.Ldate|log.Ltime|log.Lshortfile)
logger.Println("---> logger:check to make sure is works")
}
func main(){
logger.Println("開始計算時間")
t1:=time.Now()
HandleFile(baseurl)
t2:=time.Now()
logger.Println("下載所有的文件總耗時：",t2.Sub(t1))
}
//遍歷指定路徑下的所有文件，filepath.Walk("路徑",walkFunc)
func walkFunc(path string,info os.FileInfo,err error)error{
fmt.Println(path)
return nil
}
//處理文件信息
func HandleFile(url string)error{
if url==""{
err:=fmt.Errorf("url是空的")
logger.Println(err)
return err
}
logger.Println("發起get請求")
htmlData,err:= HttpGet(url)
if htmlData==""{
err:=fmt.Errorf("HttpGet中有錯啦，請檢查HttpGet數據")
logger.Println(err)
return err
}
if err!=nil{
logger.Println(err)
return err
}
cutUrl := strings.Split(url,"基礎url")[1]
logger.Println("正則匹配信息")
re:=regexp.MustCompile(<a href="(.*?)">(.*?)</a>)
result := re.FindAllStringSubmatch(htmlData,-1)
if result==nil{
err:=fmt.Errorf("正則匹配的數據為空")
logger.Println(err)
return err
}
logger.Println("正則匹配數據")
for i:=0;i<len(result);i++{
publicPath := cutUrl
if len(result[i])!=3{
err:=fmt.Errorf("正則出來的東西不是我想要的")
logger.Println(err)
return err
}
fmt.Println(result[i][2])
if strings.Contains(result[i][2],"/"){
logger.Println(result[i][2]+"是個文件夾")
floderName:=strings.Replace(result[i][2],"/","",-1)
if floderName!=result[i][2]{
publicPath = publicPath+"/"+floderName
fmt.Println("publicPath=============>",publicPath)
logger.Println("創建文件夾")
err=CreateFloder(publicPath)
if err!=nil{
logger.Println("CreateFloder error",err)
return err
}
urlPath := baseurl+publicPath+"/"
fmt.Println("urlPath============>",urlPath)
logger.Println("遞歸")
HandleFile(urlPath)
}
}else{
publicPath = publicPath+result[i][2]
logger.Println("遇到是文件，就將數據寫入文件")
err:=WriteFile(publicPath)
if err!=nil{
logger.Println(err)
return err
}
}
}
return nil
}
//發起get請求
func HttpGet(url string)(string,error){
if url==""{
err:= fmt.Errorf("傳入的url為空")
logger.Println(err)
return "",err
}
if !strings.Contains(url,"http:"){
err:=fmt.Errorf("傳入的url不正確")
logger.Println(err)
return "",err
}
resp,err:=http.Get(url)
if err!=nil{
err1:= fmt.Errorf("http.Get error===========>%v",err)
logger.Println(err1)
return "",err1
}
defer resp.Body.Close()
body,err := ioutil.ReadAll(resp.Body)
if err!=nil{
err1:=fmt.Errorf("ioutil.ReadAll error===========>%v",err)
logger.Println(err1)
return "",err1
}
if strings.Contains(string(body),"404 page not found"){
err:=fmt.Errorf("找不到該網頁")
logger.Println(err)
return "",err
}
return string(body),nil
}
//判斷文件是否存在
func IsExist(path string)(bool){
if path==""{
return false
}
_,err:=os.Stat(path)
if err!=nil{
if os.IsExist(err){
return true
}else{
return false
}
}
return true
}
//創建文件夾
func CreateFloder(publicPath string)(error){
if publicPath==""{
err:=fmt.Errorf("傳入urlpath的地址空")
logger.Println(err)
return err
}
logger.Println(publicPath,"創建文件夾")
err:=os.MkdirAll(basePath+"//"+publicPath,os.ModePerm)
if err!=nil{
err:=fmt.Errorf("創建文件錯誤啦:%v",err)
logger.Println(err)
return err
}
return nil
}
//寫入文件
func WriteFile(publicPath string)error{
if publicPath==""{
err:=fmt.Errorf("傳入的參數為空，請注意！！！")
logger.Println(err)
return err
}
htmlData,err:=HttpGet(baseurl+publicPath)
if htmlData==""{
err:=fmt.Errorf("HttpGet中有錯啦，請檢查HttpGet數據")
logger.Println(err)
return err
}
if err!=nil{
logger.Println(err)
return err
}
file,err:=os.Create(basePath+publicPath)
defer file.Close()
if err!=nil{
err:=fmt.Errorf("os.Open失敗,err=%v",err)
logger.Println(err)
return err
}
if file==nil{
err:=fmt.Errorf("創建文件失敗")
logger.Println(err)
return err
}
file.WriteString(htmlData)
return nil
}
