利用HtmlAgilityPack抓取网站图片并下载~~~~~~邪恶完善版

本文转载自查看原文 2012-02-16 21:31 4017 C# WinForms

今日看博客园发现一个不错的抓取贴（主要是那个url。。。你懂的），花几分钟改了下，代码增加了按年月日建立目录，按文章建立子目录，图片都保存于内，命令行方式运行，增加了全站的参数。。。

原始版本：

利用HtmlAgilityPack抓取XX网站图片并下载~~~~~~邪恶版。。。。

新版本代码：

#region Using namespace

using System;
using System.IO;
using System.Linq;
using System.Net;
using HtmlAgilityPack;

#endregion

namespace DownloadImages
{
     internal class Program
    {
         private static readonly WebClient Wc = new WebClient();
         private static readonly char[] InvalidFileNameChars = new[]
                                                                  {
                                                                       ' " ',
                                                                       ' < ',
                                                                       ' > ',
                                                                       ' | ',
                                                                       ' \0 ',
                                                                       ' \u0001 ',
                                                                       ' \u0002 ',
                                                                       ' \u0003 ',
                                                                       ' \u0004 ',
                                                                       ' \u0005 ',
                                                                       ' \u0006 ',
                                                                       ' \a ',
                                                                       ' \b ',
                                                                       ' \t ',
                                                                       ' \n ',
                                                                       ' \v ',
                                                                       ' \f ',
                                                                       ' \r ',
                                                                       ' \u000e ',
                                                                       ' \u000f ',
                                                                       ' \u0010 ',
                                                                       ' \u0011 ',
                                                                       ' \u0012 ',
                                                                       ' \u0013 ',
                                                                       ' \u0014 ',
                                                                       ' \u0015 ',
                                                                       ' \u0016 ',
                                                                       ' \u0017 ',
                                                                       ' \u0018 ',
                                                                       ' \u0019 ',
                                                                       ' \u001a ',
                                                                       ' \u001b ',
                                                                       ' \u001c ',
                                                                       ' \u001d ',
                                                                       ' \u001e ',
                                                                       ' \u001f ',
                                                                       ' : ',
                                                                       ' * ',
                                                                       ' ? ',
                                                                       ' \\ ',
                                                                       ' / '
                                                                  };
         public static string CleanInvalidFileName( string fileName)
        {
            fileName = fileName + "";
            fileName = InvalidFileNameChars.Aggregate(fileName, (current, c) => current.Replace(c + "", ""));

             if (fileName.Length > 1)
                 if (fileName[ 0] == ' . ')
                    fileName = " dot " + fileName.TrimStart( ' . ');

             return fileName;
        }
         private static void Main( string[] args)
        {
            Start();
        }

         private static void Start()
        {
             var web = new HtmlWeb();
             var startDate = int.Parse(DateTime.Parse( " 2010-08-18 ").ToString( " yyyyMMdd "));
             var endDate = int.Parse(DateTime.Now.ToString( " yyyyMMdd "));
             const int startPageId = 49430;
             const int endPageId = 124621;
             for ( int k = startDate; k <= endDate; k++)
            {
                 for ( int j = startPageId; j <= endPageId; j++)
                {
                     string cnblogs = http://xxxxxxxx/ + k + " / " + j + " .html "; //此处省略……源码内详
                    HtmlDocument doc = web.Load(cnblogs);
                     var titles = doc.DocumentNode.SelectNodes( " //title ");
                     var titleName = j.ToString();
                     if( titles!= null && titles.Count> 0)
                        titleName = titles[ 0].InnerText;
                    HtmlNode node = doc.GetElementbyId( " ks_xp ");
                     if (node == null)
                    {
                         continue;
                    }
                     foreach (HtmlNode child in node.SelectNodes( " //img "))
                    {
                         if (child.Attributes[ " src "] == null)
                             continue;

                         string imgurl = child.Attributes[ " src "].Value;
                        DownLoadImg(imgurl, k + "", CleanInvalidFileName(titleName));
                        Console.WriteLine( " 正在下载: " + titleName + " " + imgurl);
                    }
                }
            }
             // 善后
            CleanEmptyFolders();
        }

         private static void CleanEmptyFolders()
        {
             var rootFolders = Environment.CurrentDirectory + " \\Images\\ ";
             var folders = Directory.GetDirectories(rootFolders, " *.* ", SearchOption.AllDirectories);
             foreach( var f in folders)
            {
                 if (Directory.GetFiles(f, " *.* ", SearchOption.AllDirectories).Length == 0)
                    Directory.Delete(f);
            }
        }

         private static void DownLoadImg( string url, string folderName, string subFolderName)
        {
             var fileName = CleanInvalidFileName(url.Substring(url.LastIndexOf( " / ") + 1));
             var fileFolder = Environment.CurrentDirectory + " \\Images\\ " + folderName + " \\ " + subFolderName + " \\ " ;
             if (!Directory.Exists(fileFolder))
                Directory.CreateDirectory(fileFolder);
            fileName = fileFolder + fileName;
             try
            {
                Wc.DownloadFile(url, fileName);
            }
             catch (Exception ex)
            {
                Console.WriteLine(ex.Message);
            }
        }
    }
}

测试程序和源码下载：

/Files/Chinasf/DownloadImages.rar

免责声明！

本站转载的文章为个人学习借鉴使用，本站对版权不负任何法律责任。如果侵犯了您的隐私权益，请联系本站邮箱yoyou2525@163.com删除。

猜您在找 python网络爬虫抓取网站图片使用python来批量抓取网站图片使用HtmlAgilityPack抓取网页数据 scrapy爬虫学习系列五：图片的抓取和下载 HtmlAgilityPack 简单实现图片抓取下载到本地① 使用Puppeteer进行数据抓取(四)——图片下载喜迎Win10周年版，芒果TV UWP完善手机视频离线下载利用htmlunit登陆带验证码图片的网站 google、baidu、yahoo、bing这些搜索引擎网站的图片抓取方法汇总