diff --git a/EHDownloader.sln b/EHDownloader.sln new file mode 100644 index 0000000..6426fce --- /dev/null +++ b/EHDownloader.sln @@ -0,0 +1,25 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.29123.88 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EHDownloader", "EHDownloader\EHDownloader.csproj", "{824F3CE9-87F6-48E0-B9D6-D5F19DC33960}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {824F3CE9-87F6-48E0-B9D6-D5F19DC33960}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {824F3CE9-87F6-48E0-B9D6-D5F19DC33960}.Debug|Any CPU.Build.0 = Debug|Any CPU + {824F3CE9-87F6-48E0-B9D6-D5F19DC33960}.Release|Any CPU.ActiveCfg = Release|Any CPU + {824F3CE9-87F6-48E0-B9D6-D5F19DC33960}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {FCA04208-1958-4F13-82E2-1904DB013042} + EndGlobalSection +EndGlobal diff --git a/EHDownloader/App.config b/EHDownloader/App.config new file mode 100644 index 0000000..f9820cb --- /dev/null +++ b/EHDownloader/App.config @@ -0,0 +1,24 @@ + + + + +
+ + + + + + + + + D:\books + + + 1512454 + + + 6b31380b7ce97c6d455772ea3ead014f + + + + \ No newline at end of file diff --git a/EHDownloader/Book.cs b/EHDownloader/Book.cs new file mode 100644 index 0000000..952975f --- /dev/null +++ b/EHDownloader/Book.cs @@ -0,0 +1,303 @@ +using EHDownloader.FileContainer; +using HtmlAgilityPack; +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Text; +using System.Threading.Tasks; + +namespace EHDownloader +{ + enum FetchBookResults + { + LoadPageFailed, + FetchTitleFailed, + FetchPageSetFailed, + FetchPageLinkFailed, + Completed, + } + + enum DownloadBookResults + { + FetchPageSetFailed, + FetchPageLinkFailed, + Completed, + DownloadImageFailed, + } + + enum BookStates + { + [Description("等待")] + Wait, + //[Description("擷取頁面失敗")] + //FetchError, + [Description("載入首頁失敗")] + LoadFailed, + [Description("擷取標題失敗")] + FetchTitleFailed, + [Description("擷取頁集合失敗")] + FetchPageSetFailed, + [Description("擷取頁連結失敗")] + FetchPageLinkFailed, + + [Description("擷取頁面完成")] + FetchCompleted, + + + + [Description("下載完成")] + DownloadCompleted, + [Description("下載完成, 但是有些頁面出錯")] + DownloadFinishedButSomePageError, + Compressed, + } + + class Book + { + private const string PageSetXPath = @"//table[@class='ptb']//a"; + private const string SubTitleXPath = @"//*[@id='gj']"; + private const string TitleXPath = @"//*[@id='gn']"; + private const string PageLinksXPath = @"//*[@id='gdt']//a"; + static IFileContainProvider fileContainerProvider = new FolderFileContainerProvider(); + + public string ParentFolder { get; private set; } + + public string Url { get; private set; } + + public string Title { get; private set; } + + public DateTime DownloadDateTime { get; private set; } + + public List Pages { get; private set; } + + public int PageCount => Pages.Count; + + public BookStates BookState { get; set; } = BookStates.Wait; + + + public Book(string url, string parentFolder) + { + if (!url.EndsWith("/")) + url += '/'; + Url = url; + ParentFolder = parentFolder; + } + + IHtmlDocumentProvider _htmlDocProvider = new HtmlDocument_HttpClient(); + + + + public async Task FetchAsync() + { + // SortedSet pageSets = new SortedSet(); + SortedDictionary pageSets = new SortedDictionary(); + + var cookieContainer = WebHelper.GetCookieContainer(); + Uri uri = new Uri(Url); + cookieContainer.Add(new Cookie("nw", "1", uri.PathAndQuery, uri.Host)); + HtmlDocument doc; + try + { + doc = await _htmlDocProvider.GetDocumentAsync(Url); + } + catch (Exception ex) + { + BookState = BookStates.LoadFailed; + return FetchBookResults.LoadPageFailed; + } + + try + { + Title = parseTitle(doc); + } + catch + { + BookState = BookStates.FetchTitleFailed; + return FetchBookResults.FetchTitleFailed; + } + + try + { + // get pagesets + var htmlPageSets = doc.DocumentNode.SelectNodes(PageSetXPath); + if (htmlPageSets == null) + { + BookState = BookStates.FetchPageSetFailed; + return FetchBookResults.FetchPageSetFailed; + } + foreach (var htmlPageSet in htmlPageSets) + { + var pageSetUrl = htmlPageSet.Attributes["href"].Value; + if (pageSetUrl == Url) + continue; + var querys = System.Web.HttpUtility.ParseQueryString(new Uri(pageSetUrl).Query); + int pageIndex = int.Parse(querys["p"]); + if (!pageSets.ContainsKey(pageIndex)) + pageSets.Add(pageIndex, pageSetUrl); + } + //System.Web.HttpUtility. + } + catch + { + BookState = BookStates.FetchPageSetFailed; + return FetchBookResults.FetchTitleFailed; + } + + // add other page sets + if (pageSets.Count > 0) + { + int lastPageIndex = pageSets.Last().Key; + for (int i = 1; i < lastPageIndex; i++) + { + if (!pageSets.ContainsKey(i)) + { + pageSets.Add(i, $"{Url}?p={i}"); + } + } + } + + + try + { + Pages = new List(FetchPages(doc)); + foreach (var pageSetUrl in pageSets.Values) + { + Pages.AddRange(await FetchPages(pageSetUrl)); + } + } + catch + { + BookState = BookStates.FetchPageLinkFailed; + return FetchBookResults.FetchPageLinkFailed; + } + + + for (int i = 0; i < Pages.Count; i++) + { + Pages[i].PageNumber = i + 1; + } + + BookState = BookStates.FetchCompleted; + return FetchBookResults.Completed; + } + + private string parseTitle(HtmlDocument doc) + { + var bookTitleNode = doc.DocumentNode.SelectNodes(SubTitleXPath).FirstOrDefault(); + string title = bookTitleNode.InnerHtml; + if (!string.IsNullOrWhiteSpace(title)) + return WebUtility.HtmlDecode(title); + + bookTitleNode = doc.DocumentNode.SelectNodes(TitleXPath).FirstOrDefault(); + title = bookTitleNode.InnerHtml; + return WebUtility.HtmlDecode(title); + } + + public async Task DownloadBookAsync(IProgress progress = null) + { + var fileContainer = fileContainerProvider.CreateNewContainer(getFolderPath()); + DownloadDateTime = DateTime.Now; + int tryTimes = 0; + int tryLimit = 3; + while (tryTimes++ < tryLimit) + { + foreach (var page in Pages.Where(p => p.Result != DownloadPageResults.Completed)) + { + try + { + progress?.Report(new DownloadProgressInfo() { Book = this, Page = page }); + await page.DownloadToAsync(fileContainer); + } + catch (Exception) + { + } + } + + if (Pages.All(p => p.Result == DownloadPageResults.Completed)) + { + BookState = BookStates.DownloadCompleted; + return; + } + } + BookState = BookStates.DownloadFinishedButSomePageError; + } + + private string GetFolderName() + { + string folderName = Title; + foreach (var ch in System.IO.Path.GetInvalidFileNameChars()) + folderName = folderName.Replace(ch, ' '); + return folderName; + } + + private async Task> FetchPages(string url) + { + return FetchPages(await _htmlDocProvider.GetDocumentAsync(url)); + } + + private static IEnumerable FetchPages(HtmlDocument doc) + { + var htmlLinks = doc.DocumentNode.SelectNodes(PageLinksXPath); + foreach (var htmlLink in htmlLinks) + { + var linkStr = htmlLink.Attributes["href"].Value; + yield return new Page(linkStr); + } + } + + public async Task CompressAsync() + { + string folderPath = getFolderPath(); + if (BookState != BookStates.DownloadCompleted) + return; + if (!System.IO.Directory.Exists(ParentFolder)) + return; + + ZipFileContainer zipFileContainer = new ZipFileContainer(folderPath); + + foreach (var file in System.IO.Directory.GetFiles(folderPath)) + { + using (var fileStream = System.IO.File.OpenRead(file)) + using (var outStream = zipFileContainer.CreateFile(System.IO.Path.GetFileName(file))) + await fileStream.CopyToAsync(outStream); + } + + zipFileContainer.Close(); + if (Settings.Instance.DeleteDownloadFolderAfterCompressed) + { + System.IO.Directory.Delete(folderPath, true); + } + + BookState = BookStates.Compressed; + } + + private string getFolderPath() + { + return System.IO.Path.Combine(ParentFolder, GetFolderName()); + } + } + + internal class DownloadProgressInfo + { + public Book Book { get; set; } + public Page Page { get; set; } + public int PageNumber => Page.PageNumber; + public int PageCount => Book.PageCount; + + } + + static public class EnumHelper + { + public static string GetDescription(this Enum source) + { + System.Reflection.FieldInfo fi = source.GetType().GetField(source.ToString()); + DescriptionAttribute[] attributes = (DescriptionAttribute[])fi.GetCustomAttributes( + typeof(DescriptionAttribute), false); + if (attributes.Length > 0) return attributes[0].Description; + else return source.ToString(); + } + } +} diff --git a/EHDownloader/DbContext.cs b/EHDownloader/DbContext.cs new file mode 100644 index 0000000..cd750b7 --- /dev/null +++ b/EHDownloader/DbContext.cs @@ -0,0 +1,52 @@ +using Dapper; +using System; +using System.Collections.Generic; +using System.Data.SQLite; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace EHDownloader +{ + class DbContext + { + public DbContext() + { + if (File.Exists(Settings.GetDbPath())) return; + + using (var cn = new SQLiteConnection(Settings.GetConnectString())) + { + cn.Execute(@" + CREATE TABLE Book ( + Url VARCHAR(256), + Title VARNCHAR(256), + PageCount smallint, + DownloadDateTime datetime, + CONSTRAINT Book_PK PRIMARY KEY (Url) + )"); + + } + } + + public bool PageIsExists(string url) + { + using (var cn = new SQLiteConnection(Settings.GetConnectString())) + { + var query = + "select 1 from Book where Url=(@url)"; + return cn.Query(query, new { url }).FirstOrDefault() != null; + } + } + + public void InsertPage(Book book) + { + using (var cn = new SQLiteConnection(Settings.GetConnectString())) + { + var insertScript = + "INSERT INTO Book VALUES (@Url,@Title,@PageCount,@DownloadDateTime)"; + cn.Execute(insertScript, book); + } + } + } +} diff --git a/EHDownloader/EHDownloader.csproj b/EHDownloader/EHDownloader.csproj new file mode 100644 index 0000000..ea06cd0 --- /dev/null +++ b/EHDownloader/EHDownloader.csproj @@ -0,0 +1,117 @@ + + + + + Debug + AnyCPU + {824F3CE9-87F6-48E0-B9D6-D5F19DC33960} + WinExe + EHDownloader + EHDownloader + v4.7.2 + 512 + true + true + + + + + AnyCPU + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + AnyCPU + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + ..\packages\Dapper.2.0.4\lib\netstandard2.0\Dapper.dll + + + ..\packages\HtmlAgilityPack.1.11.12\lib\Net45\HtmlAgilityPack.dll + + + + + ..\packages\System.Data.SQLite.Core.1.0.111.0\lib\net46\System.Data.SQLite.dll + + + + + + + + + + + + + + + + + + + + + + + + + Form + + + Form1.cs + + + + + + + + + Form1.cs + + + ResXFileCodeGenerator + Resources.Designer.cs + Designer + + + True + Resources.resx + + + + SettingsSingleFileGenerator + Settings.Designer.cs + + + True + Settings.settings + True + + + + + + + + + + 此專案參考這部電腦上所缺少的 NuGet 套件。請啟用 NuGet 套件還原,以下載該套件。如需詳細資訊,請參閱 http://go.microsoft.com/fwlink/?LinkID=322105。缺少的檔案是 {0}。 + + + + \ No newline at end of file diff --git a/EHDownloader/FileContainer/FileContainerBase.cs b/EHDownloader/FileContainer/FileContainerBase.cs new file mode 100644 index 0000000..3342a42 --- /dev/null +++ b/EHDownloader/FileContainer/FileContainerBase.cs @@ -0,0 +1,24 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace EHDownloader.FileContainer +{ + abstract class FileContainerBase + { + public string Name { get; private set; } + + public FileContainerBase(string name) + { + Name = name; + } + + public abstract Stream CreateFile(string fileName); + + public virtual void Close() { } + + } +} diff --git a/EHDownloader/FileContainer/FolderFileContainer.cs b/EHDownloader/FileContainer/FolderFileContainer.cs new file mode 100644 index 0000000..3887def --- /dev/null +++ b/EHDownloader/FileContainer/FolderFileContainer.cs @@ -0,0 +1,22 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace EHDownloader.FileContainer +{ + class FolderFileContainer : FileContainerBase + { + public FolderFileContainer(string name) : base(name) + { + System.IO.Directory.CreateDirectory(name); + } + + public override Stream CreateFile(string fileName) + { + return File.Create(Path.Combine(Name, fileName)); + } + } +} diff --git a/EHDownloader/FileContainer/FolderFileContainerProvider.cs b/EHDownloader/FileContainer/FolderFileContainerProvider.cs new file mode 100644 index 0000000..49c5c6c --- /dev/null +++ b/EHDownloader/FileContainer/FolderFileContainerProvider.cs @@ -0,0 +1,16 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace EHDownloader.FileContainer +{ + class FolderFileContainerProvider : IFileContainProvider + { + public FileContainerBase CreateNewContainer(string name) + { + return new FolderFileContainer(name); + } + } +} diff --git a/EHDownloader/FileContainer/IFileContainProvider.cs b/EHDownloader/FileContainer/IFileContainProvider.cs new file mode 100644 index 0000000..a4a9d6c --- /dev/null +++ b/EHDownloader/FileContainer/IFileContainProvider.cs @@ -0,0 +1,13 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace EHDownloader.FileContainer +{ + interface IFileContainProvider + { + FileContainerBase CreateNewContainer(string name); + } +} diff --git a/EHDownloader/FileContainer/ZipFileContainer.cs b/EHDownloader/FileContainer/ZipFileContainer.cs new file mode 100644 index 0000000..ae70267 --- /dev/null +++ b/EHDownloader/FileContainer/ZipFileContainer.cs @@ -0,0 +1,34 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace EHDownloader.FileContainer +{ + class ZipFileContainer : FileContainerBase + { + private ZipArchive _zipArchive; + + public ZipFileContainer(string name) : base(name) + { + if (!name.EndsWith(".zip")) name += ".zip"; + _zipArchive = new ZipArchive(File.Create(name), ZipArchiveMode.Create, false, Encoding.UTF8); + } + + public override Stream CreateFile(string fileName) + { + return _zipArchive + .CreateEntry(fileName, CompressionLevel.NoCompression) + .Open(); + } + + public override void Close() + { + _zipArchive.Dispose(); + base.Close(); + } + } +} diff --git a/EHDownloader/FileContainer/ZipFileContainerProvider.cs b/EHDownloader/FileContainer/ZipFileContainerProvider.cs new file mode 100644 index 0000000..fc35aa0 --- /dev/null +++ b/EHDownloader/FileContainer/ZipFileContainerProvider.cs @@ -0,0 +1,16 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace EHDownloader.FileContainer +{ + class ZipFileContainerProvider : IFileContainProvider + { + public FileContainerBase CreateNewContainer(string name) + { + return new ZipFileContainer(name); + } + } +} diff --git a/EHDownloader/Form1.Designer.cs b/EHDownloader/Form1.Designer.cs new file mode 100644 index 0000000..7f349be --- /dev/null +++ b/EHDownloader/Form1.Designer.cs @@ -0,0 +1,139 @@ +namespace EHDownloader +{ + partial class Form1 + { + /// + /// 設計工具所需的變數。 + /// + private System.ComponentModel.IContainer components = null; + + /// + /// 清除任何使用中的資源。 + /// + /// 如果應該處置受控資源則為 true,否則為 false。 + protected override void Dispose(bool disposing) + { + if (disposing && (components != null)) + { + components.Dispose(); + } + base.Dispose(disposing); + } + + #region Windows Form 設計工具產生的程式碼 + + /// + /// 此為設計工具支援所需的方法 - 請勿使用程式碼編輯器修改 + /// 這個方法的內容。 + /// + private void InitializeComponent() + { + this.btn_Add = new System.Windows.Forms.Button(); + this.txt_url = new System.Windows.Forms.TextBox(); + this.lv_Tasks = new System.Windows.Forms.ListView(); + this.columnHeader1 = ((System.Windows.Forms.ColumnHeader)(new System.Windows.Forms.ColumnHeader())); + this.columnHeader2 = ((System.Windows.Forms.ColumnHeader)(new System.Windows.Forms.ColumnHeader())); + this.btn_Retry = new System.Windows.Forms.Button(); + this.txt_folder = new System.Windows.Forms.TextBox(); + this.btn_resetFolder = new System.Windows.Forms.Button(); + this.SuspendLayout(); + // + // btn_Add + // + this.btn_Add.Location = new System.Drawing.Point(713, 12); + this.btn_Add.Name = "btn_Add"; + this.btn_Add.Size = new System.Drawing.Size(75, 23); + this.btn_Add.TabIndex = 0; + this.btn_Add.Text = "Add"; + this.btn_Add.UseVisualStyleBackColor = true; + this.btn_Add.Click += new System.EventHandler(this.btn_Add_Click); + // + // txt_url + // + this.txt_url.Location = new System.Drawing.Point(13, 12); + this.txt_url.Name = "txt_url"; + this.txt_url.Size = new System.Drawing.Size(694, 22); + this.txt_url.TabIndex = 1; + this.txt_url.Text = "https://exhentai.org/g/2621788/e4178a8e4b/"; + // + // lv_Tasks + // + this.lv_Tasks.Columns.AddRange(new System.Windows.Forms.ColumnHeader[] { + this.columnHeader1, + this.columnHeader2}); + this.lv_Tasks.HideSelection = false; + this.lv_Tasks.Location = new System.Drawing.Point(13, 69); + this.lv_Tasks.Name = "lv_Tasks"; + this.lv_Tasks.Size = new System.Drawing.Size(775, 369); + this.lv_Tasks.TabIndex = 2; + this.lv_Tasks.UseCompatibleStateImageBehavior = false; + this.lv_Tasks.View = System.Windows.Forms.View.Details; + // + // columnHeader1 + // + this.columnHeader1.Text = "Url"; + this.columnHeader1.Width = 600; + // + // columnHeader2 + // + this.columnHeader2.Text = "State"; + this.columnHeader2.Width = 108; + // + // btn_Retry + // + this.btn_Retry.Location = new System.Drawing.Point(13, 445); + this.btn_Retry.Name = "btn_Retry"; + this.btn_Retry.Size = new System.Drawing.Size(75, 23); + this.btn_Retry.TabIndex = 3; + this.btn_Retry.Text = "Retry"; + this.btn_Retry.UseVisualStyleBackColor = true; + this.btn_Retry.Click += new System.EventHandler(this.Btn_Retry_Click); + // + // txt_folder + // + this.txt_folder.Location = new System.Drawing.Point(13, 41); + this.txt_folder.Name = "txt_folder"; + this.txt_folder.Size = new System.Drawing.Size(694, 22); + this.txt_folder.TabIndex = 4; + // + // btn_resetFolder + // + this.btn_resetFolder.Location = new System.Drawing.Point(713, 41); + this.btn_resetFolder.Name = "btn_resetFolder"; + this.btn_resetFolder.Size = new System.Drawing.Size(75, 23); + this.btn_resetFolder.TabIndex = 5; + this.btn_resetFolder.Text = "Reset"; + this.btn_resetFolder.UseVisualStyleBackColor = true; + this.btn_resetFolder.Click += new System.EventHandler(this.btn_resetFolder_Click); + // + // Form1 + // + this.AutoScaleDimensions = new System.Drawing.SizeF(6F, 12F); + this.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font; + this.ClientSize = new System.Drawing.Size(800, 486); + this.Controls.Add(this.btn_resetFolder); + this.Controls.Add(this.txt_folder); + this.Controls.Add(this.btn_Retry); + this.Controls.Add(this.lv_Tasks); + this.Controls.Add(this.txt_url); + this.Controls.Add(this.btn_Add); + this.Name = "Form1"; + this.Text = "Form1"; + this.ResumeLayout(false); + this.PerformLayout(); + + } + + #endregion + + private System.Windows.Forms.Button btn_Add; + private System.Windows.Forms.TextBox txt_url; + private System.Windows.Forms.ListView lv_Tasks; + private System.Windows.Forms.ColumnHeader columnHeader1; + private System.Windows.Forms.ColumnHeader columnHeader2; + private System.Windows.Forms.Button btn_Retry; + private System.Windows.Forms.TextBox txt_folder; + private System.Windows.Forms.Button btn_resetFolder; + } +} + diff --git a/EHDownloader/Form1.cs b/EHDownloader/Form1.cs new file mode 100644 index 0000000..1799651 --- /dev/null +++ b/EHDownloader/Form1.cs @@ -0,0 +1,219 @@ +using HtmlAgilityPack; +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Data; +using System.Drawing; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Windows.Forms; + +namespace EHDownloader +{ + public partial class Form1 : System.Windows.Forms.Form + { + string baseFolder; + string _folder; + static System.Text.RegularExpressions.Regex _regex_url = new System.Text.RegularExpressions.Regex(@"^https?://e-hentai.org/g/[^/]*/[^/]*/?$", + System.Text.RegularExpressions.RegexOptions.Compiled); + static System.Text.RegularExpressions.Regex _ex_regex_url = new System.Text.RegularExpressions.Regex(@"^https?://exhentai.org/g/[^/]*/[^/]*/?$", + System.Text.RegularExpressions.RegexOptions.Compiled); + + System.Threading.Thread _clipboardMonitorThread; + bool _clipboardMonitorThreadWorking = false; + + DbContext _dbContext; + + public Form1() + { + InitializeComponent(); + + baseFolder = Properties.Settings.Default.BaseFolder; + resetFolder(); + _dbContext = new DbContext(); + + + _clipboardMonitorThread = new System.Threading.Thread(clipboardMonitorProc); + _clipboardMonitorThread.SetApartmentState(System.Threading.ApartmentState.STA); + _clipboardMonitorThread.Start(); + } + + protected override void OnClosing(CancelEventArgs e) + { + base.OnClosing(e); + _clipboardMonitorThreadWorking = false; + + } + + bool isEhUrl(string url) + { + return _regex_url.IsMatch(url) || _ex_regex_url.IsMatch(url); + } + + + private void clipboardMonitorProc() + { + _clipboardMonitorThreadWorking = true; + string lastText = string.Empty; + while (!IsDisposed && _clipboardMonitorThreadWorking) + { + try + { + var clipboardText = Clipboard.GetText(); + if (lastText != clipboardText && + !string.IsNullOrWhiteSpace(clipboardText)) + { + lastText = clipboardText; + var lines = clipboardText.Split(new[] { "\r\n", "\r", "\n" }, StringSplitOptions.None); + foreach (var url in lines) + { + if (isEhUrl(url)) + { + BeginInvoke(new Action(() => AddTask(url))); + } + } + } + System.Threading.Thread.Sleep(1000); + } + catch (Exception) + { + + } + } + } + + private void btn_Add_Click(object sender, EventArgs e) + { + if (!string.IsNullOrEmpty(txt_url.Text)) + { + AddTask(txt_url.Text); + } + + // await Book.DownloadBookAsync("https://e-hentai.org/g/1464116/fa326cdb2b/", @"Z:\"); + //await Book.GetBookAsync("https://e-hentai.org/g/1464871/56158ca706/"); + + // await Book.GetBookAsync("https://e-hentai.org/g/1463748/3a40615ef7"); + // Book.GetBook("https://e-hentai.org/g/1411798/ac15344281/"); + + } + + bool downloading = false; + // object downloadingLck = new object(); + // System.Threading.Mutex downloadingMutex = new System.Threading.Mutex(true); + + private async void AddTask(string url) + { + if (!isEhUrl(url)) return; + if (lv_Tasks.Items.ContainsKey(url)) return; + if (_dbContext.PageIsExists(url)) return; + ListViewItem lvi = new ListViewItem(url) { Name = url }; + lvi.SubItems.Add("Wait"); + + Book book = new Book(url, _folder); + lvi.Text = url; + lvi.Tag = book; + + lv_Tasks.Items.Add(lvi); + await TryStartTask(lvi); + + + } + + private async Task TryStartTask() + => await TryStartTask(lv_Tasks.Items[0]); + + private async Task TryStartTask(ListViewItem lvi) + { + if (downloading) + return; + try + { + downloading = true; + Book book = (Book)lvi.Tag; + bool running = true; + while (running) + { + + lvi.SubItems[1].Text = book.BookState.GetDescription(); + switch (book.BookState) + { + case BookStates.Wait: + await book.FetchAsync(); + break; + case BookStates.FetchCompleted: + await book.DownloadBookAsync(new Progress( + (info) => lvi.SubItems[1].Text = $"下載中 {info.PageNumber}/{info.PageCount}")); + break; + case BookStates.DownloadCompleted: + await book.CompressAsync(); + _dbContext.InsertPage(book); + goto default; + case BookStates.Compressed: + case BookStates.LoadFailed: + case BookStates.FetchTitleFailed: + case BookStates.FetchPageSetFailed: + case BookStates.FetchPageLinkFailed: + case BookStates.DownloadFinishedButSomePageError: + default: + var i = lv_Tasks.Items.IndexOf(lvi) + 1; + if (i >= lv_Tasks.Items.Count) return; + lvi = lv_Tasks.Items[i]; + book = (Book)lvi.Tag; + break; + } + } + } + finally + { + downloading = false; + } + } + + private async void Btn_Retry_Click(object sender, EventArgs e) + { + if (downloading) + return; + for (int i = 0; i < lv_Tasks.Items.Count; i++) + { + var lvi = lv_Tasks.Items[i]; + var book = (Book)lvi.Tag; + switch (book.BookState) + { + case BookStates.Wait: + case BookStates.FetchCompleted: + case BookStates.DownloadCompleted: + case BookStates.Compressed: + default: + continue; + case BookStates.LoadFailed: + case BookStates.FetchTitleFailed: + case BookStates.FetchPageSetFailed: + case BookStates.FetchPageLinkFailed: + book.BookState = BookStates.Wait; + continue; + case BookStates.DownloadFinishedButSomePageError: + book.BookState = BookStates.FetchCompleted; + continue; + } + } + await TryStartTask(); + } + + private void setFolder(string folder) + { + _folder = folder; + txt_folder.Text = folder; + } + + void resetFolder() + { + setFolder(System.IO.Path.Combine(baseFolder, DateTime.Today.ToString("yyMMdd"))); + } + + private void btn_resetFolder_Click(object sender, EventArgs e) + { + resetFolder(); + } + } +} diff --git a/EHDownloader/Form1.resx b/EHDownloader/Form1.resx new file mode 100644 index 0000000..1af7de1 --- /dev/null +++ b/EHDownloader/Form1.resx @@ -0,0 +1,120 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + \ No newline at end of file diff --git a/EHDownloader/HtmlDocumentProvider.cs b/EHDownloader/HtmlDocumentProvider.cs new file mode 100644 index 0000000..cf75fc6 --- /dev/null +++ b/EHDownloader/HtmlDocumentProvider.cs @@ -0,0 +1,115 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Text; +using System.Threading.Tasks; +using HtmlAgilityPack; + +namespace EHDownloader +{ + interface IHtmlDocumentProvider + { + HtmlDocument GetDocument(string url); + Task GetDocumentAsync(string url); + } + + + class HtmlDocument_DirectLoader : IHtmlDocumentProvider + { + public HtmlDocument GetDocument(string url) + { + HtmlWeb htmlWeb = GetHtmlWeb(); + return htmlWeb.Load(url); + } + + public Task GetDocumentAsync(string url) + { + HtmlWeb htmlWeb = GetHtmlWeb(); + return htmlWeb.LoadFromWebAsync(url); + } + + private static HtmlWeb GetHtmlWeb() + { + HtmlWeb htmlWeb = new HtmlWeb(); + htmlWeb.PreRequest += + (request) => + { + request.CookieContainer = WebHelper.GetCookieContainer(); + request.UserAgent = WebHelper.UserAgent; + return true; + }; + return htmlWeb; + } + } + + + class HtmlDocument_HttpWebRequest : IHtmlDocumentProvider + { + public HtmlDocument GetDocument(string url) + { + HttpWebRequest request = GetRequest(url); + var response = request.GetResponse(); + var stream = response.GetResponseStream(); + var doc = new HtmlDocument(); + doc.Load(stream); + return doc; + } + + public async Task GetDocumentAsync(string url) + { + HttpWebRequest request = GetRequest(url); + var response = await request.GetResponseAsync(); + var stream = response.GetResponseStream(); + var doc = new HtmlDocument(); + doc.Load(stream); + return doc; + } + + private static HttpWebRequest GetRequest(string url) + { + HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url); + request.UserAgent = WebHelper.UserAgent; + request.CookieContainer = WebHelper.GetCookieContainer(); + return request; + } + } + + class HtmlDocument_HttpClient : IHtmlDocumentProvider + { + public HtmlDocument GetDocument(string url) + { + return GetDocumentAsync(url).Result; + } + + public async Task GetDocumentAsync(string url) + { + HttpClient client = GetHttpClient(); + //var response = await client.GetAsync(url); + //if (!response.IsSuccessStatusCode) + //{ + + //} + //String urlContents = await response.Content.ReadAsStringAsync(); + var doc = new HtmlDocument(); + doc.LoadHtml(await client.GetStringAsync(url)); + return doc; + } + + private static HttpClient GetHttpClient() + { + HttpClientHandler handler = new HttpClientHandler() + { + UseCookies = true, + CookieContainer = WebHelper.GetCookieContainer(), + }; + HttpClient client = new HttpClient(handler); + client.DefaultRequestHeaders.Add("user-agent", WebHelper.UserAgent); + return client; + } + } + +} + + diff --git a/EHDownloader/HtmlWebHelper.cs b/EHDownloader/HtmlWebHelper.cs new file mode 100644 index 0000000..91619d6 --- /dev/null +++ b/EHDownloader/HtmlWebHelper.cs @@ -0,0 +1,82 @@ +using HtmlAgilityPack; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Text; +using System.Threading.Tasks; + +namespace EHDownloader +{ + static class WebHelper + { + public static HtmlWeb HtmlWeb; + private static CookieContainer _cookieContainer; + public static HttpClient HttpClient = new HttpClient(); + + // static public HtmlWeb GetInstance() => _htmlWeb; + + static WebHelper() + { + _cookieContainer = new CookieContainer(); + HtmlWeb = new HtmlWeb() { UseCookies = false }; + HtmlWeb.PreRequest += new HtmlWeb.PreRequestHandler(preRequest); + + var handler = new HttpClientHandler() + { + CookieContainer = _cookieContainer, + UseCookies = true, + + }; + HttpClient = new HttpClient(handler); + HttpClient.DefaultRequestHeaders.Add("user-agent", UserAgent); + + _cookieContainer.Add(new Cookie("ipb_member_id", Properties.Settings.Default.ipb_member_id, "/", ".exhentai.org")); + _cookieContainer.Add(new Cookie("ipb_pass_hash", Properties.Settings.Default.ipb_pass_hash, "/", ".exhentai.org")); + //_cookieContainer.Add(new Cookie("igneous", "1794dbb8a", "/", ".exhentai.org")); + //_cookieContainer.Add(new Cookie("sk", "268qob5mp4g4vh9vbhi8l7wmme0p", "/", ".exhentai.org")); + + } + + static public CookieContainer GetCookieContainer() + { + return _cookieContainer; + } + + static public string UserAgent => "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36"; + + static private bool preRequest(HttpWebRequest request) + { + request.CookieContainer = _cookieContainer; + request.UserAgent = UserAgent; + + return true; + } + + static public HtmlDocument Load(string url) + { + return HtmlWeb.Load(url); + } + + public static async Task LoadAsync(string url) + { + + return await HtmlWeb.LoadFromWebAsync(url); + } + + static public void Download() + { + + } + + //static private WebClient _instance = new WebClient(); + //static public WebClient GetInstance() => _instance; + + //static public HtmlDocument Load(string url) + //{ + // return _instance.D(url); + //} + + } +} diff --git a/EHDownloader/Page.cs b/EHDownloader/Page.cs new file mode 100644 index 0000000..d1df858 --- /dev/null +++ b/EHDownloader/Page.cs @@ -0,0 +1,158 @@ +using HtmlAgilityPack; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Text; +using System.Threading.Tasks; + +namespace EHDownloader +{ + enum DownloadPageResults + { + WaitToDownload, + Downloading, + + Completed, + LoadPageHtmlFailed, + DownloadImageResponseError, + DownloadImageError, + } + + class Page + { + public string Url { get; private set; } + + public int PageNumber { get; internal set; } + + public string FileName => PageNumber.ToString("00000"); + + public DownloadPageResults Result { get; private set; } = DownloadPageResults.WaitToDownload; + + IHtmlDocumentProvider _htmlDocProvider = new HtmlDocument_HttpClient(); + + public Page(string url) + { + Url = url; + } + + internal async Task DownloadToAsync(FileContainer.FileContainerBase fileContainer) + { + HtmlDocument doc; + string imgUrl; + try + { + doc = await _htmlDocProvider.GetDocumentAsync(Url); + var imgElem = doc.DocumentNode.SelectNodes(@"//*[@id='img']").FirstOrDefault(); + if (imgElem == null) + { + throw new Exception($"Fetch img error: {Url}"); + } + imgUrl = imgElem.Attributes["src"].Value; + } + catch + { + Result = DownloadPageResults.LoadPageHtmlFailed; + return; + } + + + try + { + var response = await WebHelper.HttpClient.GetAsync(imgUrl); + if (response.StatusCode != System.Net.HttpStatusCode.OK) + { + System.Diagnostics.Debug.WriteLine($"Response is not OK: {Url}, {response.StatusCode}"); + Result = DownloadPageResults.DownloadImageResponseError; + return; + } + string ext = ".jpg"; + switch (response.Content.Headers.ContentType.MediaType) + { + case "image/jpeg": + ext = ".jpg"; + break; + case "image/png": + ext = ".png"; + break; + case "image/gif": + ext = ".gif"; + break; + } + using (var memoryStream = new System.IO.MemoryStream()) + { + await response.Content.CopyToAsync(memoryStream); + memoryStream.Seek(0, System.IO.SeekOrigin.Begin); + using (var fileStream = fileContainer.CreateFile(FileName + ext)) + await memoryStream.CopyToAsync(fileStream); + Result = DownloadPageResults.Completed; + } + } + catch + { + Result = DownloadPageResults.DownloadImageError; + return; + } + + } + + internal async Task DownloadToAsync(string folder) + { + int retryTimes = 0; + int retryLimit = 5; + while (retryTimes < retryLimit) + { + // "skipserver=31908-18135" + var doc = await WebHelper.LoadAsync(Url); + var imgElem = doc.DocumentNode.SelectNodes(@"//*[@id='img']").FirstOrDefault(); + if (imgElem == null) + { + System.Diagnostics.Debug.WriteLine($"Fetch img error: {Url}"); + return; + } + var imgUrl = imgElem.Attributes["src"].Value; + + try + { + + var response = await WebHelper.HttpClient.GetAsync(imgUrl); + if (response.StatusCode != System.Net.HttpStatusCode.OK) + { + System.Diagnostics.Debug.WriteLine($"Response is not OK: {Url}, {response.StatusCode}"); + + return; + } + string ext = ".jpg"; + switch (response.Content.Headers.ContentType.MediaType) + { + case "image/jpeg": + ext = ".jpg"; + break; + case "image/png": + ext = ".png"; + break; + case "image/gif": + ext = ".gif"; + break; + } + + string path = System.IO.Path.Combine(folder, FileName + ext); + using (var fs = System.IO.File.Create(path)) + await response.Content.CopyToAsync(fs); + break; + } + catch + { + if (retryTimes++ >= retryLimit) + { + System.Diagnostics.Debug.WriteLine($"Download image failed, skip: {Url}"); + } + else + { + System.Diagnostics.Debug.WriteLine($"Download image failed, retry{retryTimes}/{retryLimit}: {Url}"); + } + } + } + } + } +} diff --git a/EHDownloader/Program.cs b/EHDownloader/Program.cs new file mode 100644 index 0000000..8a8f8d6 --- /dev/null +++ b/EHDownloader/Program.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using System.Windows.Forms; + +namespace EHDownloader +{ + static class Program + { + /// + /// 應用程式的主要進入點。 + /// + [STAThread] + static void Main() + { + Application.EnableVisualStyles(); + Application.SetCompatibleTextRenderingDefault(false); + Application.Run(new Form1()); + } + } +} diff --git a/EHDownloader/Properties/AssemblyInfo.cs b/EHDownloader/Properties/AssemblyInfo.cs new file mode 100644 index 0000000..15e5535 --- /dev/null +++ b/EHDownloader/Properties/AssemblyInfo.cs @@ -0,0 +1,36 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +// 組件的一般資訊是由下列的屬性集控制。 +// 變更這些屬性的值即可修改組件的相關 +// 資訊。 +[assembly: AssemblyTitle("EHDownloader")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("EHDownloader")] +[assembly: AssemblyCopyright("Copyright © 2019")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// 將 ComVisible 設為 false 可對 COM 元件隱藏 +// 組件中的類型。若必須從 COM 存取此組件中的類型, +// 的類型,請在該類型上將 ComVisible 屬性設定為 true。 +[assembly: ComVisible(false)] + +// 下列 GUID 為專案公開 (Expose) 至 COM 時所要使用的 typelib ID +[assembly: Guid("824f3ce9-87f6-48e0-b9d6-d5f19dc33960")] + +// 組件的版本資訊由下列四個值所組成: +// +// 主要版本 +// 次要版本 +// 組建編號 +// 修訂編號 +// +// 您可以指定所有的值,也可以使用 '*' 將組建和修訂編號 +// 設為預設,如下所示: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/EHDownloader/Properties/Resources.Designer.cs b/EHDownloader/Properties/Resources.Designer.cs new file mode 100644 index 0000000..d2afefb --- /dev/null +++ b/EHDownloader/Properties/Resources.Designer.cs @@ -0,0 +1,71 @@ +//------------------------------------------------------------------------------ +// +// 這段程式碼是由工具產生的。 +// 執行階段版本:4.0.30319.42000 +// +// 變更這個檔案可能會導致不正確的行為,而且如果已重新產生 +// 程式碼,則會遺失變更。 +// +//------------------------------------------------------------------------------ + +namespace EHDownloader.Properties +{ + + + /// + /// 用於查詢當地語系化字串等的強類型資源類別 + /// + // 這個類別是自動產生的,是利用 StronglyTypedResourceBuilder + // 類別透過 ResGen 或 Visual Studio 這類工具產生。 + // 若要加入或移除成員,請編輯您的 .ResX 檔,然後重新執行 ResGen + // (利用 /str 選項),或重建您的 VS 專案。 + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "4.0.0.0")] + [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] + [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] + internal class Resources + { + + private static global::System.Resources.ResourceManager resourceMan; + + private static global::System.Globalization.CultureInfo resourceCulture; + + [global::System.Diagnostics.CodeAnalysis.SuppressMessageAttribute("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] + internal Resources() + { + } + + /// + /// 傳回這個類別使用的快取的 ResourceManager 執行個體。 + /// + [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] + internal static global::System.Resources.ResourceManager ResourceManager + { + get + { + if ((resourceMan == null)) + { + global::System.Resources.ResourceManager temp = new global::System.Resources.ResourceManager("EHDownloader.Properties.Resources", typeof(Resources).Assembly); + resourceMan = temp; + } + return resourceMan; + } + } + + /// + /// 覆寫目前執行緒的 CurrentUICulture 屬性,對象是所有 + /// 使用這個強類型資源類別的資源查閱。 + /// + [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] + internal static global::System.Globalization.CultureInfo Culture + { + get + { + return resourceCulture; + } + set + { + resourceCulture = value; + } + } + } +} diff --git a/EHDownloader/Properties/Resources.resx b/EHDownloader/Properties/Resources.resx new file mode 100644 index 0000000..af7dbeb --- /dev/null +++ b/EHDownloader/Properties/Resources.resx @@ -0,0 +1,117 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=2.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=2.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + \ No newline at end of file diff --git a/EHDownloader/Properties/Settings.Designer.cs b/EHDownloader/Properties/Settings.Designer.cs new file mode 100644 index 0000000..8f79842 --- /dev/null +++ b/EHDownloader/Properties/Settings.Designer.cs @@ -0,0 +1,53 @@ +//------------------------------------------------------------------------------ +// +// 這段程式碼是由工具產生的。 +// 執行階段版本:4.0.30319.42000 +// +// 對這個檔案所做的變更可能會造成錯誤的行為,而且如果重新產生程式碼, +// 變更將會遺失。 +// +//------------------------------------------------------------------------------ + +namespace EHDownloader.Properties { + + + [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("Microsoft.VisualStudio.Editors.SettingsDesigner.SettingsSingleFileGenerator", "16.7.0.0")] + internal sealed partial class Settings : global::System.Configuration.ApplicationSettingsBase { + + private static Settings defaultInstance = ((Settings)(global::System.Configuration.ApplicationSettingsBase.Synchronized(new Settings()))); + + public static Settings Default { + get { + return defaultInstance; + } + } + + [global::System.Configuration.ApplicationScopedSettingAttribute()] + [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] + [global::System.Configuration.DefaultSettingValueAttribute("D:\\books")] + public string BaseFolder { + get { + return ((string)(this["BaseFolder"])); + } + } + + [global::System.Configuration.ApplicationScopedSettingAttribute()] + [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] + [global::System.Configuration.DefaultSettingValueAttribute("1512454")] + public string ipb_member_id { + get { + return ((string)(this["ipb_member_id"])); + } + } + + [global::System.Configuration.ApplicationScopedSettingAttribute()] + [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] + [global::System.Configuration.DefaultSettingValueAttribute("6b31380b7ce97c6d455772ea3ead014f")] + public string ipb_pass_hash { + get { + return ((string)(this["ipb_pass_hash"])); + } + } + } +} diff --git a/EHDownloader/Properties/Settings.settings b/EHDownloader/Properties/Settings.settings new file mode 100644 index 0000000..1cdcbe6 --- /dev/null +++ b/EHDownloader/Properties/Settings.settings @@ -0,0 +1,15 @@ + + + + + + D:\books + + + 1512454 + + + 6b31380b7ce97c6d455772ea3ead014f + + + \ No newline at end of file diff --git a/EHDownloader/Settings.cs b/EHDownloader/Settings.cs new file mode 100644 index 0000000..c710e21 --- /dev/null +++ b/EHDownloader/Settings.cs @@ -0,0 +1,29 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace EHDownloader +{ + class Settings + { + static public Settings Instance { get; } = new Settings(); + public bool DeleteDownloadFolderAfterCompressed { get; set; } = true; + + + + internal static string GetDbPath() + { + return @".\db.sqlite"; + //return Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "db.sqlite"); + } + + + internal static string GetConnectString() + { + return $"data source={GetDbPath()}"; + } + } +} diff --git a/EHDownloader/packages.config b/EHDownloader/packages.config new file mode 100644 index 0000000..e0432af --- /dev/null +++ b/EHDownloader/packages.config @@ -0,0 +1,7 @@ + + + + + + + \ No newline at end of file