From fe4a4769f82db89731effede4e3930b595829447 Mon Sep 17 00:00:00 2001 From: Serraniel Date: Mon, 17 Apr 2017 21:56:53 +0200 Subject: [PATCH] Somehow implementation of downloading all the shit out of it --- .../Classes/Job.cs | 102 +++++++++- .../Classes/JobScheduler.cs | 192 ++++++++++++++++++ Discord Media Loader.Application/Core.cs | 12 +- .../DML.Application.csproj | 1 + .../MainForm.Designer.cs | 105 +++++----- Discord Media Loader.Application/MainForm.cs | 30 +-- .../Properties/AssemblyInfo.cs | 4 +- .../Properties/AssemblyInfo.cs | 4 +- 8 files changed, 380 insertions(+), 70 deletions(-) create mode 100644 Discord Media Loader.Application/Classes/JobScheduler.cs diff --git a/Discord Media Loader.Application/Classes/Job.cs b/Discord Media Loader.Application/Classes/Job.cs index ba5aabd..e55a15b 100644 --- a/Discord Media Loader.Application/Classes/Job.cs +++ b/Discord Media Loader.Application/Classes/Job.cs @@ -1,8 +1,11 @@ using System; using System.Collections.Generic; +using System.Diagnostics.Eventing.Reader; using System.Linq; using System.Text; using System.Threading.Tasks; +using Discord; +using SweetLib.Utils; using static SweetLib.Utils.Logger.Logger; namespace DML.Application.Classes @@ -12,6 +15,8 @@ namespace DML.Application.Classes public int Id { get; set; } public ulong GuildId { get; set; } public ulong ChannelId { get; set; } + public double KnownTimestamp { get; set; } = 0; + private double StopTimestamp { get; set; } = 0; internal void Store() { @@ -20,7 +25,102 @@ namespace DML.Application.Classes var jobDb = Core.Database.GetCollection("jobs"); Trace("Adding new value..."); - jobDb.Insert(this); + + if (jobDb.Find(x => x.ChannelId == ChannelId && x.GuildId == GuildId).Any()) + { + jobDb.Update(this); + } + else + { + jobDb.Insert(this); + } + } + + private Server FindServerById(ulong id) + { + Trace($"Trying to find server by Id: {id}"); + return (from s in Core.Client.Servers where s.Id == id select s).FirstOrDefault(); + } + + private Channel FindChannelById(Server server, ulong id) + { + Trace($"Trying to find channel in {server} by id: {id}"); + return (from c in server.TextChannels where c.Id == id select c).FirstOrDefault(); + } + + internal async Task> Scan() + { + Debug($"Starting scan of guild {GuildId} channel {ChannelId}..."); + var result = new List(); + + var limit = 100; + var lastId = ulong.MaxValue; + var isFirst = true; + var finished = false; + + var guild = FindServerById(GuildId); + var channel = FindChannelById(guild, ChannelId); + + if (Math.Abs(StopTimestamp) < 0.4) + StopTimestamp = KnownTimestamp; + Trace("Initialized scanning parameters."); + + while (!finished) + { + Trace("Entering scanning loop..."); + Message[] messages; + + Trace($"Downloading next {limit} messages..."); + if (isFirst) + { + messages = await channel.DownloadMessages(limit, null); + } + else + { + messages = await channel.DownloadMessages(limit, lastId); + } + Trace($"Downloaded {messages.Length} messages."); + + isFirst = false; + + foreach (var m in messages) + { + Debug($"Processing message {m.Id}"); + if (m.Id < lastId) + { + Trace($"Updating lastId ({lastId}) to {m.Id}"); + lastId = m.Id; + } + + if (SweetUtils.DateTimeToUnixTimeStamp(m.Timestamp) < StopTimestamp) + { + Debug("Found a message with a known timestamp...Stopping scan."); + finished = true; + continue; + } + + Trace($"Message {m.Id} has {m.Attachments.Length} attachments."); + if (m.Attachments.Length > 0) + { + result.Add(m); + Trace($"Added message {m.Id}"); + } + Debug($"Finished message {m.Id}"); + } + + finished = finished || messages.Length < limit; + } + Trace($"Downloaded all messages for guild {GuildId} channel {ChannelId}."); + + Trace("Sorting messages..."); + result.Sort((a, b) => DateTime.Compare(a.Timestamp, b.Timestamp)); + + Trace("Updating StopTimestamp for next scan..."); + StopTimestamp = SweetUtils.DateTimeToUnixTimeStamp(result[result.Count - 1].Timestamp); + + Debug($"Fisnished scan of guild {GuildId} channel {ChannelId}."); + + return result; } internal static IEnumerable RestoreJobs() diff --git a/Discord Media Loader.Application/Classes/JobScheduler.cs b/Discord Media Loader.Application/Classes/JobScheduler.cs new file mode 100644 index 0000000..ff599fe --- /dev/null +++ b/Discord Media Loader.Application/Classes/JobScheduler.cs @@ -0,0 +1,192 @@ +using System; +using System.Collections; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Text; +using System.Threading.Tasks; +using Discord; +using SweetLib.Utils; +using static SweetLib.Utils.Logger.Logger; + +namespace DML.Application.Classes +{ + internal class JobScheduler + { + private bool Run { get; set; } = false; + internal List JobList { get; set; } = new List(); + internal Dictionary> RunningJobs = new Dictionary>(); + internal int RunningThreads { get; set; } = 0; + + internal void Stop() + { + Run = false; + } + + internal void Start() + { + Run = true; + + Task.Run(async () => + { + Info("Started JobScheduler..."); + while (Run) + { + Debug("Entering job list handler loop..."); + foreach (var job in JobList) + { + Debug($"Checking job {job}"); + var hasJob = false; + + Trace("Locking scheduler..."); + lock (this) + { + Trace("Checking if job is already performed..."); + hasJob = RunningJobs.ContainsKey(job.Id); + } + Trace("Unlocked scheduler."); + + if (!hasJob) + { + Debug("Job is not performed yet...Performing job..."); + var queue = new Queue(); + + Trace("Locking scheduler..."); + lock (this) + { + Trace("Adding job to running jobs."); + RunningJobs.Add(job.Id, queue); + } + Trace("Unlocked scheduler."); + + Trace("Issuing job message scan..."); + var messages = await job.Scan(); + + Trace($"Adding {messages.Count} messages to queue..."); + foreach (var msg in messages) + { + queue.Enqueue(msg); + } + Trace($"Added {queue.Count} messages to queue."); + + if(messages.Count!= queue.Count) + Warn("Not all messages have been added into the queue."); + + var startedDownload = false; + + while (!startedDownload) + { + Debug("Entering loop to check thread availability"); + Trace("Locking scheduler..."); + lock (this) + { + Trace($"Checking thread limit. Running: {RunningThreads}, Max: {Core.Settings.ThreadLimit}"); + if (RunningThreads >= Core.Settings.ThreadLimit) + continue; + + RunningThreads++; + startedDownload = true; + } + Trace("Unlocked scheduler."); + } + + Trace("Start downloading job async."); + Task.Run(() => WorkQueue(job.Id)); // do not await to work parallel + } + } + } + }); + } + + private void WorkQueue(int jobId) + { + try + { + Debug("Beginning job download..."); + Trace("Finding job..."); + var job = (from j in JobList where j.Id == jobId select j).FirstOrDefault(); + + if (job == null) + { + Warn($"Associating job not found! JobId: {jobId}"); + return; + } + Trace("Found job."); + + Queue queue; + Trace("Locking scheduler..."); + lock (this) + { + Trace("Finiding queue..."); + if (!RunningJobs.TryGetValue(jobId, out queue)) + { + Warn($"Queue for job {jobId} not found!"); + return; + } + Trace("Queue found."); + } + Trace("Unlocked scheduler."); + + Debug($"Messages to process for job {jobId}: {queue.Count}"); + while (queue.Count > 0) + { + Trace("Dequeueing message..."); + var message = queue.Dequeue(); + + Debug($"Attachments for message {message.Id}: {message.Attachments.Length}"); + foreach (var a in message.Attachments) + { + var fileName = Path.Combine(Core.Settings.OperatingFolder, Core.Settings.FileNameScheme); + + Trace("Replacing filename placeholders..."); + fileName = + fileName.Replace("%guild%", message.Server.Name) + .Replace("%channel%", message.Channel.Name) + .Replace("%timestamp%", SweetUtils.DateTimeToUnixTimeStamp(message.Timestamp).ToString()) + .Replace("%name%", a.Filename); + Trace($"Detemined file name: {fileName}."); + + if (File.Exists(fileName) && new FileInfo(fileName).Length == a.Size) + { + Debug($"{fileName} already existing with its estimated size. Skipping..."); + continue; + } + + Trace("Determining directory..."); + var fileDirectory = Path.GetDirectoryName(fileName); + + if (!Directory.Exists(fileDirectory)) + { + Info($"Directory {fileDirectory} does not exist. Creating directory..."); + Directory.CreateDirectory(fileDirectory); + Debug("Created directory."); + } + + var wc = new WebClient(); + Debug($"Starting downloading of attachment {a.Id}..."); + wc.DownloadFile(new Uri(a.Url), fileName); + Debug($"Downloaded attachment {a.Id}."); + + Trace("Updating known timestamp for job..."); + job.KnownTimestamp = SweetUtils.DateTimeToUnixTimeStamp(message.Timestamp); + job.Store(); + } + } + } + finally + { + Trace("Locking scheduler..."); + lock (this) + { + Trace($"Removing {jobId} from running jobs..."); + RunningJobs.Remove(jobId); + Trace("Decreasing thread count..."); + RunningThreads--; + } + Trace("Unlocked scheduler."); + } + } + } +} diff --git a/Discord Media Loader.Application/Core.cs b/Discord Media Loader.Application/Core.cs index aa048ce..72d587d 100644 --- a/Discord Media Loader.Application/Core.cs +++ b/Discord Media Loader.Application/Core.cs @@ -23,6 +23,7 @@ namespace DML.Application internal static DiscordClient Client { get; set; } internal static LiteDatabase Database { get; set; } internal static Settings Settings { get; set; } + internal static JobScheduler Scheduler { get; } = new JobScheduler(); internal static string DataDirectory => Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), @"Serraniel\Discord Media Loader"); @@ -101,6 +102,9 @@ namespace DML.Application Settings.Store(); } + Debug("Loading jobs collection out of database..."); + Scheduler.JobList = Job.RestoreJobs().ToList(); + Info("Loaded settings."); Debug( $"Settings: Email: {Settings.Email}, password: {(string.IsNullOrEmpty(Settings.Password) ? "not set" : "is set")}, use username: {Settings.UseUserData}, loginToken: {Settings.LoginToken}"); @@ -183,8 +187,14 @@ namespace DML.Application Trace("Dialog closed."); } } - + + Info("Starting scheduler..."); + Scheduler.Start(); + System.Windows.Forms.Application.Run(new MainForm()); + + Info("Stopping scheduler..."); + Scheduler.Stop(); } catch (Exception ex) { diff --git a/Discord Media Loader.Application/DML.Application.csproj b/Discord Media Loader.Application/DML.Application.csproj index edc1796..1391ee5 100644 --- a/Discord Media Loader.Application/DML.Application.csproj +++ b/Discord Media Loader.Application/DML.Application.csproj @@ -75,6 +75,7 @@ + diff --git a/Discord Media Loader.Application/MainForm.Designer.cs b/Discord Media Loader.Application/MainForm.Designer.cs index 4b734f3..61ea9de 100644 --- a/Discord Media Loader.Application/MainForm.Designer.cs +++ b/Discord Media Loader.Application/MainForm.Designer.cs @@ -39,13 +39,13 @@ this.edOperatingFolder = new System.Windows.Forms.TextBox(); this.lbOperatingFolder = new System.Windows.Forms.Label(); this.groupBox1 = new System.Windows.Forms.GroupBox(); + this.btnAddJob = new System.Windows.Forms.Button(); + this.cbChannel = new System.Windows.Forms.ComboBox(); + this.lbChannel = new System.Windows.Forms.Label(); + this.cbGuild = new System.Windows.Forms.ComboBox(); + this.lbGuild = new System.Windows.Forms.Label(); this.statusStrip1 = new System.Windows.Forms.StatusStrip(); this.groupBox2 = new System.Windows.Forms.GroupBox(); - this.lbGuild = new System.Windows.Forms.Label(); - this.cbGuild = new System.Windows.Forms.ComboBox(); - this.lbChannel = new System.Windows.Forms.Label(); - this.cbChannel = new System.Windows.Forms.ComboBox(); - this.btnAddJob = new System.Windows.Forms.Button(); this.pnlSettings.SuspendLayout(); ((System.ComponentModel.ISupportInitialize)(this.edThreadLimit)).BeginInit(); this.groupBox1.SuspendLayout(); @@ -172,6 +172,54 @@ this.groupBox1.TabStop = false; this.groupBox1.Text = "Add a job"; // + // btnAddJob + // + this.btnAddJob.Anchor = ((System.Windows.Forms.AnchorStyles)((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Right))); + this.btnAddJob.Location = new System.Drawing.Point(481, 19); + this.btnAddJob.Name = "btnAddJob"; + this.btnAddJob.Size = new System.Drawing.Size(66, 23); + this.btnAddJob.TabIndex = 4; + this.btnAddJob.Text = "&Add"; + this.btnAddJob.UseVisualStyleBackColor = true; + this.btnAddJob.Click += new System.EventHandler(this.btnAddJob_Click); + // + // cbChannel + // + this.cbChannel.Anchor = ((System.Windows.Forms.AnchorStyles)(((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Left) + | System.Windows.Forms.AnchorStyles.Right))); + this.cbChannel.FormattingEnabled = true; + this.cbChannel.Location = new System.Drawing.Point(294, 19); + this.cbChannel.Name = "cbChannel"; + this.cbChannel.Size = new System.Drawing.Size(181, 21); + this.cbChannel.TabIndex = 3; + // + // lbChannel + // + this.lbChannel.AutoSize = true; + this.lbChannel.Location = new System.Drawing.Point(239, 22); + this.lbChannel.Name = "lbChannel"; + this.lbChannel.Size = new System.Drawing.Size(49, 13); + this.lbChannel.TabIndex = 2; + this.lbChannel.Text = "Channel:"; + // + // cbGuild + // + this.cbGuild.FormattingEnabled = true; + this.cbGuild.Location = new System.Drawing.Point(52, 19); + this.cbGuild.Name = "cbGuild"; + this.cbGuild.Size = new System.Drawing.Size(181, 21); + this.cbGuild.TabIndex = 1; + this.cbGuild.SelectedIndexChanged += new System.EventHandler(this.cbGuild_SelectedIndexChanged); + // + // lbGuild + // + this.lbGuild.AutoSize = true; + this.lbGuild.Location = new System.Drawing.Point(12, 22); + this.lbGuild.Name = "lbGuild"; + this.lbGuild.Size = new System.Drawing.Size(34, 13); + this.lbGuild.TabIndex = 0; + this.lbGuild.Text = "Guild:"; + // // statusStrip1 // this.statusStrip1.Location = new System.Drawing.Point(0, 311); @@ -190,53 +238,6 @@ this.groupBox2.TabStop = false; this.groupBox2.Text = "Jobs"; // - // lbGuild - // - this.lbGuild.AutoSize = true; - this.lbGuild.Location = new System.Drawing.Point(12, 22); - this.lbGuild.Name = "lbGuild"; - this.lbGuild.Size = new System.Drawing.Size(34, 13); - this.lbGuild.TabIndex = 0; - this.lbGuild.Text = "Guild:"; - // - // cbGuild - // - this.cbGuild.FormattingEnabled = true; - this.cbGuild.Location = new System.Drawing.Point(52, 19); - this.cbGuild.Name = "cbGuild"; - this.cbGuild.Size = new System.Drawing.Size(181, 21); - this.cbGuild.TabIndex = 1; - this.cbGuild.SelectedIndexChanged += new System.EventHandler(this.cbGuild_SelectedIndexChanged); - // - // lbChannel - // - this.lbChannel.AutoSize = true; - this.lbChannel.Location = new System.Drawing.Point(239, 22); - this.lbChannel.Name = "lbChannel"; - this.lbChannel.Size = new System.Drawing.Size(49, 13); - this.lbChannel.TabIndex = 2; - this.lbChannel.Text = "Channel:"; - // - // cbChannel - // - this.cbChannel.Anchor = ((System.Windows.Forms.AnchorStyles)(((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Left) - | System.Windows.Forms.AnchorStyles.Right))); - this.cbChannel.FormattingEnabled = true; - this.cbChannel.Location = new System.Drawing.Point(294, 19); - this.cbChannel.Name = "cbChannel"; - this.cbChannel.Size = new System.Drawing.Size(181, 21); - this.cbChannel.TabIndex = 3; - // - // btnAddJob - // - this.btnAddJob.Anchor = ((System.Windows.Forms.AnchorStyles)((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Right))); - this.btnAddJob.Location = new System.Drawing.Point(481, 19); - this.btnAddJob.Name = "btnAddJob"; - this.btnAddJob.Size = new System.Drawing.Size(66, 23); - this.btnAddJob.TabIndex = 4; - this.btnAddJob.Text = "&Add"; - this.btnAddJob.UseVisualStyleBackColor = true; - // // MainForm // this.AutoScaleDimensions = new System.Drawing.SizeF(6F, 13F); diff --git a/Discord Media Loader.Application/MainForm.cs b/Discord Media Loader.Application/MainForm.cs index de2ee9c..361f556 100644 --- a/Discord Media Loader.Application/MainForm.cs +++ b/Discord Media Loader.Application/MainForm.cs @@ -1,6 +1,7 @@ using System.Linq; using System.Windows.Forms; using Discord; +using DML.Application.Classes; using static SweetLib.Utils.Logger.Logger; namespace DML.Application @@ -101,18 +102,6 @@ namespace DML.Application return (from c in server.TextChannels where c.Name == name select c).FirstOrDefault(); } - private Server FindServerById(ulong id) - { - Trace($"Trying to find server by Id: {id}"); - return (from s in Core.Client.Servers where s.Id == id select s).FirstOrDefault(); - } - - private Channel FindChannelById(Server server, ulong id) - { - Trace($"Trying to find channel in {server} by id: {id}"); - return (from c in server.TextChannels where c.Id == id select c).FirstOrDefault(); - } - private void cbGuild_SelectedIndexChanged(object sender, System.EventArgs e) { Trace("Guild index changed."); @@ -146,5 +135,22 @@ namespace DML.Application Debug("Finished updating channel dropdown component."); } + + private void btnAddJob_Click(object sender, System.EventArgs e) + { + var job = new Job + { + GuildId = FindServerByName(cbGuild.Text).Id, + ChannelId = FindChannelByName(FindServerByName(cbGuild.Text), cbChannel.Text).Id + }; + + if (!(from j in Core.Scheduler.JobList + where j.GuildId == job.GuildId && j.ChannelId == job.ChannelId + select j).Any()) + { + Core.Scheduler.JobList.Add(job); + job.Store(); + } + } } } diff --git a/Discord Media Loader.Application/Properties/AssemblyInfo.cs b/Discord Media Loader.Application/Properties/AssemblyInfo.cs index b6220c5..e8f4156 100644 --- a/Discord Media Loader.Application/Properties/AssemblyInfo.cs +++ b/Discord Media Loader.Application/Properties/AssemblyInfo.cs @@ -32,5 +32,5 @@ using System.Runtime.InteropServices; // Sie können alle Werte angeben oder die standardmäßigen Build- und Revisionsnummern // übernehmen, indem Sie "*" eingeben: // [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("0.2.137.0")] -[assembly: AssemblyFileVersion("0.2.137.0")] +[assembly: AssemblyVersion("0.2.142.0")] +[assembly: AssemblyFileVersion("0.2.142.0")] diff --git a/Discord Media Loader/Properties/AssemblyInfo.cs b/Discord Media Loader/Properties/AssemblyInfo.cs index 0e70c7c..aeb87aa 100644 --- a/Discord Media Loader/Properties/AssemblyInfo.cs +++ b/Discord Media Loader/Properties/AssemblyInfo.cs @@ -32,5 +32,5 @@ using System.Runtime.InteropServices; // Sie können alle Werte angeben oder die standardmäßigen Build- und Revisionsnummern // übernehmen, indem Sie "*" eingeben: // [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("0.99.65.0")] -[assembly: AssemblyFileVersion("0.99.65.0")] +[assembly: AssemblyVersion("0.99.70.0")] +[assembly: AssemblyFileVersion("0.99.70.0")]