Somehow implementation of downloading all the shit out of it

Discord Media Loader.Application/Classes/Job.cs

@@ -1,8 +1,11 @@
 using System;
 using System.Collections.Generic;
+using System.Diagnostics.Eventing.Reader;
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
+using Discord;
+using SweetLib.Utils;
 using static SweetLib.Utils.Logger.Logger;
 
 namespace DML.Application.Classes
@@ -12,6 +15,8 @@ namespace DML.Application.Classes
         public int Id { get; set; }
         public ulong GuildId { get; set; }
         public ulong ChannelId { get; set; }
+        public double KnownTimestamp { get; set; } = 0;
+        private double StopTimestamp { get; set; } = 0;
 
         internal void Store()
         {
@@ -20,7 +25,102 @@ namespace DML.Application.Classes
             var jobDb = Core.Database.GetCollection<Job>("jobs");
 
             Trace("Adding new value...");
-            jobDb.Insert(this);
+
+            if (jobDb.Find(x => x.ChannelId == ChannelId && x.GuildId == GuildId).Any())
+            {
+                jobDb.Update(this);
+            }
+            else
+            {
+                jobDb.Insert(this);
+            }
+        }
+
+        private Server FindServerById(ulong id)
+        {
+            Trace($"Trying to find server by Id: {id}");
+            return (from s in Core.Client.Servers where s.Id == id select s).FirstOrDefault();
+        }
+
+        private Channel FindChannelById(Server server, ulong id)
+        {
+            Trace($"Trying to find channel in {server} by id: {id}");
+            return (from c in server.TextChannels where c.Id == id select c).FirstOrDefault();
+        }
+
+        internal async Task<List<Message>> Scan()
+        {
+            Debug($"Starting scan of guild {GuildId} channel {ChannelId}...");
+            var result = new List<Message>();
+
+            var limit = 100;
+            var lastId = ulong.MaxValue;
+            var isFirst = true;
+            var finished = false;
+
+            var guild = FindServerById(GuildId);
+            var channel = FindChannelById(guild, ChannelId);
+
+            if (Math.Abs(StopTimestamp) < 0.4)
+                StopTimestamp = KnownTimestamp;
+            Trace("Initialized scanning parameters.");
+
+            while (!finished)
+            {
+                Trace("Entering scanning loop...");
+                Message[] messages;
+
+                Trace($"Downloading next {limit} messages...");
+                if (isFirst)
+                {
+                    messages = await channel.DownloadMessages(limit, null);
+                }
+                else
+                {
+                    messages = await channel.DownloadMessages(limit, lastId);
+                }
+                Trace($"Downloaded {messages.Length} messages.");
+
+                isFirst = false;
+
+                foreach (var m in messages)
+                {
+                    Debug($"Processing message {m.Id}");
+                    if (m.Id < lastId)
+                    {
+                        Trace($"Updating lastId ({lastId}) to {m.Id}");
+                        lastId = m.Id;
+                    }
+
+                    if (SweetUtils.DateTimeToUnixTimeStamp(m.Timestamp) < StopTimestamp)
+                    {
+                        Debug("Found a message with a known timestamp...Stopping scan.");
+                        finished = true;
+                        continue;
+                    }
+
+                    Trace($"Message {m.Id} has {m.Attachments.Length} attachments.");
+                    if (m.Attachments.Length > 0)
+                    {
+                        result.Add(m);
+                        Trace($"Added message {m.Id}");
+                    }
+                    Debug($"Finished message {m.Id}");
+                }
+
+                finished = finished || messages.Length < limit;
+            }
+            Trace($"Downloaded all messages for guild {GuildId} channel {ChannelId}.");
+
+            Trace("Sorting messages...");
+            result.Sort((a, b) => DateTime.Compare(a.Timestamp, b.Timestamp));
+
+            Trace("Updating StopTimestamp for next scan...");
+            StopTimestamp = SweetUtils.DateTimeToUnixTimeStamp(result[result.Count - 1].Timestamp);
+
+            Debug($"Fisnished scan of guild {GuildId} channel {ChannelId}.");
+
+            return result;
         }
 
         internal static IEnumerable<Job> RestoreJobs()

Discord Media Loader.Application/Classes/JobScheduler.cs

@@ -0,0 +1,192 @@
+using System;
+using System.Collections;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Net;
+using System.Text;
+using System.Threading.Tasks;
+using Discord;
+using SweetLib.Utils;
+using static SweetLib.Utils.Logger.Logger;
+
+namespace DML.Application.Classes
+{
+    internal class JobScheduler
+    {
+        private bool Run { get; set; } = false;
+        internal List<Job> JobList { get; set; } = new List<Job>();
+        internal Dictionary<int, Queue<Message>> RunningJobs = new Dictionary<int, Queue<Message>>();
+        internal int RunningThreads { get; set; } = 0;
+
+        internal void Stop()
+        {
+            Run = false;
+        }
+
+        internal void Start()
+        {
+            Run = true;
+            
+            Task.Run(async () =>
+            {
+                Info("Started JobScheduler...");
+                while (Run)
+                {
+                    Debug("Entering job list handler loop...");
+                    foreach (var job in JobList)
+                    {
+                        Debug($"Checking job {job}");
+                        var hasJob = false;
+
+                        Trace("Locking scheduler...");
+                        lock (this)
+                        {
+                            Trace("Checking if job is already performed...");
+                            hasJob = RunningJobs.ContainsKey(job.Id);
+                        }
+                        Trace("Unlocked scheduler.");
+
+                        if (!hasJob)
+                        {
+                            Debug("Job is not performed yet...Performing job...");
+                            var queue = new Queue<Message>();
+
+                            Trace("Locking scheduler...");
+                            lock (this)
+                            {
+                                Trace("Adding job to running jobs.");
+                                RunningJobs.Add(job.Id, queue);
+                            }
+                            Trace("Unlocked scheduler.");
+
+                            Trace("Issuing job message scan...");
+                            var messages = await job.Scan();
+
+                            Trace($"Adding {messages.Count} messages to queue...");
+                            foreach (var msg in messages)
+                            {
+                                queue.Enqueue(msg);
+                            }
+                            Trace($"Added {queue.Count} messages to queue.");
+
+                            if(messages.Count!= queue.Count)
+                                Warn("Not all messages have been added into the queue.");
+
+                            var startedDownload = false;
+
+                            while (!startedDownload)
+                            {
+                                Debug("Entering loop to check thread availability");
+                                Trace("Locking scheduler...");
+                                lock (this)
+                                {
+                                    Trace($"Checking thread limit. Running: {RunningThreads}, Max: {Core.Settings.ThreadLimit}");
+                                    if (RunningThreads >= Core.Settings.ThreadLimit)
+                                        continue;
+
+                                    RunningThreads++;
+                                    startedDownload = true;
+                                }
+                                Trace("Unlocked scheduler.");
+                            }
+
+                            Trace("Start downloading job async.");
+                            Task.Run(() => WorkQueue(job.Id)); // do not await to work parallel
+                        }
+                    }
+                }
+            });
+        }
+
+        private void WorkQueue(int jobId)
+        {
+            try
+            {
+                Debug("Beginning job download...");
+                Trace("Finding job...");
+                var job = (from j in JobList where j.Id == jobId select j).FirstOrDefault();
+
+                if (job == null)
+                {
+                    Warn($"Associating job not found! JobId: {jobId}");
+                    return;
+                }
+                Trace("Found job.");
+
+                Queue<Message> queue;
+                Trace("Locking scheduler...");
+                lock (this)
+                {
+                    Trace("Finiding queue...");
+                    if (!RunningJobs.TryGetValue(jobId, out queue))
+                    {
+                        Warn($"Queue for job {jobId} not found!");
+                        return;
+                    }
+                    Trace("Queue found.");
+                }
+                Trace("Unlocked scheduler.");
+                
+                Debug($"Messages to process for job {jobId}: {queue.Count}");
+                while (queue.Count > 0)
+                {
+                    Trace("Dequeueing message...");
+                    var message = queue.Dequeue();
+
+                    Debug($"Attachments for message {message.Id}: {message.Attachments.Length}");
+                    foreach (var a in message.Attachments)
+                    {
+                        var fileName = Path.Combine(Core.Settings.OperatingFolder, Core.Settings.FileNameScheme);
+                        
+                        Trace("Replacing filename placeholders...");
+                        fileName =
+                            fileName.Replace("%guild%", message.Server.Name)
+                                .Replace("%channel%", message.Channel.Name)
+                                .Replace("%timestamp%", SweetUtils.DateTimeToUnixTimeStamp(message.Timestamp).ToString())
+                                .Replace("%name%", a.Filename);
+                        Trace($"Detemined file name: {fileName}.");
+
+                        if (File.Exists(fileName) && new FileInfo(fileName).Length == a.Size)
+                        {
+                            Debug($"{fileName} already existing with its estimated size. Skipping...");
+                            continue;
+                        }
+                        
+                        Trace("Determining directory...");
+                        var fileDirectory = Path.GetDirectoryName(fileName);
+
+                        if (!Directory.Exists(fileDirectory))
+                        {
+                            Info($"Directory {fileDirectory} does not exist. Creating directory...");
+                            Directory.CreateDirectory(fileDirectory);
+                            Debug("Created directory.");
+                        }
+
+                        var wc = new WebClient();
+                        Debug($"Starting downloading of attachment {a.Id}...");
+                        wc.DownloadFile(new Uri(a.Url), fileName);
+                        Debug($"Downloaded attachment {a.Id}.");
+
+                        Trace("Updating known timestamp for job...");
+                        job.KnownTimestamp = SweetUtils.DateTimeToUnixTimeStamp(message.Timestamp);
+                        job.Store();
+                    }
+                }
+            }
+            finally
+            {
+                Trace("Locking scheduler...");
+                lock (this)
+                {
+                    Trace($"Removing {jobId} from running jobs...");
+                    RunningJobs.Remove(jobId);
+                    Trace("Decreasing thread count...");
+                    RunningThreads--;
+                }
+                Trace("Unlocked scheduler.");
+            }
+        }
+    }
+}

Discord Media Loader.Application/Core.cs

@@ -23,6 +23,7 @@ namespace DML.Application
         internal static DiscordClient Client { get; set; }
         internal static LiteDatabase Database { get; set; }
         internal static Settings Settings { get; set; }
+        internal static JobScheduler Scheduler { get; } = new JobScheduler();
 
         internal static string DataDirectory
             => Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), @"Serraniel\Discord Media Loader");
@@ -101,6 +102,9 @@ namespace DML.Application
                     Settings.Store();
                 }
 
+                Debug("Loading jobs collection out of database...");
+                Scheduler.JobList = Job.RestoreJobs().ToList();
+
                 Info("Loaded settings.");
                 Debug(
                     $"Settings: Email: {Settings.Email}, password: {(string.IsNullOrEmpty(Settings.Password) ? "not set" : "is set")}, use username: {Settings.UseUserData}, loginToken: {Settings.LoginToken}");
@@ -184,7 +188,13 @@ namespace DML.Application
                     }
                 }
 
+                Info("Starting scheduler...");
+                Scheduler.Start();
+
                 System.Windows.Forms.Application.Run(new MainForm());
+
+                Info("Stopping scheduler...");
+                Scheduler.Stop();
             }
             catch (Exception ex)
             {

Discord Media Loader.Application/DML.Application.csproj

@@ -75,6 +75,7 @@
   </ItemGroup>
   <ItemGroup>
     <Compile Include="Classes\Job.cs" />
+    <Compile Include="Classes\JobScheduler.cs" />
     <Compile Include="Classes\Settings.cs" />
     <Compile Include="Core.cs" />
     <Compile Include="Dialogs\LoginDialog.cs">

Discord Media Loader.Application/MainForm.Designer.cs

@@ -39,13 +39,13 @@
             this.edOperatingFolder = new System.Windows.Forms.TextBox();
             this.lbOperatingFolder = new System.Windows.Forms.Label();
             this.groupBox1 = new System.Windows.Forms.GroupBox();
+            this.btnAddJob = new System.Windows.Forms.Button();
+            this.cbChannel = new System.Windows.Forms.ComboBox();
+            this.lbChannel = new System.Windows.Forms.Label();
+            this.cbGuild = new System.Windows.Forms.ComboBox();
+            this.lbGuild = new System.Windows.Forms.Label();
             this.statusStrip1 = new System.Windows.Forms.StatusStrip();
             this.groupBox2 = new System.Windows.Forms.GroupBox();
-            this.lbGuild = new System.Windows.Forms.Label();
-            this.cbGuild = new System.Windows.Forms.ComboBox();
-            this.lbChannel = new System.Windows.Forms.Label();
-            this.cbChannel = new System.Windows.Forms.ComboBox();
-            this.btnAddJob = new System.Windows.Forms.Button();
             this.pnlSettings.SuspendLayout();
             ((System.ComponentModel.ISupportInitialize)(this.edThreadLimit)).BeginInit();
             this.groupBox1.SuspendLayout();
@@ -172,6 +172,54 @@
             this.groupBox1.TabStop = false;
             this.groupBox1.Text = "Add a job";
             // 
+            // btnAddJob
+            // 
+            this.btnAddJob.Anchor = ((System.Windows.Forms.AnchorStyles)((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Right)));
+            this.btnAddJob.Location = new System.Drawing.Point(481, 19);
+            this.btnAddJob.Name = "btnAddJob";
+            this.btnAddJob.Size = new System.Drawing.Size(66, 23);
+            this.btnAddJob.TabIndex = 4;
+            this.btnAddJob.Text = "&Add";
+            this.btnAddJob.UseVisualStyleBackColor = true;
+            this.btnAddJob.Click += new System.EventHandler(this.btnAddJob_Click);
+            // 
+            // cbChannel
+            // 
+            this.cbChannel.Anchor = ((System.Windows.Forms.AnchorStyles)(((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Left) 
+            | System.Windows.Forms.AnchorStyles.Right)));
+            this.cbChannel.FormattingEnabled = true;
+            this.cbChannel.Location = new System.Drawing.Point(294, 19);
+            this.cbChannel.Name = "cbChannel";
+            this.cbChannel.Size = new System.Drawing.Size(181, 21);
+            this.cbChannel.TabIndex = 3;
+            // 
+            // lbChannel
+            // 
+            this.lbChannel.AutoSize = true;
+            this.lbChannel.Location = new System.Drawing.Point(239, 22);
+            this.lbChannel.Name = "lbChannel";
+            this.lbChannel.Size = new System.Drawing.Size(49, 13);
+            this.lbChannel.TabIndex = 2;
+            this.lbChannel.Text = "Channel:";
+            // 
+            // cbGuild
+            // 
+            this.cbGuild.FormattingEnabled = true;
+            this.cbGuild.Location = new System.Drawing.Point(52, 19);
+            this.cbGuild.Name = "cbGuild";
+            this.cbGuild.Size = new System.Drawing.Size(181, 21);
+            this.cbGuild.TabIndex = 1;
+            this.cbGuild.SelectedIndexChanged += new System.EventHandler(this.cbGuild_SelectedIndexChanged);
+            // 
+            // lbGuild
+            // 
+            this.lbGuild.AutoSize = true;
+            this.lbGuild.Location = new System.Drawing.Point(12, 22);
+            this.lbGuild.Name = "lbGuild";
+            this.lbGuild.Size = new System.Drawing.Size(34, 13);
+            this.lbGuild.TabIndex = 0;
+            this.lbGuild.Text = "Guild:";
+            // 
             // statusStrip1
             // 
             this.statusStrip1.Location = new System.Drawing.Point(0, 311);
@@ -190,53 +238,6 @@
             this.groupBox2.TabStop = false;
             this.groupBox2.Text = "Jobs";
             // 
-            // lbGuild
-            // 
-            this.lbGuild.AutoSize = true;
-            this.lbGuild.Location = new System.Drawing.Point(12, 22);
-            this.lbGuild.Name = "lbGuild";
-            this.lbGuild.Size = new System.Drawing.Size(34, 13);
-            this.lbGuild.TabIndex = 0;
-            this.lbGuild.Text = "Guild:";
-            // 
-            // cbGuild
-            // 
-            this.cbGuild.FormattingEnabled = true;
-            this.cbGuild.Location = new System.Drawing.Point(52, 19);
-            this.cbGuild.Name = "cbGuild";
-            this.cbGuild.Size = new System.Drawing.Size(181, 21);
-            this.cbGuild.TabIndex = 1;
-            this.cbGuild.SelectedIndexChanged += new System.EventHandler(this.cbGuild_SelectedIndexChanged);
-            // 
-            // lbChannel
-            // 
-            this.lbChannel.AutoSize = true;
-            this.lbChannel.Location = new System.Drawing.Point(239, 22);
-            this.lbChannel.Name = "lbChannel";
-            this.lbChannel.Size = new System.Drawing.Size(49, 13);
-            this.lbChannel.TabIndex = 2;
-            this.lbChannel.Text = "Channel:";
-            // 
-            // cbChannel
-            // 
-            this.cbChannel.Anchor = ((System.Windows.Forms.AnchorStyles)(((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Left) 
-            | System.Windows.Forms.AnchorStyles.Right)));
-            this.cbChannel.FormattingEnabled = true;
-            this.cbChannel.Location = new System.Drawing.Point(294, 19);
-            this.cbChannel.Name = "cbChannel";
-            this.cbChannel.Size = new System.Drawing.Size(181, 21);
-            this.cbChannel.TabIndex = 3;
-            // 
-            // btnAddJob
-            // 
-            this.btnAddJob.Anchor = ((System.Windows.Forms.AnchorStyles)((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Right)));
-            this.btnAddJob.Location = new System.Drawing.Point(481, 19);
-            this.btnAddJob.Name = "btnAddJob";
-            this.btnAddJob.Size = new System.Drawing.Size(66, 23);
-            this.btnAddJob.TabIndex = 4;
-            this.btnAddJob.Text = "&Add";
-            this.btnAddJob.UseVisualStyleBackColor = true;
-            // 
             // MainForm
             // 
             this.AutoScaleDimensions = new System.Drawing.SizeF(6F, 13F);

Discord Media Loader.Application/MainForm.cs

@@ -1,6 +1,7 @@
 using System.Linq;
 using System.Windows.Forms;
 using Discord;
+using DML.Application.Classes;
 using static SweetLib.Utils.Logger.Logger;
 
 namespace DML.Application
@@ -101,18 +102,6 @@ namespace DML.Application
             return (from c in server.TextChannels where c.Name == name select c).FirstOrDefault();
         }
 
-        private Server FindServerById(ulong id)
-        {
-            Trace($"Trying to find server by Id: {id}");
-            return (from s in Core.Client.Servers where s.Id == id select s).FirstOrDefault();
-        }
-
-        private Channel FindChannelById(Server server, ulong id)
-        {
-            Trace($"Trying to find channel in {server} by id: {id}");
-            return (from c in server.TextChannels where c.Id == id select c).FirstOrDefault();
-        }
-
         private void cbGuild_SelectedIndexChanged(object sender, System.EventArgs e)
         {
             Trace("Guild index changed.");
@@ -146,5 +135,22 @@ namespace DML.Application
 
             Debug("Finished updating channel dropdown component.");
         }
+
+        private void btnAddJob_Click(object sender, System.EventArgs e)
+        {
+            var job = new Job
+            {
+                GuildId = FindServerByName(cbGuild.Text).Id,
+                ChannelId = FindChannelByName(FindServerByName(cbGuild.Text), cbChannel.Text).Id
+            };
+
+            if (!(from j in Core.Scheduler.JobList
+                  where j.GuildId == job.GuildId && j.ChannelId == job.ChannelId
+                  select j).Any())
+            {
+                Core.Scheduler.JobList.Add(job);
+                job.Store();
+            }
+        }
     }
 }

Discord Media Loader.Application/Properties/AssemblyInfo.cs

@@ -32,5 +32,5 @@ using System.Runtime.InteropServices;
 // Sie können alle Werte angeben oder die standardmäßigen Build- und Revisionsnummern 
 // übernehmen, indem Sie "*" eingeben:
 // [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion("0.2.137.0")]
+[assembly: AssemblyVersion("0.2.142.0")]
-[assembly: AssemblyFileVersion("0.2.137.0")]
+[assembly: AssemblyFileVersion("0.2.142.0")]

Discord Media Loader/Properties/AssemblyInfo.cs

@@ -32,5 +32,5 @@ using System.Runtime.InteropServices;
 // Sie können alle Werte angeben oder die standardmäßigen Build- und Revisionsnummern 
 // übernehmen, indem Sie "*" eingeben:
 // [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion("0.99.65.0")]
+[assembly: AssemblyVersion("0.99.70.0")]
-[assembly: AssemblyFileVersion("0.99.65.0")]
+[assembly: AssemblyFileVersion("0.99.70.0")]