From b5945036fdea4630fb2769d262ff3830cc9de744 Mon Sep 17 00:00:00 2001 From: Serraniel Date: Mon, 22 Jan 2018 20:07:10 +0100 Subject: [PATCH 1/2] Contine on exception during scanning --- .../Classes/JobScheduler.cs | 104 ++++++++++-------- 1 file changed, 56 insertions(+), 48 deletions(-) diff --git a/Discord Media Loader.Application/Classes/JobScheduler.cs b/Discord Media Loader.Application/Classes/JobScheduler.cs index 156a131..0d48177 100644 --- a/Discord Media Loader.Application/Classes/JobScheduler.cs +++ b/Discord Media Loader.Application/Classes/JobScheduler.cs @@ -91,73 +91,81 @@ namespace DML.AppCore.Classes Logger.Info("Started JobScheduler..."); while (Run) { - Logger.Debug("Entering job list handler loop..."); - //foreach (var job in JobList) - for (var i = JobList.Count - 1; i >= 0; i--) + try { - var job = JobList[i]; - Logger.Debug($"Checking job {job}"); - var hasJob = false; - - Logger.Trace("Locking scheduler..."); - lock (this) + Logger.Debug("Entering job list handler loop..."); + //foreach (var job in JobList) + for (var i = JobList.Count - 1; i >= 0; i--) { - Logger.Trace("Checking if job is already performed..."); - hasJob = RunningJobs.ContainsKey(job.Id); - } - Logger.Trace("Unlocked scheduler."); - - if (!hasJob) - { - Logger.Debug("Job is not performed yet...Performing job..."); - var queue = new Queue(); + var job = JobList[i]; + Logger.Debug($"Checking job {job}"); + var hasJob = false; Logger.Trace("Locking scheduler..."); lock (this) { - Logger.Trace("Adding job to running jobs."); - RunningJobs.Add(job.Id, queue); + Logger.Trace("Checking if job is already performed..."); + hasJob = RunningJobs.ContainsKey(job.Id); } Logger.Trace("Unlocked scheduler."); - Logger.Trace("Issuing job message scan..."); - var messages = await job.Scan(); - - if (messages == null) - continue; - - Logger.Trace($"Adding {messages.Count} messages to queue..."); - foreach (var msg in messages) + if (!hasJob) { - queue.Enqueue(msg); - } - Logger.Trace($"Added {queue.Count} messages to queue."); + Logger.Debug("Job is not performed yet...Performing job..."); + var queue = new Queue(); - if (messages.Count != queue.Count) - Logger.Warn("Not all messages have been added into the queue."); - - var startedDownload = false; - - while (!startedDownload) - { - Logger.Debug("Entering loop to check thread availability"); Logger.Trace("Locking scheduler..."); lock (this) { - Logger.Trace($"Checking thread limit. Running: {RunningThreads}, Max: {Core.Settings.ThreadLimit}"); - if (RunningThreads >= Core.Settings.ThreadLimit) - continue; - - RunningThreads++; - startedDownload = true; + Logger.Trace("Adding job to running jobs."); + RunningJobs.Add(job.Id, queue); } Logger.Trace("Unlocked scheduler."); - } - Logger.Trace("Start downloading job async."); - Task.Run(() => WorkQueue(job.Id)); // do not await to work parallel + Logger.Trace("Issuing job message scan..."); + var messages = await job.Scan(); + + if (messages == null) + continue; + + Logger.Trace($"Adding {messages.Count} messages to queue..."); + foreach (var msg in messages) + { + queue.Enqueue(msg); + } + Logger.Trace($"Added {queue.Count} messages to queue."); + + if (messages.Count != queue.Count) + Logger.Warn("Not all messages have been added into the queue."); + + var startedDownload = false; + + while (!startedDownload) + { + Logger.Debug("Entering loop to check thread availability"); + Logger.Trace("Locking scheduler..."); + lock (this) + { + Logger.Trace( + $"Checking thread limit. Running: {RunningThreads}, Max: {Core.Settings.ThreadLimit}"); + if (RunningThreads >= Core.Settings.ThreadLimit) + continue; + + RunningThreads++; + startedDownload = true; + } + Logger.Trace("Unlocked scheduler."); + } + + Logger.Trace("Start downloading job async."); + Task.Run(() => WorkQueue(job.Id)); // do not await to work parallel + } } } + catch (Exception ex) + { + Logger.Error(ex.Message); + } } }); } From ef21b6ed051404f501762905081b3707fd685468 Mon Sep 17 00:00:00 2001 From: Serraniel Date: Mon, 22 Jan 2018 20:08:08 +0100 Subject: [PATCH 2/2] Fixed an error which caused null pointer exceptions in discord framework and interrupted the scanning queue permanently Changed message count to acutally count scanned messages even if they already have scanned before --- Discord Media Loader.Application/Classes/Job.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Discord Media Loader.Application/Classes/Job.cs b/Discord Media Loader.Application/Classes/Job.cs index 0a6d57c..a2ea874 100644 --- a/Discord Media Loader.Application/Classes/Job.cs +++ b/Discord Media Loader.Application/Classes/Job.cs @@ -74,7 +74,8 @@ namespace DML.AppCore.Classes var channel = FindChannelById(guild, ChannelId); Debug("Checking channel access"); - if (!channel.Users.Contains(channel.Guild.CurrentUser)) + //channel.GetUser(channel.Guild.CurrentUser.Id); + if (channel.GetUser(channel.Guild.CurrentUser.Id) == null) { Info("Skipping channel without access"); return result; @@ -126,6 +127,8 @@ namespace DML.AppCore.Classes if (!IsValid) return null; + Core.Scheduler.MessagesScanned++; + Debug($"Processing message {m.Id}"); if (m.Id < lastId) { @@ -148,8 +151,6 @@ namespace DML.AppCore.Classes Trace($"Added message {m.Id}"); } Debug($"Finished message {m.Id}"); - - Core.Scheduler.MessagesScanned++; } finished = finished || messages.Count < limit;