From 7a90260207cf7c75d53e1bf1359a274c38cace58 Mon Sep 17 00:00:00 2001 From: Serraniel Date: Fri, 15 Feb 2019 20:08:11 +0100 Subject: [PATCH] #27 Prepared a migration from timestmaps to message ids in the jobs and relocated the Core.cs classfile. --- .../Classes/Job.cs | 27 +++++--- .../Classes/JobScheduler.cs | 16 +---- .../Classes/MediaData.cs | 14 ++-- .../Classes/Settings.cs | 2 +- .../{Classes => Core}/Core.cs | 7 +- .../Core/Migrator.cs | 67 +++++++++++++++++++ .../DML.Application.csproj | 5 +- .../Dialogs/LoginDialog.cs | 6 +- .../MainForm.cs | 54 +++++++-------- src/Discord Media Loader/Program.cs | 6 +- 10 files changed, 136 insertions(+), 68 deletions(-) rename src/Discord Media Loader.Application/{Classes => Core}/Core.cs (99%) create mode 100644 src/Discord Media Loader.Application/Core/Migrator.cs diff --git a/src/Discord Media Loader.Application/Classes/Job.cs b/src/Discord Media Loader.Application/Classes/Job.cs index 5ed93d5..7ef2e1b 100644 --- a/src/Discord Media Loader.Application/Classes/Job.cs +++ b/src/Discord Media Loader.Application/Classes/Job.cs @@ -13,16 +13,16 @@ **********************************************************************************************/ #endregion +using Discord; +using Discord.WebSocket; +using DML.Application.Classes; +using DML.Application.Core; +using DML.Client; +using SweetLib.Utils.Extensions; using System; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; -using Discord; -using Discord.WebSocket; -using DML.Application.Classes; -using DML.Client; -using SweetLib.Utils; -using SweetLib.Utils.Extensions; using static SweetLib.Utils.Logger.Logger; namespace DML.AppCore.Classes @@ -49,7 +49,7 @@ namespace DML.AppCore.Classes { Debug("Storing job to database..."); Trace("Getting jobs collection..."); - var jobDb = Core.Database.GetCollection("jobs"); + var jobDb = Application.Core.Core.Database.GetCollection("jobs"); Trace("Adding new value..."); @@ -67,7 +67,7 @@ namespace DML.AppCore.Classes { Debug("Deleting job from database..."); Trace("Getting jobs collection..."); - var jobDb = Core.Database.GetCollection("jobs"); + var jobDb = Application.Core.Core.Database.GetCollection("jobs"); Trace("Deleting value..."); jobDb.Delete(Id); @@ -111,7 +111,10 @@ namespace DML.AppCore.Classes } if (Math.Abs(StopTimestamp) < 0.4) + { StopTimestamp = KnownTimestamp; + } + Trace("Initialized scanning parameters."); while (!finished) @@ -139,9 +142,11 @@ namespace DML.AppCore.Classes foreach (var m in messages) { if (!IsValid) + { return false; + } - Core.Scheduler.MessagesScanned++; + Application.Core.Core.Scheduler.MessagesScanned++; Debug($"Processing message {m.Id}"); if (m.Id < lastId) @@ -161,7 +166,7 @@ namespace DML.AppCore.Classes if (m.Attachments.Count > 0) { result.Add(m); - Core.Scheduler.TotalAttachments += (ulong)m.Attachments.Count; + Application.Core.Core.Scheduler.TotalAttachments += (ulong)m.Attachments.Count; Trace($"Added message {m.Id}"); } Debug($"Finished message {m.Id}"); @@ -226,7 +231,7 @@ namespace DML.AppCore.Classes { Debug("Restoring jobs..."); Trace("Getting jobs collection..."); - var jobDb = Core.Database.GetCollection("jobs"); + var jobDb = Application.Core.Core.Database.GetCollection("jobs"); Trace("Creating new empty job list"); return jobDb.FindAll(); diff --git a/src/Discord Media Loader.Application/Classes/JobScheduler.cs b/src/Discord Media Loader.Application/Classes/JobScheduler.cs index 6d8b97f..c999141 100644 --- a/src/Discord Media Loader.Application/Classes/JobScheduler.cs +++ b/src/Discord Media Loader.Application/Classes/JobScheduler.cs @@ -13,22 +13,12 @@ **********************************************************************************************/ #endregion -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Threading.Tasks; using Discord; using DML.AppCore.Classes; -using System.Globalization; -using System.IO; -using System.Linq; -using System.Net; -using Discord; -using Discord.WebSocket; -using DML.Application.Classes; -using SweetLib.Utils; -using SweetLib.Utils.Extensions; using SweetLib.Utils.Logger; +using System; +using System.Collections.Generic; +using System.Threading.Tasks; namespace DML.Application.Classes { diff --git a/src/Discord Media Loader.Application/Classes/MediaData.cs b/src/Discord Media Loader.Application/Classes/MediaData.cs index 04dfdfc..2e18e2e 100644 --- a/src/Discord Media Loader.Application/Classes/MediaData.cs +++ b/src/Discord Media Loader.Application/Classes/MediaData.cs @@ -13,13 +13,13 @@ **********************************************************************************************/ #endregion +using DML.Client; +using SweetLib.Utils.Logger; using System; using System.IO; using System.Linq; using System.Net; using System.Threading.Tasks; -using DML.Client; -using SweetLib.Utils.Logger; using static SweetLib.Utils.Logger.Logger; namespace DML.Application.Classes @@ -38,7 +38,7 @@ namespace DML.Application.Classes { Debug("Storing job to database..."); Trace("Getting jobs collection..."); - var db = Core.Database.GetCollection("media"); + var db = Core.Core.Database.GetCollection("media"); Trace("Adding new value..."); @@ -56,7 +56,7 @@ namespace DML.Application.Classes { Debug("Deleting job from database..."); Trace("Getting jobs collection..."); - var db = Core.Database.GetCollection("media"); + var db = Core.Core.Database.GetCollection("media"); Trace("Deleting value..."); db.Delete(Id); @@ -67,7 +67,7 @@ namespace DML.Application.Classes Trace("Beginning attachment download..."); Debug("Building filename..."); - var fileName = Path.Combine(Core.Settings.OperatingFolder, Core.Settings.FileNameScheme); + var fileName = Path.Combine(Core.Core.Settings.OperatingFolder, Core.Core.Settings.FileNameScheme); Debug($"Base filename: {fileName}"); @@ -88,7 +88,9 @@ namespace DML.Application.Classes Trace("Adding extension if required"); if (extensionRequired) + { fileName += Path.GetExtension(Filename); + } Debug($"Final filename: {fileName}"); @@ -104,7 +106,7 @@ namespace DML.Application.Classes Logger.Debug($"Downloaded attachment {Id}."); - Core.Scheduler.AttachmentsDownloaded++; + Core.Core.Scheduler.AttachmentsDownloaded++; } } } diff --git a/src/Discord Media Loader.Application/Classes/Settings.cs b/src/Discord Media Loader.Application/Classes/Settings.cs index 6398a8f..f9fa783 100644 --- a/src/Discord Media Loader.Application/Classes/Settings.cs +++ b/src/Discord Media Loader.Application/Classes/Settings.cs @@ -37,7 +37,7 @@ namespace DML.Application.Classes public void Store() { Logger.Trace("Getting settings collection..."); - var settingsDB = Core.Database.GetCollection("settings"); + var settingsDB = Core.Core.Database.GetCollection("settings"); Logger.Debug("Storing settings to database..."); diff --git a/src/Discord Media Loader.Application/Classes/Core.cs b/src/Discord Media Loader.Application/Core/Core.cs similarity index 99% rename from src/Discord Media Loader.Application/Classes/Core.cs rename to src/Discord Media Loader.Application/Core/Core.cs index 5939f81..77e1a75 100644 --- a/src/Discord Media Loader.Application/Classes/Core.cs +++ b/src/Discord Media Loader.Application/Core/Core.cs @@ -17,6 +17,7 @@ using Discord; using Discord.Net; using Discord.WebSocket; using DML.AppCore.Classes; +using DML.Application.Classes; using DML.Application.Classes.RPC; using DML.Application.Dialogs; using DML.Client; @@ -35,7 +36,7 @@ using System.Threading.Tasks; using System.Windows.Forms; using Logger = SweetLib.Utils.Logger.Logger; -namespace DML.Application.Classes +namespace DML.Application.Core { public static class Core { @@ -118,6 +119,8 @@ namespace DML.Application.Classes Database = new LiteDatabase(databasePath); Database.Log.Logging += (message) => Logger.Trace($"LiteDB: {message}"); + Migrator.CheckMigrations(); + Logger.Debug("Loading settings collection out of database..."); var settingsDB = Database.GetCollection("settings"); if (settingsDB.Count() > 1) @@ -276,7 +279,7 @@ namespace DML.Application.Classes job.Store(); } } - + Settings.RescanRequired = false; Settings.Store(); diff --git a/src/Discord Media Loader.Application/Core/Migrator.cs b/src/Discord Media Loader.Application/Core/Migrator.cs new file mode 100644 index 0000000..f0f5566 --- /dev/null +++ b/src/Discord Media Loader.Application/Core/Migrator.cs @@ -0,0 +1,67 @@ +#region LICENSE +/********************************************************************************************** + * Copyright (C) 2017-2019 - All Rights Reserved + * + * This file is part of "DML.Application". + * The official repository is hosted at https://github.com/Serraniel/DiscordMediaLoader + * + * "DML.Application" is licensed under Apache 2.0. + * Full license is included in the project repository. + * + * Users who edited Migrator.cs under the condition of the used license: + * - Serraniel (https://github.com/Serraniel) + **********************************************************************************************/ +#endregion + + +using LiteDB; +using SweetLib.Utils.Extensions; + +namespace DML.Application.Core +{ + internal static class Migrator + { + internal static ushort Version => 1; + + internal static void CheckMigrations() + { + var baseVersion = Core.Database.Engine.UserVersion; + + for (var step = baseVersion; step <= Version; step++) + { + Migrate(step); + } + } + + private static void Migrate(ushort step) + { + switch (step) + { + case 0: + // base database + break; + case 1: + foreach (var jobDoc in Core.Database.Engine.Find("jobs", Query.All())) + { + // pseudo datetime snowflake conversion https://discordapp.com/developers/docs/reference#convert-snowflake-to-datetime + var pseudoId = 0UL; + var timestamp = (ulong)jobDoc["KnownTimestamp"].AsDouble * 1000; // milliseconds have not been stored + + if (timestamp > 0) + { + pseudoId = timestamp - 1420070400000 << 22; + pseudoId -= (1000 * 60 * 60 * 24) << 22; // substract one random day of pseudo id just in case the timestamp has errors + } + + jobDoc["LastMessageId"] = pseudoId; + + Core.Database.Engine.Update("jobs", jobDoc); + } + + break; + } + + Core.Database.Engine.UserVersion = step; + } + } +} diff --git a/src/Discord Media Loader.Application/DML.Application.csproj b/src/Discord Media Loader.Application/DML.Application.csproj index ac6a707..fabb2c2 100644 --- a/src/Discord Media Loader.Application/DML.Application.csproj +++ b/src/Discord Media Loader.Application/DML.Application.csproj @@ -164,13 +164,14 @@ - + - + + Form diff --git a/src/Discord Media Loader.Application/Dialogs/LoginDialog.cs b/src/Discord Media Loader.Application/Dialogs/LoginDialog.cs index 51ef097..9180ff6 100644 --- a/src/Discord Media Loader.Application/Dialogs/LoginDialog.cs +++ b/src/Discord Media Loader.Application/Dialogs/LoginDialog.cs @@ -30,7 +30,7 @@ namespace DML.Application.Dialogs private void LoginDialog_Shown(object sender, EventArgs e) { Trace("Login dialog shown."); - edToken.Text = Core.Settings.LoginToken; + edToken.Text = Core.Core.Settings.LoginToken; } private void LoginDialog_FormClosing(object sender, FormClosingEventArgs e) @@ -40,9 +40,9 @@ namespace DML.Application.Dialogs return; Debug("Adjusting login settings..."); - Core.Settings.LoginToken = edToken.Text; + Core.Core.Settings.LoginToken = edToken.Text; - Core.Settings.Store(); + Core.Core.Settings.Store(); } private void btnOk_Click(object sender, EventArgs e) diff --git a/src/Discord Media Loader.Application/MainForm.cs b/src/Discord Media Loader.Application/MainForm.cs index f70655f..8781e64 100644 --- a/src/Discord Media Loader.Application/MainForm.cs +++ b/src/Discord Media Loader.Application/MainForm.cs @@ -64,16 +64,16 @@ namespace DML.Application lbVersion.Text = $"v{Assembly.GetExecutingAssembly().GetName().Version} Copyright © by Serraniel"; Trace("Refreshing operating folder component..."); - edOperatingFolder.Text = Core.Settings.OperatingFolder; + edOperatingFolder.Text = Core.Core.Settings.OperatingFolder; Trace("Refreshing name scheme component..."); - edNameScheme.Text = Core.Settings.FileNameScheme; + edNameScheme.Text = Core.Core.Settings.FileNameScheme; Trace("Refreshing skip existing files component..."); - cbSkipExisting.Checked = Core.Settings.SkipExistingFiles; + cbSkipExisting.Checked = Core.Core.Settings.SkipExistingFiles; Trace("Refreshing thread limit component..."); - edThreadLimit.Value = Core.Settings.ThreadLimit; + edThreadLimit.Value = Core.Core.Settings.ThreadLimit; if (cbGuild.Items.Count == 0) { @@ -88,7 +88,7 @@ namespace DML.Application Trace("Refreshing job list component..."); var oldIndex = lbxJobs.SelectedIndex; lbxJobs.Items.Clear(); - foreach (var job in Core.Scheduler.JobList) + foreach (var job in Core.Core.Scheduler.JobList) { lbxJobs.Items.Add(new IdentifiedString(job.Id, $"{FindServerById(job.GuildId)?.Name}:{FindChannelById(FindServerById(job.GuildId), job.ChannelId)?.Name}")); } @@ -107,19 +107,19 @@ namespace DML.Application } Trace("Updating operating folder..."); - Core.Settings.OperatingFolder = edOperatingFolder.Text; + Core.Core.Settings.OperatingFolder = edOperatingFolder.Text; Trace("Updating name scheme..."); - Core.Settings.FileNameScheme = edNameScheme.Text; + Core.Core.Settings.FileNameScheme = edNameScheme.Text; Trace("Updating skip existing files..."); - Core.Settings.SkipExistingFiles = cbSkipExisting.Checked; + Core.Core.Settings.SkipExistingFiles = cbSkipExisting.Checked; Trace("Updating thread limit..."); - Core.Settings.ThreadLimit = (int)edThreadLimit.Value; + Core.Core.Settings.ThreadLimit = (int)edThreadLimit.Value; Trace("Storing new settings..."); - Core.Settings.Store(); + Core.Core.Settings.Store(); Info("New settings have been saved."); } @@ -211,12 +211,12 @@ namespace DML.Application ChannelId = ((IdentifiedString)cbChannel.SelectedItem).Id }; - if (!(from j in Core.Scheduler.JobList + if (!(from j in Core.Core.Scheduler.JobList where j.GuildId == job.GuildId && j.ChannelId == job.ChannelId select j).Any()) { job.Store(); - Core.Scheduler.JobList.Add(job); + Core.Core.Scheduler.JobList.Add(job); } RefreshComponents(); @@ -234,11 +234,11 @@ namespace DML.Application var jobId = ((IdentifiedString)lbxJobs.SelectedItem).Id; - var job = Core.Scheduler.JobList.FirstOrDefault(j => j.Id == jobId); + var job = Core.Core.Scheduler.JobList.FirstOrDefault(j => j.Id == jobId); if (job != null) { - Core.Scheduler.JobList.Remove(job); - Core.Scheduler.RunningJobs.Remove(job.Id); + Core.Core.Scheduler.JobList.Remove(job); + Core.Core.Scheduler.RunningJobs.Remove(job.Id); job.Stop(); job.Delete(); } @@ -249,9 +249,9 @@ namespace DML.Application private void tmrRefreshProgress_Tick(object sender, System.EventArgs e) { - var scanned = Core.Scheduler.MessagesScanned; - var totalAttachments = Core.Scheduler.TotalAttachments; - var done = Core.Scheduler.AttachmentsDownloaded; + var scanned = Core.Core.Scheduler.MessagesScanned; + var totalAttachments = Core.Core.Scheduler.TotalAttachments; + var done = Core.Core.Scheduler.AttachmentsDownloaded; var progress = totalAttachments > 0 ? (int)(100 * done / totalAttachments) : 0; progress = Math.Min(Math.Max(0, progress), 100); @@ -260,16 +260,16 @@ namespace DML.Application lbProgress.Text = $"Scanned: {scanned} Downloaded: {done} Open: {totalAttachments - done}"; - if (Core.Settings.UseRPC) + if (Core.Core.Settings.UseRPC) { - Core.RpcPresence.details = "Downloading media files"; - Core.RpcPresence.state = $"{done} / {totalAttachments} ({pgbProgress.Value}%)"; - Core.RpcPresence.largeImageKey = "main"; - Core.RpcPresence.largeImageText = "Visit discordmedialoader.net"; - Core.RpcPresence.smallImageKey = "author"; - Core.RpcPresence.smallImageText = "Made by Serraniel"; + Core.Core.RpcPresence.details = "Downloading media files"; + Core.Core.RpcPresence.state = $"{done} / {totalAttachments} ({pgbProgress.Value}%)"; + Core.Core.RpcPresence.largeImageKey = "main"; + Core.Core.RpcPresence.largeImageText = "Visit discordmedialoader.net"; + Core.Core.RpcPresence.smallImageKey = "author"; + Core.Core.RpcPresence.smallImageText = "Made by Serraniel"; - Core.RpcUpdatePresence(); + Core.Core.RpcUpdatePresence(); } } @@ -315,7 +315,7 @@ namespace DML.Application private void showTokenToolStripMenuItem_Click(object sender, EventArgs e) { - Clipboard.SetText(Core.Settings.LoginToken); + Clipboard.SetText(Core.Core.Settings.LoginToken); MessageBox.Show(this, "Your login token has been copied to your clipboard.", "Discord Media Loader", MessageBoxButtons.OK); } diff --git a/src/Discord Media Loader/Program.cs b/src/Discord Media Loader/Program.cs index f733eb4..dadbac2 100644 --- a/src/Discord Media Loader/Program.cs +++ b/src/Discord Media Loader/Program.cs @@ -13,10 +13,10 @@ **********************************************************************************************/ #endregion +using DML.Application.Core; +using Nito.AsyncEx; using System; using System.Windows.Forms; -using DML.Application.Classes; -using Nito.AsyncEx; namespace Discord_Media_Loader { @@ -31,7 +31,7 @@ namespace Discord_Media_Loader var splashScreen = new FrmSplash(); if (splashScreen.ShowDialog() == DialogResult.OK) { - DoLaunch(paramStrings); + DoLaunch(paramStrings); } else {