Helload of shit i have done and reversed 👀
This commit is contained in:
parent
eca0e337c5
commit
df0980c86e
17
DML.AppCore/DML.AppCore.csproj
Normal file
17
DML.AppCore/DML.AppCore.csproj
Normal file
|
@ -0,0 +1,17 @@
|
||||||
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
|
<PropertyGroup>
|
||||||
|
<TargetFramework>netstandard1.4</TargetFramework>
|
||||||
|
</PropertyGroup>
|
||||||
|
|
||||||
|
<ItemGroup>
|
||||||
|
<PackageReference Include="Discord.Net" Version="1.0.2" />
|
||||||
|
<PackageReference Include="Discord.Net.Commands" Version="1.0.2" />
|
||||||
|
<PackageReference Include="LiteDB" Version="3.1.0" />
|
||||||
|
</ItemGroup>
|
||||||
|
|
||||||
|
<ItemGroup>
|
||||||
|
<ProjectReference Include="..\..\SweetLib\SweetLib\SweetLib.csproj" />
|
||||||
|
</ItemGroup>
|
||||||
|
|
||||||
|
</Project>
|
268
Discord Media Loader.Application/Classes/Core.cs
Normal file
268
Discord Media Loader.Application/Classes/Core.cs
Normal file
|
@ -0,0 +1,268 @@
|
||||||
|
using System;
|
||||||
|
using System.Globalization;
|
||||||
|
using System.IO;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Runtime;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
using System.Windows.Forms;
|
||||||
|
using Discord;
|
||||||
|
using Discord.Net;
|
||||||
|
using Discord.WebSocket;
|
||||||
|
using DML.AppCore.Classes;
|
||||||
|
using DML.Application.Dialogs;
|
||||||
|
using LiteDB;
|
||||||
|
using SweetLib.Utils;
|
||||||
|
using SweetLib.Utils.Logger;
|
||||||
|
using SweetLib.Utils.Logger.Memory;
|
||||||
|
using Logger = SweetLib.Utils.Logger.Logger;
|
||||||
|
|
||||||
|
namespace DML.Application.Classes
|
||||||
|
{
|
||||||
|
public static class Core
|
||||||
|
{
|
||||||
|
internal static DiscordSocketClient Client { get; set; }
|
||||||
|
internal static LiteDatabase Database { get; set; }
|
||||||
|
internal static Settings Settings { get; set; }
|
||||||
|
internal static JobScheduler Scheduler { get; } = new JobScheduler();
|
||||||
|
|
||||||
|
internal static string DataDirectory
|
||||||
|
=> Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), @"Serraniel\Discord Media Loader");
|
||||||
|
|
||||||
|
public static async Task Run(string[] paramStrings)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var splash = new FrmInternalSplash();
|
||||||
|
splash.Show();
|
||||||
|
System.Windows.Forms.Application.DoEvents();
|
||||||
|
|
||||||
|
Logger.Info("Starting up Discord Media Loader application...");
|
||||||
|
var useTrace = false;
|
||||||
|
#if DEBUG
|
||||||
|
//temporary add debug log level if debugging...
|
||||||
|
Logger.GlobalLogLevel |= LogLevel.Debug;
|
||||||
|
Logger.Debug("Running in debug configuartion. Added log level debug.");
|
||||||
|
#endif
|
||||||
|
|
||||||
|
Logger.Debug($"Parameters: {string.Join(", ", paramStrings)}");
|
||||||
|
if (paramStrings.Contains("--trace") || paramStrings.Contains("-t"))
|
||||||
|
{
|
||||||
|
useTrace = true;
|
||||||
|
Logger.GlobalLogLevel |= LogLevel.Trace;
|
||||||
|
Logger.Trace("Trace parameter found. Added log level trace.");
|
||||||
|
}
|
||||||
|
|
||||||
|
Logger.Debug($"Application data folder: {DataDirectory}");
|
||||||
|
|
||||||
|
Logger.Trace("Checking application data folder...");
|
||||||
|
if (!Directory.Exists(DataDirectory))
|
||||||
|
{
|
||||||
|
Logger.Debug("Creating application data folder...");
|
||||||
|
Directory.CreateDirectory(DataDirectory);
|
||||||
|
Logger.Trace("Creating application data folder.");
|
||||||
|
}
|
||||||
|
|
||||||
|
Logger.Trace("Initializing profile optimizations...");
|
||||||
|
ProfileOptimization.SetProfileRoot(System.Windows.Forms.Application.UserAppDataPath);
|
||||||
|
ProfileOptimization.StartProfile("profile.opt");
|
||||||
|
Logger.Trace("Finished initializing profile optimizations.");
|
||||||
|
|
||||||
|
Logger.Trace("Trying to identify log memory...");
|
||||||
|
var logMemory = Logger.DefaultLogMemory as ArchivableConsoleLogMemory;
|
||||||
|
if (logMemory != null)
|
||||||
|
{
|
||||||
|
var logFolder = Path.Combine(DataDirectory, "logs");
|
||||||
|
if (!Directory.Exists(logFolder))
|
||||||
|
{
|
||||||
|
Logger.Debug("Creating log folder...");
|
||||||
|
Directory.CreateDirectory(logFolder);
|
||||||
|
Logger.Trace("Created log folder.");
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
var logFile = Path.Combine(logFolder,
|
||||||
|
SweetUtils.LegalizeFilename($"{DateTime.Now.ToString(CultureInfo.CurrentCulture.DateTimeFormat.SortableDateTimePattern)}.log.zip"));
|
||||||
|
|
||||||
|
Logger.Trace($"Setting log file: {logFile}");
|
||||||
|
logMemory.AutoArchiveOnDispose = true;
|
||||||
|
logMemory.ArchiveFile = logFile;
|
||||||
|
}
|
||||||
|
|
||||||
|
Logger.Debug("Loading database...");
|
||||||
|
Database = new LiteDatabase(Path.Combine(DataDirectory, "config.db"));
|
||||||
|
Database.Log.Logging += (message) => Logger.Trace($"LiteDB: {message}");
|
||||||
|
|
||||||
|
Logger.Debug("Loading settings collection out of database...");
|
||||||
|
var settingsDB = Database.GetCollection<Settings>("settings");
|
||||||
|
if (settingsDB.Count() > 1)
|
||||||
|
{
|
||||||
|
Logger.Warn("Found more than one setting. Loading first one...");
|
||||||
|
}
|
||||||
|
Settings = settingsDB.FindAll().FirstOrDefault();
|
||||||
|
if (Settings == null)
|
||||||
|
{
|
||||||
|
Logger.Warn("Settings not found. Creating new one. This is normal on first start up...");
|
||||||
|
Settings = new Settings();
|
||||||
|
Settings.Store();
|
||||||
|
}
|
||||||
|
|
||||||
|
Logger.Debug("Loading jobs collection out of database...");
|
||||||
|
Scheduler.JobList = Job.RestoreJobs().ToList();
|
||||||
|
|
||||||
|
Logger.Info("Loaded settings.");
|
||||||
|
Logger.Debug(
|
||||||
|
$"Settings: Email: {Settings.Email}, password: {(string.IsNullOrEmpty(Settings.Password) ? "not set" : "is set")}, use username: {Settings.UseUserData}, loginToken: {Settings.LoginToken}");
|
||||||
|
|
||||||
|
Logger.Trace("Updating log level...");
|
||||||
|
Logger.GlobalLogLevel = Settings.ApplicactionLogLevel;
|
||||||
|
#if DEBUG
|
||||||
|
//temporary add debug log level if debugging...
|
||||||
|
Logger.GlobalLogLevel |= LogLevel.Debug;
|
||||||
|
Logger.Debug("Running in debug configuartion. Added log level debug.");
|
||||||
|
#endif
|
||||||
|
if (useTrace)
|
||||||
|
{
|
||||||
|
Logger.GlobalLogLevel |= LogLevel.Trace;
|
||||||
|
Logger.Trace("Creating application data folder.");
|
||||||
|
}
|
||||||
|
|
||||||
|
Logger.Debug("Creating discord client...");
|
||||||
|
|
||||||
|
Client = new DiscordSocketClient();
|
||||||
|
Client.Log += (arg) =>
|
||||||
|
{
|
||||||
|
var logMessage = $"DiscordClient: {arg.Message}";
|
||||||
|
switch (arg.Severity)
|
||||||
|
{
|
||||||
|
case LogSeverity.Verbose:
|
||||||
|
Logger.Trace(logMessage);
|
||||||
|
break;
|
||||||
|
case LogSeverity.Debug:
|
||||||
|
Logger.Trace(logMessage);
|
||||||
|
break;
|
||||||
|
case LogSeverity.Info:
|
||||||
|
Logger.Info(logMessage);
|
||||||
|
break;
|
||||||
|
case LogSeverity.Warning:
|
||||||
|
Logger.Warn(logMessage);
|
||||||
|
break;
|
||||||
|
case LogSeverity.Error:
|
||||||
|
Logger.Error(logMessage);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
return Task.CompletedTask;
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
Logger.Info("Trying to log into discord...");
|
||||||
|
var abort = false;
|
||||||
|
|
||||||
|
Client.Connected += Client_Connected;
|
||||||
|
|
||||||
|
while (Client.LoginState != LoginState.LoggedIn && !abort)
|
||||||
|
{
|
||||||
|
Logger.Debug(Client.ConnectionState.ToString());
|
||||||
|
Logger.Debug(Client.LoginState.ToString());
|
||||||
|
|
||||||
|
Logger.Trace("Entering login loop.");
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
if (Client.ConnectionState == ConnectionState.Connecting)
|
||||||
|
continue;
|
||||||
|
|
||||||
|
if (!string.IsNullOrEmpty(Settings.LoginToken))
|
||||||
|
{
|
||||||
|
Logger.Debug("Trying to login with last known token...");
|
||||||
|
await Client.LoginAsync(TokenType.User, Settings.LoginToken);
|
||||||
|
await Task.Delay(1000);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
catch (HttpException ex)
|
||||||
|
{
|
||||||
|
Logger.Warn($"Login seems to have failed or gone wrong: {ex.GetType().Name} - {ex.Message}");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (Client.LoginState == LoginState.LoggedOut)
|
||||||
|
{
|
||||||
|
Settings.Password = string.Empty;
|
||||||
|
Logger.Debug("Showing dialog for username and password...");
|
||||||
|
var loginDlg = new LoginDialog();
|
||||||
|
loginDlg.ShowDialog();
|
||||||
|
Logger.Trace("Dialog closed.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Logger.Debug("Start checking for invalid jobs...");
|
||||||
|
|
||||||
|
//Client
|
||||||
|
|
||||||
|
while (Client.Guilds.Count == 0)
|
||||||
|
{
|
||||||
|
// wait until guilds are loaded
|
||||||
|
}
|
||||||
|
|
||||||
|
for (var i = Scheduler.JobList.Count - 1; i >= 0; i--)
|
||||||
|
{
|
||||||
|
var job = Scheduler.JobList[i];
|
||||||
|
var isError = false;
|
||||||
|
var guild = FindServerById(job.GuildId);
|
||||||
|
if (guild == null)
|
||||||
|
isError = true;
|
||||||
|
else
|
||||||
|
{
|
||||||
|
var channel = FindChannelById(guild, job.ChannelId);
|
||||||
|
if (channel == null)
|
||||||
|
isError = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (isError)
|
||||||
|
{
|
||||||
|
MessageBox.Show($"Invalid job for guild {job.GuildId}, channel {job.ChannelId} found. Guild or channel may not exist any more. This job will be deleted...", "Invalid job",
|
||||||
|
MessageBoxButtons.OK, MessageBoxIcon.Warning);
|
||||||
|
|
||||||
|
Scheduler.JobList.Remove(job);
|
||||||
|
Scheduler.RunningJobs.Remove(job.Id);
|
||||||
|
job.Stop();
|
||||||
|
job.Delete();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
splash.Close();
|
||||||
|
|
||||||
|
Logger.Info("Starting scheduler...");
|
||||||
|
Scheduler.Start();
|
||||||
|
|
||||||
|
System.Windows.Forms.Application.Run(new MainForm());
|
||||||
|
|
||||||
|
Logger.Info("Stopping scheduler...");
|
||||||
|
Scheduler.Stop();
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
Logger.Error($"{ex.Message} occured at: {ex.StackTrace}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static Task Client_Connected()
|
||||||
|
{
|
||||||
|
Logger.Debug("Connected");
|
||||||
|
return Task.CompletedTask;
|
||||||
|
}
|
||||||
|
|
||||||
|
//TODO: this is thrid time we implement this.....this has to be fixed!!!
|
||||||
|
private static SocketGuild FindServerById(ulong id)
|
||||||
|
{
|
||||||
|
Logger.Trace($"Trying to find server by Id: {id}");
|
||||||
|
return (from s in Core.Client.Guilds where s.Id == id select s).FirstOrDefault();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static SocketTextChannel FindChannelById(SocketGuild server, ulong id)
|
||||||
|
{
|
||||||
|
Logger.Trace($"Trying to find channel in {server} by id: {id}");
|
||||||
|
return (from c in server.TextChannels where c.Id == id select c).FirstOrDefault();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
161
Discord Media Loader.Application/Classes/Job.cs
Normal file
161
Discord Media Loader.Application/Classes/Job.cs
Normal file
|
@ -0,0 +1,161 @@
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
using Discord;
|
||||||
|
using Discord.WebSocket;
|
||||||
|
using DML.Application.Classes;
|
||||||
|
using SweetLib.Utils;
|
||||||
|
using static SweetLib.Utils.Logger.Logger;
|
||||||
|
|
||||||
|
namespace DML.AppCore.Classes
|
||||||
|
{
|
||||||
|
public class Job
|
||||||
|
{
|
||||||
|
public int Id { get; set; }
|
||||||
|
public ulong GuildId { get; set; }
|
||||||
|
public ulong ChannelId { get; set; }
|
||||||
|
public double KnownTimestamp { get; set; } = 0;
|
||||||
|
private double StopTimestamp { get; set; } = 0;
|
||||||
|
private bool IsValid { get; set; } = true;
|
||||||
|
|
||||||
|
internal void Store()
|
||||||
|
{
|
||||||
|
Debug("Storing job to database...");
|
||||||
|
Trace("Getting jobs collection...");
|
||||||
|
var jobDb = Core.Database.GetCollection<Job>("jobs");
|
||||||
|
|
||||||
|
Trace("Adding new value...");
|
||||||
|
|
||||||
|
if (jobDb.Find(x => x.ChannelId == ChannelId && x.GuildId == GuildId).Any())
|
||||||
|
{
|
||||||
|
jobDb.Update(this);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
jobDb.Insert(this);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public void Delete()
|
||||||
|
{
|
||||||
|
Debug("Deleting job from database...");
|
||||||
|
Trace("Getting jobs collection...");
|
||||||
|
var jobDb = Core.Database.GetCollection<Job>("jobs");
|
||||||
|
|
||||||
|
Trace("Deleting value...");
|
||||||
|
jobDb.Delete(Id);
|
||||||
|
}
|
||||||
|
|
||||||
|
private SocketGuild FindServerById(ulong id)
|
||||||
|
{
|
||||||
|
Trace($"Trying to find server by Id: {id}");
|
||||||
|
return (from s in Core.Client.Guilds where s.Id == id select s).FirstOrDefault();
|
||||||
|
}
|
||||||
|
|
||||||
|
private SocketTextChannel FindChannelById(SocketGuild server, ulong id)
|
||||||
|
{
|
||||||
|
Trace($"Trying to find channel in {server} by id: {id}");
|
||||||
|
return (from c in server.TextChannels where c.Id == id select c).FirstOrDefault();
|
||||||
|
}
|
||||||
|
|
||||||
|
internal async Task<List<SocketMessage>> Scan()
|
||||||
|
{
|
||||||
|
Debug($"Starting scan of guild {GuildId} channel {ChannelId}...");
|
||||||
|
var result = new List<SocketMessage>();
|
||||||
|
|
||||||
|
var limit = 100;
|
||||||
|
var lastId = ulong.MaxValue;
|
||||||
|
var isFirst = true;
|
||||||
|
var finished = false;
|
||||||
|
|
||||||
|
var guild = FindServerById(GuildId);
|
||||||
|
var channel = FindChannelById(guild, ChannelId);
|
||||||
|
|
||||||
|
if (Math.Abs(StopTimestamp) < 0.4)
|
||||||
|
StopTimestamp = KnownTimestamp;
|
||||||
|
Trace("Initialized scanning parameters.");
|
||||||
|
|
||||||
|
while (!finished)
|
||||||
|
{
|
||||||
|
Trace("Entering scanning loop...");
|
||||||
|
SocketMessage[] messages;
|
||||||
|
|
||||||
|
Trace($"Downloading next {limit} messages...");
|
||||||
|
if (isFirst)
|
||||||
|
{
|
||||||
|
messages = await channel.GetMessagesAsync(limit).ToArray() as SocketMessage[];
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
messages = await channel.GetMessagesAsync(lastId, Direction.Before, limit).ToArray() as SocketMessage[];
|
||||||
|
}
|
||||||
|
Trace($"Downloaded {messages.Length} messages.");
|
||||||
|
|
||||||
|
isFirst = false;
|
||||||
|
|
||||||
|
foreach (var m in messages)
|
||||||
|
{
|
||||||
|
if (!IsValid)
|
||||||
|
return null;
|
||||||
|
|
||||||
|
Debug($"Processing message {m.Id}");
|
||||||
|
if (m.Id < lastId)
|
||||||
|
{
|
||||||
|
Trace($"Updating lastId ({lastId}) to {m.Id}");
|
||||||
|
lastId = m.Id;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (SweetUtils.DateTimeToUnixTimeStamp(m.CreatedAt.UtcDateTime) <= StopTimestamp)
|
||||||
|
{
|
||||||
|
Debug("Found a message with a known timestamp...Stopping scan.");
|
||||||
|
finished = true;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
Trace($"Message {m.Id} has {m.Attachments.Count} attachments.");
|
||||||
|
if (m.Attachments.Count > 0)
|
||||||
|
{
|
||||||
|
result.Add(m);
|
||||||
|
Core.Scheduler.TotalAttachments++;
|
||||||
|
Trace($"Added message {m.Id}");
|
||||||
|
}
|
||||||
|
Debug($"Finished message {m.Id}");
|
||||||
|
|
||||||
|
Core.Scheduler.MessagesScanned++;
|
||||||
|
}
|
||||||
|
|
||||||
|
finished = finished || messages.Length < limit;
|
||||||
|
}
|
||||||
|
Trace($"Downloaded all messages for guild {GuildId} channel {ChannelId}.");
|
||||||
|
|
||||||
|
Trace("Sorting messages...");
|
||||||
|
result.Sort((a, b) => DateTime.Compare(a.CreatedAt.UtcDateTime, b.CreatedAt.UtcDateTime));
|
||||||
|
|
||||||
|
if (result.Count > 0)
|
||||||
|
{
|
||||||
|
Trace("Updating StopTimestamp for next scan...");
|
||||||
|
StopTimestamp = SweetUtils.DateTimeToUnixTimeStamp(result[result.Count - 1].CreatedAt.UtcDateTime);
|
||||||
|
}
|
||||||
|
|
||||||
|
Debug($"Fisnished scan of guild {GuildId} channel {ChannelId}.");
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void Stop()
|
||||||
|
{
|
||||||
|
IsValid = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static IEnumerable<Job> RestoreJobs()
|
||||||
|
{
|
||||||
|
Debug("Restoring jobs...");
|
||||||
|
Trace("Getting jobs collection...");
|
||||||
|
var jobDb = Core.Database.GetCollection<Job>("jobs");
|
||||||
|
|
||||||
|
Trace("Creating new empty job list");
|
||||||
|
return jobDb.FindAll();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
295
Discord Media Loader.Application/Classes/JobScheduler.cs
Normal file
295
Discord Media Loader.Application/Classes/JobScheduler.cs
Normal file
|
@ -0,0 +1,295 @@
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Diagnostics;
|
||||||
|
using System.IO;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Net;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
using Discord.WebSocket;
|
||||||
|
using DML.Application.Classes;
|
||||||
|
using SweetLib.Utils;
|
||||||
|
using SweetLib.Utils.Logger;
|
||||||
|
|
||||||
|
namespace DML.AppCore.Classes
|
||||||
|
{
|
||||||
|
public class JobScheduler
|
||||||
|
{
|
||||||
|
private ulong messageScanned = 0;
|
||||||
|
private ulong totalAttachments = 0;
|
||||||
|
private ulong attachmentsDownloaded = 0;
|
||||||
|
|
||||||
|
private bool Run { get; set; } = false;
|
||||||
|
public List<Job> JobList { get; set; } = new List<Job>();
|
||||||
|
public Dictionary<int, Queue<SocketMessage>> RunningJobs = new Dictionary<int, Queue<SocketMessage>>();
|
||||||
|
internal int RunningThreads { get; set; } = 0;
|
||||||
|
|
||||||
|
internal ulong MessagesScanned
|
||||||
|
{
|
||||||
|
get
|
||||||
|
{
|
||||||
|
lock (this)
|
||||||
|
{
|
||||||
|
return messageScanned;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
set
|
||||||
|
{
|
||||||
|
lock (this)
|
||||||
|
{
|
||||||
|
messageScanned = value;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
internal ulong TotalAttachments
|
||||||
|
{
|
||||||
|
get
|
||||||
|
{
|
||||||
|
lock (this)
|
||||||
|
{
|
||||||
|
return totalAttachments;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
set
|
||||||
|
{
|
||||||
|
lock (this)
|
||||||
|
{
|
||||||
|
totalAttachments = value;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
internal ulong AttachmentsDownloaded
|
||||||
|
{
|
||||||
|
get
|
||||||
|
{
|
||||||
|
lock (this)
|
||||||
|
{
|
||||||
|
return attachmentsDownloaded;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
set
|
||||||
|
{
|
||||||
|
lock (this)
|
||||||
|
{
|
||||||
|
attachmentsDownloaded = value;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
internal ulong AttachmentsToDownload => TotalAttachments - AttachmentsDownloaded;
|
||||||
|
|
||||||
|
public void Stop()
|
||||||
|
{
|
||||||
|
Run = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void Start()
|
||||||
|
{
|
||||||
|
Run = true;
|
||||||
|
|
||||||
|
Task.Run(async () =>
|
||||||
|
{
|
||||||
|
Logger.Info("Started JobScheduler...");
|
||||||
|
while (Run)
|
||||||
|
{
|
||||||
|
Logger.Debug("Entering job list handler loop...");
|
||||||
|
//foreach (var job in JobList)
|
||||||
|
for (var i = JobList.Count - 1; i >= 0; i--)
|
||||||
|
{
|
||||||
|
var job = JobList[i];
|
||||||
|
Logger.Debug($"Checking job {job}");
|
||||||
|
var hasJob = false;
|
||||||
|
|
||||||
|
Logger.Trace("Locking scheduler...");
|
||||||
|
lock (this)
|
||||||
|
{
|
||||||
|
Logger.Trace("Checking if job is already performed...");
|
||||||
|
hasJob = RunningJobs.ContainsKey(job.Id);
|
||||||
|
}
|
||||||
|
Logger.Trace("Unlocked scheduler.");
|
||||||
|
|
||||||
|
if (!hasJob)
|
||||||
|
{
|
||||||
|
Logger.Debug("Job is not performed yet...Performing job...");
|
||||||
|
var queue = new Queue<SocketMessage>();
|
||||||
|
|
||||||
|
Logger.Trace("Locking scheduler...");
|
||||||
|
lock (this)
|
||||||
|
{
|
||||||
|
Logger.Trace("Adding job to running jobs.");
|
||||||
|
RunningJobs.Add(job.Id, queue);
|
||||||
|
}
|
||||||
|
Logger.Trace("Unlocked scheduler.");
|
||||||
|
|
||||||
|
Logger.Trace("Issuing job message scan...");
|
||||||
|
var messages = await job.Scan();
|
||||||
|
|
||||||
|
if (messages == null)
|
||||||
|
continue;
|
||||||
|
|
||||||
|
Logger.Trace($"Adding {messages.Count} messages to queue...");
|
||||||
|
foreach (var msg in messages)
|
||||||
|
{
|
||||||
|
queue.Enqueue(msg);
|
||||||
|
}
|
||||||
|
Logger.Trace($"Added {queue.Count} messages to queue.");
|
||||||
|
|
||||||
|
if (messages.Count != queue.Count)
|
||||||
|
Logger.Warn("Not all messages have been added into the queue.");
|
||||||
|
|
||||||
|
var startedDownload = false;
|
||||||
|
|
||||||
|
while (!startedDownload)
|
||||||
|
{
|
||||||
|
Logger.Debug("Entering loop to check thread availability");
|
||||||
|
Logger.Trace("Locking scheduler...");
|
||||||
|
lock (this)
|
||||||
|
{
|
||||||
|
Logger.Trace($"Checking thread limit. Running: {RunningThreads}, Max: {Core.Settings.ThreadLimit}");
|
||||||
|
if (RunningThreads >= Core.Settings.ThreadLimit)
|
||||||
|
continue;
|
||||||
|
|
||||||
|
RunningThreads++;
|
||||||
|
startedDownload = true;
|
||||||
|
}
|
||||||
|
Logger.Trace("Unlocked scheduler.");
|
||||||
|
}
|
||||||
|
|
||||||
|
Logger.Trace("Start downloading job async.");
|
||||||
|
Task.Run(() => WorkQueue(job.Id)); // do not await to work parallel
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private void WorkQueue(int jobId)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
Logger.Debug("Beginning job download...");
|
||||||
|
Logger.Trace("Finding job...");
|
||||||
|
var job = (from j in JobList where j.Id == jobId select j).FirstOrDefault();
|
||||||
|
|
||||||
|
if (job == null)
|
||||||
|
{
|
||||||
|
Logger.Warn($"Associating job not found! JobId: {jobId}");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
Logger.Trace("Found job.");
|
||||||
|
|
||||||
|
Queue<SocketMessage> queue;
|
||||||
|
Logger.Trace("Locking scheduler...");
|
||||||
|
lock (this)
|
||||||
|
{
|
||||||
|
Logger.Trace("Finiding queue...");
|
||||||
|
if (!RunningJobs.TryGetValue(jobId, out queue))
|
||||||
|
{
|
||||||
|
Logger.Warn($"Queue for job {jobId} not found!");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
Logger.Trace("Queue found.");
|
||||||
|
}
|
||||||
|
Logger.Trace("Unlocked scheduler.");
|
||||||
|
|
||||||
|
Logger.Debug($"Messages to process for job {jobId}: {queue.Count}");
|
||||||
|
while (queue.Count > 0)
|
||||||
|
{
|
||||||
|
Logger.Trace("Locking scheduler...");
|
||||||
|
lock (this)
|
||||||
|
{
|
||||||
|
Logger.Trace("Checking if still a job...");
|
||||||
|
if (!RunningJobs.ContainsKey(jobId))
|
||||||
|
{
|
||||||
|
Logger.Warn($"Queue for job {jobId} not found!");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
Logger.Trace("Continue working...");
|
||||||
|
}
|
||||||
|
Logger.Trace("Unlocked scheduler.");
|
||||||
|
|
||||||
|
Logger.Trace("Dequeueing message...");
|
||||||
|
var message = queue.Dequeue();
|
||||||
|
|
||||||
|
Logger.Debug($"Attachments for message {message.Id}: {message.Attachments.Count}");
|
||||||
|
foreach (var a in message.Attachments)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var fileName = Path.Combine(Core.Settings.OperatingFolder, Core.Settings.FileNameScheme);
|
||||||
|
|
||||||
|
Logger.Trace("Replacing filename placeholders...");
|
||||||
|
|
||||||
|
var extensionRequired = !fileName.EndsWith("%name%");
|
||||||
|
|
||||||
|
var serverName = "unknown";
|
||||||
|
|
||||||
|
var socketTextChannel = message.Channel as SocketTextChannel;
|
||||||
|
if (socketTextChannel != null)
|
||||||
|
{
|
||||||
|
serverName = socketTextChannel.Guild.Name.Replace(":", "").Replace("/", "")
|
||||||
|
.Replace("\\", "");
|
||||||
|
}
|
||||||
|
|
||||||
|
fileName =
|
||||||
|
fileName.Replace("%guild%", serverName)
|
||||||
|
.Replace("%channel%", message.Channel.Name)
|
||||||
|
.Replace("%timestamp%", SweetUtils.DateTimeToUnixTimeStamp(message.CreatedAt.UtcDateTime).ToString())
|
||||||
|
.Replace("%name%", a.Filename)
|
||||||
|
.Replace("%id%", a.Id.ToString());
|
||||||
|
|
||||||
|
if (extensionRequired)
|
||||||
|
fileName += Path.GetExtension(a.Filename);
|
||||||
|
|
||||||
|
Logger.Trace($"Detemined file name: {fileName}.");
|
||||||
|
|
||||||
|
|
||||||
|
if (File.Exists(fileName) && new FileInfo(fileName).Length == a.Size)
|
||||||
|
{
|
||||||
|
Logger.Debug($"{fileName} already existing with its estimated size. Skipping...");
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
Logger.Trace("Determining directory...");
|
||||||
|
var fileDirectory = Path.GetDirectoryName(fileName);
|
||||||
|
|
||||||
|
if (!Directory.Exists(fileDirectory))
|
||||||
|
{
|
||||||
|
Logger.Info($"Directory {fileDirectory} does not exist. Creating directory...");
|
||||||
|
Directory.CreateDirectory(fileDirectory);
|
||||||
|
Logger.Debug("Created directory.");
|
||||||
|
}
|
||||||
|
|
||||||
|
var wc = new WebClient();
|
||||||
|
Logger.Debug($"Starting downloading of attachment {a.Id}...");
|
||||||
|
|
||||||
|
wc.DownloadFile(new Uri(a.Url), fileName);
|
||||||
|
Logger.Debug($"Downloaded attachment {a.Id}.");
|
||||||
|
|
||||||
|
Logger.Trace("Updating known timestamp for job...");
|
||||||
|
job.KnownTimestamp = SweetUtils.DateTimeToUnixTimeStamp(message.CreatedAt.UtcDateTime);
|
||||||
|
job.Store();
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
AttachmentsDownloaded++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
Logger.Trace("Locking scheduler...");
|
||||||
|
lock (this)
|
||||||
|
{
|
||||||
|
Logger.Trace($"Removing {jobId} from running jobs...");
|
||||||
|
RunningJobs.Remove(jobId);
|
||||||
|
Logger.Trace("Decreasing thread count...");
|
||||||
|
RunningThreads--;
|
||||||
|
}
|
||||||
|
Logger.Trace("Unlocked scheduler.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
39
Discord Media Loader.Application/Classes/Settings.cs
Normal file
39
Discord Media Loader.Application/Classes/Settings.cs
Normal file
|
@ -0,0 +1,39 @@
|
||||||
|
using System.Diagnostics;
|
||||||
|
using SweetLib.Utils.Logger;
|
||||||
|
|
||||||
|
namespace DML.Application.Classes
|
||||||
|
{
|
||||||
|
public class Settings
|
||||||
|
{
|
||||||
|
public int Id { get; } = 1; // using always unique ID
|
||||||
|
public string Email { get; set; }
|
||||||
|
public string Password { get; set; }
|
||||||
|
public string LoginToken { get; set; }
|
||||||
|
public bool UseUserData { get; set; } = false;
|
||||||
|
public bool SavePassword { get; set; } = false;
|
||||||
|
public LogLevel ApplicactionLogLevel { get; set; } = LogLevel.Info | LogLevel.Warn | LogLevel.Error;
|
||||||
|
public string OperatingFolder { get; set; }
|
||||||
|
public string FileNameScheme { get; set; } = @"%guild%\%channel%\%id%";
|
||||||
|
public bool SkipExistingFiles { get; set; } = true;
|
||||||
|
public int ThreadLimit { get; set; } = 50;
|
||||||
|
|
||||||
|
public void Store()
|
||||||
|
{
|
||||||
|
Logger.Trace("Getting settings collection...");
|
||||||
|
var settingsDB = Core.Database.GetCollection<Settings>("settings");
|
||||||
|
|
||||||
|
Logger.Debug("Storing settings to database...");
|
||||||
|
|
||||||
|
if (settingsDB.Exists(_setting => _setting.Id == Id))
|
||||||
|
{
|
||||||
|
Logger.Trace("Updating existing value...");
|
||||||
|
settingsDB.Update(this);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
Logger.Trace("Adding new value...");
|
||||||
|
settingsDB.Insert(this);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -155,6 +155,9 @@
|
||||||
</Reference>
|
</Reference>
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
|
<Compile Include="Classes\Core.cs" />
|
||||||
|
<Compile Include="Classes\Job.cs" />
|
||||||
|
<Compile Include="Classes\JobScheduler.cs" />
|
||||||
<Compile Include="Dialogs\LoginDialog.cs">
|
<Compile Include="Dialogs\LoginDialog.cs">
|
||||||
<SubType>Form</SubType>
|
<SubType>Form</SubType>
|
||||||
</Compile>
|
</Compile>
|
||||||
|
@ -179,6 +182,7 @@
|
||||||
<DesignTime>True</DesignTime>
|
<DesignTime>True</DesignTime>
|
||||||
<DependentUpon>Resources.resx</DependentUpon>
|
<DependentUpon>Resources.resx</DependentUpon>
|
||||||
</Compile>
|
</Compile>
|
||||||
|
<Compile Include="Classes\Settings.cs" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<EmbeddedResource Include="Dialogs\LoginDialog.resx">
|
<EmbeddedResource Include="Dialogs\LoginDialog.resx">
|
||||||
|
@ -204,6 +208,10 @@
|
||||||
<Project>{02c1f8ef-32f2-4e77-a36d-79129402af37}</Project>
|
<Project>{02c1f8ef-32f2-4e77-a36d-79129402af37}</Project>
|
||||||
<Name>SweetLib</Name>
|
<Name>SweetLib</Name>
|
||||||
</ProjectReference>
|
</ProjectReference>
|
||||||
|
<ProjectReference Include="..\DML.AppCore\DML.AppCore.csproj">
|
||||||
|
<Project>{4db264a7-2352-4057-b3de-61fcfec01ae5}</Project>
|
||||||
|
<Name>DML.AppCore</Name>
|
||||||
|
</ProjectReference>
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<None Include="Resources\Serraniel-Logo4-NO-BG.png" />
|
<None Include="Resources\Serraniel-Logo4-NO-BG.png" />
|
||||||
|
|
|
@ -7,6 +7,7 @@ using System.Linq;
|
||||||
using System.Text;
|
using System.Text;
|
||||||
using System.Threading.Tasks;
|
using System.Threading.Tasks;
|
||||||
using System.Windows.Forms;
|
using System.Windows.Forms;
|
||||||
|
using DML.Application.Classes;
|
||||||
using static SweetLib.Utils.Logger.Logger;
|
using static SweetLib.Utils.Logger.Logger;
|
||||||
|
|
||||||
namespace DML.Application.Dialogs
|
namespace DML.Application.Dialogs
|
||||||
|
|
|
@ -5,6 +5,8 @@ using System.Reflection;
|
||||||
using System.Windows.Forms;
|
using System.Windows.Forms;
|
||||||
using Discord;
|
using Discord;
|
||||||
using Discord.WebSocket;
|
using Discord.WebSocket;
|
||||||
|
using DML.AppCore;
|
||||||
|
using DML.AppCore.Classes;
|
||||||
using DML.Application.Classes;
|
using DML.Application.Classes;
|
||||||
using static SweetLib.Utils.Logger.Logger;
|
using static SweetLib.Utils.Logger.Logger;
|
||||||
|
|
||||||
|
|
|
@ -1,6 +1,5 @@
|
||||||
<?xml version="1.0" encoding="utf-8"?>
|
<?xml version="1.0" encoding="utf-8"?>
|
||||||
<packages>
|
<packages>
|
||||||
<package id="Discord.Net" version="1.0.2" targetFramework="net461" />
|
|
||||||
<package id="Discord.Net.Commands" version="1.0.2" targetFramework="net461" />
|
<package id="Discord.Net.Commands" version="1.0.2" targetFramework="net461" />
|
||||||
<package id="Discord.Net.Core" version="1.0.2" targetFramework="net461" />
|
<package id="Discord.Net.Core" version="1.0.2" targetFramework="net461" />
|
||||||
<package id="Discord.Net.Rest" version="1.0.2" targetFramework="net461" />
|
<package id="Discord.Net.Rest" version="1.0.2" targetFramework="net461" />
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
|
|
||||||
Microsoft Visual Studio Solution File, Format Version 12.00
|
Microsoft Visual Studio Solution File, Format Version 12.00
|
||||||
# Visual Studio 15
|
# Visual Studio 15
|
||||||
VisualStudioVersion = 15.0.26730.10
|
VisualStudioVersion = 15.0.26730.16
|
||||||
MinimumVisualStudioVersion = 10.0.40219.1
|
MinimumVisualStudioVersion = 10.0.40219.1
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Discord Media Loader", "Discord Media Loader\Discord Media Loader.csproj", "{EDC92554-DBC1-4F9C-9317-379A8BF441E8}"
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Discord Media Loader", "Discord Media Loader\Discord Media Loader.csproj", "{EDC92554-DBC1-4F9C-9317-379A8BF441E8}"
|
||||||
EndProject
|
EndProject
|
||||||
|
@ -9,7 +9,7 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DML.Application", "Discord
|
||||||
EndProject
|
EndProject
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SweetLib", "..\SweetLib\SweetLib\SweetLib.csproj", "{02C1F8EF-32F2-4E77-A36D-79129402AF37}"
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SweetLib", "..\SweetLib\SweetLib\SweetLib.csproj", "{02C1F8EF-32F2-4E77-A36D-79129402AF37}"
|
||||||
EndProject
|
EndProject
|
||||||
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DML.Core", "DML.Core\DML.Core.csproj", "{BADDA2C1-BF8E-40DC-A2B2-80B5D609C4ED}"
|
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DML.AppCore", "DML.AppCore\DML.AppCore.csproj", "{4DB264A7-2352-4057-B3DE-61FCFEC01AE5}"
|
||||||
EndProject
|
EndProject
|
||||||
Global
|
Global
|
||||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||||
|
@ -29,10 +29,10 @@ Global
|
||||||
{02C1F8EF-32F2-4E77-A36D-79129402AF37}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
{02C1F8EF-32F2-4E77-A36D-79129402AF37}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||||
{02C1F8EF-32F2-4E77-A36D-79129402AF37}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
{02C1F8EF-32F2-4E77-A36D-79129402AF37}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||||
{02C1F8EF-32F2-4E77-A36D-79129402AF37}.Release|Any CPU.Build.0 = Release|Any CPU
|
{02C1F8EF-32F2-4E77-A36D-79129402AF37}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||||
{BADDA2C1-BF8E-40DC-A2B2-80B5D609C4ED}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
{4DB264A7-2352-4057-B3DE-61FCFEC01AE5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||||
{BADDA2C1-BF8E-40DC-A2B2-80B5D609C4ED}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
{4DB264A7-2352-4057-B3DE-61FCFEC01AE5}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||||
{BADDA2C1-BF8E-40DC-A2B2-80B5D609C4ED}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
{4DB264A7-2352-4057-B3DE-61FCFEC01AE5}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||||
{BADDA2C1-BF8E-40DC-A2B2-80B5D609C4ED}.Release|Any CPU.Build.0 = Release|Any CPU
|
{4DB264A7-2352-4057-B3DE-61FCFEC01AE5}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||||
EndGlobalSection
|
EndGlobalSection
|
||||||
GlobalSection(SolutionProperties) = preSolution
|
GlobalSection(SolutionProperties) = preSolution
|
||||||
HideSolutionNode = FALSE
|
HideSolutionNode = FALSE
|
||||||
|
|
|
@ -214,6 +214,10 @@
|
||||||
<Project>{c130de6a-3237-42b5-be9f-783d1cd104c6}</Project>
|
<Project>{c130de6a-3237-42b5-be9f-783d1cd104c6}</Project>
|
||||||
<Name>DML.Application</Name>
|
<Name>DML.Application</Name>
|
||||||
</ProjectReference>
|
</ProjectReference>
|
||||||
|
<ProjectReference Include="..\DML.AppCore\DML.AppCore.csproj">
|
||||||
|
<Project>{4db264a7-2352-4057-b3de-61fcfec01ae5}</Project>
|
||||||
|
<Name>DML.AppCore</Name>
|
||||||
|
</ProjectReference>
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
|
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
|
||||||
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
|
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
|
||||||
|
|
|
@ -6,6 +6,7 @@ using System.Threading.Tasks;
|
||||||
using System.Windows.Forms;
|
using System.Windows.Forms;
|
||||||
using Discord_Media_Loader.Helper;
|
using Discord_Media_Loader.Helper;
|
||||||
using DML.Application;
|
using DML.Application;
|
||||||
|
using DML.Application.Classes;
|
||||||
using Nito.AsyncEx;
|
using Nito.AsyncEx;
|
||||||
|
|
||||||
namespace Discord_Media_Loader
|
namespace Discord_Media_Loader
|
||||||
|
|
Loading…
Reference in a new issue