Serraniel/DiscordMediaLoader
1
0
Fork 0
Code Issues 30 Pull requests 16 Packages Projects Releases 18 Wiki Activity

Somehow implementation of downloading all the shit out of it

Browse source
This commit is contained in:
Serraniel 2017-04-17 21:56:53 +02:00
parent 53f49b20c5
commit fe4a4769f8
8 changed files with 380 additions and 70 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

102
Discord Media Loader.Application/Classes/Job.cs
View file

@ -1,8 +1,11 @@
using System;
using System.Collections.Generic;
using System.Diagnostics.Eventing.Reader;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Discord;
using SweetLib.Utils;
using static SweetLib.Utils.Logger.Logger;
namespace DML.Application.Classes
@ -12,6 +15,8 @@ namespace DML.Application.Classes
public int Id { get; set; }
public ulong GuildId { get; set; }
public ulong ChannelId { get; set; }
public double KnownTimestamp { get; set; } = 0;
private double StopTimestamp { get; set; } = 0;
internal void Store()
{
@ -20,7 +25,102 @@ namespace DML.Application.Classes
var jobDb = Core.Database.GetCollection<Job>("jobs");
Trace("Adding new value...");
jobDb.Insert(this);
if (jobDb.Find(x => x.ChannelId == ChannelId && x.GuildId == GuildId).Any())
{
jobDb.Update(this);
}
else
{
jobDb.Insert(this);
}
}
private Server FindServerById(ulong id)
{
Trace($"Trying to find server by Id: {id}");
return (from s in Core.Client.Servers where s.Id == id select s).FirstOrDefault();
}
private Channel FindChannelById(Server server, ulong id)
{
Trace($"Trying to find channel in {server} by id: {id}");
return (from c in server.TextChannels where c.Id == id select c).FirstOrDefault();
}
internal async Task<List<Message>> Scan()
{
Debug($"Starting scan of guild {GuildId} channel {ChannelId}...");
var result = new List<Message>();
var limit = 100;
var lastId = ulong.MaxValue;
var isFirst = true;
var finished = false;
var guild = FindServerById(GuildId);
var channel = FindChannelById(guild, ChannelId);
if (Math.Abs(StopTimestamp) < 0.4)
StopTimestamp = KnownTimestamp;
Trace("Initialized scanning parameters.");
while (!finished)
{
Trace("Entering scanning loop...");
Message[] messages;
Trace($"Downloading next {limit} messages...");
if (isFirst)
{
messages = await channel.DownloadMessages(limit, null);
}
else
{
messages = await channel.DownloadMessages(limit, lastId);
}
Trace($"Downloaded {messages.Length} messages.");
isFirst = false;
foreach (var m in messages)
{
Debug($"Processing message {m.Id}");
if (m.Id < lastId)
{
Trace($"Updating lastId ({lastId}) to {m.Id}");
lastId = m.Id;
}
if (SweetUtils.DateTimeToUnixTimeStamp(m.Timestamp) < StopTimestamp)
{
Debug("Found a message with a known timestamp...Stopping scan.");
finished = true;
continue;
}
Trace($"Message {m.Id} has {m.Attachments.Length} attachments.");
if (m.Attachments.Length > 0)
{
result.Add(m);
Trace($"Added message {m.Id}");
}
Debug($"Finished message {m.Id}");
}
finished = finished || messages.Length < limit;
}
Trace($"Downloaded all messages for guild {GuildId} channel {ChannelId}.");
Trace("Sorting messages...");
result.Sort((a, b) => DateTime.Compare(a.Timestamp, b.Timestamp));
Trace("Updating StopTimestamp for next scan...");
StopTimestamp = SweetUtils.DateTimeToUnixTimeStamp(result[result.Count - 1].Timestamp);
Debug($"Fisnished scan of guild {GuildId} channel {ChannelId}.");
return result;
}
internal static IEnumerable<Job> RestoreJobs()

192
Discord Media Loader.Application/Classes/JobScheduler.cs Normal file
View file

@ -0,0 +1,192 @@
using System;
using System.Collections;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Threading.Tasks;
using Discord;
using SweetLib.Utils;
using static SweetLib.Utils.Logger.Logger;
namespace DML.Application.Classes
{
internal class JobScheduler
{
private bool Run { get; set; } = false;
internal List<Job> JobList { get; set; } = new List<Job>();
internal Dictionary<int, Queue<Message>> RunningJobs = new Dictionary<int, Queue<Message>>();
internal int RunningThreads { get; set; } = 0;
internal void Stop()
{
Run = false;
}
internal void Start()
{
Run = true;
Task.Run(async () =>
{
Info("Started JobScheduler...");
while (Run)
{
Debug("Entering job list handler loop...");
foreach (var job in JobList)
{
Debug($"Checking job {job}");
var hasJob = false;
Trace("Locking scheduler...");
lock (this)
{
Trace("Checking if job is already performed...");
hasJob = RunningJobs.ContainsKey(job.Id);
}
Trace("Unlocked scheduler.");
if (!hasJob)
{
Debug("Job is not performed yet...Performing job...");
var queue = new Queue<Message>();
Trace("Locking scheduler...");
lock (this)
{
Trace("Adding job to running jobs.");
RunningJobs.Add(job.Id, queue);
}
Trace("Unlocked scheduler.");
Trace("Issuing job message scan...");
var messages = await job.Scan();
Trace($"Adding {messages.Count} messages to queue...");
foreach (var msg in messages)
{
queue.Enqueue(msg);
}
Trace($"Added {queue.Count} messages to queue.");
if(messages.Count!= queue.Count)
Warn("Not all messages have been added into the queue.");
var startedDownload = false;
while (!startedDownload)
{
Debug("Entering loop to check thread availability");
Trace("Locking scheduler...");
lock (this)
{
Trace($"Checking thread limit. Running: {RunningThreads}, Max: {Core.Settings.ThreadLimit}");
if (RunningThreads >= Core.Settings.ThreadLimit)
continue;
RunningThreads++;
startedDownload = true;
}
Trace("Unlocked scheduler.");
}
Trace("Start downloading job async.");
Task.Run(() => WorkQueue(job.Id)); // do not await to work parallel
}
}
}
});
}
private void WorkQueue(int jobId)
{
try
{
Debug("Beginning job download...");
Trace("Finding job...");
var job = (from j in JobList where j.Id == jobId select j).FirstOrDefault();
if (job == null)
{
Warn($"Associating job not found! JobId: {jobId}");
return;
}
Trace("Found job.");
Queue<Message> queue;
Trace("Locking scheduler...");
lock (this)
{
Trace("Finiding queue...");
if (!RunningJobs.TryGetValue(jobId, out queue))
{
Warn($"Queue for job {jobId} not found!");
return;
}
Trace("Queue found.");
}
Trace("Unlocked scheduler.");
Debug($"Messages to process for job {jobId}: {queue.Count}");
while (queue.Count > 0)
{
Trace("Dequeueing message...");
var message = queue.Dequeue();
Debug($"Attachments for message {message.Id}: {message.Attachments.Length}");
foreach (var a in message.Attachments)
{
var fileName = Path.Combine(Core.Settings.OperatingFolder, Core.Settings.FileNameScheme);
Trace("Replacing filename placeholders...");
fileName =
fileName.Replace("%guild%", message.Server.Name)
.Replace("%channel%", message.Channel.Name)
.Replace("%timestamp%", SweetUtils.DateTimeToUnixTimeStamp(message.Timestamp).ToString())
.Replace("%name%", a.Filename);
Trace($"Detemined file name: {fileName}.");
if (File.Exists(fileName) && new FileInfo(fileName).Length == a.Size)
{
Debug($"{fileName} already existing with its estimated size. Skipping...");
continue;
}
Trace("Determining directory...");
var fileDirectory = Path.GetDirectoryName(fileName);
if (!Directory.Exists(fileDirectory))
{
Info($"Directory {fileDirectory} does not exist. Creating directory...");
Directory.CreateDirectory(fileDirectory);
Debug("Created directory.");
}
var wc = new WebClient();
Debug($"Starting downloading of attachment {a.Id}...");
wc.DownloadFile(new Uri(a.Url), fileName);
Debug($"Downloaded attachment {a.Id}.");
Trace("Updating known timestamp for job...");
job.KnownTimestamp = SweetUtils.DateTimeToUnixTimeStamp(message.Timestamp);
job.Store();
}
}
}
finally
{
Trace("Locking scheduler...");
lock (this)
{
Trace($"Removing {jobId} from running jobs...");
RunningJobs.Remove(jobId);
Trace("Decreasing thread count...");
RunningThreads--;
}
Trace("Unlocked scheduler.");
}
}
}
}

12
Discord Media Loader.Application/Core.cs
View file

@ -23,6 +23,7 @@ namespace DML.Application
internal static DiscordClient Client { get; set; }
internal static LiteDatabase Database { get; set; }
internal static Settings Settings { get; set; }
internal static JobScheduler Scheduler { get; } = new JobScheduler();
internal static string DataDirectory
=> Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), @"Serraniel\Discord Media Loader");
@ -101,6 +102,9 @@ namespace DML.Application
Settings.Store();
}
Debug("Loading jobs collection out of database...");
Scheduler.JobList = Job.RestoreJobs().ToList();
Info("Loaded settings.");
Debug(
$"Settings: Email: {Settings.Email}, password: {(string.IsNullOrEmpty(Settings.Password) ? "not set" : "is set")}, use username: {Settings.UseUserData}, loginToken: {Settings.LoginToken}");
@ -183,8 +187,14 @@ namespace DML.Application
Trace("Dialog closed.");
}
}
Info("Starting scheduler...");
Scheduler.Start();
System.Windows.Forms.Application.Run(new MainForm());
Info("Stopping scheduler...");
Scheduler.Stop();
}
catch (Exception ex)
{

1
Discord Media Loader.Application/DML.Application.csproj
View file

@ -75,6 +75,7 @@
</ItemGroup>
<ItemGroup>
<Compile Include="Classes\Job.cs" />
<Compile Include="Classes\JobScheduler.cs" />
<Compile Include="Classes\Settings.cs" />
<Compile Include="Core.cs" />
<Compile Include="Dialogs\LoginDialog.cs">

105
Discord Media Loader.Application/MainForm.Designer.cs generated
View file

@ -39,13 +39,13 @@
this.edOperatingFolder = new System.Windows.Forms.TextBox();
this.lbOperatingFolder = new System.Windows.Forms.Label();
this.groupBox1 = new System.Windows.Forms.GroupBox();
this.btnAddJob = new System.Windows.Forms.Button();
this.cbChannel = new System.Windows.Forms.ComboBox();
this.lbChannel = new System.Windows.Forms.Label();
this.cbGuild = new System.Windows.Forms.ComboBox();
this.lbGuild = new System.Windows.Forms.Label();
this.statusStrip1 = new System.Windows.Forms.StatusStrip();
this.groupBox2 = new System.Windows.Forms.GroupBox();
this.lbGuild = new System.Windows.Forms.Label();
this.cbGuild = new System.Windows.Forms.ComboBox();
this.lbChannel = new System.Windows.Forms.Label();
this.cbChannel = new System.Windows.Forms.ComboBox();
this.btnAddJob = new System.Windows.Forms.Button();
this.pnlSettings.SuspendLayout();
((System.ComponentModel.ISupportInitialize)(this.edThreadLimit)).BeginInit();
this.groupBox1.SuspendLayout();
@ -172,6 +172,54 @@
this.groupBox1.TabStop = false;
this.groupBox1.Text = "Add a job";
//
// btnAddJob
//
this.btnAddJob.Anchor = ((System.Windows.Forms.AnchorStyles)((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Right)));
this.btnAddJob.Location = new System.Drawing.Point(481, 19);
this.btnAddJob.Name = "btnAddJob";
this.btnAddJob.Size = new System.Drawing.Size(66, 23);
this.btnAddJob.TabIndex = 4;
this.btnAddJob.Text = "&Add";
this.btnAddJob.UseVisualStyleBackColor = true;
this.btnAddJob.Click += new System.EventHandler(this.btnAddJob_Click);
//
// cbChannel
//
this.cbChannel.Anchor = ((System.Windows.Forms.AnchorStyles)(((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Left)
| System.Windows.Forms.AnchorStyles.Right)));
this.cbChannel.FormattingEnabled = true;
this.cbChannel.Location = new System.Drawing.Point(294, 19);
this.cbChannel.Name = "cbChannel";
this.cbChannel.Size = new System.Drawing.Size(181, 21);
this.cbChannel.TabIndex = 3;
//
// lbChannel
//
this.lbChannel.AutoSize = true;
this.lbChannel.Location = new System.Drawing.Point(239, 22);
this.lbChannel.Name = "lbChannel";
this.lbChannel.Size = new System.Drawing.Size(49, 13);
this.lbChannel.TabIndex = 2;
this.lbChannel.Text = "Channel:";
//
// cbGuild
//
this.cbGuild.FormattingEnabled = true;
this.cbGuild.Location = new System.Drawing.Point(52, 19);
this.cbGuild.Name = "cbGuild";
this.cbGuild.Size = new System.Drawing.Size(181, 21);
this.cbGuild.TabIndex = 1;
this.cbGuild.SelectedIndexChanged += new System.EventHandler(this.cbGuild_SelectedIndexChanged);
//
// lbGuild
//
this.lbGuild.AutoSize = true;
this.lbGuild.Location = new System.Drawing.Point(12, 22);
this.lbGuild.Name = "lbGuild";
this.lbGuild.Size = new System.Drawing.Size(34, 13);
this.lbGuild.TabIndex = 0;
this.lbGuild.Text = "Guild:";
//
// statusStrip1
//
this.statusStrip1.Location = new System.Drawing.Point(0, 311);
@ -190,53 +238,6 @@
this.groupBox2.TabStop = false;
this.groupBox2.Text = "Jobs";
//
// lbGuild
//
this.lbGuild.AutoSize = true;
this.lbGuild.Location = new System.Drawing.Point(12, 22);
this.lbGuild.Name = "lbGuild";
this.lbGuild.Size = new System.Drawing.Size(34, 13);
this.lbGuild.TabIndex = 0;
this.lbGuild.Text = "Guild:";
//
// cbGuild
//
this.cbGuild.FormattingEnabled = true;
this.cbGuild.Location = new System.Drawing.Point(52, 19);
this.cbGuild.Name = "cbGuild";
this.cbGuild.Size = new System.Drawing.Size(181, 21);
this.cbGuild.TabIndex = 1;
this.cbGuild.SelectedIndexChanged += new System.EventHandler(this.cbGuild_SelectedIndexChanged);
//
// lbChannel
//
this.lbChannel.AutoSize = true;
this.lbChannel.Location = new System.Drawing.Point(239, 22);
this.lbChannel.Name = "lbChannel";
this.lbChannel.Size = new System.Drawing.Size(49, 13);
this.lbChannel.TabIndex = 2;
this.lbChannel.Text = "Channel:";
//
// cbChannel
//
this.cbChannel.Anchor = ((System.Windows.Forms.AnchorStyles)(((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Left)
| System.Windows.Forms.AnchorStyles.Right)));
this.cbChannel.FormattingEnabled = true;
this.cbChannel.Location = new System.Drawing.Point(294, 19);
this.cbChannel.Name = "cbChannel";
this.cbChannel.Size = new System.Drawing.Size(181, 21);
this.cbChannel.TabIndex = 3;
//
// btnAddJob
//
this.btnAddJob.Anchor = ((System.Windows.Forms.AnchorStyles)((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Right)));
this.btnAddJob.Location = new System.Drawing.Point(481, 19);
this.btnAddJob.Name = "btnAddJob";
this.btnAddJob.Size = new System.Drawing.Size(66, 23);
this.btnAddJob.TabIndex = 4;
this.btnAddJob.Text = "&Add";
this.btnAddJob.UseVisualStyleBackColor = true;
//
// MainForm
//
this.AutoScaleDimensions = new System.Drawing.SizeF(6F, 13F);

30
Discord Media Loader.Application/MainForm.cs
View file

@ -1,6 +1,7 @@
using System.Linq;
using System.Windows.Forms;
using Discord;
using DML.Application.Classes;
using static SweetLib.Utils.Logger.Logger;
namespace DML.Application
@ -101,18 +102,6 @@ namespace DML.Application
return (from c in server.TextChannels where c.Name == name select c).FirstOrDefault();
}
private Server FindServerById(ulong id)
{
Trace($"Trying to find server by Id: {id}");
return (from s in Core.Client.Servers where s.Id == id select s).FirstOrDefault();
}
private Channel FindChannelById(Server server, ulong id)
{
Trace($"Trying to find channel in {server} by id: {id}");
return (from c in server.TextChannels where c.Id == id select c).FirstOrDefault();
}
private void cbGuild_SelectedIndexChanged(object sender, System.EventArgs e)
{
Trace("Guild index changed.");
@ -146,5 +135,22 @@ namespace DML.Application
Debug("Finished updating channel dropdown component.");
}
private void btnAddJob_Click(object sender, System.EventArgs e)
{
var job = new Job
{
GuildId = FindServerByName(cbGuild.Text).Id,
ChannelId = FindChannelByName(FindServerByName(cbGuild.Text), cbChannel.Text).Id
};
if (!(from j in Core.Scheduler.JobList
where j.GuildId == job.GuildId && j.ChannelId == job.ChannelId
select j).Any())
{
Core.Scheduler.JobList.Add(job);
job.Store();
}
}
}
}

4
Discord Media Loader.Application/Properties/AssemblyInfo.cs
View file

@ -32,5 +32,5 @@ using System.Runtime.InteropServices;
// Sie können alle Werte angeben oder die standardmäßigen Build- und Revisionsnummern
// übernehmen, indem Sie "*" eingeben:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("0.2.137.0")]
[assembly: AssemblyFileVersion("0.2.137.0")]
[assembly: AssemblyVersion("0.2.142.0")]
[assembly: AssemblyFileVersion("0.2.142.0")]

4
Discord Media Loader/Properties/AssemblyInfo.cs
View file

@ -32,5 +32,5 @@ using System.Runtime.InteropServices;
// Sie können alle Werte angeben oder die standardmäßigen Build- und Revisionsnummern
// übernehmen, indem Sie "*" eingeben:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("0.99.65.0")]
[assembly: AssemblyFileVersion("0.99.65.0")]
[assembly: AssemblyVersion("0.99.70.0")]
[assembly: AssemblyFileVersion("0.99.70.0")]