DiscordMediaLoader/Discord Media Loader.Application/Classes/Job.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Discord;
using Discord.WebSocket;
using DML.Application.Classes;
using DML.Client;
using SweetLib.Utils;
using SweetLib.Utils.Extensions;
using static SweetLib.Utils.Logger.Logger;

namespace DML.AppCore.Classes
{
    public class Job
    {
        public int Id { get; set; }
        public ulong GuildId { get; set; }
        public ulong ChannelId { get; set; }
        public double KnownTimestamp { get; set; } = 0;
        private double StopTimestamp { get; set; } = 0;
        private bool IsValid { get; set; } = true;

        internal void Store()
        {
            Debug("Storing job to database...");
            Trace("Getting jobs collection...");
            var jobDb = Core.Database.GetCollection<Job>("jobs");

            Trace("Adding new value...");

            if (jobDb.Find(x => x.ChannelId == ChannelId && x.GuildId == GuildId).Any())
            {
                jobDb.Update(this);
            }
            else
            {
                jobDb.Insert(this);
            }
        }

        public void Delete()
        {
            Debug("Deleting job from database...");
            Trace("Getting jobs collection...");
            var jobDb = Core.Database.GetCollection<Job>("jobs");

            Trace("Deleting value...");
            jobDb.Delete(Id);
        }

        private SocketGuild FindServerById(ulong id)
        {
            Trace($"Trying to find server by Id: {id}");
            return (from s in DMLClient.Client.Guilds where s.Id == id select s).FirstOrDefault();
        }

        private SocketTextChannel FindChannelById(SocketGuild server, ulong id)
        {
            Trace($"Trying to find channel in {server} by id: {id}");
            return (from c in server.TextChannels where c.Id == id select c).FirstOrDefault();
        }

        internal async Task<bool> Scan()
        {
            Debug($"Starting scan of guild {GuildId} channel {ChannelId}...");
            var result = new List<IMessage>();

            var limit = 100;
            var lastId = ulong.MaxValue;
            var isFirst = true;
            var finished = false;
            var scanStartTimeStamp = DateTime.UtcNow;

            var guild = FindServerById(GuildId);
            var channel = FindChannelById(guild, ChannelId);

            Debug("Checking channel access");
            if (channel.GetUser(channel.Guild.CurrentUser.Id) == null)
            {
                Info("Skipping channel without access");
                return true;
            }

            if (Math.Abs(StopTimestamp) < 0.4)
                StopTimestamp = KnownTimestamp;
            Trace("Initialized scanning parameters.");

            while (!finished)
            {
                Trace("Entering scanning loop...");
                var messages = new List<IMessage>();

                Trace($"Downloading next {limit} messages...");
                if (isFirst)
                {
                    var realMessages = await channel.GetMessagesAsync(limit).ToArray();

                    messages.AddRange(realMessages.SelectMany(realMessageArray => realMessageArray));
                }
                else
                {
                    var realMessages = await channel.GetMessagesAsync(lastId, Direction.Before, limit).ToArray();

                    messages.AddRange(realMessages.SelectMany(realMessageArray => realMessageArray));
                }

                Trace($"Downloaded {messages.Count} messages.");
                isFirst = false;

                foreach (var m in messages)
                {
                    if (!IsValid)
                        return false;

                    Core.Scheduler.MessagesScanned++;

                    Debug($"Processing message {m.Id}");
                    if (m.Id < lastId)
                    {
                        Trace($"Updating lastId ({lastId}) to {m.Id}");
                        lastId = m.Id;
                    }

                    if (m.CreatedAt.UtcDateTime.ToUnixTimeStamp() <= StopTimestamp)
                    {
                        Debug("Found a message with a known timestamp...Stopping scan.");
                        finished = true;
                        continue;
                    }

                    Trace($"Message {m.Id} has {m.Attachments.Count} attachments.");
                    if (m.Attachments.Count > 0)
                    {
                        result.Add(m);
                        Core.Scheduler.TotalAttachments += (ulong)m.Attachments.Count;
                        Trace($"Added message {m.Id}");
                    }
                    Debug($"Finished message {m.Id}");
                }

                finished = finished || messages.Count < limit;
            }

            Trace($"Downloaded all messages for guild {GuildId} channel {ChannelId}.");

            Trace("Sorting messages...");
            result.Sort((a, b) => DateTime.Compare(a.CreatedAt.UtcDateTime, b.CreatedAt.UtcDateTime));

            foreach (var r in result)
            {
                foreach (var a in r.Attachments)
                {
                    var mediaData = new MediaData
                    {
                        Id = a.Id,
                        GuildId = (r.Channel as SocketTextChannel)?.Guild?.Id ?? 0,
                        ChannelId = r.Channel.Id,
                        DownloadSource = a.Url,
                        Filename = a.Filename,
                        TimeStamp = r.CreatedAt.UtcDateTime.ToUnixTimeStamp(),
                        FileSize = a.Size
                    };
                    mediaData.Store();
                }
            }

            if (result.Count > 0)
            {
                Trace("Updating StopTimestamp for next scan...");
                StopTimestamp = result[result.Count - 1].CreatedAt.UtcDateTime.ToUnixTimeStamp();
                KnownTimestamp = StopTimestamp;
                Store();
                return false;
            }
            else
            {
                // if we found any messages we remember the timestamp of starting so we don´t have to scan all past messages....
                StopTimestamp = scanStartTimeStamp.ToUnixTimeStamp();
                KnownTimestamp = StopTimestamp;
                Store();

                var realLastMessage = await channel.GetMessagesAsync(1).ToArray();
                return scanStartTimeStamp > (realLastMessage.SelectMany(realLastMessageArray => realLastMessageArray)
                                                 .FirstOrDefault()?.CreatedAt.UtcDateTime ??
                                             scanStartTimeStamp);
            }

            Debug($"Fisnished scan of guild {GuildId} channel {ChannelId}.");
        }

        public void Stop()
        {
            IsValid = false;
        }

        public static IEnumerable<Job> RestoreJobs()
        {
            Debug("Restoring jobs...");
            Trace("Getting jobs collection...");
            var jobDb = Core.Database.GetCollection<Job>("jobs");

            Trace("Creating new empty job list");
            return jobDb.FindAll();
        }
    }
}