From 9155af6711bc83a910b66abb396fbe7cdd66344f Mon Sep 17 00:00:00 2001 From: Maksym Pavlenko Date: Tue, 8 Nov 2016 22:51:03 -0800 Subject: [PATCH] Implement redis storage --- src/Podsync/Controllers/StatusController.cs | 24 +++++ src/Podsync/Services/PodsyncConfiguration.cs | 2 + src/Podsync/Services/Storage/FeedMetadata.cs | 13 +++ .../Services/Storage/IStorageService.cs | 14 +++ src/Podsync/Services/Storage/RedisStorage.cs | 101 ++++++++++++++++++ src/Podsync/Startup.cs | 2 + src/Podsync/appsettings.json | 3 +- src/Podsync/project.json | 5 +- .../Services/Storage/RedisStorageTests.cs | 77 +++++++++++++ 9 files changed, 239 insertions(+), 2 deletions(-) create mode 100644 src/Podsync/Controllers/StatusController.cs create mode 100644 src/Podsync/Services/Storage/FeedMetadata.cs create mode 100644 src/Podsync/Services/Storage/IStorageService.cs create mode 100644 src/Podsync/Services/Storage/RedisStorage.cs create mode 100644 test/Podsync.Tests/Services/Storage/RedisStorageTests.cs diff --git a/src/Podsync/Controllers/StatusController.cs b/src/Podsync/Controllers/StatusController.cs new file mode 100644 index 0000000..eb22e38 --- /dev/null +++ b/src/Podsync/Controllers/StatusController.cs @@ -0,0 +1,24 @@ +using System.Threading.Tasks; +using Microsoft.AspNetCore.Mvc; +using Podsync.Services.Storage; + +namespace Podsync.Controllers +{ + public class StatusController : Controller + { + private readonly IStorageService _storageService; + + public StatusController(IStorageService storageService) + { + _storageService = storageService; + } + + public async Task Index() + { + var time = await _storageService.Ping(); + + return $"Path: {Request.Path}\r\n" + + $"Redis: {time}"; + } + } +} \ No newline at end of file diff --git a/src/Podsync/Services/PodsyncConfiguration.cs b/src/Podsync/Services/PodsyncConfiguration.cs index f827ff3..5dd2fe5 100644 --- a/src/Podsync/Services/PodsyncConfiguration.cs +++ b/src/Podsync/Services/PodsyncConfiguration.cs @@ -3,5 +3,7 @@ public class PodsyncConfiguration { public string YouTubeApiKey { get; set; } + + public string RedisConnectionString { get; set; } } } \ No newline at end of file diff --git a/src/Podsync/Services/Storage/FeedMetadata.cs b/src/Podsync/Services/Storage/FeedMetadata.cs new file mode 100644 index 0000000..a61e8ed --- /dev/null +++ b/src/Podsync/Services/Storage/FeedMetadata.cs @@ -0,0 +1,13 @@ +using Podsync.Services.Links; + +namespace Podsync.Services.Storage +{ + public struct FeedMetadata + { + public Provider Provider { get; set; } + + public LinkType LinkType { get; set; } + + public string Id { get; set; } + } +} \ No newline at end of file diff --git a/src/Podsync/Services/Storage/IStorageService.cs b/src/Podsync/Services/Storage/IStorageService.cs new file mode 100644 index 0000000..3ff2bdd --- /dev/null +++ b/src/Podsync/Services/Storage/IStorageService.cs @@ -0,0 +1,14 @@ +using System; +using System.Threading.Tasks; + +namespace Podsync.Services.Storage +{ + public interface IStorageService : IDisposable + { + Task Ping(); + + Task Save(FeedMetadata metadata); + + Task Load(string key); + } +} \ No newline at end of file diff --git a/src/Podsync/Services/Storage/RedisStorage.cs b/src/Podsync/Services/Storage/RedisStorage.cs new file mode 100644 index 0000000..89b2ceb --- /dev/null +++ b/src/Podsync/Services/Storage/RedisStorage.cs @@ -0,0 +1,101 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using HashidsNet; +using Microsoft.Extensions.Options; +using Podsync.Services.Links; +using StackExchange.Redis; + +namespace Podsync.Services.Storage +{ + public class RedisStorage : IStorageService + { + private const string IdKey = "keygen"; + private const string IdSalt = "65fce519433f4218aa0cee6394225eea"; + private const int IdLength = 4; + + // Store all fields manually for backward compatibility with existing implementation + private const string ProviderField = "provider"; + private const string TypeField = "type"; + private const string IdField = "id"; + + private static readonly IHashids HashIds = new Hashids(IdSalt, IdLength); + + private readonly IDatabase _db; + + public RedisStorage(IOptions configuration) + { + var cs = configuration.Value.RedisConnectionString; + var connection = ConnectionMultiplexer.ConnectAsync(cs).GetAwaiter().GetResult(); + + _db = connection.GetDatabase(); + } + + public void Dispose() + { + _db.Multiplexer.Dispose(); + } + + public Task Ping() + { + return _db.PingAsync(); + } + + public async Task Save(FeedMetadata metadata) + { + var id = await MakeId(); + + await _db.HashSetAsync(id, new[] + { + new HashEntry(ProviderField, metadata.Provider.ToString()), + new HashEntry(TypeField, metadata.LinkType.ToString()), + new HashEntry(IdField, metadata.Id) + }); + + await _db.KeyExpireAsync(id, TimeSpan.FromDays(1)); + + return id; + } + + public async Task Load(string key) + { + if (string.IsNullOrWhiteSpace(key)) + { + throw new ArgumentException("Feed key can't be empty"); + } + + var entries = await _db.HashGetAllAsync(key); + + if (entries.Length == 0) + { + throw new KeyNotFoundException("Invaid key"); + } + + var metadata = new FeedMetadata + { + Id = entries.Single(x => x.Name == IdField).Value, + LinkType = ToEnum(entries.Single(x => x.Name == TypeField).Value), + Provider = ToEnum(entries.Single(x => x.Name == ProviderField).Value) + }; + + return metadata; + } + + public Task ResetCounter() + { + return _db.KeyDeleteAsync(IdKey); + } + + public async Task MakeId() + { + var id = await _db.StringIncrementAsync(IdKey); + return HashIds.EncodeLong(id); + } + + private static T ToEnum(string key) + { + return (T)Enum.Parse(typeof(T), key, true); + } + } +} \ No newline at end of file diff --git a/src/Podsync/Startup.cs b/src/Podsync/Startup.cs index 63781ff..05c573b 100644 --- a/src/Podsync/Startup.cs +++ b/src/Podsync/Startup.cs @@ -6,6 +6,7 @@ using Microsoft.Extensions.Logging; using Podsync.Services; using Podsync.Services.Links; using Podsync.Services.Resolver; +using Podsync.Services.Storage; using Podsync.Services.Videos.YouTube; namespace Podsync @@ -39,6 +40,7 @@ namespace Podsync services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); // Add framework services. services.AddMvc(); diff --git a/src/Podsync/appsettings.json b/src/Podsync/appsettings.json index 0c7b871..6fb93ab 100644 --- a/src/Podsync/appsettings.json +++ b/src/Podsync/appsettings.json @@ -9,6 +9,7 @@ }, "Podsync": { - "YouTubeApiKey": "" + "YouTubeApiKey": "", + "RedisConnectionString": "localhost" } } diff --git a/src/Podsync/project.json b/src/Podsync/project.json index 404bf7d..255e325 100644 --- a/src/Podsync/project.json +++ b/src/Podsync/project.json @@ -1,6 +1,7 @@ { "dependencies": { "Google.Apis.YouTube.v3": "1.16.0.582", + "Hashids.net": "1.2.2", "Microsoft.AspNetCore.Diagnostics": "1.0.0", "Microsoft.AspNetCore.Mvc": "1.0.0", "Microsoft.AspNetCore.Razor.Tools": { @@ -23,12 +24,14 @@ "type": "platform" }, "Microsoft.VisualStudio.Web.BrowserLink.Loader": "14.0.0", + "StackExchange.Redis": "1.1.608", "System.Xml.XmlSerializer": "4.0.11" }, "tools": { "BundlerMinifier.Core": "2.0.238", "Microsoft.AspNetCore.Razor.Tools": "1.0.0-preview2-final", - "Microsoft.AspNetCore.Server.IISIntegration.Tools": "1.0.0-preview2-final" + "Microsoft.AspNetCore.Server.IISIntegration.Tools": "1.0.0-preview2-final", + "Microsoft.Extensions.SecretManager.Tools": "1.0.0-preview2-final" }, "frameworks": { "netcoreapp1.0": { diff --git a/test/Podsync.Tests/Services/Storage/RedisStorageTests.cs b/test/Podsync.Tests/Services/Storage/RedisStorageTests.cs new file mode 100644 index 0000000..6e0c799 --- /dev/null +++ b/test/Podsync.Tests/Services/Storage/RedisStorageTests.cs @@ -0,0 +1,77 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Podsync.Services.Links; +using Podsync.Services.Storage; +using Xunit; + +namespace Podsync.Tests.Services.Storage +{ + public class RedisStorageTests : TestBase, IDisposable + { + private readonly RedisStorage _storage; + + public RedisStorageTests() + { + _storage = new RedisStorage(Options); + } + + [Fact] + public async Task PingTest() + { + var time = await _storage.Ping(); + Assert.True(time.TotalMilliseconds > 0); + } + + [Fact] + public void MakeIdTest() + { + const int idCount = 50; + + try + { + var results = new string[idCount]; + Parallel.For(0, results.Length, (i, _) => results[i] = _storage.MakeId().GetAwaiter().GetResult()); + + Assert.Equal(results.Length, results.Distinct().Count()); + } + finally + { + _storage.ResetCounter(); + } + } + + [Fact] + public async Task SaveLoadFeedTest() + { + var feed = new FeedMetadata + { + Id = "123", + LinkType = LinkType.Channel, + Provider = Provider.Vimeo + }; + + var id = await _storage.Save(feed); + Assert.NotEmpty(id); + Assert.Equal(4, id.Length); + + var loaded = await _storage.Load(id); + + Assert.Equal(feed.Id, loaded.Id); + Assert.Equal(feed.LinkType, loaded.LinkType); + Assert.Equal(feed.Provider, loaded.Provider); + } + + [Fact] + public Task LoadInvalidFeedTest() + { + return Assert.ThrowsAsync(() => _storage.Load("test")); + } + + public void Dispose() + { + _storage.Dispose(); + } + } +} \ No newline at end of file