From fa868e96a595df435627714855212f11ffb35b5f Mon Sep 17 00:00:00 2001 From: Bruno Massa Date: Fri, 30 Jun 2023 17:08:28 -0300 Subject: [PATCH 1/6] perf: removed the yaml parser regex --- source/Parser/YAMLParser.cs | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/source/Parser/YAMLParser.cs b/source/Parser/YAMLParser.cs index f25235f..1fc27f8 100644 --- a/source/Parser/YAMLParser.cs +++ b/source/Parser/YAMLParser.cs @@ -2,7 +2,7 @@ using System; using System.Collections.Generic; using System.IO; using System.Linq; -using System.Text.RegularExpressions; +using System.Text; using SuCoS.Helper; using SuCoS.Models; using YamlDotNet.Serialization; @@ -14,9 +14,6 @@ namespace SuCoS.Parser; /// public partial class YAMLParser : IFrontmatterParser { - [GeneratedRegex(@"^---\s*[\r\n](?.*?)[\r\n]---\s*", RegexOptions.Singleline)] - private partial Regex YAMLRegex(); - /// /// YamlDotNet parser, strictly set to allow automatically parse only known fields /// @@ -43,15 +40,24 @@ public partial class YAMLParser : IFrontmatterParser throw new ArgumentNullException(nameof(filePath)); } - var match = YAMLRegex().Match(fileContent); - if (match.Success) + using var sr = new StringReader(fileContent); + var frontmatterBuilder = new StringBuilder(); + string? line; + + while ((line = sr.ReadLine()) != null && line != "---") { } + while ((line = sr.ReadLine()) != null && line != "---") { - var yaml = match.Groups["frontmatter"].Value; - var rawContent = fileContent[match.Length..].TrimStart('\n'); - var frontmatter = ParseYAML(ref site, filePath, yaml, rawContent); - return frontmatter; + frontmatterBuilder.AppendLine(line); } - return null; + + // Join the read lines to form the frontmatter + var yamlFrontmatter = frontmatterBuilder.ToString(); + + // Now, you can parse the YAML frontmatter + var parser = new YamlDotNet.Core.Parser(new StringReader(yamlFrontmatter)); + var frontmatter = ParseYAML(ref site, filePath, yamlFrontmatter, sr.ReadToEnd()); + + return frontmatter; } private Frontmatter ParseYAML(ref Site site, in string filePath, string frontmatter, string rawContent) -- GitLab From bc3d761851aade1de55357249df9ca91345ed295 Mon Sep 17 00:00:00 2001 From: Bruno Massa Date: Fri, 30 Jun 2023 18:14:04 -0300 Subject: [PATCH 2/6] reafctor: remove ReadSourceFrontmatter --- .build.Nuke/Build.Test.cs | 13 +------- .nuke/build.schema.json | 2 -- source/Helpers/SiteHelper.cs | 4 ++- source/Models/Site.cs | 51 +++-------------------------- source/Parser/IFrontmatterParser.cs | 8 +++++ source/Parser/YAMLParser.cs | 11 +++++++ test/Models/SiteTests.cs | 31 +++++++++--------- test/Parser/YAMLParserTests.cs | 12 +++++-- 8 files changed, 52 insertions(+), 80 deletions(-) diff --git a/.build.Nuke/Build.Test.cs b/.build.Nuke/Build.Test.cs index 937b3f6..c1d0c37 100644 --- a/.build.Nuke/Build.Test.cs +++ b/.build.Nuke/Build.Test.cs @@ -6,7 +6,6 @@ using Nuke.Common.Tools.ReportGenerator; using Nuke.Common.IO; using Nuke.Common.Tools.Coverlet; using static Nuke.Common.Tools.Coverlet.CoverletTasks; -using static Nuke.Common.IO.FileSystemTasks; using Serilog; using System; @@ -20,8 +19,6 @@ sealed partial class Build : NukeBuild { AbsolutePath testDirectory => RootDirectory / "test"; AbsolutePath testDLLDirectory => testDirectory / "bin" / "Debug" / "net7.0"; - AbsolutePath testSiteSourceDirectory => RootDirectory / "test" / ".TestSites"; - AbsolutePath testSiteDestinationDirectory => testDLLDirectory / ".TestSites"; AbsolutePath testAssembly => testDLLDirectory / "test.dll"; AbsolutePath coverageDirectory => RootDirectory / "coverage-results"; AbsolutePath coverageResultDirectory => coverageDirectory / "coverage"; @@ -29,16 +26,8 @@ sealed partial class Build : NukeBuild AbsolutePath coverageReportDirectory => coverageDirectory / "report"; AbsolutePath coverageReportSummaryDirectory => coverageReportDirectory / "Summary.txt"; - Target PrepareTestFiles => _ => _ - .After(Clean) - .Executes(() => - { - testSiteDestinationDirectory.CreateOrCleanDirectory(); - CopyDirectoryRecursively(testSiteSourceDirectory, testSiteDestinationDirectory, DirectoryExistsPolicy.Merge); - }); - Target Test => _ => _ - .DependsOn(Compile, PrepareTestFiles) + .DependsOn(Compile) .Executes(() => { coverageResultDirectory.CreateDirectory(); diff --git a/.nuke/build.schema.json b/.nuke/build.schema.json index 9dbaef2..7def812 100644 --- a/.nuke/build.schema.json +++ b/.nuke/build.schema.json @@ -115,7 +115,6 @@ "CreatePackage", "GitLabCreateRelease", "GitLabCreateTag", - "PrepareTestFiles", "Publish", "Restore", "ShowCurrentVersion", @@ -141,7 +140,6 @@ "CreatePackage", "GitLabCreateRelease", "GitLabCreateTag", - "PrepareTestFiles", "Publish", "Restore", "ShowCurrentVersion", diff --git a/source/Helpers/SiteHelper.cs b/source/Helpers/SiteHelper.cs index 9993b2e..ecaa537 100644 --- a/source/Helpers/SiteHelper.cs +++ b/source/Helpers/SiteHelper.cs @@ -91,7 +91,9 @@ public static class SiteHelper site.ResetCache(); - site.ScanAllMarkdownFiles(); + // Scan content files + var markdownFiles = FileUtils.GetAllMarkdownFiles(site.SourceContentPath); + site.ContentPaths.AddRange(markdownFiles); site.ParseSourceFiles(stopwatch); diff --git a/source/Models/Site.cs b/source/Models/Site.cs index 4ec4c88..409afb3 100644 --- a/source/Models/Site.cs +++ b/source/Models/Site.cs @@ -123,7 +123,7 @@ public class Site : IParams /// List of all content to be scanned and processed. /// [YamlIgnore] - public List<(string filePath, string content)> RawPages { get; set; } = new(); + public List ContentPaths { get; set; } = new(); /// /// Command line options @@ -217,22 +217,6 @@ public class Site : IParams this.Clock = clock; } - /// - /// Scans all markdown files in the source directory. - /// - public void ScanAllMarkdownFiles() - { - // Scan content files - var markdownFiles = FileUtils.GetAllMarkdownFiles(SourceContentPath); - - foreach (var fileAbsolutePath in markdownFiles) - { - var content = File.ReadAllText(fileAbsolutePath); - var relativePath = Path.GetRelativePath(SourceContentPath, fileAbsolutePath); - RawPages.Add((relativePath, content)); - } - } - /// /// Resets the template cache to force a reload of all templates. /// @@ -321,11 +305,12 @@ public class Site : IParams // Process the source files, extracting the frontmatter var filesParsed = 0; // counter to keep track of the number of files processed - _ = Parallel.ForEach(RawPages, file => + _ = Parallel.ForEach(ContentPaths, filePath => { try { - var frontmatter = ReadSourceFrontmatter(file.filePath, file.content, frontmatterParser); + var frontmatter = frontmatterParser.ParseFrontmatter(this, filePath) + ?? throw new FormatException($"Error parsing frontmatter for {filePath}"); if (frontmatter.IsValidDate(options)) { @@ -334,7 +319,7 @@ public class Site : IParams } catch (Exception ex) { - Logger?.Error(ex, "Error parsing file {file}", file.filePath); + Logger?.Error(ex, "Error parsing file {file}", filePath); } // Use interlocked to safely increment the counter in a multi-threaded environment @@ -435,30 +420,4 @@ public class Site : IParams CreateAutomaticFrontmatter(contentTemplate, frontmatter); } } - - /// - /// Reads the frontmatter from the source file. - /// - /// The file path. - /// The file content. - /// The frontmatter parser. - /// The parsed frontmatter. - private Frontmatter ReadSourceFrontmatter(string filePath, string content, IFrontmatterParser frontmatterParser) - { - // test if filePath or config is null - if (filePath is null) - { - throw new ArgumentNullException(nameof(filePath)); - } - if (frontmatterParser is null) - { - throw new ArgumentNullException(nameof(frontmatterParser)); - } - - // Separate the YAML frontmatter from the file content - var frontmatter = frontmatterParser.ParseFrontmatter(this, filePath, content) - ?? throw new FormatException($"Error parsing frontmatter for {filePath}"); - - return frontmatter; - } } diff --git a/source/Parser/IFrontmatterParser.cs b/source/Parser/IFrontmatterParser.cs index 045d9c5..5d3b9d7 100644 --- a/source/Parser/IFrontmatterParser.cs +++ b/source/Parser/IFrontmatterParser.cs @@ -11,6 +11,14 @@ public interface IFrontmatterParser /// Extract the frontmatter from the content file. /// /// + /// + /// + Frontmatter? ParseFrontmatter(Site site, in string filePath); + + /// + /// Extract the frontmatter from the content. + /// + /// /// /// /// diff --git a/source/Parser/YAMLParser.cs b/source/Parser/YAMLParser.cs index 1fc27f8..629b939 100644 --- a/source/Parser/YAMLParser.cs +++ b/source/Parser/YAMLParser.cs @@ -28,6 +28,13 @@ public partial class YAMLParser : IFrontmatterParser readonly IDeserializer yamlDeserializer = new DeserializerBuilder() .Build(); + /// + public Frontmatter? ParseFrontmatter(Site site, in string filePath) + { + var fileContent = File.ReadAllText(filePath); + return ParseFrontmatter(site, filePath, fileContent); + } + /// public Frontmatter? ParseFrontmatter(Site site, in string filePath, in string fileContent) { @@ -63,6 +70,10 @@ public partial class YAMLParser : IFrontmatterParser private Frontmatter ParseYAML(ref Site site, in string filePath, string frontmatter, string rawContent) { var page = yamlDeserializerRigid.Deserialize(frontmatter); + if (page is null) + { + throw new FormatException("Error parsing frontmatter"); + } var sourceFileNameWithoutExtension = Path.GetFileNameWithoutExtension(filePath) ?? string.Empty; var section = SiteHelper.GetSection(filePath); page.RawContent = rawContent; diff --git a/test/Models/SiteTests.cs b/test/Models/SiteTests.cs index 74489e6..f38bb9c 100644 --- a/test/Models/SiteTests.cs +++ b/test/Models/SiteTests.cs @@ -2,6 +2,7 @@ using Xunit; using Moq; using SuCoS.Models; using System.Globalization; +using SuCoS.Helper; namespace SuCoS.Tests; @@ -14,6 +15,10 @@ public class SiteTests private readonly Mock systemClockMock; readonly string testSite1Path = ".TestSites/01"; + // based on the compiled test.dll path + // that is typically "bin/Debug/netX.0/test.dll" + readonly string testSitesPath = "../../.."; + public SiteTests() { systemClockMock = new Mock(); @@ -23,24 +28,18 @@ public class SiteTests } [Theory] - [InlineData("test01.md", @"--- -Title: Test Content 1 ---- - -Test Content 1 -")] - [InlineData("test02.md", @"--- -Title: Test Content 2 ---- - -Test Content 2 -")] - public void Test_ScanAllMarkdownFiles(string fileName, string fileContent) + [InlineData("test01.md")] + [InlineData("test02.md")] + public void Test_ScanAllMarkdownFiles(string fileName) { - site.SourceDirectoryPath = testSite1Path; - site.ScanAllMarkdownFiles(); + var fullPath = Path.GetFullPath(Path.Combine(testSitesPath, testSite1Path)); + + // Act + var ContentPaths = FileUtils.GetAllMarkdownFiles(fullPath); + var filePath = Path.Combine(fullPath, "content", fileName); - Assert.Contains(site.RawPages, rp => rp.filePath == fileName && rp.content == fileContent); + // Assert + Assert.Contains(ContentPaths, rp => rp == filePath); } [Theory] diff --git a/test/Parser/YAMLParserTests.cs b/test/Parser/YAMLParserTests.cs index 06eb559..02166d6 100644 --- a/test/Parser/YAMLParserTests.cs +++ b/test/Parser/YAMLParserTests.cs @@ -12,8 +12,12 @@ public class YAMLParserTests private readonly YAMLParser parser; private readonly Mock site; private readonly string fileContent = @"--- -Title: Real Data Title +Title: Test Title +Type: post Date: 2023-07-01 +LastMod: 2023-06-01 +PublishDate: 2023-06-01 +ExpiryDate: 2024-06-01 Categories: ['Test', 'Real Data'] Tags: ['Test', 'Real Data'] --- @@ -47,8 +51,9 @@ Title: Test Title --- ", "Test Title")] [InlineData(@"--- +Date: 2023-04-01 --- -", null)] +", "")] public void ParseFrontmatter_ShouldParseTitleCorrectly(string fileContent, string expectedTitle) { var filePath = "test.md"; @@ -173,13 +178,14 @@ customParam: Custom Value [Fact] public void ParseFrontmatter_ShouldParseContentInSiteFolder() { + var date = DateTime.Parse("2023-07-01", CultureInfo.InvariantCulture); var frontmatter = parser.ParseFrontmatter(site.Object, "", fileContent); // Act site.Object.PostProcessFrontMatter(frontmatter!); // Asset - Assert.Equal(DateTime.Parse("2023-07-01", CultureInfo.InvariantCulture), frontmatter?.Date); + Assert.Equal(date, frontmatter?.Date); } [Fact] -- GitLab From 08aedbae6dd0723e3cecf3b61fa0110a9b2d5ba2 Mon Sep 17 00:00:00 2001 From: Bruno Massa Date: Fri, 30 Jun 2023 20:15:24 -0300 Subject: [PATCH 3/6] chore: ParseFrontmatterAndMarkdownFromFile --- source/Helpers/FileUtils.cs | 12 +++-- source/Models/Frontmatter.cs | 1 + source/Models/Site.cs | 2 +- source/Parser/IFrontmatterParser.cs | 5 ++- source/Parser/YAMLParser.cs | 69 ++++++++++++++++++----------- 5 files changed, 56 insertions(+), 33 deletions(-) diff --git a/source/Helpers/FileUtils.cs b/source/Helpers/FileUtils.cs index a098144..3425713 100644 --- a/source/Helpers/FileUtils.cs +++ b/source/Helpers/FileUtils.cs @@ -14,17 +14,23 @@ public static class FileUtils /// Gets all Markdown files in the specified directory. /// /// The directory path. + /// The initial directory path. /// The list of Markdown file paths. - public static List GetAllMarkdownFiles(string directory) + public static List GetAllMarkdownFiles(string directory, string? basePath = null) { + basePath ??= directory; var markdownFiles = new List(); + var files = Directory.GetFiles(directory, "*.md"); - markdownFiles.AddRange(files); + foreach (var filePath in files) + { + markdownFiles.Add(Path.GetRelativePath(basePath, filePath)); + } var subdirectories = Directory.GetDirectories(directory); foreach (var subdirectory in subdirectories) { - markdownFiles.AddRange(GetAllMarkdownFiles(subdirectory)); + markdownFiles.AddRange(GetAllMarkdownFiles(subdirectory, basePath)); } return markdownFiles; diff --git a/source/Models/Frontmatter.cs b/source/Models/Frontmatter.cs index 33cd623..466ea35 100644 --- a/source/Models/Frontmatter.cs +++ b/source/Models/Frontmatter.cs @@ -316,6 +316,7 @@ public class Frontmatter : IBaseContent, IParams URLforce ??= URL ?? (isIndex ? "{{ page.SourcePathDirectory }}" : "{{ page.SourcePathDirectory }}/{{ page.Title }}"); + try { if (Site.FluidParser.TryParse(URLforce, out var template, out var error)) diff --git a/source/Models/Site.cs b/source/Models/Site.cs index 409afb3..3c59960 100644 --- a/source/Models/Site.cs +++ b/source/Models/Site.cs @@ -309,7 +309,7 @@ public class Site : IParams { try { - var frontmatter = frontmatterParser.ParseFrontmatter(this, filePath) + var frontmatter = frontmatterParser.ParseFrontmatterAndMarkdownFromFile(this, filePath, SourceContentPath) ?? throw new FormatException($"Error parsing frontmatter for {filePath}"); if (frontmatter.IsValidDate(options)) diff --git a/source/Parser/IFrontmatterParser.cs b/source/Parser/IFrontmatterParser.cs index 5d3b9d7..0277a60 100644 --- a/source/Parser/IFrontmatterParser.cs +++ b/source/Parser/IFrontmatterParser.cs @@ -12,8 +12,9 @@ public interface IFrontmatterParser /// /// /// + /// /// - Frontmatter? ParseFrontmatter(Site site, in string filePath); + Frontmatter? ParseFrontmatterAndMarkdownFromFile(Site site, in string filePath, in string sourceContentPath); /// /// Extract the frontmatter from the content. @@ -22,7 +23,7 @@ public interface IFrontmatterParser /// /// /// - Frontmatter? ParseFrontmatter(Site site, in string filePath, in string fileContent); + Frontmatter? ParseFrontmatterAndMarkdown(Site site, in string filePath, in string fileContent); /// /// Parse the app config file. diff --git a/source/Parser/YAMLParser.cs b/source/Parser/YAMLParser.cs index 629b939..b011adb 100644 --- a/source/Parser/YAMLParser.cs +++ b/source/Parser/YAMLParser.cs @@ -29,14 +29,32 @@ public partial class YAMLParser : IFrontmatterParser .Build(); /// - public Frontmatter? ParseFrontmatter(Site site, in string filePath) + public Frontmatter? ParseFrontmatterAndMarkdownFromFile(Site site, in string filePath, in string? sourceContentPath = null) { - var fileContent = File.ReadAllText(filePath); - return ParseFrontmatter(site, filePath, fileContent); + if (site is null) + { + throw new ArgumentNullException(nameof(site)); + } + if (filePath is null) + { + throw new ArgumentNullException(nameof(filePath)); + } + + string? fileContent; + try + { + fileContent = File.ReadAllText(Path.GetFullPath(Path.Combine(sourceContentPath ?? string.Empty, filePath))); + } + catch (Exception ex) + { + throw new FileNotFoundException(filePath, ex); + } + + return ParseFrontmatterAndMarkdown(site, filePath, fileContent); } /// - public Frontmatter? ParseFrontmatter(Site site, in string filePath, in string fileContent) + public Frontmatter? ParseFrontmatterAndMarkdown(Site site, in string filePath, in string fileContent) { if (site is null) { @@ -47,33 +65,29 @@ public partial class YAMLParser : IFrontmatterParser throw new ArgumentNullException(nameof(filePath)); } - using var sr = new StringReader(fileContent); + using var content = new StringReader(fileContent); var frontmatterBuilder = new StringBuilder(); string? line; - while ((line = sr.ReadLine()) != null && line != "---") { } - while ((line = sr.ReadLine()) != null && line != "---") + while ((line = content.ReadLine()) != null && line != "---") { } + while ((line = content.ReadLine()) != null && line != "---") { frontmatterBuilder.AppendLine(line); } // Join the read lines to form the frontmatter - var yamlFrontmatter = frontmatterBuilder.ToString(); + var yaml = frontmatterBuilder.ToString(); + var rawContent = content.ReadToEnd(); // Now, you can parse the YAML frontmatter - var parser = new YamlDotNet.Core.Parser(new StringReader(yamlFrontmatter)); - var frontmatter = ParseYAML(ref site, filePath, yamlFrontmatter, sr.ReadToEnd()); + var page = ParseYAML(ref site, filePath, yaml, rawContent); - return frontmatter; + return page; } - private Frontmatter ParseYAML(ref Site site, in string filePath, string frontmatter, string rawContent) + private Frontmatter ParseYAML(ref Site site, in string filePath, string yaml, in string rawContent) { - var page = yamlDeserializerRigid.Deserialize(frontmatter); - if (page is null) - { - throw new FormatException("Error parsing frontmatter"); - } + var page = yamlDeserializerRigid.Deserialize(new StringReader(yaml)) ?? throw new FormatException("Error parsing frontmatter"); var sourceFileNameWithoutExtension = Path.GetFileNameWithoutExtension(filePath) ?? string.Empty; var section = SiteHelper.GetSection(filePath); page.RawContent = rawContent; @@ -85,13 +99,14 @@ public partial class YAMLParser : IFrontmatterParser page.Title ??= sourceFileNameWithoutExtension; page.Type ??= section; - var yamlObject = yamlDeserializer.Deserialize(new StringReader(frontmatter)); + var yamlObject = yamlDeserializer.Deserialize(new StringReader(yaml)); if (yamlObject is Dictionary yamlDictionary) { - if (yamlDictionary.TryGetValue("Tags", out var tags) && tags is List tagsValues) + if (yamlDictionary.TryGetValue("Tags", out var tags) && tags is List tagsValues) { - foreach (var tagName in tagsValues) + foreach (var tagObj in tagsValues) { + var tagName = (string)tagObj; var contentTemplate = new BasicContent( title: tagName, section: "tags", @@ -101,16 +116,16 @@ public partial class YAMLParser : IFrontmatterParser _ = site.CreateAutomaticFrontmatter(contentTemplate, page); } } - ParseParams(page, typeof(Frontmatter), frontmatter, yamlDictionary); + ParseParams(page, typeof(Frontmatter), yaml, yamlDictionary); } return page; } /// - public Site ParseSiteSettings(string content) + public Site ParseSiteSettings(string yaml) { - var settings = yamlDeserializerRigid.Deserialize(content); - ParseParams(settings, typeof(Site), content); + var settings = yamlDeserializerRigid.Deserialize(yaml); + ParseParams(settings, typeof(Site), yaml); return settings; } @@ -119,9 +134,9 @@ public partial class YAMLParser : IFrontmatterParser /// /// Site or Frontmatter object, that implements IParams /// The type (Site or Frontmatter) - /// YAML content + /// YAML content /// yamlObject already parsed if available - public void ParseParams(IParams settings, Type type, string content, object? yamlObject = null) + public void ParseParams(IParams settings, Type type, string yaml, object? yamlObject = null) { if (settings is null) { @@ -132,7 +147,7 @@ public partial class YAMLParser : IFrontmatterParser throw new ArgumentNullException(nameof(type)); } - yamlObject ??= yamlDeserializer.Deserialize(new StringReader(content)); + yamlObject ??= yamlDeserializer.Deserialize(new StringReader(yaml)); if (yamlObject is Dictionary yamlDictionary) { foreach (var key in yamlDictionary.Keys.Cast()) -- GitLab From 696364b1142ed3ce2d0085ad94cecdadc2a80ec8 Mon Sep 17 00:00:00 2001 From: Bruno Massa Date: Fri, 30 Jun 2023 20:45:05 -0300 Subject: [PATCH 4/6] test: better YAMLParserTesting --- test/Models/SiteTests.cs | 7 +- test/Parser/YAMLParserTests.cs | 181 +++++++++++++++++++++++++-------- 2 files changed, 139 insertions(+), 49 deletions(-) diff --git a/test/Models/SiteTests.cs b/test/Models/SiteTests.cs index f38bb9c..da1f012 100644 --- a/test/Models/SiteTests.cs +++ b/test/Models/SiteTests.cs @@ -32,14 +32,13 @@ public class SiteTests [InlineData("test02.md")] public void Test_ScanAllMarkdownFiles(string fileName) { - var fullPath = Path.GetFullPath(Path.Combine(testSitesPath, testSite1Path)); + var siteFullPath = Path.GetFullPath(Path.Combine(testSitesPath, testSite1Path)); // Act - var ContentPaths = FileUtils.GetAllMarkdownFiles(fullPath); - var filePath = Path.Combine(fullPath, "content", fileName); + var ContentPaths = FileUtils.GetAllMarkdownFiles(Path.Combine(siteFullPath, "content")); // Assert - Assert.Contains(ContentPaths, rp => rp == filePath); + Assert.Contains(ContentPaths, rp => rp == fileName); } [Theory] diff --git a/test/Parser/YAMLParserTests.cs b/test/Parser/YAMLParserTests.cs index 02166d6..6815017 100644 --- a/test/Parser/YAMLParserTests.cs +++ b/test/Parser/YAMLParserTests.cs @@ -11,26 +11,49 @@ public class YAMLParserTests { private readonly YAMLParser parser; private readonly Mock site; - private readonly string fileContent = @"--- -Title: Test Title + + private readonly string pageFrontmater = @"Title: Test Title Type: post Date: 2023-07-01 LastMod: 2023-06-01 PublishDate: 2023-06-01 ExpiryDate: 2024-06-01 -Categories: ['Test', 'Real Data'] -Tags: ['Test', 'Real Data'] ---- - +Tags: + - Test + - Real Data +Categories: + - Test + - Real Data +NestedData: + Level2: + - Test + - Real Data +customParam: Custom Value +"; + private readonly string pageMarkdown = @" # Real Data Test This is a test using real data. Real Data Test "; + private readonly string siteContent = @" +Title: My Site +BaseUrl: https://www.example.com/ +customParam: Custom Value +NestedData: + Level2: + - Test + - Real Data +"; + private readonly string pageContent; public YAMLParserTests() { parser = new YAMLParser(); site = new Mock(); + pageContent = @$"--- +{pageFrontmater} +--- +{pageMarkdown}"; } [Fact] @@ -59,7 +82,7 @@ Date: 2023-04-01 var filePath = "test.md"; // Act - var frontmatter = parser.ParseFrontmatter(site.Object, filePath, fileContent); + var frontmatter = parser.ParseFrontmatterAndMarkdown(site.Object, filePath, fileContent); // Asset Assert.Equal(expectedTitle, frontmatter?.Title); @@ -68,13 +91,8 @@ Date: 2023-04-01 [Fact] public void ParseFrontmatter_ShouldThrowException_WhenSiteIsNull() { - var fileContent = @"--- -Title: Test Title ---- -"; - // Asset - Assert.Throws(() => parser.ParseFrontmatter(null!, "test.md", fileContent)); + Assert.Throws(() => parser.ParseFrontmatterAndMarkdown(null!, "test.md", pageContent)); } [Theory] @@ -92,7 +110,7 @@ Date: 2023/01/01 var expectedDate = DateTime.Parse(expectedDateString, CultureInfo.InvariantCulture); // Act - var frontmatter = parser.ParseFrontmatter(site.Object, filePath, fileContent); + var frontmatter = parser.ParseFrontmatterAndMarkdown(site.Object, filePath, fileContent); // Asset Assert.Equal(expectedDate, frontmatter?.Date); @@ -102,22 +120,13 @@ Date: 2023/01/01 public void ParseFrontmatter_ShouldParseOtherFieldsCorrectly() { var filePath = "test.md"; - var fileContent = @"--- -Title: Test Title -Type: post -Date: 2023-01-01 -LastMod: 2023-06-01 -PublishDate: 2023-06-01 -ExpiryDate: 2024-06-01 ---- -"; - var expectedDate = DateTime.Parse("2023-01-01", CultureInfo.InvariantCulture); + var expectedDate = DateTime.Parse("2023-07-01", CultureInfo.InvariantCulture); var expectedLastMod = DateTime.Parse("2023-06-01", CultureInfo.InvariantCulture); var expectedPublishDate = DateTime.Parse("2023-06-01", CultureInfo.InvariantCulture); var expectedExpiryDate = DateTime.Parse("2024-06-01", CultureInfo.InvariantCulture); // Act - var frontmatter = parser.ParseFrontmatter(site.Object, filePath, fileContent); + var frontmatter = parser.ParseFrontmatterAndMarkdown(site.Object, filePath, pageContent); // Asset Assert.Equal("Test Title", frontmatter?.Title); @@ -138,17 +147,12 @@ Title var filePath = "test.md"; // Asset - Assert.Throws(() => parser.ParseFrontmatter(site.Object, filePath, fileContent)); + Assert.Throws(() => parser.ParseFrontmatterAndMarkdown(site.Object, filePath, fileContent)); } [Fact] public void ParseSiteSettings_ShouldReturnSiteWithCorrectSettings() { - var siteContent = @" -BaseUrl: https://www.example.com/ -Title: My Site -"; - // Act var siteSettings = parser.ParseSiteSettings(siteContent); @@ -160,26 +164,21 @@ Title: My Site [Fact] public void ParseParams_ShouldFillParamsWithNonMatchingFields() { - var settings = new Frontmatter("Test Title", "/test.md", site.Object); - var content = @" -Title: Test Title -customParam: Custom Value -"; + var page = new Frontmatter("Test Title", "/test.md", site.Object); // Act - parser.ParseParams(settings, typeof(Frontmatter), content); + parser.ParseParams(page, typeof(Frontmatter), pageFrontmater); // Asset - Assert.True(settings.Params.ContainsKey("customParam")); - Assert.Equal("Custom Value", settings.Params["customParam"]); + Assert.True(page.Params.ContainsKey("customParam")); + Assert.Equal("Custom Value", page.Params["customParam"]); } - [Fact] public void ParseFrontmatter_ShouldParseContentInSiteFolder() { var date = DateTime.Parse("2023-07-01", CultureInfo.InvariantCulture); - var frontmatter = parser.ParseFrontmatter(site.Object, "", fileContent); + var frontmatter = parser.ParseFrontmatterAndMarkdown(site.Object, "", pageContent); // Act site.Object.PostProcessFrontMatter(frontmatter!); @@ -189,14 +188,106 @@ customParam: Custom Value } [Fact] - public void ParseFrontmatter_ShouldParseCategoriesCorrectly() + public void ParseFrontmatter_ShouldCreateTags() { - var frontmatter = parser.ParseFrontmatter(site.Object, "", fileContent); + var frontmatter = parser.ParseFrontmatterAndMarkdown(site.Object, "", pageContent); - // Act - site.Object.PostProcessFrontMatter(frontmatter!); + // Asset + Assert.Equal(2, frontmatter!.Tags?.Count); + } + + [Fact] + public void ParseFrontmatter_ShouldParseCategoriesCorrectly() + { + var frontmatter = parser.ParseFrontmatterAndMarkdown(site.Object, "fakeFilePath", pageContent); // Asset Assert.Equal(new[] { "Test", "Real Data" }, frontmatter?.Params["Categories"]); + Assert.Equal(new[] { "Test", "Real Data" }, (frontmatter?.Params["NestedData"] as Dictionary)?["Level2"]); + Assert.Equal("Test", ((frontmatter?.Params["NestedData"] as Dictionary)?["Level2"] as List)?[0]); + } + + [Fact] + public void ParseFrontmatter_ShouldThrowExceptionWhenSiteIsNull() + { + Assert.Throws(() => parser.ParseFrontmatterAndMarkdownFromFile(null!, "fakeFilePath")); + } + + [Fact] + public void ParseFrontmatter_ShouldThrowExceptionWhenFilePathIsNull() + { + Assert.Throws(() => parser.ParseFrontmatterAndMarkdownFromFile(site.Object, null!)); + } + + [Fact] + public void ParseFrontmatter_ShouldThrowExceptionWhenFilePathDoesNotExist() + { + Assert.Throws(() => parser.ParseFrontmatterAndMarkdownFromFile(site.Object, "fakePath")); + } + + [Fact] + public void ParseFrontmatter_ShouldThrowExceptionWhenFilePathDoesNotExist2() + { + Assert.Throws(() => parser.ParseFrontmatterAndMarkdown(site.Object, null!, "fakeContent")); + } + + [Fact] + public void ParseFrontmatter_ShouldHandleEmptyFileContent() + { + Assert.Throws(() => parser.ParseFrontmatterAndMarkdown(site.Object, "fakeFilePath", "")); + } + + [Fact] + public void ParseYAML_ShouldThrowExceptionWhenFrontmatterIsInvalid() + { + Assert.Throws(() => parser.ParseFrontmatterAndMarkdown(site.Object, "fakeFilePath", "invalidFrontmatter")); + } + + [Fact] + public void ParseSiteSettings_ShouldReturnSiteSettings() + { + var site = parser.ParseSiteSettings(siteContent); + Assert.NotNull(site); + Assert.Equal("My Site", site.Title); + Assert.Equal("https://www.example.com/", site.BaseUrl); + } + + [Fact] + public void ParseSiteSettings_ShouldReturnContent() + { + var frontmatter = parser.ParseFrontmatterAndMarkdown(site.Object, "fakeFilePath", pageContent); + + Assert.Equal(pageMarkdown, frontmatter?.RawContent); + } + + [Fact] + public void SiteParams_ShouldThrowExceptionWhenSettingsIsNull() + { + Assert.Throws(() => parser.ParseParams(null!, typeof(Site), siteContent)); + } + + [Fact] + public void SiteParams_ShouldThrowExceptionWhenTypeIsNull() + { + var site = new Site(); + Assert.Throws(() => parser.ParseParams(site, null!, siteContent)); + } + + [Fact] + public void SiteParams_ShouldHandleEmptyContent() + { + parser.ParseParams(site.Object, typeof(Site), string.Empty); + Assert.Empty(site.Object.Params); + } + + [Fact] + public void SiteParams_ShouldPopulateParamsWithExtraFields() + { + parser.ParseParams(site.Object, typeof(Site), siteContent); + Assert.NotEmpty(site.Object.Params); + Assert.True(site.Object.Params.ContainsKey("customParam")); + Assert.Equal("Custom Value", site.Object.Params["customParam"]); + Assert.Equal(new[] { "Test", "Real Data" }, ((Dictionary)site.Object.Params["NestedData"])["Level2"]); + Assert.Equal("Test", ((site.Object?.Params["NestedData"] as Dictionary)?["Level2"] as List)?[0]); } } -- GitLab From e8ec02ea26d25ffbddc91f5857585b8a4587f332 Mon Sep 17 00:00:00 2001 From: Bruno Massa Date: Fri, 30 Jun 2023 20:45:37 -0300 Subject: [PATCH 5/6] chore: minor logging improvements --- source/BuildCommand.cs | 2 +- source/ServeCommand.cs | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/source/BuildCommand.cs b/source/BuildCommand.cs index 5776640..8f6c16e 100644 --- a/source/BuildCommand.cs +++ b/source/BuildCommand.cs @@ -65,7 +65,7 @@ public class BuildCommand : BaseGeneratorCommand File.WriteAllText(outputAbsolutePath, result); // Log - logger.Debug("Page created: {Permalink}", frontmatter.Permalink); + logger.Debug("Page created: {Permalink}", outputAbsolutePath); // Use interlocked to safely increment the counter in a multi-threaded environment _ = Interlocked.Increment(ref pagesCreated); diff --git a/source/ServeCommand.cs b/source/ServeCommand.cs index 3d97d0b..42e24df 100644 --- a/source/ServeCommand.cs +++ b/source/ServeCommand.cs @@ -207,36 +207,42 @@ public class ServeCommand : BaseGeneratorCommand, IDisposable var fileAbsolutePath = Path.Combine(site.SourceStaticPath, requestPath.TrimStart('/')); var fileAbsoluteThemePath = Path.Combine(site.SourceThemeStaticPath, requestPath.TrimStart('/')); - logger.Debug("Request received for {RequestPath}", requestPath); + string? resultType; // Return the server startup timestamp as the response if (requestPath == "/ping") { + resultType = "ping"; await HandlePingRequest(context); } // Check if it is one of the Static files (serve the actual file) else if (File.Exists(fileAbsolutePath)) { + resultType = "static"; await HandleStaticFileRequest(context, fileAbsolutePath); } // Check if it is one of the Static files (serve the actual file) else if (File.Exists(fileAbsoluteThemePath)) { + resultType = "themestatic"; await HandleStaticFileRequest(context, fileAbsoluteThemePath); } // Check if the requested file path corresponds to a registered page else if (site.PagesDict.TryGetValue(requestPath, out var frontmatter)) { + resultType = "dict"; await HandleRegisteredPageRequest(context, frontmatter); } else { + resultType = "404"; await HandleNotFoundRequest(context); } + logger.Debug("Request {type}\tfor {RequestPath}", resultType, requestPath); } private Task HandlePingRequest(HttpContext context) -- GitLab From 61ff9fbd6f7c2a08a32c3e316ce33fcfb513d9f7 Mon Sep 17 00:00:00 2001 From: Bruno Massa Date: Fri, 30 Jun 2023 20:52:42 -0300 Subject: [PATCH 6/6] refactor: simplify the markdown search and parsing --- source/Helpers/FileUtils.cs | 13 ++++--------- source/Parser/YAMLParser.cs | 14 ++++++++------ test/Models/SiteTests.cs | 3 ++- 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/source/Helpers/FileUtils.cs b/source/Helpers/FileUtils.cs index 3425713..d69f404 100644 --- a/source/Helpers/FileUtils.cs +++ b/source/Helpers/FileUtils.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using SuCoS.Models; namespace SuCoS.Helper; @@ -19,21 +20,15 @@ public static class FileUtils public static List GetAllMarkdownFiles(string directory, string? basePath = null) { basePath ??= directory; - var markdownFiles = new List(); - - var files = Directory.GetFiles(directory, "*.md"); - foreach (var filePath in files) - { - markdownFiles.Add(Path.GetRelativePath(basePath, filePath)); - } + var files = Directory.GetFiles(directory, "*.md").ToList(); var subdirectories = Directory.GetDirectories(directory); foreach (var subdirectory in subdirectories) { - markdownFiles.AddRange(GetAllMarkdownFiles(subdirectory, basePath)); + files.AddRange(GetAllMarkdownFiles(subdirectory, basePath)); } - return markdownFiles; + return files; } /// diff --git a/source/Parser/YAMLParser.cs b/source/Parser/YAMLParser.cs index b011adb..897c271 100644 --- a/source/Parser/YAMLParser.cs +++ b/source/Parser/YAMLParser.cs @@ -41,28 +41,30 @@ public partial class YAMLParser : IFrontmatterParser } string? fileContent; + string? fileRelativePath; try { - fileContent = File.ReadAllText(Path.GetFullPath(Path.Combine(sourceContentPath ?? string.Empty, filePath))); + fileContent = File.ReadAllText(filePath); + fileRelativePath = Path.GetRelativePath(sourceContentPath ?? string.Empty, filePath); } catch (Exception ex) { throw new FileNotFoundException(filePath, ex); } - return ParseFrontmatterAndMarkdown(site, filePath, fileContent); + return ParseFrontmatterAndMarkdown(site, fileRelativePath, fileContent); } /// - public Frontmatter? ParseFrontmatterAndMarkdown(Site site, in string filePath, in string fileContent) + public Frontmatter? ParseFrontmatterAndMarkdown(Site site, in string fileRelativePath, in string fileContent) { if (site is null) { throw new ArgumentNullException(nameof(site)); } - if (filePath is null) + if (fileRelativePath is null) { - throw new ArgumentNullException(nameof(filePath)); + throw new ArgumentNullException(nameof(fileRelativePath)); } using var content = new StringReader(fileContent); @@ -80,7 +82,7 @@ public partial class YAMLParser : IFrontmatterParser var rawContent = content.ReadToEnd(); // Now, you can parse the YAML frontmatter - var page = ParseYAML(ref site, filePath, yaml, rawContent); + var page = ParseYAML(ref site, fileRelativePath, yaml, rawContent); return page; } diff --git a/test/Models/SiteTests.cs b/test/Models/SiteTests.cs index da1f012..402b278 100644 --- a/test/Models/SiteTests.cs +++ b/test/Models/SiteTests.cs @@ -36,9 +36,10 @@ public class SiteTests // Act var ContentPaths = FileUtils.GetAllMarkdownFiles(Path.Combine(siteFullPath, "content")); + var fileFullPath = Path.Combine(siteFullPath, "content", fileName); // Assert - Assert.Contains(ContentPaths, rp => rp == fileName); + Assert.Contains(ContentPaths, rp => rp == fileFullPath); } [Theory] -- GitLab