From 3dab6c47830716d2e5c551cfd76fb6de6e396d0f Mon Sep 17 00:00:00 2001 From: Bruno Massa Date: Mon, 3 Jul 2023 21:30:36 -0300 Subject: [PATCH 1/2] feat: scan and parse in the same step --- source/Helpers/FileUtils.cs | 20 ------------ source/Helpers/SiteHelper.cs | 8 ++--- source/Models/Site.cs | 60 ++++++++++++++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 24 deletions(-) diff --git a/source/Helpers/FileUtils.cs b/source/Helpers/FileUtils.cs index 3fbf30f..963b662 100644 --- a/source/Helpers/FileUtils.cs +++ b/source/Helpers/FileUtils.cs @@ -11,26 +11,6 @@ namespace SuCoS.Helpers; /// public static class FileUtils { - /// - /// Gets all Markdown files in the specified directory. - /// - /// The directory path. - /// The initial directory path. - /// The list of Markdown file paths. - public static IEnumerable GetAllMarkdownFiles(string directory, string? basePath = null) - { - basePath ??= directory; - var files = Directory.GetFiles(directory, "*.md").ToList(); - - var subdirectories = Directory.GetDirectories(directory); - foreach (var subdirectory in subdirectories) - { - files.AddRange(GetAllMarkdownFiles(subdirectory, basePath)); - } - - return files; - } - /// /// Gets the content of a template file based on the frontmatter and the theme path. /// diff --git a/source/Helpers/SiteHelper.cs b/source/Helpers/SiteHelper.cs index 0207a5f..4c33c18 100644 --- a/source/Helpers/SiteHelper.cs +++ b/source/Helpers/SiteHelper.cs @@ -36,11 +36,11 @@ public static class SiteHelper site.ResetCache(); - // Scan content files - var markdownFiles = FileUtils.GetAllMarkdownFiles(site.SourceContentPath); - site.ContentPaths.AddRange(markdownFiles); + stopwatch.Start("Parse"); - site.ParseSourceFiles(stopwatch); + var filesParsed = site.ParseAndScanSourceFiles(site.SourceContentPath); + + stopwatch.Stop("Parse", filesParsed); site.TemplateOptions.FileProvider = new PhysicalFileProvider(Path.GetFullPath(site.SourceThemePath)); diff --git a/source/Models/Site.cs b/source/Models/Site.cs index a3159fb..57de5bf 100644 --- a/source/Models/Site.cs +++ b/source/Models/Site.cs @@ -229,6 +229,54 @@ public class Site : IParams IgnoreCacheBefore = DateTime.Now; } + public int ParseAndScanSourceFiles(string? directory) + { + // Process the source files, extracting the frontmatter + var filesParsed = 0; // counter to keep track of the number of files processed + directory ??= SourceContentPath; + + var markdownFiles = Directory.GetFiles(directory, "*.md"); + + _ = Parallel.ForEach(markdownFiles, filePath => + { + try + { + var frontmatter = frontmatterParser.ParseFrontmatterAndMarkdownFromFile(this, filePath, SourceContentPath) + ?? throw new FormatException($"Error parsing frontmatter for {filePath}"); + + if (frontmatter.IsValidDate(options)) + { + PostProcessFrontMatter(frontmatter, true); + } + } + catch (Exception ex) + { + Logger?.Error(ex, "Error parsing file {file}", filePath); + } + + // Use interlocked to safely increment the counter in a multi-threaded environment + _ = Interlocked.Increment(ref filesParsed); + }); + + Logger.Error("{asdf}", filesParsed); + + // If the home page is not yet created, create it! + if (!PagesDict.TryGetValue("/", out var home)) + { + home = CreateIndexPage(string.Empty); + } + Home = home; + home.Kind = Kind.index; + + var subdirectories = Directory.GetDirectories(directory); + foreach (var subdirectory in subdirectories) + { + filesParsed += ParseAndScanSourceFiles(subdirectory); + } + + return filesParsed; + } + /// /// Create a page not from the content folder, but as part of the process. /// It's used to create tag pages, section list pages, etc. @@ -333,6 +381,18 @@ public class Site : IParams stopwatch.Stop("Parse", filesParsed); } + public void GetAllMarkdownFiles(string? directory) + { + directory ??= SourceContentPath; + ContentPaths.AddRange(Directory.GetFiles(directory, "*.md").ToList()); + + var subdirectories = Directory.GetDirectories(directory); + foreach (var subdirectory in subdirectories) + { + GetAllMarkdownFiles(subdirectory); + } + } + /// /// Creates the frontmatter for the index page. /// -- GitLab From c362cd582771b6c53754b65c8eefb1b9ff49528a Mon Sep 17 00:00:00 2001 From: Bruno Massa Date: Wed, 5 Jul 2023 15:56:13 -0300 Subject: [PATCH 2/2] feat: page parent --- source/Helpers/SiteHelper.cs | 6 +- source/Models/Frontmatter.cs | 31 ++++- source/Models/Site.cs | 197 +++++++++++++------------------- test/Models/FrontmatterTests.cs | 6 +- test/Models/SiteTests.cs | 7 +- 5 files changed, 118 insertions(+), 129 deletions(-) diff --git a/source/Helpers/SiteHelper.cs b/source/Helpers/SiteHelper.cs index 4c33c18..5dcc818 100644 --- a/source/Helpers/SiteHelper.cs +++ b/source/Helpers/SiteHelper.cs @@ -38,9 +38,9 @@ public static class SiteHelper stopwatch.Start("Parse"); - var filesParsed = site.ParseAndScanSourceFiles(site.SourceContentPath); + site.ParseAndScanSourceFiles(site.SourceContentPath); - stopwatch.Stop("Parse", filesParsed); + stopwatch.Stop("Parse", site.filesParsedToReport); site.TemplateOptions.FileProvider = new PhysicalFileProvider(Path.GetFullPath(site.SourceThemePath)); @@ -109,7 +109,7 @@ public static class SiteHelper site.Logger = logger; site.options = options; site.SourceDirectoryPath = options.Source; - site.OutputPath = options.Output; + site.OutputPath = options.Output!; // Liquid template options, needed to theme the content // but also parse URLs diff --git a/source/Models/Frontmatter.cs b/source/Models/Frontmatter.cs index 621f4c2..378e378 100644 --- a/source/Models/Frontmatter.cs +++ b/source/Models/Frontmatter.cs @@ -89,6 +89,12 @@ public class Frontmatter : IBaseContent, IParams [YamlIgnore] public string? SourcePathDirectory { get; set; } + /// + /// The source directory of the file. + /// + [YamlIgnore] + public string? SourcePathLastDirectory => Path.GetDirectoryName(SourcePathDirectory ?? string.Empty); + /// /// Point to the site configuration. /// @@ -120,6 +126,13 @@ public class Frontmatter : IBaseContent, IParams [YamlIgnore] public ConcurrentBag? PagesReferences { get; set; } + /// + /// Other content that mention this content. + /// Used to create the tags list and Related Posts section. + /// + [YamlIgnore] + public Frontmatter? Parent { get; set; } + /// /// A list of tags, if any. /// @@ -315,8 +328,22 @@ public class Frontmatter : IBaseContent, IParams URLforce ??= URL ?? (isIndex - ? "{{ page.SourcePathDirectory }}" - : @"{{ page.SourcePathDirectory }}/{%- liquid + ? @"{%- liquid +if page.Parent +echo page.Parent.Permalink +echo '/' +endif +if page.Title != '' +echo page.Title +else +echo page.SourcePathLastDirectory +endif +-%}" + : @"{%- liquid +if page.Parent +echo page.Parent.Permalink +echo '/' +endif if page.Title != '' echo page.Title else diff --git a/source/Models/Site.cs b/source/Models/Site.cs index 57de5bf..da1b576 100644 --- a/source/Models/Site.cs +++ b/source/Models/Site.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Globalization; using System.IO; using System.Linq; using System.Threading; @@ -7,7 +8,6 @@ using System.Threading.Tasks; using Fluid; using Markdig; using Serilog; -using SuCoS.Helpers; using SuCoS.Parser; using YamlDotNet.Serialization; @@ -119,12 +119,6 @@ public class Site : IParams /// public Frontmatter? Home { get; private set; } - /// - /// List of all content to be scanned and processed. - /// - [YamlIgnore] - public List ContentPaths { get; } = new(); - /// /// Command line options /// @@ -189,6 +183,11 @@ public class Site : IParams private List? regularPagesCache; + /// + /// Number of files parsed, used in the report. + /// + public int filesParsedToReport; + /// /// Markdig 20+ built-in extensions /// @@ -229,68 +228,99 @@ public class Site : IParams IgnoreCacheBefore = DateTime.Now; } - public int ParseAndScanSourceFiles(string? directory) + /// + /// Search recursively for all markdown files in the content folder, then + /// parse their content for front matter meta data and markdown. + /// + /// Folder to scan + /// Folder recursive level + /// Page of the upper directory + /// + public void ParseAndScanSourceFiles(string directory, int level = 0, Frontmatter? pageParent = null) { - // Process the source files, extracting the frontmatter - var filesParsed = 0; // counter to keep track of the number of files processed directory ??= SourceContentPath; var markdownFiles = Directory.GetFiles(directory, "*.md"); - _ = Parallel.ForEach(markdownFiles, filePath => + var indexPath = markdownFiles.FirstOrDefault(file => Path.GetFileName(file).ToUpperInvariant() == "INDEX.MD"); + if (indexPath != null) { - try + markdownFiles = markdownFiles.Where(file => file != indexPath).ToArray(); + var frontmatter = ParseSourceFile(pageParent, indexPath); + if (level == 0) { - var frontmatter = frontmatterParser.ParseFrontmatterAndMarkdownFromFile(this, filePath, SourceContentPath) - ?? throw new FormatException($"Error parsing frontmatter for {filePath}"); - - if (frontmatter.IsValidDate(options)) - { - PostProcessFrontMatter(frontmatter, true); - } + Home = frontmatter; + frontmatter!.Permalink = "/"; + PagesDict.Remove(frontmatter.Permalink); + PagesDict.Add(frontmatter.Permalink, frontmatter); } - catch (Exception ex) + else { - Logger?.Error(ex, "Error parsing file {file}", filePath); + pageParent = frontmatter; } + } + else if (level == 0) + { + Home = CreateIndexPage(string.Empty); + } + else if (level == 1) + { + var section = directory; + var contentTemplate = new BasicContent( + title: section, + section: "section", + type: "section", + url: section + ); + pageParent = CreateAutomaticFrontmatter(contentTemplate, null); + } - // Use interlocked to safely increment the counter in a multi-threaded environment - _ = Interlocked.Increment(ref filesParsed); + _ = Parallel.ForEach(markdownFiles, filePath => + { + ParseSourceFile(pageParent, filePath); }); - Logger.Error("{asdf}", filesParsed); - - // If the home page is not yet created, create it! - if (!PagesDict.TryGetValue("/", out var home)) + var subdirectories = Directory.GetDirectories(directory); + foreach (var subdirectory in subdirectories) { - home = CreateIndexPage(string.Empty); + ParseAndScanSourceFiles(subdirectory, level + 1, pageParent); } - Home = home; - home.Kind = Kind.index; + } - var subdirectories = Directory.GetDirectories(directory); - foreach (var subdirectory in subdirectories) + private Frontmatter? ParseSourceFile(Frontmatter? pageParent, string filePath) + { + Frontmatter? frontmatter = null; + try { - filesParsed += ParseAndScanSourceFiles(subdirectory); + frontmatter = frontmatterParser.ParseFrontmatterAndMarkdownFromFile(this, filePath, SourceContentPath) + ?? throw new FormatException($"Error parsing frontmatter for {filePath}"); + + if (frontmatter.IsValidDate(options)) + { + PostProcessFrontMatter(frontmatter, pageParent, true); + } } + catch (Exception ex) + { + Logger?.Error(ex, "Error parsing file {file}", filePath); + } + + // Use interlocked to safely increment the counter in a multi-threaded environment + _ = Interlocked.Increment(ref filesParsedToReport); - return filesParsed; + return frontmatter; } /// /// Create a page not from the content folder, but as part of the process. /// It's used to create tag pages, section list pages, etc. /// - public Frontmatter CreateAutomaticFrontmatter(BasicContent baseContent, Frontmatter originalFrontmatter) + public Frontmatter CreateAutomaticFrontmatter(BasicContent baseContent, Frontmatter? originalFrontmatter) { if (baseContent is null) { throw new ArgumentNullException(nameof(baseContent)); } - if (originalFrontmatter is null) - { - throw new ArgumentNullException(nameof(originalFrontmatter)); - } var id = baseContent.URL; Frontmatter? frontmatter; @@ -318,15 +348,17 @@ public class Site : IParams } } - if (frontmatter.Kind != Kind.index && originalFrontmatter.Permalink is not null) + if (frontmatter.Kind != Kind.index && originalFrontmatter?.Permalink is not null) { frontmatter.PagesReferences!.Add(originalFrontmatter.Permalink!); } + // TODO: still too hardcoded - if (frontmatter.Type != "tags") + if (frontmatter.Type != "tags" || originalFrontmatter is null) + { return frontmatter; - + } lock (originalFrontmatter) { originalFrontmatter.Tags ??= new(); @@ -335,64 +367,6 @@ public class Site : IParams return frontmatter; } - /// - /// Parses the source files and extracts the frontmatter. - /// - public void ParseSourceFiles(StopwatchReporter stopwatch) - { - if (stopwatch is null) - { - throw new ArgumentNullException(nameof(stopwatch)); - } - - stopwatch.Start("Parse"); - - // Process the source files, extracting the frontmatter - var filesParsed = 0; // counter to keep track of the number of files processed - _ = Parallel.ForEach(ContentPaths, filePath => - { - try - { - var frontmatter = frontmatterParser.ParseFrontmatterAndMarkdownFromFile(this, filePath, SourceContentPath) - ?? throw new FormatException($"Error parsing frontmatter for {filePath}"); - - if (frontmatter.IsValidDate(options)) - { - PostProcessFrontMatter(frontmatter, true); - } - } - catch (Exception ex) - { - Logger?.Error(ex, "Error parsing file {file}", filePath); - } - - // Use interlocked to safely increment the counter in a multi-threaded environment - _ = Interlocked.Increment(ref filesParsed); - }); - - // If the home page is not yet created, create it! - if (!PagesDict.TryGetValue("/", out var home)) - { - home = CreateIndexPage(string.Empty); - } - Home = home; - home.Kind = Kind.index; - - stopwatch.Stop("Parse", filesParsed); - } - - public void GetAllMarkdownFiles(string? directory) - { - directory ??= SourceContentPath; - ContentPaths.AddRange(Directory.GetFiles(directory, "*.md").ToList()); - - var subdirectories = Directory.GetDirectories(directory); - foreach (var subdirectory in subdirectories) - { - GetAllMarkdownFiles(subdirectory); - } - } - /// /// Creates the frontmatter for the index page. /// @@ -419,15 +393,17 @@ public class Site : IParams /// /// Extra calculation and automatic data for each frontmatter. /// - /// + /// The given page to be processed + /// The parent page, if any /// - public void PostProcessFrontMatter(Frontmatter frontmatter, bool overwrite = false) + public void PostProcessFrontMatter(Frontmatter frontmatter, Frontmatter? pageParent = null, bool overwrite = false) { if (frontmatter is null) { throw new ArgumentNullException(nameof(frontmatter)); } + frontmatter.Parent = pageParent; frontmatter.Permalink = frontmatter.CreatePermalink(); lock (syncLockPostProcess) { @@ -458,20 +434,5 @@ public class Site : IParams } } } - - // Create a section page when due - if (frontmatter.Type == "section" - || string.IsNullOrEmpty(frontmatter.Permalink) - || string.IsNullOrEmpty(frontmatter.Section)) - { - return; - } - var contentTemplate = new BasicContent( - title: frontmatter.Section, - section: "section", - type: "section", - url: frontmatter.Section - ); - CreateAutomaticFrontmatter(contentTemplate, frontmatter); } -} +} \ No newline at end of file diff --git a/test/Models/FrontmatterTests.cs b/test/Models/FrontmatterTests.cs index d94e08b..b6dcb78 100644 --- a/test/Models/FrontmatterTests.cs +++ b/test/Models/FrontmatterTests.cs @@ -135,8 +135,8 @@ public class FrontmatterTests } [Theory] - [InlineData("/test/path", "/test/path/test-title")] - [InlineData("/another/path", "/another/path/test-title")] + [InlineData("/test/path", "/test-title")] + [InlineData("/another/path", "/test-title")] public void CreatePermalink_ShouldReturnCorrectUrl_WhenUrlIsNull(string sourcePathDirectory, string expectedUrl) { var frontmatter = new Frontmatter(titleCONST, sourcePathCONST, site) @@ -149,7 +149,7 @@ public class FrontmatterTests } [Theory] - [InlineData(null, "/path/to/test-title")] + [InlineData(null, "/test-title")] [InlineData("{{ page.Title }}/{{ page.SourceFileNameWithoutExtension }}", "/test-title/file")] public void Permalink_CreateWithDefaultOrCustomURLTemplate(string urlTemplate, string expectedPermalink) { diff --git a/test/Models/SiteTests.cs b/test/Models/SiteTests.cs index da57db3..f5364b8 100644 --- a/test/Models/SiteTests.cs +++ b/test/Models/SiteTests.cs @@ -31,14 +31,15 @@ public class SiteTests [InlineData("test02.md")] public void Test_ScanAllMarkdownFiles(string fileName) { + var fileNameWithoutExtension = Path.GetFileNameWithoutExtension(fileName); var siteFullPath = Path.GetFullPath(Path.Combine(testSitesPath, testSite1PathCONST)); // Act - var ContentPaths = FileUtils.GetAllMarkdownFiles(Path.Combine(siteFullPath, "content")); - var fileFullPath = Path.Combine(siteFullPath, "content", fileName); + site.ParseAndScanSourceFiles(siteFullPath); // Assert - Assert.Contains(ContentPaths, rp => rp == fileFullPath); + Assert.Contains(site.Pages, page => page.SourcePathDirectory?.Length == 0); + Assert.Contains(site.Pages, page => page.SourceFileNameWithoutExtension == fileNameWithoutExtension); } [Theory] -- GitLab