Add support for Git external sources

src/DebianTarballBuilder.cs

@@ -1,18 +1,18 @@
 using System.Diagnostics.CodeAnalysis;
+using Flamenco.ExternalSources;
 
 namespace Flamenco;
 
 public class DebianTarballBuilder
 {
     public required BuildTarget BuildTarget { get; set; }
     public required SourceDirectoryInfo SourceDirectory { get; set; }
-
     public required DirectoryInfo DestinationDirectory { get; set; }
-    
+
     public async Task<bool> BuildDebianDirectoryAsync(CancellationToken cancellationToken = default)
     {
         return await ProcessDirectoryAsync(
-            sourceDirectory: SourceDirectory.DirectoryInfo, 
+            sourceDirectory: SourceDirectory.DirectoryInfo,
             destinationDirectory: DestinationDirectory,
             cancellationToken);
     }
@@ -23,27 +23,27 @@ private async Task<bool> ProcessDirectoryAsync(
         CancellationToken cancellationToken = default)
     {
         // TODO: rename this method. "process" is a term that is to generic.
-        
+
         bool errorDetected = false;
-        var destinationFiles = new Dictionary<string, (FileInfo Info, int BuildTargetSpecificity)>();
-        
+        var destinationFiles = new Dictionary<string, (FileInfo Info, int BuildTargetSpecificity, bool IsFlamencoFile)>();
+
         foreach (var file in sourceDirectory.EnumerateFiles())
         {
             cancellationToken.ThrowIfCancellationRequested();
-            
-            if (!TryInspectFileExtension(file, out var fileName, out var buildTargetSpecificity))
+
+            if (!TryInspectFileExtension(file, out var fileName, out var buildTargetSpecificity, out var isFlamencoFile))
             {
                 errorDetected = true;
                 continue;
             }
 
             if (buildTargetSpecificity < 0) continue;
-            
+
             if (destinationFiles.TryGetValue(fileName, out var otherFile))
             {
                 if (buildTargetSpecificity.Value > otherFile.BuildTargetSpecificity)
                 {
-                    destinationFiles[fileName] = (file, buildTargetSpecificity.Value);
+                    destinationFiles[fileName] = (file, buildTargetSpecificity.Value, isFlamencoFile);
                 }
                 else if (buildTargetSpecificity.Value == otherFile.BuildTargetSpecificity)
                 {
@@ -53,7 +53,7 @@ private async Task<bool> ProcessDirectoryAsync(
             }
             else
             {
-                destinationFiles[fileName] = (file, buildTargetSpecificity.Value);
+                destinationFiles[fileName] = (file, buildTargetSpecificity.Value, isFlamencoFile);
             }
         }
 
@@ -83,54 +83,96 @@ private async Task<bool> ProcessDirectoryAsync(
                 return false;
             }
         }
-        
-        foreach (var (fileName, (fileInfo, _)) in destinationFiles)
+
+        foreach (var (fileName, (fileInfo, _, isFlamencoFile)) in destinationFiles)
         {
             cancellationToken.ThrowIfCancellationRequested();
-
-            var destinationPath = Path.Combine(destinationDirectory.FullName, fileName);
 
-            try
+            if (isFlamencoFile)
             {
-                fileInfo.CopyTo(destinationPath);
+                switch (fileName)
+                {
+                    case ValidDescriptorFileNames.External:
+                        try
+                        {
+                            var externalSource = ExternalSourceBase.Create(fileInfo.FullName);
+                            await externalSource.Download(destinationDirectory.FullName);
+                            break;
+                        }
+                        catch (Exception exception)
+                        {
+                            Log.Error($"Failed to reference external source with {fileInfo.FullName}");
+                            Log.Debug($"Exception Message: {exception.Message}");
+                            return false;
+                        }
+                }
             }
-            catch (Exception exception)
+            else
             {
-                Log.Error($"Failed to create destination file '{destinationPath}'");
-                Log.Debug($"Exception Message: {exception.Message}");
-                return false;
+                var destinationPath = Path.Combine(destinationDirectory.FullName, fileName);
+
+                try
+                {
+                    fileInfo.CopyTo(destinationPath);
+                }
+                catch (Exception exception)
+                {
+                    Log.Error($"Failed to create destination file '{destinationPath}'");
+                    Log.Debug($"Exception Message: {exception.Message}");
+                    return false;
+                }
             }
         }
-        
+
         var tasks = new List<Task<bool>>();
         foreach (var sourceSubDirectory in sourceDirectory.EnumerateDirectories())
         {
             var destinationSubDirectoryPath = Path.Combine(destinationDirectory.FullName, sourceSubDirectory.Name);
-                
+
             tasks.Add(ProcessDirectoryAsync(
-                sourceDirectory: sourceSubDirectory, 
-                destinationDirectory: new DirectoryInfo(destinationSubDirectoryPath), 
+                sourceDirectory: sourceSubDirectory,
+                destinationDirectory: new DirectoryInfo(destinationSubDirectoryPath),
                 cancellationToken));
         }
 
         foreach (var result in await Task.WhenAll(tasks))
         {
             errorDetected = errorDetected && result;
         }
-        
+
         return !errorDetected;
     }
-
+
+    /// <summary>
+    /// Inspects the file name and determines which Flamenco parameters are set based on the file extensions.
+    /// </summary>
+    /// <param name="file">The file information.</param>
+    /// <param name="fileName">The final file name without any Flamenco parameters.</param>
+    /// <param name="buildTargetSpecificity">
+    /// A specificity score that determines how specific a file is to a source package or series.
+    /// A specificity of <c>-1</c> means that the file is not specific to current build target.
+    /// A specificity of <c>0</c> means that the file is not specific to a source package or series.
+    /// A specificity of <c>1</c> means that the file is specific to either a source package or a series.
+    /// A specificity of <c>2</c> means that the file is specific to both a source package and a series.
+    /// </param>
+    /// <param name="isFlamencoFile">
+    /// Flags whether the current file is a Flamenco internal file that does not get directly copied to the
+    /// destination directory, such as a Flamenco external link or orig file.
+    /// </param>
+    /// <returns></returns>
     private bool TryInspectFileExtension(
-        FileInfo file, 
+        FileInfo file,
         [NotNullWhen(returnValue: true)]
-        out string? fileName, 
+        out string? fileName,
         [NotNullWhen(returnValue: true)]
-        out int? buildTargetSpecificity)
+        out int? buildTargetSpecificity,
+        out bool isFlamencoFile)
     {
         string[] fileExtensions = file.Name.Split('.');
         // Remember, that fileExtensions[0] is just the file name!
 
+        isFlamencoFile = ValidDescriptorFileNames.All.Any(f => file.Name.Contains(f));
+
         if (fileExtensions.Length < 2)
         {
             fileName = file.Name;
@@ -141,15 +183,15 @@ private bool TryInspectFileExtension(
         bool matchesTarget = true;
         bool packageNameIsDefined = false;
         bool seriesNameIsDefined = false;
-        
+
         string extensionName;
         int totalExtensionLength = 0;
-        
+
         if (fileExtensions.Length >= 2)
         {
             extensionName = fileExtensions[^1];
             totalExtensionLength = extensionName.Length + 1;
-            
+
             if (SourceDirectory.BuildableTargets.PackageNames.Contains(extensionName))
             {
                 packageNameIsDefined = true;
@@ -167,7 +209,7 @@ private bool TryInspectFileExtension(
                 return true;
             }
         }
-        
+
         if (fileExtensions.Length >= 3)
         {
             extensionName = fileExtensions[^2];
@@ -177,7 +219,7 @@ private bool TryInspectFileExtension(
                 if (packageNameIsDefined)
                 {
                     Log.Error(message: $"The file extensions of '{file.FullName}' specifies two package names.");
-                    
+
                     fileName = null;
                     buildTargetSpecificity = null;
                     return false;
@@ -192,12 +234,12 @@ private bool TryInspectFileExtension(
                 if (seriesNameIsDefined)
                 {
                     Log.Error(message: $"The file extensions of '{file.FullName}' specifies two series names.");
-                    
+
                     fileName = null;
                     buildTargetSpecificity = null;
                     return false;
                 }
-                
+
                 Log.Warning(message: $"The file extensions of '{file.FullName}' has the format '*.SERIES.PACKAGE' instead of '*.PACKAGE.SERIES'.");
                 seriesNameIsDefined = true;
                 matchesTarget = matchesTarget && extensionName.Equals(BuildTarget.SeriesName);
@@ -219,7 +261,7 @@ private bool TryInspectFileExtension(
         {
             buildTargetSpecificity = 1;
         }
-        
+
         return true;
     }
-}
+}

src/ExternalSources/ExternalSourceBase.cs

@@ -0,0 +1,42 @@
+namespace Flamenco.ExternalSources;
+
+public abstract class ExternalSourceBase
+{
+    public abstract Task Download(string destinationDirectory);
+
+    public static ExternalSourceBase Create(string descriptorFilePath)
+    {
+        if (!File.Exists(descriptorFilePath))
+        {
+            throw new FileNotFoundException("Descriptor file not found.", descriptorFilePath);
+        }
+
+        var fileContentLines = File.ReadAllLines(descriptorFilePath);
+        var type = fileContentLines.FirstOrDefault(l => l.StartsWith("type"))?.Split("=").Last();
+
+        if (type is null)
+        {
+            throw new ApplicationException($"{descriptorFilePath} is missing a type.");
+        }
+
+        switch (type)
+        {
+            case "git":
+                var repositoryUrl = fileContentLines.FirstOrDefault(l => l.StartsWith("repo"))?.Split('=').Last();
+                var branch = fileContentLines.FirstOrDefault(l => l.StartsWith("branch"))?.Split('=').Last();
+                var tag = fileContentLines.FirstOrDefault(l => l.StartsWith("tag"))?.Split('=').Last();
+                var commit = fileContentLines.FirstOrDefault(l => l.StartsWith("commit"))?.Split('=').Last();
+
+                if (repositoryUrl is null)
+                {
+                    throw new ApplicationException($"Repository URL is required when descriptor file is git.");
+                }
+
+                // Reference precedence is commit, then tag, then branch. If all of them are null, the default
+                // behavior is to check out the repo's default branch.
+                return new GitExternalSource(new Uri(repositoryUrl), commit ?? (tag ?? branch));
+            default:
+                throw new ApplicationException($"Unknown external source type: {type}.");
+        }
+    }
+}

src/ExternalSources/GitExternalSource.cs

@@ -0,0 +1,22 @@
+using LibGit2Sharp;
+
+namespace Flamenco.ExternalSources;
+
+public class GitExternalSource(Uri repositoryUrl, string? reference = null) : ExternalSourceBase
+{
+    public Uri RepositoryUrl { get; } = repositoryUrl;
+    public string? Reference { get; } = reference;
+
+    public override async Task Download(string destinationDirectory)
+    {
+        await Task.Run(() => Repository.Clone(RepositoryUrl.ToString(), destinationDirectory));
+
+        if (Reference is not null)
+        {
+            using var repo = new Repository(destinationDirectory);
+            LibGit2Sharp.Commands.Checkout(repo, Reference);
+        }
+
+        Directory.Delete(Path.Join(destinationDirectory, ".git"), recursive: true);
+    }
+}

src/Flamenco.csproj

@@ -12,6 +12,7 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="LibGit2Sharp" Version="0.30.0" />
     <PackageReference Include="System.CommandLine" Version="2.0.0-beta4.22272.1" />
     <PackageReference Include="System.CommandLine.NamingConventionBinder" Version="2.0.0-beta4.22272.1" />
   </ItemGroup>

src/NamingConventions.cs

@@ -2,15 +2,25 @@
 
 namespace Flamenco;
 
+public static class ValidDescriptorFileNames
+{
+    public static IEnumerable<string> All =>
+    [
+        External
+    ];
+
+    public const string External = "flamenco.external";
+}
+
 /*
 public record Series
 {
     // .NET Regex Language reference:
     // https://learn.microsoft.com/en-us/dotnet/standard/base-types/regular-expression-language-quick-reference#anchors
     private static readonly Regex SeriesNamePattern = new Regex(
-        pattern: "\A([a-z]+)\z", 
+        pattern: "\A([a-z]+)\z",
         options: RegexOptions.Compiled);
-    
+
     public string Name { get; private init; }
 
     public Series? Parse(string value)
@@ -20,7 +30,7 @@ public record Series
             Log.Error($"'{value}' is an invalid series name.");
             return null;
         }
-        
+
         return new Series
         {
             Name = value
@@ -36,7 +46,7 @@ public record Package
     // .NET Regex Language reference:
     // https://learn.microsoft.com/en-us/dotnet/standard/base-types/regular-expression-language-quick-reference#anchors
     private static readonly Regex PackageNamePattern = new Regex(
-        pattern: "\A([a-z0-9][-a-z0-9.+]+)\z", 
+        pattern: "\A([a-z0-9][-a-z0-9.+]+)\z",
         options: RegexOptions.Compiled);
     public string Name { get; private init; }
 
@@ -47,11 +57,11 @@ public record Package
             Log.Error($"'{value}' is an invalid package name.");
             return null;
         }
-        
+
         return new Package
         {
             Name = value
         };
     }
 }
-*/
+*/