add common helpers

2020-05-29 15:24:03 +03:00
parent 7c335a2ccc
commit 00f276c05e
18 changed files with 548 additions and 175 deletions
--- a/helpers/packages-generation/generate-versions-manifest.ps1
+++ b/helpers/packages-generation/generate-versions-manifest.ps1
@@ -1,158 +0,0 @@
-<#
-.SYNOPSIS
-Generate versions manifest based on repository releases
-
-.DESCRIPTION
-Versions manifest is needed to find the latest assets for particular version of tool
-.PARAMETER GitHubRepositoryOwner
-Required parameter. The organization which tool repository belongs
-.PARAMETER GitHubRepositoryName
-Optional parameter. The name of tool repository
-.PARAMETER GitHubAccessToken
-Required parameter. PAT Token to overcome GitHub API Rate limit
-.PARAMETER OutputFile
-Required parameter. File "*.json" where generated results will be saved
-.PARAMETER PlatformMapFile
-Optional parameter. Path to the json file with platform map
-Structure example:
-{
-    "macos-1014": [
-        {
-            "platform": "darwin",
-            "platform_version": "10.14"
-        }, ...
-    ], ...
-}
-#>
-
-param (
-    [Parameter(Mandatory)] [string] $GitHubRepositoryOwner,
-    [Parameter(Mandatory)] [string] $GitHubRepositoryName,
-    [Parameter(Mandatory)] [string] $GitHubAccessToken,
-    [Parameter(Mandatory)] [string] $OutputFile,
-    [string] $PlatformMapFile
-)
-
-Import-Module (Join-Path $PSScriptRoot "../github/github-api.psm1")
-
-if ($PlatformMapFile -and (Test-Path $PlatformMapFile)) {
-    $PlatformMap = Get-Content $PlatformMapFile -Raw | ConvertFrom-Json -AsHashtable
-} else {
-    $PlatformMap = @{}
-}
-
-function Get-FileNameWithoutExtension {
-    param (
-        [Parameter(Mandatory)][string]$Filename
-    )
-
-    if ($Filename.EndsWith(".tar.gz")) {
-        $Filename = [IO.path]::GetFileNameWithoutExtension($Filename)
-    }
-
-    return [IO.path]::GetFileNameWithoutExtension($Filename)
-}
-
-function New-AssetItem {
-    param (
-        [Parameter(Mandatory)][string]$Filename,
-        [Parameter(Mandatory)][string]$DownloadUrl,
-        [Parameter(Mandatory)][string]$Arch,
-        [Parameter(Mandatory)][string]$Platform,
-        [string]$PlatformVersion
-    )
-    $asset = New-Object PSObject
-
-    $asset | Add-Member -Name "filename" -Value $Filename -MemberType NoteProperty
-    $asset | Add-Member -Name "arch" -Value $Arch -MemberType NoteProperty
-    $asset | Add-Member -Name "platform" -Value $Platform -MemberType NoteProperty
-    if ($PlatformVersion) { $asset | Add-Member -Name "platform_version" -Value $PlatformVersion -MemberType NoteProperty }
-    $asset | Add-Member -Name "download_url" -Value $DownloadUrl -MemberType NoteProperty
-
-    return $asset
-}
-
-function Build-AssetsList {
-    param (
-        [AllowEmptyCollection()]
-        [Parameter(Mandatory)][array]$ReleaseAssets
-    )
-
-    
-    $assets = @()
-    foreach($releaseAsset in $ReleaseAssets) {
-        $filename = Get-FileNameWithoutExtension -Filename $releaseAsset.name
-        $parts = $filename.Split("-")
-        $arch = $parts[-1]
-        $buildPlatform = [string]::Join("-", $parts[2..($parts.Length-2)])
-
-        if ($PlatformMap[$buildPlatform]) {
-            $PlatformMap[$buildPlatform] | ForEach-Object {
-                $assets += New-AssetItem -Filename $releaseAsset.name `
-                                         -DownloadUrl $releaseAsset.browser_download_url `
-                                         -Arch $arch `
-                                         -Platform $_.platform `
-                                         -PlatformVersion $_.platform_version
-            }
-
-        } else {
-            $assets += New-AssetItem -Filename $releaseAsset.name `
-                                     -DownloadUrl $releaseAsset.browser_download_url `
-                                     -Arch $arch `
-                                     -Platform $buildPlatform
-        }
-    }
-
-    return $assets
-}
-
-function Get-VersionFromRelease {
-    param (
-        [Parameter(Mandatory)][object]$Release
-    )
-    # Release name can contain additional information after ':' so filter it
-    [string]$releaseName = $Release.name.Split(':')[0]
-    [Version]$version = $null
-    if (![Version]::TryParse($releaseName, [ref]$version)) {
-        throw "Release '$($Release.id)' has invalid title '$($Release.name)'. It can't be parsed as version. ( $($Release.html_url) )"
-    }
-
-    return $version
-}
-
-function Build-VersionsManifest {
-    param (
-        [Parameter(Mandatory)][array]$Releases
-    )
-
-    $Releases = $Releases | Sort-Object -Property "published_at" -Descending
-
-    $versionsHash = @{}
-    foreach ($release in $Releases) {
-        if (($release.draft -eq $true) -or ($release.prerelease -eq $true)) {
-            continue
-        }
-
-        [Version]$version = Get-VersionFromRelease $release
-        $versionKey = $version.ToString()
-
-        if ($versionsHash.ContainsKey($versionKey)) {
-            continue
-        }
-
-        $versionsHash.Add($versionKey, [PSCustomObject]@{
-            version = $versionKey
-            stable = $true
-            release_url = $release.html_url
-            files = Build-AssetsList $release.assets
-        })
-    }
-
-    # Sort versions by descending
-    return $versionsHash.Values | Sort-Object -Property @{ Expression = { [Version]$_.version }; Descending = $true }
-}
-
-$gitHubApi = Get-GitHubApi -AccountName $GitHubRepositoryOwner -ProjectName $GitHubRepositoryName -AccessToken $GitHubAccessToken
-$releases = $gitHubApi.GetReleases()
-$versionIndex = Build-VersionsManifest $releases
-$versionIndex | ConvertTo-Json -Depth 5 | Out-File $OutputFile -Encoding UTF8NoBOM -Force
--- a/helpers/packages-generation/manifest-generator.ps1
+++ b/helpers/packages-generation/manifest-generator.ps1
@@ -0,0 +1,35 @@
+<#
+.SYNOPSIS
+Generate versions manifest based on repository releases
+
+.DESCRIPTION
+Versions manifest is needed to find the latest assets for particular version of tool
+.PARAMETER GitHubRepositoryOwner
+Required parameter. The organization which tool repository belongs
+.PARAMETER GitHubRepositoryName
+Optional parameter. The name of tool repository
+.PARAMETER GitHubAccessToken
+Required parameter. PAT Token to overcome GitHub API Rate limit
+.PARAMETER OutputFile
+Required parameter. File "*.json" where generated results will be saved
+.PARAMETER ConfigurationFile
+Path to the json file with parsing configuration
+#>
+
+param (
+    [Parameter(Mandatory)] [string] $GitHubRepositoryOwner,
+    [Parameter(Mandatory)] [string] $GitHubRepositoryName,
+    [Parameter(Mandatory)] [string] $GitHubAccessToken,
+    [Parameter(Mandatory)] [string] $OutputFile,
+    [Parameter(Mandatory)] [string] $ConfigurationFile
+)
+
+Import-Module (Join-Path $PSScriptRoot "../github/github-api.psm1")
+Import-Module (Join-Path $PSScriptRoot "manifest-utils.psm1") -Force
+
+$configuration = Read-ConfigurationFile -Filepath $ConfigurationFile
+
+$gitHubApi = Get-GitHubApi -AccountName $GitHubRepositoryOwner -ProjectName $GitHubRepositoryName -AccessToken $GitHubAccessToken
+$releases = $gitHubApi.GetReleases()
+$versionIndex = Build-VersionsManifest -Releases $releases -Configuration $configuration
+$versionIndex | ConvertTo-Json -Depth 5 | Out-File $OutputFile -Encoding UTF8NoBOM -Force
--- a/helpers/packages-generation/manifest-utils.Tests.ps1
+++ b/helpers/packages-generation/manifest-utils.Tests.ps1
@@ -0,0 +1,116 @@
+Import-Module (Join-Path $PSScriptRoot "manifest-utils.psm1") -Force
+
+Describe "New-AssetItem" {
+    It "use regex to parse all values in correct order" {
+        $githubAsset = @{ name = "python-3.8.3-linux-16.04-x64.tar.gz"; browser_download_url = "long_url"; }
+        $configuration = @{
+            regex = "python-\d+\.\d+\.\d+-(\w+)-([\w\.]+)?-?(x\d+)";
+            groups = [PSCustomObject]@{ platform = 1; platform_version = 2; arch = 3; };
+        }
+        $expectedOutput = [PSCustomObject]@{
+            filename = "python-3.8.3-linux-16.04-x64.tar.gz"; platform = "linux"; platform_version = "16.04";
+            arch = "x64"; download_url = "long_url";
+        }
+
+        $actualOutput = New-AssetItem -ReleaseAsset $githubAsset -Configuration $configuration
+        Assert-Equivalent -Actual $actualOutput -Expected $expectedOutput
+    }
+
+    It "support constant values in groups" {
+        $githubAsset = @{ name = "python-3.8.3-linux-16.04-x64.tar.gz"; browser_download_url = "long_url"; }
+        $configuration = @{
+            regex = "python-\d+\.\d+\.\d+-(\w+)-([\w\.]+)?-?(x\d+)";
+            groups = [PSCustomObject]@{ platform = 1; platform_version = 2; arch = "x64"; }
+        }
+        $expectedOutput = [PSCustomObject]@{
+            filename = "python-3.8.3-linux-16.04-x64.tar.gz"; platform = "linux"; platform_version = "16.04";
+            arch = "x64"; download_url = "long_url";
+        }
+
+        $actualOutput = New-AssetItem -ReleaseAsset $githubAsset -Configuration $configuration
+        Assert-Equivalent -Actual $actualOutput -Expected $expectedOutput
+    }
+
+    It "Skip empty groups" {
+        $githubAsset = @{ name = "python-3.8.3-win32-x64.zip"; browser_download_url = "long_url"; }
+        $configuration = @{
+            regex = "python-\d+\.\d+\.\d+-(\w+)-([\w\.]+)?-?(x\d+)";
+            groups = [PSCustomObject]@{ platform = 1; platform_version = 2; arch = 3; }
+        }
+        $expectedOutput = [PSCustomObject]@{
+            filename = "python-3.8.3-win32-x64.zip"; platform = "win32";
+            arch = "x64"; download_url = "long_url";
+        }
+
+        $actualOutput = New-AssetItem -ReleaseAsset $githubAsset -Configuration $configuration
+        Assert-Equivalent -Actual $actualOutput -Expected $expectedOutput
+    }
+}
+
+Describe "Get-VersionFromRelease" {
+    It "clear version" {
+        $release = @{ name = "3.8.3" }
+        Get-VersionFromRelease -Release $release | Should -Be "3.8.3"
+    }
+
+    It "version with title" {
+        $release = @{ name = "3.8.3: Release title" }
+        Get-VersionFromRelease -Release $release | Should -Be "3.8.3"
+    }
+}
+
+Describe "Build-VersionsManifest" {
+    $assets = @(
+        @{ name = "python-3.8.3-linux-16.04-x64.tar.gz"; browser_download_url = "fake_url"; }
+        @{ name = "python-3.8.3-linux-18.04-x64.tar.gz"; browser_download_url = "fake_url"; }
+    )
+    $configuration = @{
+        regex = "python-\d+\.\d+\.\d+-(\w+)-([\w\.]+)?-?(x\d+)";
+        groups = [PSCustomObject]@{ platform = 1; platform_version = 2; arch = "x64"; }
+    }
+    $expectedManifestFiles = @(
+        [PSCustomObject]@{ filename = "python-3.8.3-linux-16.04-x64.tar.gz"; arch = "x64"; platform = "linux"; platform_version = "16.04"; download_url = "fake_url" },
+        [PSCustomObject]@{ filename = "python-3.8.3-linux-18.04-x64.tar.gz"; arch = "x64"; platform = "linux"; platform_version = "18.04"; download_url = "fake_url" }
+    )
+
+    It "build manifest with correct version order" {
+        $releases = @(
+            @{ name = "3.8.1"; draft = $false; prerelease = $false; html_url = "fake_html_url"; published_at = "2020-05-14T09:54:06Z"; assets = $assets },
+            @{ name = "3.5.2: Hello"; draft = $false; prerelease = $false; html_url = "fake_html_url"; published_at = "2020-05-06T11:45:36Z"; assets = $assets },
+            @{ name = "3.8.3: Release title"; draft = $false; prerelease = $false; html_url = "fake_html_url"; published_at = "2020-05-06T11:43:38Z"; assets = $assets }
+        )
+        $expectedManifest = @(
+            [PSCustomObject]@{ version = "3.8.3"; stable = $true; release_url = "fake_html_url"; files = $expectedManifestFiles },
+            [PSCustomObject]@{ version = "3.8.1"; stable = $true; release_url = "fake_html_url"; files = $expectedManifestFiles },
+            [PSCustomObject]@{ version = "3.5.2"; stable = $true; release_url = "fake_html_url"; files = $expectedManifestFiles }
+        )
+        $actualManifest = Build-VersionsManifest -Releases $releases -Configuration $configuration
+        Assert-Equivalent -Actual $actualManifest -Expected $expectedManifest
+    }
+
+    It "Skip draft and prerelease" {
+        $releases = @(
+            @{ name = "3.8.1"; draft = $true; prerelease = $false; html_url = "fake_html_url"; published_at = "2020-05-14T09:54:06Z"; assets = $assets },
+            @{ name = "3.5.2"; draft = $false; prerelease = $true; html_url = "fake_html_url"; published_at = "2020-05-06T11:45:36Z"; assets = $assets },
+            @{ name = "3.8.3"; draft = $false; prerelease = $false; html_url = "fake_html_url"; published_at = "2020-05-06T11:43:38Z"; assets = $assets }
+        )
+        $expectedManifest = @(
+            [PSCustomObject]@{ version = "3.8.3"; stable = $true; release_url = "fake_html_url"; files = $expectedManifestFiles }
+        )
+        [array]$actualManifest = Build-VersionsManifest -Releases $releases -Configuration $configuration
+        Assert-Equivalent -Actual $actualManifest -Expected $expectedManifest 
+    }
+
+    It "take latest published release for each version" {
+        $releases = @(
+            @{ name = "3.8.1"; draft = $false; prerelease = $false; html_url = "fake_html_url1"; published_at = "2020-05-06T11:45:36Z"; assets = $assets },
+            @{ name = "3.8.1"; draft = $false; prerelease = $false; html_url = "fake_html_url2"; published_at = "2020-05-14T09:54:06Z"; assets = $assets },
+            @{ name = "3.8.1"; draft = $false; prerelease = $false; html_url = "fake_html_url3"; published_at = "2020-05-06T11:43:38Z"; assets = $assets }
+        )
+        $expectedManifest = @(
+            [PSCustomObject]@{ version = "3.8.1"; stable = $true; release_url = "fake_html_url2"; files = $expectedManifestFiles }
+        )
+        [array]$actualManifest = Build-VersionsManifest -Releases $releases -Configuration $configuration
+        Assert-Equivalent -Actual $actualManifest -Expected $expectedManifest
+    }
+}
--- a/helpers/packages-generation/manifest-utils.psm1
+++ b/helpers/packages-generation/manifest-utils.psm1
@@ -0,0 +1,77 @@
+function Read-ConfigurationFile {
+    param ([Parameter(Mandatory)][string]$Filepath)
+    return Get-Content $Filepath -Raw | ConvertFrom-Json
+}
+
+function New-AssetItem {
+    param (
+        [Parameter(Mandatory)][object]$ReleaseAsset,
+        [Parameter(Mandatory)][object]$Configuration
+    )
+    $regexResult = [regex]::Match($ReleaseAsset.name, $Configuration.regex)
+    if (-not $regexResult.Success) { throw "Can't match asset filename '$($_.name)' to regex" }
+
+    $result = New-Object PSObject
+    $result | Add-Member -Name "filename" -Value $ReleaseAsset.name -MemberType NoteProperty
+    $Configuration.groups.PSObject.Properties | ForEach-Object {
+        if (($_.Value).GetType().Name.StartsWith("Int")) {
+            $value = $regexResult.Groups[$_.Value].Value
+        } else {
+            $value = $_.Value
+        }
+
+        if (-not ([string]::IsNullOrEmpty($value))) {
+            $result | Add-Member -Name $_.Name -Value $value -MemberType NoteProperty
+        }
+    }
+
+    $result | Add-Member -Name "download_url" -Value $ReleaseAsset.browser_download_url -MemberType NoteProperty
+    return $result
+}
+
+function Get-VersionFromRelease {
+    param (
+        [Parameter(Mandatory)][object]$Release
+    )
+    # Release name can contain additional information after ':' so filter it
+    [string]$releaseName = $Release.name.Split(':')[0]
+    [Version]$version = $null
+    if (![Version]::TryParse($releaseName, [ref]$version)) {
+        throw "Release '$($Release.id)' has invalid title '$($Release.name)'. It can't be parsed as version. ( $($Release.html_url) )"
+    }
+
+    return $version
+}
+
+function Build-VersionsManifest {
+    param (
+        [Parameter(Mandatory)][array]$Releases,
+        [Parameter(Mandatory)][object]$Configuration
+    )
+
+    $Releases = $Releases | Sort-Object -Property "published_at" -Descending
+
+    $versionsHash = @{}
+    foreach ($release in $Releases) {
+        if (($release.draft -eq $true) -or ($release.prerelease -eq $true)) {
+            continue
+        }
+
+        [Version]$version = Get-VersionFromRelease $release
+        $versionKey = $version.ToString()
+
+        if ($versionsHash.ContainsKey($versionKey)) {
+            continue
+        }
+
+        $versionsHash.Add($versionKey, [PSCustomObject]@{
+            version = $versionKey
+            stable = $true
+            release_url = $release.html_url
+            files = $release.assets | ForEach-Object { New-AssetItem -ReleaseAsset $_ -Configuration $Configuration }
+        })
+    }
+
+    # Sort versions by descending
+    return $versionsHash.Values | Sort-Object -Property @{ Expression = { [Version]$_.version }; Descending = $true }
+}
--- a/helpers/packages-generation/pester-extensions.psm1
+++ b/helpers/packages-generation/pester-extensions.psm1
@@ -1,33 +0,0 @@
-<#
-.SYNOPSIS
-Pester extension that allows to run command and validate exit code
-.EXAMPLE
-"python file.py" | Should -ReturnZeroExitCode
-#>
-function ShouldReturnZeroExitCode {
-    Param(
-        [Parameter (Mandatory = $true)] [ValidateNotNullOrEmpty()]
-        [String]$ActualValue,
-        [switch]$Negate
-    )
-
-    Write-Host "Run command '${ActualValue}'"
-    Invoke-Expression -Command $ActualValue | ForEach-Object { Write-Host $_ }
-    $actualExitCode = $LASTEXITCODE
-
-    [bool]$succeeded = $actualExitCode -eq 0
-    if ($Negate) { $succeeded = -not $succeeded }
-
-    if (-not $succeeded)
-    {
-        $failureMessage = "Command '${ActualValue}' has finished with exit code ${actualExitCode}"
-    }
-
-    return New-Object PSObject -Property @{
-        Succeeded      = $succeeded
-        FailureMessage = $failureMessage
-    }
-}
-
-Add-AssertionOperator -Name ReturnZeroExitCode `
-                    -Test  $function:ShouldReturnZeroExitCode