From 00f276c05e2f99a26ff0a9fc9f47bfe3ec7c71c3 Mon Sep 17 00:00:00 2001 From: Dmitry Shibanov Date: Fri, 29 May 2020 15:24:03 +0300 Subject: [PATCH 1/5] add common helpers --- azure-pipelines/templates/test-job.yml | 9 +- config/node-manifest-config.json | 7 + helpers/CODE_OF_CONDUCT.md | 76 +++++++++ helpers/CONTRIBUTING.md | 31 ++++ helpers/LICENSE | 21 +++ helpers/README.md | 5 + helpers/SECURITY.md | 3 + helpers/clean-toolcache.ps1 | 13 ++ helpers/common-helpers.psm1 | 85 ++++++++++ helpers/nix-helpers.psm1 | 20 ++- .../generate-versions-manifest.ps1 | 158 ------------------ .../manifest-generator.ps1 | 35 ++++ .../manifest-utils.Tests.ps1 | 116 +++++++++++++ .../packages-generation/manifest-utils.psm1 | 77 +++++++++ .../pester-extensions.psm1 | 0 helpers/win-helpers.psm1 | 17 +- helpers/win-vs-env.psm1 | 48 ++++++ tests/Node.Tests.ps1 | 2 +- 18 files changed, 548 insertions(+), 175 deletions(-) create mode 100644 config/node-manifest-config.json create mode 100644 helpers/CODE_OF_CONDUCT.md create mode 100644 helpers/CONTRIBUTING.md create mode 100644 helpers/LICENSE create mode 100644 helpers/README.md create mode 100644 helpers/SECURITY.md create mode 100644 helpers/clean-toolcache.ps1 create mode 100644 helpers/common-helpers.psm1 delete mode 100644 helpers/packages-generation/generate-versions-manifest.ps1 create mode 100644 helpers/packages-generation/manifest-generator.ps1 create mode 100644 helpers/packages-generation/manifest-utils.Tests.ps1 create mode 100644 helpers/packages-generation/manifest-utils.psm1 rename helpers/{packages-generation => }/pester-extensions.psm1 (100%) create mode 100644 helpers/win-vs-env.psm1 diff --git a/azure-pipelines/templates/test-job.yml b/azure-pipelines/templates/test-job.yml index d823162..1205bcb 100644 --- a/azure-pipelines/templates/test-job.yml +++ b/azure-pipelines/templates/test-job.yml @@ -10,12 +10,9 @@ jobs: - task: PowerShell@2 displayName: Fully cleanup the toolcache directory before testing inputs: - TargetType: inline - script: | - $NodeToolcachePath = Join-Path -Path $env:AGENT_TOOLSDIRECTORY -ChildPath "node" - if (Test-Path $NodeToolcachePath) { - Remove-Item -Path $NodeToolcachePath -Recurse -Force - } + targetType: filePath + filePath: helpers/clean-toolcache.ps1 + arguments: -ToolName "node" - task: DownloadPipelineArtifact@2 inputs: diff --git a/config/node-manifest-config.json b/config/node-manifest-config.json new file mode 100644 index 0000000..765f52d --- /dev/null +++ b/config/node-manifest-config.json @@ -0,0 +1,7 @@ +{ + "regex": "node-\\d+\\.\\d+\\.\\d+-(\\w+)-(x\\d+)", + "groups": { + "arch": 2, + "platform": 1 + } +} \ No newline at end of file diff --git a/helpers/CODE_OF_CONDUCT.md b/helpers/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..517657b --- /dev/null +++ b/helpers/CODE_OF_CONDUCT.md @@ -0,0 +1,76 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to make participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, sex characteristics, gender identity and expression, +level of experience, education, socio-economic status, nationality, personal +appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies within all project spaces, and it also applies when +an individual is representing the project or its community in public spaces. +Examples of representing a project or community include using an official +project e-mail address, posting via an official social media account, or acting +as an appointed representative at an online or offline event. Representation of +a project may be further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at opensource@github.com. All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see +https://www.contributor-covenant.org/faq \ No newline at end of file diff --git a/helpers/CONTRIBUTING.md b/helpers/CONTRIBUTING.md new file mode 100644 index 0000000..fbc5fc5 --- /dev/null +++ b/helpers/CONTRIBUTING.md @@ -0,0 +1,31 @@ +## Contributing + +[fork]: https://github.com/actions/versions-package-tools/fork +[pr]: https://github.com/actions/versions-package-tools/compare +[code-of-conduct]: CODE_OF_CONDUCT.md + +Hi there! We're thrilled that you'd like to contribute to this project. Your help is essential for keeping it great. + +Contributions to this project are [released](https://help.github.com/articles/github-terms-of-service/#6-contributions-under-repository-license) to the public under the [MIT](LICENSE.md). + +Please note that this project is released with a [Contributor Code of Conduct][code-of-conduct]. By participating in this project you agree to abide by its terms. + +## Submitting a pull request + +1. [Fork][fork] and clone the repository +1. Create a new branch: `git checkout -b my-branch-name` +1. Make your changes +1. Push to your fork and [submit a pull request][pr] +1. Make sure that checks in your pull request are green + +Here are a few things you can do that will increase the likelihood of your pull request being accepted: + +- Please include a summary of the change and which issue is fixed. Also include relevant motivation and context. +- Follow the style guide for [PowerShell](https://github.com/PoshCode/PowerShellPracticeAndStyle). +- Write [good commit messages](http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html). + +## Resources + +- [How to Contribute to Open Source](https://opensource.guide/how-to-contribute/) +- [Using Pull Requests](https://help.github.com/articles/about-pull-requests/) +- [GitHub Help](https://help.github.com) diff --git a/helpers/LICENSE b/helpers/LICENSE new file mode 100644 index 0000000..f42f5ab --- /dev/null +++ b/helpers/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 GitHub + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/helpers/README.md b/helpers/README.md new file mode 100644 index 0000000..d9d6ccf --- /dev/null +++ b/helpers/README.md @@ -0,0 +1,5 @@ +# Common tools for generation of packages in the actions/*-versions repositories +This repository contains PowerShell modules that are used to generate packages for Actions. The packages are consumed by the images generated through [actions/virtual-environments](https://github.com/actions/virtual-environments) and some of the setup-* Actions + +## Contribution +Contributions are welcome! See [Contributor's Guide](./CONTRIBUTING.md) for more details about contribution process and code structure diff --git a/helpers/SECURITY.md b/helpers/SECURITY.md new file mode 100644 index 0000000..f0b196f --- /dev/null +++ b/helpers/SECURITY.md @@ -0,0 +1,3 @@ +If you discover a security issue in this repo, please submit it through the [GitHub Security Bug Bounty](https://hackerone.com/github) + +Thanks for helping make GitHub Actions safe for everyone. diff --git a/helpers/clean-toolcache.ps1 b/helpers/clean-toolcache.ps1 new file mode 100644 index 0000000..67796de --- /dev/null +++ b/helpers/clean-toolcache.ps1 @@ -0,0 +1,13 @@ +param ( + [string] $ToolName +) + +$targetPath = $env:AGENT_TOOLSDIRECTORY +if ($ToolName) { + $targetPath = Join-Path $targetPath $ToolName +} + +if (Test-Path $targetPath) { + Get-ChildItem -Path $targetPath -Recurse | Where-Object { $_.LinkType -eq "SymbolicLink" } | ForEach-Object { $_.Delete() } + Remove-Item -Path $targetPath -Recurse -Force +} diff --git a/helpers/common-helpers.psm1 b/helpers/common-helpers.psm1 new file mode 100644 index 0000000..cbc7665 --- /dev/null +++ b/helpers/common-helpers.psm1 @@ -0,0 +1,85 @@ +<# +.SYNOPSIS +The execute command and print all output to the logs +#> +function Execute-Command { + [CmdletBinding()] + param( + [Parameter(Mandatory=$true)][string] $Command + ) + + Write-Debug "Execute $Command" + + try { + Invoke-Expression $Command | ForEach-Object { Write-Host $_ } + if ($LASTEXITCODE -ne 0) { throw "Exit code: $LASTEXITCODE"} + } + catch { + $errorMessage = "Error happened during command execution: $Command `n $_" + Write-Host $errorMessage + if ($ErrorActionPreference -ne "Continue") { + # avoid logging Azure DevOps issues in case of $ErrorActionPreference -eq Continue + Write-Host "##vso[task.logissue type=error;] $errorMessage" + } + } +} + +<# +.SYNOPSIS +Download file from url and return local path to file +#> +function Download-File { + param( + [Parameter(Mandatory=$true)] + [Uri]$Uri, + [Parameter(Mandatory=$true)] + [String]$OutputFolder + ) + + $targetFilename = [IO.Path]::GetFileName($Uri) + $targetFilepath = Join-Path $OutputFolder $targetFilename + + Write-Debug "Download source from $Uri to $OutFile" + try { + (New-Object System.Net.WebClient).DownloadFile($Uri, $targetFilepath) + return $targetFilepath + } catch { + Write-Host "Error during downloading file from '$Uri'" + "$_" + exit 1 + } +} + +<# +.SYNOPSIS +Generate file that contains the list of all files in particular directory +#> +function New-ToolStructureDump { + param( + [Parameter(Mandatory=$true)] + [String]$ToolPath, + [Parameter(Mandatory=$true)] + [String]$OutputFolder + ) + + $outputFile = Join-Path $OutputFolder "tools_structure.txt" + + $folderContent = Get-ChildItem -Path $ToolPath -Recurse | Sort-Object | Select-Object -Property FullName, Length + $folderContent | ForEach-Object { + $relativePath = $_.FullName.Replace($ToolPath, ""); + return "${relativePath}" + } | Out-File -FilePath $outputFile +} + +<# +.SYNOPSIS +Check if it is macOS / Ubuntu platform +#> +function IsNixPlatform { + param( + [Parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] + [String]$Platform + ) + + return ($Platform -match "macos") -or ($Platform -match "ubuntu") -or ($Platform -match "linux") +} \ No newline at end of file diff --git a/helpers/nix-helpers.psm1 b/helpers/nix-helpers.psm1 index e6e3441..0f00e21 100644 --- a/helpers/nix-helpers.psm1 +++ b/helpers/nix-helpers.psm1 @@ -20,13 +20,25 @@ function Create-TarArchive { [String]$SourceFolder, [Parameter(Mandatory=$true)] [String]$ArchivePath, - [string]$CompressionType = "gz" + [string]$CompressionType = "gz", + [switch]$DereferenceSymlinks ) - $CompressionTypeArgument = If ([string]::IsNullOrWhiteSpace($CompressionType)) { "" } else { "--${CompressionType}" } + $arguments = @( + "-c", + "-f", $ArchivePath, + "." + ) + If ($CompressionType) { + $arguments += "--${CompressionType}" + } + + if ($DereferenceSymlinks) { + $arguments += "-h" + } Push-Location $SourceFolder - Write-Debug "tar -c $CompressionTypeArgument -f $ArchivePath ." - tar -c $CompressionTypeArgument -f $ArchivePath . + Write-Debug "tar $arguments" + tar @arguments Pop-Location } \ No newline at end of file diff --git a/helpers/packages-generation/generate-versions-manifest.ps1 b/helpers/packages-generation/generate-versions-manifest.ps1 deleted file mode 100644 index 52b652d..0000000 --- a/helpers/packages-generation/generate-versions-manifest.ps1 +++ /dev/null @@ -1,158 +0,0 @@ -<# -.SYNOPSIS -Generate versions manifest based on repository releases - -.DESCRIPTION -Versions manifest is needed to find the latest assets for particular version of tool -.PARAMETER GitHubRepositoryOwner -Required parameter. The organization which tool repository belongs -.PARAMETER GitHubRepositoryName -Optional parameter. The name of tool repository -.PARAMETER GitHubAccessToken -Required parameter. PAT Token to overcome GitHub API Rate limit -.PARAMETER OutputFile -Required parameter. File "*.json" where generated results will be saved -.PARAMETER PlatformMapFile -Optional parameter. Path to the json file with platform map -Structure example: -{ - "macos-1014": [ - { - "platform": "darwin", - "platform_version": "10.14" - }, ... - ], ... -} -#> - -param ( - [Parameter(Mandatory)] [string] $GitHubRepositoryOwner, - [Parameter(Mandatory)] [string] $GitHubRepositoryName, - [Parameter(Mandatory)] [string] $GitHubAccessToken, - [Parameter(Mandatory)] [string] $OutputFile, - [string] $PlatformMapFile -) - -Import-Module (Join-Path $PSScriptRoot "../github/github-api.psm1") - -if ($PlatformMapFile -and (Test-Path $PlatformMapFile)) { - $PlatformMap = Get-Content $PlatformMapFile -Raw | ConvertFrom-Json -AsHashtable -} else { - $PlatformMap = @{} -} - -function Get-FileNameWithoutExtension { - param ( - [Parameter(Mandatory)][string]$Filename - ) - - if ($Filename.EndsWith(".tar.gz")) { - $Filename = [IO.path]::GetFileNameWithoutExtension($Filename) - } - - return [IO.path]::GetFileNameWithoutExtension($Filename) -} - -function New-AssetItem { - param ( - [Parameter(Mandatory)][string]$Filename, - [Parameter(Mandatory)][string]$DownloadUrl, - [Parameter(Mandatory)][string]$Arch, - [Parameter(Mandatory)][string]$Platform, - [string]$PlatformVersion - ) - $asset = New-Object PSObject - - $asset | Add-Member -Name "filename" -Value $Filename -MemberType NoteProperty - $asset | Add-Member -Name "arch" -Value $Arch -MemberType NoteProperty - $asset | Add-Member -Name "platform" -Value $Platform -MemberType NoteProperty - if ($PlatformVersion) { $asset | Add-Member -Name "platform_version" -Value $PlatformVersion -MemberType NoteProperty } - $asset | Add-Member -Name "download_url" -Value $DownloadUrl -MemberType NoteProperty - - return $asset -} - -function Build-AssetsList { - param ( - [AllowEmptyCollection()] - [Parameter(Mandatory)][array]$ReleaseAssets - ) - - - $assets = @() - foreach($releaseAsset in $ReleaseAssets) { - $filename = Get-FileNameWithoutExtension -Filename $releaseAsset.name - $parts = $filename.Split("-") - $arch = $parts[-1] - $buildPlatform = [string]::Join("-", $parts[2..($parts.Length-2)]) - - if ($PlatformMap[$buildPlatform]) { - $PlatformMap[$buildPlatform] | ForEach-Object { - $assets += New-AssetItem -Filename $releaseAsset.name ` - -DownloadUrl $releaseAsset.browser_download_url ` - -Arch $arch ` - -Platform $_.platform ` - -PlatformVersion $_.platform_version - } - - } else { - $assets += New-AssetItem -Filename $releaseAsset.name ` - -DownloadUrl $releaseAsset.browser_download_url ` - -Arch $arch ` - -Platform $buildPlatform - } - } - - return $assets -} - -function Get-VersionFromRelease { - param ( - [Parameter(Mandatory)][object]$Release - ) - # Release name can contain additional information after ':' so filter it - [string]$releaseName = $Release.name.Split(':')[0] - [Version]$version = $null - if (![Version]::TryParse($releaseName, [ref]$version)) { - throw "Release '$($Release.id)' has invalid title '$($Release.name)'. It can't be parsed as version. ( $($Release.html_url) )" - } - - return $version -} - -function Build-VersionsManifest { - param ( - [Parameter(Mandatory)][array]$Releases - ) - - $Releases = $Releases | Sort-Object -Property "published_at" -Descending - - $versionsHash = @{} - foreach ($release in $Releases) { - if (($release.draft -eq $true) -or ($release.prerelease -eq $true)) { - continue - } - - [Version]$version = Get-VersionFromRelease $release - $versionKey = $version.ToString() - - if ($versionsHash.ContainsKey($versionKey)) { - continue - } - - $versionsHash.Add($versionKey, [PSCustomObject]@{ - version = $versionKey - stable = $true - release_url = $release.html_url - files = Build-AssetsList $release.assets - }) - } - - # Sort versions by descending - return $versionsHash.Values | Sort-Object -Property @{ Expression = { [Version]$_.version }; Descending = $true } -} - -$gitHubApi = Get-GitHubApi -AccountName $GitHubRepositoryOwner -ProjectName $GitHubRepositoryName -AccessToken $GitHubAccessToken -$releases = $gitHubApi.GetReleases() -$versionIndex = Build-VersionsManifest $releases -$versionIndex | ConvertTo-Json -Depth 5 | Out-File $OutputFile -Encoding UTF8NoBOM -Force diff --git a/helpers/packages-generation/manifest-generator.ps1 b/helpers/packages-generation/manifest-generator.ps1 new file mode 100644 index 0000000..85434d0 --- /dev/null +++ b/helpers/packages-generation/manifest-generator.ps1 @@ -0,0 +1,35 @@ +<# +.SYNOPSIS +Generate versions manifest based on repository releases + +.DESCRIPTION +Versions manifest is needed to find the latest assets for particular version of tool +.PARAMETER GitHubRepositoryOwner +Required parameter. The organization which tool repository belongs +.PARAMETER GitHubRepositoryName +Optional parameter. The name of tool repository +.PARAMETER GitHubAccessToken +Required parameter. PAT Token to overcome GitHub API Rate limit +.PARAMETER OutputFile +Required parameter. File "*.json" where generated results will be saved +.PARAMETER ConfigurationFile +Path to the json file with parsing configuration +#> + +param ( + [Parameter(Mandatory)] [string] $GitHubRepositoryOwner, + [Parameter(Mandatory)] [string] $GitHubRepositoryName, + [Parameter(Mandatory)] [string] $GitHubAccessToken, + [Parameter(Mandatory)] [string] $OutputFile, + [Parameter(Mandatory)] [string] $ConfigurationFile +) + +Import-Module (Join-Path $PSScriptRoot "../github/github-api.psm1") +Import-Module (Join-Path $PSScriptRoot "manifest-utils.psm1") -Force + +$configuration = Read-ConfigurationFile -Filepath $ConfigurationFile + +$gitHubApi = Get-GitHubApi -AccountName $GitHubRepositoryOwner -ProjectName $GitHubRepositoryName -AccessToken $GitHubAccessToken +$releases = $gitHubApi.GetReleases() +$versionIndex = Build-VersionsManifest -Releases $releases -Configuration $configuration +$versionIndex | ConvertTo-Json -Depth 5 | Out-File $OutputFile -Encoding UTF8NoBOM -Force diff --git a/helpers/packages-generation/manifest-utils.Tests.ps1 b/helpers/packages-generation/manifest-utils.Tests.ps1 new file mode 100644 index 0000000..f1aa785 --- /dev/null +++ b/helpers/packages-generation/manifest-utils.Tests.ps1 @@ -0,0 +1,116 @@ +Import-Module (Join-Path $PSScriptRoot "manifest-utils.psm1") -Force + +Describe "New-AssetItem" { + It "use regex to parse all values in correct order" { + $githubAsset = @{ name = "python-3.8.3-linux-16.04-x64.tar.gz"; browser_download_url = "long_url"; } + $configuration = @{ + regex = "python-\d+\.\d+\.\d+-(\w+)-([\w\.]+)?-?(x\d+)"; + groups = [PSCustomObject]@{ platform = 1; platform_version = 2; arch = 3; }; + } + $expectedOutput = [PSCustomObject]@{ + filename = "python-3.8.3-linux-16.04-x64.tar.gz"; platform = "linux"; platform_version = "16.04"; + arch = "x64"; download_url = "long_url"; + } + + $actualOutput = New-AssetItem -ReleaseAsset $githubAsset -Configuration $configuration + Assert-Equivalent -Actual $actualOutput -Expected $expectedOutput + } + + It "support constant values in groups" { + $githubAsset = @{ name = "python-3.8.3-linux-16.04-x64.tar.gz"; browser_download_url = "long_url"; } + $configuration = @{ + regex = "python-\d+\.\d+\.\d+-(\w+)-([\w\.]+)?-?(x\d+)"; + groups = [PSCustomObject]@{ platform = 1; platform_version = 2; arch = "x64"; } + } + $expectedOutput = [PSCustomObject]@{ + filename = "python-3.8.3-linux-16.04-x64.tar.gz"; platform = "linux"; platform_version = "16.04"; + arch = "x64"; download_url = "long_url"; + } + + $actualOutput = New-AssetItem -ReleaseAsset $githubAsset -Configuration $configuration + Assert-Equivalent -Actual $actualOutput -Expected $expectedOutput + } + + It "Skip empty groups" { + $githubAsset = @{ name = "python-3.8.3-win32-x64.zip"; browser_download_url = "long_url"; } + $configuration = @{ + regex = "python-\d+\.\d+\.\d+-(\w+)-([\w\.]+)?-?(x\d+)"; + groups = [PSCustomObject]@{ platform = 1; platform_version = 2; arch = 3; } + } + $expectedOutput = [PSCustomObject]@{ + filename = "python-3.8.3-win32-x64.zip"; platform = "win32"; + arch = "x64"; download_url = "long_url"; + } + + $actualOutput = New-AssetItem -ReleaseAsset $githubAsset -Configuration $configuration + Assert-Equivalent -Actual $actualOutput -Expected $expectedOutput + } +} + +Describe "Get-VersionFromRelease" { + It "clear version" { + $release = @{ name = "3.8.3" } + Get-VersionFromRelease -Release $release | Should -Be "3.8.3" + } + + It "version with title" { + $release = @{ name = "3.8.3: Release title" } + Get-VersionFromRelease -Release $release | Should -Be "3.8.3" + } +} + +Describe "Build-VersionsManifest" { + $assets = @( + @{ name = "python-3.8.3-linux-16.04-x64.tar.gz"; browser_download_url = "fake_url"; } + @{ name = "python-3.8.3-linux-18.04-x64.tar.gz"; browser_download_url = "fake_url"; } + ) + $configuration = @{ + regex = "python-\d+\.\d+\.\d+-(\w+)-([\w\.]+)?-?(x\d+)"; + groups = [PSCustomObject]@{ platform = 1; platform_version = 2; arch = "x64"; } + } + $expectedManifestFiles = @( + [PSCustomObject]@{ filename = "python-3.8.3-linux-16.04-x64.tar.gz"; arch = "x64"; platform = "linux"; platform_version = "16.04"; download_url = "fake_url" }, + [PSCustomObject]@{ filename = "python-3.8.3-linux-18.04-x64.tar.gz"; arch = "x64"; platform = "linux"; platform_version = "18.04"; download_url = "fake_url" } + ) + + It "build manifest with correct version order" { + $releases = @( + @{ name = "3.8.1"; draft = $false; prerelease = $false; html_url = "fake_html_url"; published_at = "2020-05-14T09:54:06Z"; assets = $assets }, + @{ name = "3.5.2: Hello"; draft = $false; prerelease = $false; html_url = "fake_html_url"; published_at = "2020-05-06T11:45:36Z"; assets = $assets }, + @{ name = "3.8.3: Release title"; draft = $false; prerelease = $false; html_url = "fake_html_url"; published_at = "2020-05-06T11:43:38Z"; assets = $assets } + ) + $expectedManifest = @( + [PSCustomObject]@{ version = "3.8.3"; stable = $true; release_url = "fake_html_url"; files = $expectedManifestFiles }, + [PSCustomObject]@{ version = "3.8.1"; stable = $true; release_url = "fake_html_url"; files = $expectedManifestFiles }, + [PSCustomObject]@{ version = "3.5.2"; stable = $true; release_url = "fake_html_url"; files = $expectedManifestFiles } + ) + $actualManifest = Build-VersionsManifest -Releases $releases -Configuration $configuration + Assert-Equivalent -Actual $actualManifest -Expected $expectedManifest + } + + It "Skip draft and prerelease" { + $releases = @( + @{ name = "3.8.1"; draft = $true; prerelease = $false; html_url = "fake_html_url"; published_at = "2020-05-14T09:54:06Z"; assets = $assets }, + @{ name = "3.5.2"; draft = $false; prerelease = $true; html_url = "fake_html_url"; published_at = "2020-05-06T11:45:36Z"; assets = $assets }, + @{ name = "3.8.3"; draft = $false; prerelease = $false; html_url = "fake_html_url"; published_at = "2020-05-06T11:43:38Z"; assets = $assets } + ) + $expectedManifest = @( + [PSCustomObject]@{ version = "3.8.3"; stable = $true; release_url = "fake_html_url"; files = $expectedManifestFiles } + ) + [array]$actualManifest = Build-VersionsManifest -Releases $releases -Configuration $configuration + Assert-Equivalent -Actual $actualManifest -Expected $expectedManifest + } + + It "take latest published release for each version" { + $releases = @( + @{ name = "3.8.1"; draft = $false; prerelease = $false; html_url = "fake_html_url1"; published_at = "2020-05-06T11:45:36Z"; assets = $assets }, + @{ name = "3.8.1"; draft = $false; prerelease = $false; html_url = "fake_html_url2"; published_at = "2020-05-14T09:54:06Z"; assets = $assets }, + @{ name = "3.8.1"; draft = $false; prerelease = $false; html_url = "fake_html_url3"; published_at = "2020-05-06T11:43:38Z"; assets = $assets } + ) + $expectedManifest = @( + [PSCustomObject]@{ version = "3.8.1"; stable = $true; release_url = "fake_html_url2"; files = $expectedManifestFiles } + ) + [array]$actualManifest = Build-VersionsManifest -Releases $releases -Configuration $configuration + Assert-Equivalent -Actual $actualManifest -Expected $expectedManifest + } +} \ No newline at end of file diff --git a/helpers/packages-generation/manifest-utils.psm1 b/helpers/packages-generation/manifest-utils.psm1 new file mode 100644 index 0000000..2d4ef00 --- /dev/null +++ b/helpers/packages-generation/manifest-utils.psm1 @@ -0,0 +1,77 @@ +function Read-ConfigurationFile { + param ([Parameter(Mandatory)][string]$Filepath) + return Get-Content $Filepath -Raw | ConvertFrom-Json +} + +function New-AssetItem { + param ( + [Parameter(Mandatory)][object]$ReleaseAsset, + [Parameter(Mandatory)][object]$Configuration + ) + $regexResult = [regex]::Match($ReleaseAsset.name, $Configuration.regex) + if (-not $regexResult.Success) { throw "Can't match asset filename '$($_.name)' to regex" } + + $result = New-Object PSObject + $result | Add-Member -Name "filename" -Value $ReleaseAsset.name -MemberType NoteProperty + $Configuration.groups.PSObject.Properties | ForEach-Object { + if (($_.Value).GetType().Name.StartsWith("Int")) { + $value = $regexResult.Groups[$_.Value].Value + } else { + $value = $_.Value + } + + if (-not ([string]::IsNullOrEmpty($value))) { + $result | Add-Member -Name $_.Name -Value $value -MemberType NoteProperty + } + } + + $result | Add-Member -Name "download_url" -Value $ReleaseAsset.browser_download_url -MemberType NoteProperty + return $result +} + +function Get-VersionFromRelease { + param ( + [Parameter(Mandatory)][object]$Release + ) + # Release name can contain additional information after ':' so filter it + [string]$releaseName = $Release.name.Split(':')[0] + [Version]$version = $null + if (![Version]::TryParse($releaseName, [ref]$version)) { + throw "Release '$($Release.id)' has invalid title '$($Release.name)'. It can't be parsed as version. ( $($Release.html_url) )" + } + + return $version +} + +function Build-VersionsManifest { + param ( + [Parameter(Mandatory)][array]$Releases, + [Parameter(Mandatory)][object]$Configuration + ) + + $Releases = $Releases | Sort-Object -Property "published_at" -Descending + + $versionsHash = @{} + foreach ($release in $Releases) { + if (($release.draft -eq $true) -or ($release.prerelease -eq $true)) { + continue + } + + [Version]$version = Get-VersionFromRelease $release + $versionKey = $version.ToString() + + if ($versionsHash.ContainsKey($versionKey)) { + continue + } + + $versionsHash.Add($versionKey, [PSCustomObject]@{ + version = $versionKey + stable = $true + release_url = $release.html_url + files = $release.assets | ForEach-Object { New-AssetItem -ReleaseAsset $_ -Configuration $Configuration } + }) + } + + # Sort versions by descending + return $versionsHash.Values | Sort-Object -Property @{ Expression = { [Version]$_.version }; Descending = $true } +} \ No newline at end of file diff --git a/helpers/packages-generation/pester-extensions.psm1 b/helpers/pester-extensions.psm1 similarity index 100% rename from helpers/packages-generation/pester-extensions.psm1 rename to helpers/pester-extensions.psm1 diff --git a/helpers/win-helpers.psm1 b/helpers/win-helpers.psm1 index 74a16f3..beac0b9 100644 --- a/helpers/win-helpers.psm1 +++ b/helpers/win-helpers.psm1 @@ -21,14 +21,19 @@ function Create-SevenZipArchive { [Parameter(Mandatory=$true)] [String]$ArchivePath, [String]$ArchiveType = "zip", - [String]$CompressionLevel = 5 + [String]$CompressionLevel = 5, + [switch]$IncludeSymlinks ) - $ArchiveTypeArgument = "-t${ArchiveType}" - $CompressionLevelArgument = "-mx=${CompressionLevel}" - + $ArchiveTypeArguments = @( + "-t${ArchiveType}", + "-mx=${CompressionLevel}" + ) + if ($IncludeSymlinks) { + $ArchiveTypeArguments += "-snl" + } Push-Location $SourceFolder - Write-Debug "7z a $ArchiveTypeArgument $CompressionLevelArgument $ArchivePath @$SourceFolder" - 7z a $ArchiveTypeArgument $CompressionLevelArgument $ArchivePath $SourceFolder\* + Write-Debug "7z a $ArchiveTypeArgument $ArchivePath @$SourceFolder" + 7z a @ArchiveTypeArguments $ArchivePath $SourceFolder\* Pop-Location } \ No newline at end of file diff --git a/helpers/win-vs-env.psm1 b/helpers/win-vs-env.psm1 new file mode 100644 index 0000000..1eb9093 --- /dev/null +++ b/helpers/win-vs-env.psm1 @@ -0,0 +1,48 @@ +### +# Visual Studio helper functions +### + +function Get-VSWhere { + $vswhere = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe"; + + if (-not (Test-Path $vswhere )) { + [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12 + $vswhere = ".\vswhere.exe" + $vswhereApiUri = "https://api.github.com/repos/Microsoft/vswhere/releases/latest" + $tag = (Invoke-RestMethod -Uri $vswhereApiUri)[0].tag_name + $vswhereUri = "https://github.com/Microsoft/vswhere/releases/download/$tag/vswhere.exe" + Invoke-WebRequest -Uri $vswhereUri -OutFile $vswhere | Out-Null + } + + return $vswhere +} + +function Invoke-Environment +{ + Param + ( + [Parameter(Mandatory)] + [string] + $Command + ) + + & "${env:COMSPEC}" /s /c "`"$Command`" -no_logo && set" | Foreach-Object { + if ($_ -match '^([^=]+)=(.*)') { + [System.Environment]::SetEnvironmentVariable($matches[1], $matches[2]) + } + } +} + +function Get-VSInstallationPath { + $vswhere = Get-VSWhere + $installationPath = & $vswhere -prerelease -legacy -latest -property installationPath + + return $installationPath +} + +function Invoke-VSDevEnvironment { + Write-Host "Invoke-VSDevEnvironment had been invoked" + $installationPath = Get-VSInstallationPath + $envFilepath = Join-Path $installationPath "Common7\Tools\vsdevcmd.bat" + Invoke-Environment -Command $envFilepath +} \ No newline at end of file diff --git a/tests/Node.Tests.ps1 b/tests/Node.Tests.ps1 index 8ff9cb7..3df4c21 100644 --- a/tests/Node.Tests.ps1 +++ b/tests/Node.Tests.ps1 @@ -3,7 +3,7 @@ param ( $Version ) -Import-Module (Join-Path $PSScriptRoot "../helpers/packages-generation/pester-extensions.psm1") +Import-Module (Join-Path $PSScriptRoot "../helpers/pester-extensions.psm1") function Get-UseNodeLogs { $logsFolderPath = Join-Path -Path $env:AGENT_HOMEDIRECTORY -ChildPath "_diag" | Join-Path -ChildPath "pages" -- 2.49.1 From e76e666987342d3cdb41686e06729d44849f5c97 Mon Sep 17 00:00:00 2001 From: Dmitry Shibanov Date: Mon, 1 Jun 2020 16:03:49 +0300 Subject: [PATCH 2/5] minor changes --- helpers/nix-helpers.psm1 | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/helpers/nix-helpers.psm1 b/helpers/nix-helpers.psm1 index 0f00e21..ed50056 100644 --- a/helpers/nix-helpers.psm1 +++ b/helpers/nix-helpers.psm1 @@ -25,18 +25,15 @@ function Create-TarArchive { ) $arguments = @( - "-c", - "-f", $ArchivePath, - "." + "-c", "--$CompressionType" ) - If ($CompressionType) { - $arguments += "--${CompressionType}" - } if ($DereferenceSymlinks) { $arguments += "-h" } + $arguments += @("-f", $ArchivePath, ".") + Push-Location $SourceFolder Write-Debug "tar $arguments" tar @arguments -- 2.49.1 From 1f65b4dbe709602971a375dea2abccb61dab0d5a Mon Sep 17 00:00:00 2001 From: Dmitry Shibanov Date: Mon, 1 Jun 2020 17:56:43 +0300 Subject: [PATCH 3/5] remove from git helpers --- helpers/CODE_OF_CONDUCT.md | 76 ----------- helpers/CONTRIBUTING.md | 31 ----- helpers/LICENSE | 21 --- helpers/README.md | 5 - helpers/SECURITY.md | 3 - helpers/azure-devops/azure-devops-api.ps1 | 89 ------------- helpers/azure-devops/build-info.ps1 | 44 ------ helpers/azure-devops/run-ci-builds.ps1 | 94 ------------- helpers/clean-toolcache.ps1 | 13 -- helpers/common-helpers.psm1 | 85 ------------ helpers/github/create-pull-request.ps1 | 106 --------------- helpers/github/git.psm1 | 81 ----------- helpers/github/github-api.psm1 | 126 ------------------ helpers/nix-helpers.psm1 | 41 ------ .../manifest-generator.ps1 | 35 ----- .../manifest-utils.Tests.ps1 | 116 ---------------- .../packages-generation/manifest-utils.psm1 | 77 ----------- helpers/pester-extensions.psm1 | 33 ----- helpers/win-helpers.psm1 | 39 ------ helpers/win-vs-env.psm1 | 48 ------- 20 files changed, 1163 deletions(-) delete mode 100644 helpers/CODE_OF_CONDUCT.md delete mode 100644 helpers/CONTRIBUTING.md delete mode 100644 helpers/LICENSE delete mode 100644 helpers/README.md delete mode 100644 helpers/SECURITY.md delete mode 100644 helpers/azure-devops/azure-devops-api.ps1 delete mode 100644 helpers/azure-devops/build-info.ps1 delete mode 100644 helpers/azure-devops/run-ci-builds.ps1 delete mode 100644 helpers/clean-toolcache.ps1 delete mode 100644 helpers/common-helpers.psm1 delete mode 100644 helpers/github/create-pull-request.ps1 delete mode 100644 helpers/github/git.psm1 delete mode 100644 helpers/github/github-api.psm1 delete mode 100644 helpers/nix-helpers.psm1 delete mode 100644 helpers/packages-generation/manifest-generator.ps1 delete mode 100644 helpers/packages-generation/manifest-utils.Tests.ps1 delete mode 100644 helpers/packages-generation/manifest-utils.psm1 delete mode 100644 helpers/pester-extensions.psm1 delete mode 100644 helpers/win-helpers.psm1 delete mode 100644 helpers/win-vs-env.psm1 diff --git a/helpers/CODE_OF_CONDUCT.md b/helpers/CODE_OF_CONDUCT.md deleted file mode 100644 index 517657b..0000000 --- a/helpers/CODE_OF_CONDUCT.md +++ /dev/null @@ -1,76 +0,0 @@ -# Contributor Covenant Code of Conduct - -## Our Pledge - -In the interest of fostering an open and welcoming environment, we as -contributors and maintainers pledge to make participation in our project and -our community a harassment-free experience for everyone, regardless of age, body -size, disability, ethnicity, sex characteristics, gender identity and expression, -level of experience, education, socio-economic status, nationality, personal -appearance, race, religion, or sexual identity and orientation. - -## Our Standards - -Examples of behavior that contributes to creating a positive environment -include: - -* Using welcoming and inclusive language -* Being respectful of differing viewpoints and experiences -* Gracefully accepting constructive criticism -* Focusing on what is best for the community -* Showing empathy towards other community members - -Examples of unacceptable behavior by participants include: - -* The use of sexualized language or imagery and unwelcome sexual attention or - advances -* Trolling, insulting/derogatory comments, and personal or political attacks -* Public or private harassment -* Publishing others' private information, such as a physical or electronic - address, without explicit permission -* Other conduct which could reasonably be considered inappropriate in a - professional setting - -## Our Responsibilities - -Project maintainers are responsible for clarifying the standards of acceptable -behavior and are expected to take appropriate and fair corrective action in -response to any instances of unacceptable behavior. - -Project maintainers have the right and responsibility to remove, edit, or -reject comments, commits, code, wiki edits, issues, and other contributions -that are not aligned to this Code of Conduct, or to ban temporarily or -permanently any contributor for other behaviors that they deem inappropriate, -threatening, offensive, or harmful. - -## Scope - -This Code of Conduct applies within all project spaces, and it also applies when -an individual is representing the project or its community in public spaces. -Examples of representing a project or community include using an official -project e-mail address, posting via an official social media account, or acting -as an appointed representative at an online or offline event. Representation of -a project may be further defined and clarified by project maintainers. - -## Enforcement - -Instances of abusive, harassing, or otherwise unacceptable behavior may be -reported by contacting the project team at opensource@github.com. All -complaints will be reviewed and investigated and will result in a response that -is deemed necessary and appropriate to the circumstances. The project team is -obligated to maintain confidentiality with regard to the reporter of an incident. -Further details of specific enforcement policies may be posted separately. - -Project maintainers who do not follow or enforce the Code of Conduct in good -faith may face temporary or permanent repercussions as determined by other -members of the project's leadership. - -## Attribution - -This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, -available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html - -[homepage]: https://www.contributor-covenant.org - -For answers to common questions about this code of conduct, see -https://www.contributor-covenant.org/faq \ No newline at end of file diff --git a/helpers/CONTRIBUTING.md b/helpers/CONTRIBUTING.md deleted file mode 100644 index fbc5fc5..0000000 --- a/helpers/CONTRIBUTING.md +++ /dev/null @@ -1,31 +0,0 @@ -## Contributing - -[fork]: https://github.com/actions/versions-package-tools/fork -[pr]: https://github.com/actions/versions-package-tools/compare -[code-of-conduct]: CODE_OF_CONDUCT.md - -Hi there! We're thrilled that you'd like to contribute to this project. Your help is essential for keeping it great. - -Contributions to this project are [released](https://help.github.com/articles/github-terms-of-service/#6-contributions-under-repository-license) to the public under the [MIT](LICENSE.md). - -Please note that this project is released with a [Contributor Code of Conduct][code-of-conduct]. By participating in this project you agree to abide by its terms. - -## Submitting a pull request - -1. [Fork][fork] and clone the repository -1. Create a new branch: `git checkout -b my-branch-name` -1. Make your changes -1. Push to your fork and [submit a pull request][pr] -1. Make sure that checks in your pull request are green - -Here are a few things you can do that will increase the likelihood of your pull request being accepted: - -- Please include a summary of the change and which issue is fixed. Also include relevant motivation and context. -- Follow the style guide for [PowerShell](https://github.com/PoshCode/PowerShellPracticeAndStyle). -- Write [good commit messages](http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html). - -## Resources - -- [How to Contribute to Open Source](https://opensource.guide/how-to-contribute/) -- [Using Pull Requests](https://help.github.com/articles/about-pull-requests/) -- [GitHub Help](https://help.github.com) diff --git a/helpers/LICENSE b/helpers/LICENSE deleted file mode 100644 index f42f5ab..0000000 --- a/helpers/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2020 GitHub - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. \ No newline at end of file diff --git a/helpers/README.md b/helpers/README.md deleted file mode 100644 index d9d6ccf..0000000 --- a/helpers/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# Common tools for generation of packages in the actions/*-versions repositories -This repository contains PowerShell modules that are used to generate packages for Actions. The packages are consumed by the images generated through [actions/virtual-environments](https://github.com/actions/virtual-environments) and some of the setup-* Actions - -## Contribution -Contributions are welcome! See [Contributor's Guide](./CONTRIBUTING.md) for more details about contribution process and code structure diff --git a/helpers/SECURITY.md b/helpers/SECURITY.md deleted file mode 100644 index f0b196f..0000000 --- a/helpers/SECURITY.md +++ /dev/null @@ -1,3 +0,0 @@ -If you discover a security issue in this repo, please submit it through the [GitHub Security Bug Bounty](https://hackerone.com/github) - -Thanks for helping make GitHub Actions safe for everyone. diff --git a/helpers/azure-devops/azure-devops-api.ps1 b/helpers/azure-devops/azure-devops-api.ps1 deleted file mode 100644 index dd2feba..0000000 --- a/helpers/azure-devops/azure-devops-api.ps1 +++ /dev/null @@ -1,89 +0,0 @@ -class AzureDevOpsApi -{ - [string] $BaseUrl - [string] $RepoOwner - [object] $AuthHeader - - AzureDevOpsApi( - [string] $TeamFoundationCollectionUri, - [string] $ProjectName, - [string] $AccessToken - ) { - $this.BaseUrl = $this.BuildBaseUrl($TeamFoundationCollectionUri, $ProjectName) - $this.AuthHeader = $this.BuildAuth($AccessToken) - } - - [object] hidden BuildAuth([string]$AccessToken) { - if ([string]::IsNullOrEmpty($AccessToken)) { - return $null - } - return @{ - Authorization = "Bearer $AccessToken" - } - } - - [string] hidden BuildBaseUrl([string]$TeamFoundationCollectionUri, [string]$ProjectName) { - return "${TeamFoundationCollectionUri}/${ProjectName}/_apis" - } - - [object] QueueBuild([string]$ToolVersion, [string]$SourceBranch, [string]$SourceVersion, [UInt32]$DefinitionId){ - $url = "build/builds" - - # The content of parameters field should be a json string - $buildParameters = @{ VERSION = $ToolVersion } | ConvertTo-Json - - $body = @{ - definition = @{ - id = $DefinitionId - } - sourceBranch = $SourceBranch - sourceVersion = $SourceVersion - parameters = $buildParameters - } | ConvertTo-Json - - return $this.InvokeRestMethod($url, 'POST', $body) - } - - [object] GetBuildInfo([UInt32]$BuildId){ - $url = "build/builds/$BuildId" - - return $this.InvokeRestMethod($url, 'GET', $null) - } - - [string] hidden BuildUrl([string]$Url) { - return "$($this.BaseUrl)/${Url}/?api-version=5.1" - } - - [object] hidden InvokeRestMethod( - [string] $Url, - [string] $Method, - [string] $Body - ) { - $requestUrl = $this.BuildUrl($Url) - $params = @{ - Method = $Method - ContentType = "application/json" - Uri = $requestUrl - Headers = @{} - } - if ($this.AuthHeader) { - $params.Headers += $this.AuthHeader - } - if (![string]::IsNullOrEmpty($body)) { - $params.Body = $Body - } - - return Invoke-RestMethod @params - } - -} - -function Get-AzureDevOpsApi { - param ( - [string] $TeamFoundationCollectionUri, - [string] $ProjectName, - [string] $AccessToken - ) - - return [AzureDevOpsApi]::New($TeamFoundationCollectionUri, $ProjectName, $AccessToken) -} \ No newline at end of file diff --git a/helpers/azure-devops/build-info.ps1 b/helpers/azure-devops/build-info.ps1 deleted file mode 100644 index 9622d8c..0000000 --- a/helpers/azure-devops/build-info.ps1 +++ /dev/null @@ -1,44 +0,0 @@ -Import-Module (Join-Path $PSScriptRoot "azure-devops-api.ps1") - -class BuildInfo -{ - [AzureDevOpsApi] $AzureDevOpsApi - [String] $Name - [UInt32] $Id - [String] $Status - [String] $Result - [String] $Link - - BuildInfo([AzureDevOpsApi] $AzureDevOpsApi, [object] $Build) - { - $this.AzureDevOpsApi = $AzureDevOpsApi - $this.Id = $Build.id - $this.Name = $Build.buildNumber - $this.Link = $Build._links.web.href - $this.Status = $Build.status - $this.Result = $Build.result - } - - [boolean] IsFinished() { - return ($this.Status -eq "completed") -or ($this.Status -eq "cancelling") - } - - [boolean] IsSuccess() { - return $this.Result -eq "succeeded" - } - - [void] UpdateBuildInfo() { - $buildInfo = $this.AzureDevOpsApi.GetBuildInfo($this.Id) - $this.Status = $buildInfo.status - $this.Result = $buildInfo.result - } -} - -function Get-BuildInfo { - param ( - [AzureDevOpsApi] $AzureDevOpsApi, - [object] $Build - ) - - return [BuildInfo]::New($AzureDevOpsApi, $Build) -} \ No newline at end of file diff --git a/helpers/azure-devops/run-ci-builds.ps1 b/helpers/azure-devops/run-ci-builds.ps1 deleted file mode 100644 index bf11295..0000000 --- a/helpers/azure-devops/run-ci-builds.ps1 +++ /dev/null @@ -1,94 +0,0 @@ -param ( - [Parameter(Mandatory)] [string] $TeamFoundationCollectionUri, - [Parameter(Mandatory)] [string] $AzureDevOpsProjectName, - [Parameter(Mandatory)] [string] $AzureDevOpsAccessToken, - [Parameter(Mandatory)] [string] $SourceBranch, - [Parameter(Mandatory)] [string] $ToolVersions, - [Parameter(Mandatory)] [UInt32] $DefinitionId, - [string] $SourceVersion -) - -Import-Module (Join-Path $PSScriptRoot "azure-devops-api.ps1") -Import-Module (Join-Path $PSScriptRoot "build-info.ps1") - -function Queue-Builds { - param ( - [Parameter(Mandatory)] [AzureDevOpsApi] $AzureDevOpsApi, - [Parameter(Mandatory)] [string] $ToolVersions, - [Parameter(Mandatory)] [string] $SourceBranch, - [Parameter(Mandatory)] [string] $SourceVersion, - [Parameter(Mandatory)] [string] $DefinitionId - ) - - [BuildInfo[]]$queuedBuilds = @() - - $ToolVersions.Split(',') | ForEach-Object { - $version = $_.Trim() - Write-Host "Queue build for $version..." - $queuedBuild = $AzureDevOpsApi.QueueBuild($version, $SourceBranch, $SourceVersion, $DefinitionId) - $buildInfo = Get-BuildInfo -AzureDevOpsApi $AzureDevOpsApi -Build $queuedBuild - Write-Host "Queued build: $($buildInfo.Link)" - $queuedBuilds += $buildInfo - } - - return $queuedBuilds -} - -function Wait-Builds { - param ( - [Parameter(Mandatory)] [BuildInfo[]] $Builds - ) - - $timeoutBetweenRefreshSec = 30 - - do { - # If build is still running - refresh its status - foreach($build in $builds) { - if (!$build.IsFinished()) { - $build.UpdateBuildInfo() - - if ($build.IsFinished()) { - Write-Host "The $($build.Name) build was completed: $($build.Link)" - } - } - } - - $runningBuildsCount = ($builds | Where-Object { !$_.IsFinished() }).Length - - Start-Sleep -Seconds $timeoutBetweenRefreshSec - } while($runningBuildsCount -gt 0) -} - -function Make-BuildsOutput { - param ( - [Parameter(Mandatory)] [BuildInfo[]] $Builds - ) - - Write-Host "Builds info:" - $builds | Format-Table -AutoSize -Property Name,Id,Status,Result,Link | Out-String -Width 10000 - - # Return exit code based on status of builds - $failedBuilds = ($builds | Where-Object { !$_.IsSuccess() }) - if ($failedBuilds.Length -ne 0) { - Write-Host "##vso[task.logissue type=error;]Builds failed" - $failedBuilds | ForEach-Object -Process { Write-Host "##vso[task.logissue type=error;]Name: $($_.Name); Link: $($_.Link)" } - Write-Host "##vso[task.complete result=Failed]" - } else { - Write-host "##[section] All builds have been passed successfully" - } -} - -$azureDevOpsApi = Get-AzureDevOpsApi -TeamFoundationCollectionUri $TeamFoundationCollectionUri ` - -ProjectName $AzureDevOpsProjectName ` - -AccessToken $AzureDevOpsAccessToken - -$queuedBuilds = Queue-Builds -AzureDevOpsApi $azureDevOpsApi ` - -ToolVersions $ToolVersions ` - -SourceBranch $SourceBranch ` - -SourceVersion $SourceVersion ` - -DefinitionId $DefinitionId - -Write-Host "Waiting results of builds ..." -Wait-Builds -Builds $queuedBuilds - -Make-BuildsOutput -Builds $queuedBuilds diff --git a/helpers/clean-toolcache.ps1 b/helpers/clean-toolcache.ps1 deleted file mode 100644 index 67796de..0000000 --- a/helpers/clean-toolcache.ps1 +++ /dev/null @@ -1,13 +0,0 @@ -param ( - [string] $ToolName -) - -$targetPath = $env:AGENT_TOOLSDIRECTORY -if ($ToolName) { - $targetPath = Join-Path $targetPath $ToolName -} - -if (Test-Path $targetPath) { - Get-ChildItem -Path $targetPath -Recurse | Where-Object { $_.LinkType -eq "SymbolicLink" } | ForEach-Object { $_.Delete() } - Remove-Item -Path $targetPath -Recurse -Force -} diff --git a/helpers/common-helpers.psm1 b/helpers/common-helpers.psm1 deleted file mode 100644 index cbc7665..0000000 --- a/helpers/common-helpers.psm1 +++ /dev/null @@ -1,85 +0,0 @@ -<# -.SYNOPSIS -The execute command and print all output to the logs -#> -function Execute-Command { - [CmdletBinding()] - param( - [Parameter(Mandatory=$true)][string] $Command - ) - - Write-Debug "Execute $Command" - - try { - Invoke-Expression $Command | ForEach-Object { Write-Host $_ } - if ($LASTEXITCODE -ne 0) { throw "Exit code: $LASTEXITCODE"} - } - catch { - $errorMessage = "Error happened during command execution: $Command `n $_" - Write-Host $errorMessage - if ($ErrorActionPreference -ne "Continue") { - # avoid logging Azure DevOps issues in case of $ErrorActionPreference -eq Continue - Write-Host "##vso[task.logissue type=error;] $errorMessage" - } - } -} - -<# -.SYNOPSIS -Download file from url and return local path to file -#> -function Download-File { - param( - [Parameter(Mandatory=$true)] - [Uri]$Uri, - [Parameter(Mandatory=$true)] - [String]$OutputFolder - ) - - $targetFilename = [IO.Path]::GetFileName($Uri) - $targetFilepath = Join-Path $OutputFolder $targetFilename - - Write-Debug "Download source from $Uri to $OutFile" - try { - (New-Object System.Net.WebClient).DownloadFile($Uri, $targetFilepath) - return $targetFilepath - } catch { - Write-Host "Error during downloading file from '$Uri'" - "$_" - exit 1 - } -} - -<# -.SYNOPSIS -Generate file that contains the list of all files in particular directory -#> -function New-ToolStructureDump { - param( - [Parameter(Mandatory=$true)] - [String]$ToolPath, - [Parameter(Mandatory=$true)] - [String]$OutputFolder - ) - - $outputFile = Join-Path $OutputFolder "tools_structure.txt" - - $folderContent = Get-ChildItem -Path $ToolPath -Recurse | Sort-Object | Select-Object -Property FullName, Length - $folderContent | ForEach-Object { - $relativePath = $_.FullName.Replace($ToolPath, ""); - return "${relativePath}" - } | Out-File -FilePath $outputFile -} - -<# -.SYNOPSIS -Check if it is macOS / Ubuntu platform -#> -function IsNixPlatform { - param( - [Parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] - [String]$Platform - ) - - return ($Platform -match "macos") -or ($Platform -match "ubuntu") -or ($Platform -match "linux") -} \ No newline at end of file diff --git a/helpers/github/create-pull-request.ps1 b/helpers/github/create-pull-request.ps1 deleted file mode 100644 index 9c01042..0000000 --- a/helpers/github/create-pull-request.ps1 +++ /dev/null @@ -1,106 +0,0 @@ -<# -.SYNOPSIS -Create commit with all unstaged changes in repository and create pull-request - -.PARAMETER RepositoryOwner -Required parameter. The organization which tool repository belongs -.PARAMETER RepositoryName -Optional parameter. The name of tool repository -.PARAMETER AccessToken -Required parameter. PAT Token to authorize -.PARAMETER BranchName -Required parameter. The name of branch where changes will be pushed -.PARAMETER CommitMessage -Required parameter. The commit message to push changes -.PARAMETER PullRequestTitle -Required parameter. The title of pull-request -.PARAMETER PullRequestBody -Required parameter. The description of pull-request -#> -param ( - [Parameter(Mandatory)] [string] $RepositoryOwner, - [Parameter(Mandatory)] [string] $RepositoryName, - [Parameter(Mandatory)] [string] $AccessToken, - [Parameter(Mandatory)] [string] $BranchName, - [Parameter(Mandatory)] [string] $CommitMessage, - [Parameter(Mandatory)] [string] $PullRequestTitle, - [Parameter(Mandatory)] [string] $PullRequestBody -) - -Import-Module (Join-Path $PSScriptRoot "github-api.psm1") -Import-Module (Join-Path $PSScriptRoot "git.psm1") - -function Update-PullRequest { - Param ( - [Parameter(Mandatory=$true)] - [object] $GitHubApi, - [Parameter(Mandatory=$true)] - [string] $Title, - [Parameter(Mandatory=$true)] - [string] $Body, - [Parameter(Mandatory=$true)] - [string] $BranchName, - [Parameter(Mandatory=$true)] - [object] $PullRequest - ) - - $updatedPullRequest = $GitHubApi.UpdatePullRequest($Title, $Body, $BranchName, $PullRequest.number) - - if (($updatedPullRequest -eq $null) -or ($updatedPullRequest.html_url -eq $null)) { - Write-Host "##vso[task.logissue type=error;] Unexpected error occurs while updating pull request." - exit 1 - } - Write-host "##[section] Pull request updated: $($updatedPullRequest.html_url)" -} - -function Create-PullRequest { - Param ( - [Parameter(Mandatory=$true)] - [object] $GitHubApi, - [Parameter(Mandatory=$true)] - [string] $Title, - [Parameter(Mandatory=$true)] - [string] $Body, - [Parameter(Mandatory=$true)] - [string] $BranchName - ) - - $createdPullRequest = $GitHubApi.CreateNewPullRequest($Title, $Body, $BranchName) - - if (($createdPullRequest -eq $null) -or ($createdPullRequest.html_url -eq $null)) { - Write-Host "##vso[task.logissue type=error;] Unexpected error occurs while creating pull request." - exit 1 - } - - Write-host "##[section] Pull request created: $($createdPullRequest.html_url)" -} - -Write-Host "Configure local git preferences" -Git-ConfigureUser -Name "Service account" -Email "no-reply@microsoft.com" - -Write-Host "Create branch: $BranchName" -Git-CreateBranch -Name $BranchName - -Write-Host "Create commit" -Git-CommitAllChanges -Message $CommitMessage - -Write-Host "Push branch: $BranchName" -Git-PushBranch -Name $BranchName -Force $true - -$gitHubApi = Get-GitHubApi -AccountName $RepositoryOwner -ProjectName $RepositoryName -AccessToken $AccessToken -$pullRequest = $gitHubApi.GetPullRequest($BranchName, $RepositoryOwner) - -if ($pullRequest.Count -gt 0) { - Write-Host "Update pull request" - Update-PullRequest -GitHubApi $gitHubApi ` - -Title $PullRequestTitle ` - -Body $PullRequestBody ` - -BranchName $BranchName ` - -PullRequest $pullRequest[0] -} else { - Write-Host "Create pull request" - Create-PullRequest -GitHubApi $gitHubApi ` - -Title $PullRequestTitle ` - -Body $PullRequestBody ` - -BranchName $BranchName -} diff --git a/helpers/github/git.psm1 b/helpers/github/git.psm1 deleted file mode 100644 index 2383a79..0000000 --- a/helpers/github/git.psm1 +++ /dev/null @@ -1,81 +0,0 @@ -<# -.SYNOPSIS -Configure git credentials to use with commits -#> -function Git-ConfigureUser { - Param ( - [Parameter(Mandatory=$true)] - [string] $Name, - [Parameter(Mandatory=$true)] - [string] $Email - ) - - git config --global user.name $Name | Out-Host - git config --global user.email $Email | Out-Host - - if ($LASTEXITCODE -ne 0) { - Write-Host "##vso[task.logissue type=error;] Unexpected failure occurs while configuring git preferences." - exit 1 - } -} - -<# -.SYNOPSIS -Create new branch -#> -function Git-CreateBranch { - Param ( - [Parameter(Mandatory=$true)] - [string] $Name - ) - - git checkout -b $Name | Out-Host - - if ($LASTEXITCODE -ne 0) { - Write-Host "##vso[task.logissue type=error;] Unexpected failure occurs while creating new branch: $Name." - exit 1 - } -} - -<# -.SYNOPSIS -Commit all staged and unstaged changes -#> -function Git-CommitAllChanges { - Param ( - [Parameter(Mandatory=$true)] - [string] $Message - ) - - git add -A | Out-Host - git commit -m "$Message" | Out-Host - - if ($LASTEXITCODE -ne 0) { - Write-Host "##vso[task.logissue type=error;] Unexpected failure occurs while commiting changes." - exit 1 - } -} - -<# -.SYNOPSIS -Push branch to remote repository -#> -function Git-PushBranch { - Param ( - [Parameter(Mandatory=$true)] - [string] $Name, - [Parameter(Mandatory=$true)] - [boolean] $Force - ) - - if ($Force) { - git push --set-upstream origin $Name --force | Out-Host - } else { - git push --set-upstream origin $Name | Out-Host - } - - if ($LASTEXITCODE -ne 0) { - Write-Host "##vso[task.logissue type=error;] Unexpected failure occurs while pushing changes." - exit 1 - } -} \ No newline at end of file diff --git a/helpers/github/github-api.psm1 b/helpers/github/github-api.psm1 deleted file mode 100644 index 2634423..0000000 --- a/helpers/github/github-api.psm1 +++ /dev/null @@ -1,126 +0,0 @@ -<# -.SYNOPSIS -The module that contains a bunch of methods to interact with GitHub API V3 -#> -class GitHubApi -{ - [string] $BaseUrl - [string] $RepoOwner - [object] $AuthHeader - - GitHubApi( - [string] $AccountName, - [string] $ProjectName, - [string] $AccessToken - ) { - $this.BaseUrl = $this.BuildBaseUrl($AccountName, $ProjectName) - $this.AuthHeader = $this.BuildAuth($AccessToken) - } - - [object] hidden BuildAuth([string]$AccessToken) { - if ([string]::IsNullOrEmpty($AccessToken)) { - return $null - } - $base64AuthInfo = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes("'':${AccessToken}")) - return @{ - Authorization = "Basic ${base64AuthInfo}" - } - } - - [string] hidden BuildBaseUrl([string]$RepositoryOwner, [string]$RepositoryName) { - return "https://api.github.com/repos/$RepositoryOwner/$RepositoryName" - } - - [object] CreateNewPullRequest([string]$Title, [string]$Body, [string]$BranchName){ - $requestBody = @{ - title = $Title - body = $Body - head = $BranchName - base = "master" - } | ConvertTo-Json - - $url = "pulls" - return $this.InvokeRestMethod($url, 'Post', $null, $requestBody) - } - - [object] GetPullRequest([string]$BranchName, [string]$RepositoryOwner){ - $url = "pulls" - return $this.InvokeRestMethod($url, 'GET', "head=${RepositoryOwner}:$BranchName&base=master", $null) - } - - [object] UpdatePullRequest([string]$Title, [string]$Body, [string]$BranchName, [string]$PullRequestNumber){ - $requestBody = @{ - title = $Title - body = $Body - head = $BranchName - base = "master" - } | ConvertTo-Json - - $url = "pulls/$PullRequestNumber" - return $this.InvokeRestMethod($url, 'Post', $null, $requestBody) - } - - [array] GetReleases(){ - $url = "releases" - $releases = @() - $pageNumber = 1 - $releaseNumberLimit = 10000 - - while ($releases.Count -le $releaseNumberLimit) - { - $requestParams = "page=${pageNumber}&per_page=100" - [array] $response = $this.InvokeRestMethod($url, 'GET', $requestParams, $null) - - if ($response.Count -eq 0) { - break - } else { - $releases += $response - $pageNumber++ - } - } - - return $releases - } - - [string] hidden BuildUrl([string]$Url, [string]$RequestParams) { - if ([string]::IsNullOrEmpty($RequestParams)) { - return "$($this.BaseUrl)/$($Url)" - } else { - return "$($this.BaseUrl)/$($Url)?$($RequestParams)" - } - } - - [object] hidden InvokeRestMethod( - [string] $Url, - [string] $Method, - [string] $RequestParams, - [string] $Body - ) { - $requestUrl = $this.BuildUrl($Url, $RequestParams) - $params = @{ - Method = $Method - ContentType = "application/json" - Uri = $requestUrl - Headers = @{} - } - if ($this.AuthHeader) { - $params.Headers += $this.AuthHeader - } - if (![string]::IsNullOrEmpty($Body)) { - $params.Body = $Body - } - - return Invoke-RestMethod @params - } - -} - -function Get-GitHubApi { - param ( - [string] $AccountName, - [string] $ProjectName, - [string] $AccessToken - ) - - return [GitHubApi]::New($AccountName, $ProjectName, $AccessToken) -} \ No newline at end of file diff --git a/helpers/nix-helpers.psm1 b/helpers/nix-helpers.psm1 deleted file mode 100644 index ed50056..0000000 --- a/helpers/nix-helpers.psm1 +++ /dev/null @@ -1,41 +0,0 @@ -<# -.SYNOPSIS -Unpack *.tar file -#> -function Extract-TarArchive { - param( - [Parameter(Mandatory=$true)] - [String]$ArchivePath, - [Parameter(Mandatory=$true)] - [String]$OutputDirectory - ) - - Write-Debug "Extract $ArchivePath to $OutputDirectory" - tar -C $OutputDirectory -xzf $ArchivePath --strip 1 -} - -function Create-TarArchive { - param( - [Parameter(Mandatory=$true)] - [String]$SourceFolder, - [Parameter(Mandatory=$true)] - [String]$ArchivePath, - [string]$CompressionType = "gz", - [switch]$DereferenceSymlinks - ) - - $arguments = @( - "-c", "--$CompressionType" - ) - - if ($DereferenceSymlinks) { - $arguments += "-h" - } - - $arguments += @("-f", $ArchivePath, ".") - - Push-Location $SourceFolder - Write-Debug "tar $arguments" - tar @arguments - Pop-Location -} \ No newline at end of file diff --git a/helpers/packages-generation/manifest-generator.ps1 b/helpers/packages-generation/manifest-generator.ps1 deleted file mode 100644 index 85434d0..0000000 --- a/helpers/packages-generation/manifest-generator.ps1 +++ /dev/null @@ -1,35 +0,0 @@ -<# -.SYNOPSIS -Generate versions manifest based on repository releases - -.DESCRIPTION -Versions manifest is needed to find the latest assets for particular version of tool -.PARAMETER GitHubRepositoryOwner -Required parameter. The organization which tool repository belongs -.PARAMETER GitHubRepositoryName -Optional parameter. The name of tool repository -.PARAMETER GitHubAccessToken -Required parameter. PAT Token to overcome GitHub API Rate limit -.PARAMETER OutputFile -Required parameter. File "*.json" where generated results will be saved -.PARAMETER ConfigurationFile -Path to the json file with parsing configuration -#> - -param ( - [Parameter(Mandatory)] [string] $GitHubRepositoryOwner, - [Parameter(Mandatory)] [string] $GitHubRepositoryName, - [Parameter(Mandatory)] [string] $GitHubAccessToken, - [Parameter(Mandatory)] [string] $OutputFile, - [Parameter(Mandatory)] [string] $ConfigurationFile -) - -Import-Module (Join-Path $PSScriptRoot "../github/github-api.psm1") -Import-Module (Join-Path $PSScriptRoot "manifest-utils.psm1") -Force - -$configuration = Read-ConfigurationFile -Filepath $ConfigurationFile - -$gitHubApi = Get-GitHubApi -AccountName $GitHubRepositoryOwner -ProjectName $GitHubRepositoryName -AccessToken $GitHubAccessToken -$releases = $gitHubApi.GetReleases() -$versionIndex = Build-VersionsManifest -Releases $releases -Configuration $configuration -$versionIndex | ConvertTo-Json -Depth 5 | Out-File $OutputFile -Encoding UTF8NoBOM -Force diff --git a/helpers/packages-generation/manifest-utils.Tests.ps1 b/helpers/packages-generation/manifest-utils.Tests.ps1 deleted file mode 100644 index f1aa785..0000000 --- a/helpers/packages-generation/manifest-utils.Tests.ps1 +++ /dev/null @@ -1,116 +0,0 @@ -Import-Module (Join-Path $PSScriptRoot "manifest-utils.psm1") -Force - -Describe "New-AssetItem" { - It "use regex to parse all values in correct order" { - $githubAsset = @{ name = "python-3.8.3-linux-16.04-x64.tar.gz"; browser_download_url = "long_url"; } - $configuration = @{ - regex = "python-\d+\.\d+\.\d+-(\w+)-([\w\.]+)?-?(x\d+)"; - groups = [PSCustomObject]@{ platform = 1; platform_version = 2; arch = 3; }; - } - $expectedOutput = [PSCustomObject]@{ - filename = "python-3.8.3-linux-16.04-x64.tar.gz"; platform = "linux"; platform_version = "16.04"; - arch = "x64"; download_url = "long_url"; - } - - $actualOutput = New-AssetItem -ReleaseAsset $githubAsset -Configuration $configuration - Assert-Equivalent -Actual $actualOutput -Expected $expectedOutput - } - - It "support constant values in groups" { - $githubAsset = @{ name = "python-3.8.3-linux-16.04-x64.tar.gz"; browser_download_url = "long_url"; } - $configuration = @{ - regex = "python-\d+\.\d+\.\d+-(\w+)-([\w\.]+)?-?(x\d+)"; - groups = [PSCustomObject]@{ platform = 1; platform_version = 2; arch = "x64"; } - } - $expectedOutput = [PSCustomObject]@{ - filename = "python-3.8.3-linux-16.04-x64.tar.gz"; platform = "linux"; platform_version = "16.04"; - arch = "x64"; download_url = "long_url"; - } - - $actualOutput = New-AssetItem -ReleaseAsset $githubAsset -Configuration $configuration - Assert-Equivalent -Actual $actualOutput -Expected $expectedOutput - } - - It "Skip empty groups" { - $githubAsset = @{ name = "python-3.8.3-win32-x64.zip"; browser_download_url = "long_url"; } - $configuration = @{ - regex = "python-\d+\.\d+\.\d+-(\w+)-([\w\.]+)?-?(x\d+)"; - groups = [PSCustomObject]@{ platform = 1; platform_version = 2; arch = 3; } - } - $expectedOutput = [PSCustomObject]@{ - filename = "python-3.8.3-win32-x64.zip"; platform = "win32"; - arch = "x64"; download_url = "long_url"; - } - - $actualOutput = New-AssetItem -ReleaseAsset $githubAsset -Configuration $configuration - Assert-Equivalent -Actual $actualOutput -Expected $expectedOutput - } -} - -Describe "Get-VersionFromRelease" { - It "clear version" { - $release = @{ name = "3.8.3" } - Get-VersionFromRelease -Release $release | Should -Be "3.8.3" - } - - It "version with title" { - $release = @{ name = "3.8.3: Release title" } - Get-VersionFromRelease -Release $release | Should -Be "3.8.3" - } -} - -Describe "Build-VersionsManifest" { - $assets = @( - @{ name = "python-3.8.3-linux-16.04-x64.tar.gz"; browser_download_url = "fake_url"; } - @{ name = "python-3.8.3-linux-18.04-x64.tar.gz"; browser_download_url = "fake_url"; } - ) - $configuration = @{ - regex = "python-\d+\.\d+\.\d+-(\w+)-([\w\.]+)?-?(x\d+)"; - groups = [PSCustomObject]@{ platform = 1; platform_version = 2; arch = "x64"; } - } - $expectedManifestFiles = @( - [PSCustomObject]@{ filename = "python-3.8.3-linux-16.04-x64.tar.gz"; arch = "x64"; platform = "linux"; platform_version = "16.04"; download_url = "fake_url" }, - [PSCustomObject]@{ filename = "python-3.8.3-linux-18.04-x64.tar.gz"; arch = "x64"; platform = "linux"; platform_version = "18.04"; download_url = "fake_url" } - ) - - It "build manifest with correct version order" { - $releases = @( - @{ name = "3.8.1"; draft = $false; prerelease = $false; html_url = "fake_html_url"; published_at = "2020-05-14T09:54:06Z"; assets = $assets }, - @{ name = "3.5.2: Hello"; draft = $false; prerelease = $false; html_url = "fake_html_url"; published_at = "2020-05-06T11:45:36Z"; assets = $assets }, - @{ name = "3.8.3: Release title"; draft = $false; prerelease = $false; html_url = "fake_html_url"; published_at = "2020-05-06T11:43:38Z"; assets = $assets } - ) - $expectedManifest = @( - [PSCustomObject]@{ version = "3.8.3"; stable = $true; release_url = "fake_html_url"; files = $expectedManifestFiles }, - [PSCustomObject]@{ version = "3.8.1"; stable = $true; release_url = "fake_html_url"; files = $expectedManifestFiles }, - [PSCustomObject]@{ version = "3.5.2"; stable = $true; release_url = "fake_html_url"; files = $expectedManifestFiles } - ) - $actualManifest = Build-VersionsManifest -Releases $releases -Configuration $configuration - Assert-Equivalent -Actual $actualManifest -Expected $expectedManifest - } - - It "Skip draft and prerelease" { - $releases = @( - @{ name = "3.8.1"; draft = $true; prerelease = $false; html_url = "fake_html_url"; published_at = "2020-05-14T09:54:06Z"; assets = $assets }, - @{ name = "3.5.2"; draft = $false; prerelease = $true; html_url = "fake_html_url"; published_at = "2020-05-06T11:45:36Z"; assets = $assets }, - @{ name = "3.8.3"; draft = $false; prerelease = $false; html_url = "fake_html_url"; published_at = "2020-05-06T11:43:38Z"; assets = $assets } - ) - $expectedManifest = @( - [PSCustomObject]@{ version = "3.8.3"; stable = $true; release_url = "fake_html_url"; files = $expectedManifestFiles } - ) - [array]$actualManifest = Build-VersionsManifest -Releases $releases -Configuration $configuration - Assert-Equivalent -Actual $actualManifest -Expected $expectedManifest - } - - It "take latest published release for each version" { - $releases = @( - @{ name = "3.8.1"; draft = $false; prerelease = $false; html_url = "fake_html_url1"; published_at = "2020-05-06T11:45:36Z"; assets = $assets }, - @{ name = "3.8.1"; draft = $false; prerelease = $false; html_url = "fake_html_url2"; published_at = "2020-05-14T09:54:06Z"; assets = $assets }, - @{ name = "3.8.1"; draft = $false; prerelease = $false; html_url = "fake_html_url3"; published_at = "2020-05-06T11:43:38Z"; assets = $assets } - ) - $expectedManifest = @( - [PSCustomObject]@{ version = "3.8.1"; stable = $true; release_url = "fake_html_url2"; files = $expectedManifestFiles } - ) - [array]$actualManifest = Build-VersionsManifest -Releases $releases -Configuration $configuration - Assert-Equivalent -Actual $actualManifest -Expected $expectedManifest - } -} \ No newline at end of file diff --git a/helpers/packages-generation/manifest-utils.psm1 b/helpers/packages-generation/manifest-utils.psm1 deleted file mode 100644 index 2d4ef00..0000000 --- a/helpers/packages-generation/manifest-utils.psm1 +++ /dev/null @@ -1,77 +0,0 @@ -function Read-ConfigurationFile { - param ([Parameter(Mandatory)][string]$Filepath) - return Get-Content $Filepath -Raw | ConvertFrom-Json -} - -function New-AssetItem { - param ( - [Parameter(Mandatory)][object]$ReleaseAsset, - [Parameter(Mandatory)][object]$Configuration - ) - $regexResult = [regex]::Match($ReleaseAsset.name, $Configuration.regex) - if (-not $regexResult.Success) { throw "Can't match asset filename '$($_.name)' to regex" } - - $result = New-Object PSObject - $result | Add-Member -Name "filename" -Value $ReleaseAsset.name -MemberType NoteProperty - $Configuration.groups.PSObject.Properties | ForEach-Object { - if (($_.Value).GetType().Name.StartsWith("Int")) { - $value = $regexResult.Groups[$_.Value].Value - } else { - $value = $_.Value - } - - if (-not ([string]::IsNullOrEmpty($value))) { - $result | Add-Member -Name $_.Name -Value $value -MemberType NoteProperty - } - } - - $result | Add-Member -Name "download_url" -Value $ReleaseAsset.browser_download_url -MemberType NoteProperty - return $result -} - -function Get-VersionFromRelease { - param ( - [Parameter(Mandatory)][object]$Release - ) - # Release name can contain additional information after ':' so filter it - [string]$releaseName = $Release.name.Split(':')[0] - [Version]$version = $null - if (![Version]::TryParse($releaseName, [ref]$version)) { - throw "Release '$($Release.id)' has invalid title '$($Release.name)'. It can't be parsed as version. ( $($Release.html_url) )" - } - - return $version -} - -function Build-VersionsManifest { - param ( - [Parameter(Mandatory)][array]$Releases, - [Parameter(Mandatory)][object]$Configuration - ) - - $Releases = $Releases | Sort-Object -Property "published_at" -Descending - - $versionsHash = @{} - foreach ($release in $Releases) { - if (($release.draft -eq $true) -or ($release.prerelease -eq $true)) { - continue - } - - [Version]$version = Get-VersionFromRelease $release - $versionKey = $version.ToString() - - if ($versionsHash.ContainsKey($versionKey)) { - continue - } - - $versionsHash.Add($versionKey, [PSCustomObject]@{ - version = $versionKey - stable = $true - release_url = $release.html_url - files = $release.assets | ForEach-Object { New-AssetItem -ReleaseAsset $_ -Configuration $Configuration } - }) - } - - # Sort versions by descending - return $versionsHash.Values | Sort-Object -Property @{ Expression = { [Version]$_.version }; Descending = $true } -} \ No newline at end of file diff --git a/helpers/pester-extensions.psm1 b/helpers/pester-extensions.psm1 deleted file mode 100644 index 46d2a75..0000000 --- a/helpers/pester-extensions.psm1 +++ /dev/null @@ -1,33 +0,0 @@ -<# -.SYNOPSIS -Pester extension that allows to run command and validate exit code -.EXAMPLE -"python file.py" | Should -ReturnZeroExitCode -#> -function ShouldReturnZeroExitCode { - Param( - [Parameter (Mandatory = $true)] [ValidateNotNullOrEmpty()] - [String]$ActualValue, - [switch]$Negate - ) - - Write-Host "Run command '${ActualValue}'" - Invoke-Expression -Command $ActualValue | ForEach-Object { Write-Host $_ } - $actualExitCode = $LASTEXITCODE - - [bool]$succeeded = $actualExitCode -eq 0 - if ($Negate) { $succeeded = -not $succeeded } - - if (-not $succeeded) - { - $failureMessage = "Command '${ActualValue}' has finished with exit code ${actualExitCode}" - } - - return New-Object PSObject -Property @{ - Succeeded = $succeeded - FailureMessage = $failureMessage - } -} - -Add-AssertionOperator -Name ReturnZeroExitCode ` - -Test $function:ShouldReturnZeroExitCode diff --git a/helpers/win-helpers.psm1 b/helpers/win-helpers.psm1 deleted file mode 100644 index beac0b9..0000000 --- a/helpers/win-helpers.psm1 +++ /dev/null @@ -1,39 +0,0 @@ -<# -.SYNOPSIS -Unpack *.7z file -#> -function Extract-SevenZipArchive { - param( - [Parameter(Mandatory=$true)] - [String]$ArchivePath, - [Parameter(Mandatory=$true)] - [String]$OutputDirectory - ) - - Write-Debug "Extract $ArchivePath to $OutputDirectory" - 7z x $ArchivePath -o"$OutputDirectory" -y | Out-Null -} - -function Create-SevenZipArchive { - param( - [Parameter(Mandatory=$true)] - [String]$SourceFolder, - [Parameter(Mandatory=$true)] - [String]$ArchivePath, - [String]$ArchiveType = "zip", - [String]$CompressionLevel = 5, - [switch]$IncludeSymlinks - ) - - $ArchiveTypeArguments = @( - "-t${ArchiveType}", - "-mx=${CompressionLevel}" - ) - if ($IncludeSymlinks) { - $ArchiveTypeArguments += "-snl" - } - Push-Location $SourceFolder - Write-Debug "7z a $ArchiveTypeArgument $ArchivePath @$SourceFolder" - 7z a @ArchiveTypeArguments $ArchivePath $SourceFolder\* - Pop-Location -} \ No newline at end of file diff --git a/helpers/win-vs-env.psm1 b/helpers/win-vs-env.psm1 deleted file mode 100644 index 1eb9093..0000000 --- a/helpers/win-vs-env.psm1 +++ /dev/null @@ -1,48 +0,0 @@ -### -# Visual Studio helper functions -### - -function Get-VSWhere { - $vswhere = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe"; - - if (-not (Test-Path $vswhere )) { - [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12 - $vswhere = ".\vswhere.exe" - $vswhereApiUri = "https://api.github.com/repos/Microsoft/vswhere/releases/latest" - $tag = (Invoke-RestMethod -Uri $vswhereApiUri)[0].tag_name - $vswhereUri = "https://github.com/Microsoft/vswhere/releases/download/$tag/vswhere.exe" - Invoke-WebRequest -Uri $vswhereUri -OutFile $vswhere | Out-Null - } - - return $vswhere -} - -function Invoke-Environment -{ - Param - ( - [Parameter(Mandatory)] - [string] - $Command - ) - - & "${env:COMSPEC}" /s /c "`"$Command`" -no_logo && set" | Foreach-Object { - if ($_ -match '^([^=]+)=(.*)') { - [System.Environment]::SetEnvironmentVariable($matches[1], $matches[2]) - } - } -} - -function Get-VSInstallationPath { - $vswhere = Get-VSWhere - $installationPath = & $vswhere -prerelease -legacy -latest -property installationPath - - return $installationPath -} - -function Invoke-VSDevEnvironment { - Write-Host "Invoke-VSDevEnvironment had been invoked" - $installationPath = Get-VSInstallationPath - $envFilepath = Join-Path $installationPath "Common7\Tools\vsdevcmd.bat" - Invoke-Environment -Command $envFilepath -} \ No newline at end of file -- 2.49.1 From 5664c18e556ba8ff882cdd919837958a76cf1291 Mon Sep 17 00:00:00 2001 From: Dmitry Shibanov Date: Mon, 1 Jun 2020 19:22:43 +0300 Subject: [PATCH 4/5] add submodules --- .gitmodules | 4 ++++ helpers | 1 + 2 files changed, 5 insertions(+) create mode 100644 .gitmodules create mode 160000 helpers diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..b343909 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,4 @@ +[submodule "helpers"] + path = helpers + url = https://github.com/actions/versions-package-tools + branch = master diff --git a/helpers b/helpers new file mode 160000 index 0000000..d8c3ce7 --- /dev/null +++ b/helpers @@ -0,0 +1 @@ +Subproject commit d8c3ce72eebad99e5377a15501ae2135ee4b2427 -- 2.49.1 From 7e91c03b0be147fb18fb3faa7f5e144a3945ce77 Mon Sep 17 00:00:00 2001 From: Dmitry Shibanov Date: Tue, 2 Jun 2020 14:40:56 +0300 Subject: [PATCH 5/5] setup 4 version for pester --- azure-pipelines/templates/test-job.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines/templates/test-job.yml b/azure-pipelines/templates/test-job.yml index 1205bcb..82df386 100644 --- a/azure-pipelines/templates/test-job.yml +++ b/azure-pipelines/templates/test-job.yml @@ -53,7 +53,7 @@ jobs: inputs: TargetType: inline script: | - Install-Module Pester -Force -Scope CurrentUser + Install-Module Pester -Force -Scope CurrentUser -RequiredVersion 4.10.1 Import-Module Pester $pesterParams = @{ Path="./Node.Tests.ps1"; -- 2.49.1