diff --git a/.vsts-ci/linux.yml b/.vsts-ci/linux.yml index 886f748..bfb3233 100644 --- a/.vsts-ci/linux.yml +++ b/.vsts-ci/linux.yml @@ -9,12 +9,12 @@ variables: resources: - repo: self clean: true -phases: -- phase: Build - - queue: - name: Hosted Ubuntu 1604 - parallel: 2 +jobs: +- job: Build + pool: + vmImage: ubuntu-latest + displayName: Linux + strategy: matrix: Linux ARM Native: buildName: ubuntu.16.04-arm diff --git a/.vsts-ci/windows.yml b/.vsts-ci/windows.yml index b5da8f4..1ccfab5 100644 --- a/.vsts-ci/windows.yml +++ b/.vsts-ci/windows.yml @@ -14,7 +14,9 @@ jobs: - job: Build pool: - vmImage: vs2017-win2016 + name: PS-PowerShell-x64 + demands: + - ImageOverride -equals PSMMS2019-PS-Native-Secure strategy: matrix: @@ -28,10 +30,6 @@ jobs: buildName: x64_arm64 steps: - - powershell: | - choco install cmake.install --installargs 'ADD_CMAKE_TO_PATH=System' - displayName: Install cmake - condition: succeeded() - powershell: | $vsPath = ${Env:ProgramFiles(x86)} + '\Microsoft Visual Studio\2017\BuildTools\MSBuild\15.0\Bin' Write-Host "##vso[task.prependpath]$vsPath" diff --git a/build.psm1 b/build.psm1 index 3c759c8..d4f090b 100644 --- a/build.psm1 +++ b/build.psm1 @@ -464,7 +464,7 @@ function Start-BuildNativeUnixBinaries { } elseif ($IsMacOS) { Start-NativeExecution { cmake -DCMAKE_TOOLCHAIN_FILE="./macos.toolchain.cmake" . } - Start-NativeExecution { make -j } + Start-NativeExecution { make -j } Start-NativeExecution { ctest --verbose } } else { @@ -533,12 +533,7 @@ function Start-BuildPowerShellNativePackage [Parameter(Mandatory = $true)] [ValidateScript({Test-Path $_ -PathType Leaf})] - [string] $psrpZipPath, - - [Parameter(Mandatory = $true)] - [string] $NuGetOutputPath, - - [switch] $SkipCleanup = $false + [string] $psrpZipPath ) if(-not (Test-Path $PackageRoot)) @@ -605,24 +600,32 @@ function Start-BuildPowerShellNativePackage Copy-Item $iconPath (Join-Path $PackageRoot -ChildPath $iconFileName) -Verbose $Nuspec -f $Version, $iconFileName | Out-File -FilePath (Join-Path $PackageRoot -ChildPath 'Microsoft.PowerShell.Native.nuspec') -Force +} - if(-not (Test-Path $NuGetOutputPath)) - { - $null = New-Item $NuGetOutputPath -Force -Verbose -ItemType Directory - } +<# +.SYNOPSIS + Pack the new nuget package from af folder. +#> +function New-NugetPackage { + param ( + [Parameter(Mandatory)] + [string] $PackageRoot, + + [Parameter(Mandatory)] + [string] $NuGetOutputPath + ) try { + if (-not (Test-Path $NuGetOutputPath)) { + $null = New-Item $NuGetOutputPath -Force -Verbose -ItemType Directory + } + Push-Location $PackageRoot nuget.exe pack . -OutputDirectory $NuGetOutputPath } finally { Pop-Location } - - if(-not $SkipCleanup -and (Test-Path $tempExtractionPath)) - { - Remove-Item $tempExtractionPath -Recurse -Force -ErrorAction SilentlyContinue - } } <# @@ -2153,63 +2156,19 @@ function Start-PSBootstrap { $cmakePresent = precheck 'cmake' $null $sdkPresent = Test-Win10SDK - # Install chocolatey - $chocolateyPath = "$env:AllUsersProfile\chocolatey\bin" - - if(precheck 'choco' $null) { - Write-Log "Chocolatey is already installed. Skipping installation." - } - elseif(($cmakePresent -eq $false) -or ($sdkPresent -eq $false)) { - Write-Log "Chocolatey not present. Installing chocolatey." - if ($Force -or "Install chocolatey via https://chocolatey.org/install.ps1") { - Invoke-Expression ((new-object net.webclient).DownloadString('https://chocolatey.org/install.ps1')) - if (-not ($machinePath.ToLower().Contains($chocolateyPath.ToLower()))) { - Write-Log "Adding $chocolateyPath to Path environment variable" - $env:Path += ";$chocolateyPath" - $newMachineEnvironmentPath += ";$chocolateyPath" - } else { - Write-Log "$chocolateyPath already present in Path environment variable" - } - } else { - Write-Error "Chocolatey is required to install missing dependencies. Please install it from https://chocolatey.org/ manually. Alternatively, install cmake and Windows 10 SDK." - return - } - } else { - Write-Log "Skipping installation of chocolatey, cause both cmake and Win 10 SDK are present." - } - # Install cmake - $cmakePath = "${env:ProgramFiles}\CMake\bin" - if($cmakePresent -and !($force.IsPresent)) { - Write-Log "Cmake is already installed. Skipping installation." + #$cmakePath = "${env:ProgramFiles}\CMake\bin" + if($cmakePresent) { + Write-Log "Cmake is already installed." } else { - Write-Log "Cmake not present or -Force used. Installing cmake." - Start-NativeExecution { choco install cmake -y --version 3.10.0 } - if (-not ($machinePath.ToLower().Contains($cmakePath.ToLower()))) { - Write-Log "Adding $cmakePath to Path environment variable" - $env:Path += ";$cmakePath" - $newMachineEnvironmentPath = "$cmakePath;$newMachineEnvironmentPath" - } else { - Write-Log "$cmakePath already present in Path environment variable" - } + throw "Cmake not present." } - # Install Windows 10 SDK - $packageName = "windows-sdk-10.0" - - if (-not $sdkPresent) { - Write-Log "Windows 10 SDK not present. Installing $packageName." - Start-NativeExecution { choco install windows-sdk-10.0 -y } - } else { - Write-Log "Windows 10 SDK present. Skipping installation." + if ($sdkPresent) { + Write-Log "Windows 10 SDK is already installed." } - - # Update path machine environment variable - if ($newMachineEnvironmentPath -ne $machinePath) { - Write-Log "Updating Path machine environment variable" - if ($Force -or $PSCmdlet.ShouldProcess("Update Path machine environment variable to $newMachineEnvironmentPath")) { - [Environment]::SetEnvironmentVariable('Path', $newMachineEnvironmentPath, 'MACHINE') - } + else { + throw "Windows 10 SDK not present." } } } diff --git a/src/libpsl-native/src/CMakeLists.txt b/src/libpsl-native/src/CMakeLists.txt index cc81571..b389977 100644 --- a/src/libpsl-native/src/CMakeLists.txt +++ b/src/libpsl-native/src/CMakeLists.txt @@ -41,4 +41,4 @@ configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/pal_config.h.in ${CMAKE_CURRENT_BINARY_DIR}/pal_config.h) -target_include_directories(psl-native PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) +target_include_directories(psl-native PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/src/powershell-native/nativemsh/pwrshcommon/pwrshcommon.cpp b/src/powershell-native/nativemsh/pwrshcommon/pwrshcommon.cpp index 0218874..43b0f7f 100644 --- a/src/powershell-native/nativemsh/pwrshcommon/pwrshcommon.cpp +++ b/src/powershell-native/nativemsh/pwrshcommon/pwrshcommon.cpp @@ -668,12 +668,14 @@ namespace NativeMsh // NOTE: The names must not include the .dll extension because it will be added programmatically. static PCSTR trustedAssemblies[] = { + "getfilesiginforedistwrapper", "Markdig.Signed", "Microsoft.ApplicationInsights", + "Microsoft.Bcl.AsyncInterfaces", "Microsoft.CodeAnalysis", "Microsoft.CodeAnalysis.CSharp", "Microsoft.CSharp", - "Microsoft.DiaSymReader.Native.amd64", + "Microsoft.Extensions.ObjectPool", "Microsoft.Management.Infrastructure", "Microsoft.Management.Infrastructure.CimCmdlets", "Microsoft.Management.Infrastructure.Native", @@ -698,7 +700,6 @@ namespace NativeMsh "netstandard", "Newtonsoft.Json", "NJsonSchema", - "PowerShell.Core.Instrumentation", "System", "System.AppContext", "System.Buffers", @@ -730,6 +731,7 @@ namespace NativeMsh "System.Diagnostics.Debug", "System.Diagnostics.DiagnosticSource", "System.Diagnostics.EventLog", + "System.Diagnostics.EventLog.Messages", "System.Diagnostics.FileVersionInfo", "System.Diagnostics.PerformanceCounter", "System.Diagnostics.Process", @@ -746,6 +748,7 @@ namespace NativeMsh "System.Drawing.Primitives", "System.Dynamic.Runtime", "System.Formats.Asn1", + "System.Formats.Tar", "System.Globalization", "System.Globalization.Calendars", "System.Globalization.Extensions", @@ -774,7 +777,6 @@ namespace NativeMsh "System.Management.Automation", "System.Memory", "System.Net", - "System.Net.Connections", "System.Net.Http", "System.Net.Http.Json", "System.Net.Http.WinHttpHandler", @@ -784,6 +786,7 @@ namespace NativeMsh "System.Net.NetworkInformation", "System.Net.Ping", "System.Net.Primitives", + "System.Net.Quic", "System.Net.Requests", "System.Net.Security", "System.Net.ServicePoint", @@ -822,8 +825,8 @@ namespace NativeMsh "System.Runtime.Extensions", "System.Runtime.Handles", "System.Runtime.InteropServices", + "System.Runtime.InteropServices.JavaScript", "System.Runtime.InteropServices.RuntimeInformation", - "System.Runtime.InteropServices.WindowsRuntime", "System.Runtime.Intrinsics", "System.Runtime.Loader", "System.Runtime.Numerics", @@ -835,6 +838,7 @@ namespace NativeMsh "System.Security", "System.Security.AccessControl", "System.Security.Claims", + "System.Security.Cryptography", "System.Security.Cryptography.Algorithms", "System.Security.Cryptography.Cng", "System.Security.Cryptography.Csp", @@ -859,6 +863,7 @@ namespace NativeMsh "System.ServiceModel.Web", "System.ServiceProcess", "System.ServiceProcess.ServiceController", + "System.Speech", "System.Text.Encoding", "System.Text.Encoding.CodePages", "System.Text.Encoding.Extensions", @@ -881,9 +886,9 @@ namespace NativeMsh "System.ValueTuple", "System.Web", "System.Web.HttpUtility", + "System.Web.Services.Description", "System.Windows", "System.Windows.Extensions", - "System.Windows.Forms", "System.Xml", "System.Xml.Linq", "System.Xml.ReaderWriter", diff --git a/tools/releaseBuild/yaml/nuget.yml b/tools/releaseBuild/yaml/nuget.yml index 2e011ad..3635da0 100644 --- a/tools/releaseBuild/yaml/nuget.yml +++ b/tools/releaseBuild/yaml/nuget.yml @@ -82,9 +82,23 @@ steps: $macOSZipPath = Join-Path "$(System.ArtifactsDirectory)/release" 'osx-symbols.zip' $psrpZipPath = Join-Path "$(System.ArtifactsDirectory)/release" 'psrp.zip' - Start-BuildPowerShellNativePackage -PackageRoot $PackageRoot -Version $(PackageVersion) -WindowsX64ZipPath $WindowsX64ZipPath -WindowsX86ZipPath $WindowsX86ZipPath -WindowsARMZipPath $WindowsARMZipPath -WindowsARM64ZipPath $WindowsARM64ZipPath -LinuxZipPath $LinuxZipPath -LinuxARMZipPath $LinuxARMZipPath -LinuxARM64ZipPath $LinuxARM64ZipPath -LinuxAlpineZipPath $LinuxAlpineZipPath -macOSZipPath $macOSZipPath -psrpZipPath $psrpZipPath -NuGetOutputPath $(NuGetPackagePath) - - displayName: 'Build NuGet package' + Start-BuildPowerShellNativePackage -PackageRoot $PackageRoot -Version $(PackageVersion) -WindowsX64ZipPath $WindowsX64ZipPath -WindowsX86ZipPath $WindowsX86ZipPath -WindowsARMZipPath $WindowsARMZipPath -WindowsARM64ZipPath $WindowsARM64ZipPath -LinuxZipPath $LinuxZipPath -LinuxARMZipPath $LinuxARMZipPath -LinuxARM64ZipPath $LinuxARM64ZipPath -LinuxAlpineZipPath $LinuxAlpineZipPath -macOSZipPath $macOSZipPath -psrpZipPath $psrpZipPath + + displayName: 'Collate NuGet package structure' + +- ${{ if ne(variables['SkipSigning'], 'True') }}: + - template: Sbom.yml@ComplianceRepo + parameters: + BuildDropPath: '$(System.ArtifactsDirectory)\NugetPackage' + Build_Repository_Uri: https://github.com/PowerShell/PowerShell-Native + PackageName: Microsoft.PowerShell.Native + PackageVersion: $(PackageVersion) + +- pwsh: | + Import-Module $(Build.SourcesDirectory)\build.psm1 + $PackageRoot = "$(System.ArtifactsDirectory)\NugetPackage" + New-NugetPackage -PackageRoot $PackageRoot -NuGetOutputPath $(NuGetPackagePath) + displayName: Build NuGet package - powershell: 'Get-Childitem $(NuGetPackagePath)' displayName: 'Capture nuget package' diff --git a/tools/releaseBuild/yaml/releaseBuild.yml b/tools/releaseBuild/yaml/releaseBuild.yml index fa6eeed..ae405ae 100644 --- a/tools/releaseBuild/yaml/releaseBuild.yml +++ b/tools/releaseBuild/yaml/releaseBuild.yml @@ -1,10 +1,15 @@ trigger: none variables: - AuthenticodeSignType: '400' - BuildConfiguration: 'Release' - NuGetPackagePath: '$(System.ArtifactsDirectory)/NuGetRelease' - PackageRoot: '$(System.ArtifactsDirectory)/Packages' + - name: BuildConfiguration + value: 'Release' + - name: NuGetPackagePath + value: '$(System.ArtifactsDirectory)/NuGetRelease' + - name: PackageRoot + value: '$(System.ArtifactsDirectory)/Packages' + - group: PSNativeAPIScan + - group: SymbolPublish + - group: Azure Blob variable group resources: repositories: @@ -20,7 +25,9 @@ stages: jobs: - job: BuildWin pool: - vmImage: vs2017-win2016 + name: PowerShell1ES + demands: + - ImageOverride -equals PSMMS2019-PS-Native-Secure displayName: Windows strategy: matrix: @@ -36,7 +43,11 @@ stages: - template: windows-build.yml - job: SignWin - pool: Package ES Standard Build + pool: + name: PowerShell1ES + demands: + - ImageOverride -equals PSMMS2019-PS-Native-Secure + displayName: Sign Windows variables: - group: ESRP @@ -57,7 +68,9 @@ stages: - job: BuildLinux displayName: Linux pool: - vmImage: ubuntu-16.04 + name: PowerShell1ES + demands: + - ImageOverride -equals PSMMSUbuntu20.04-Secure strategy: matrix: UbuntuArm: @@ -84,20 +97,73 @@ stages: - SignWin - BuildLinux - BuildMac - pool: Package ES Standard Build + pool: + name: PowerShell1ES + demands: + - ImageOverride -equals PSMMS2019-PS-Native-Secure variables: - group: ESRP steps: - template: nuget.yml +- stage: PublishSymbol + displayName: Publish pwrshplugin symbols + dependsOn: Build + jobs: + - job: Symbols_Job + pool: + name: PowerShell1ES + demands: + - ImageOverride -equals PSMMS2019-PS-Native-Secure + steps: + - download: current + artifact: release + + - pwsh: | + Write-Verbose -Verbose "Enumerating $(Pipeline.Workspace)/release" + $downloadedArtifacts = Get-ChildItem -Recurse "$(Pipeline.Workspace)/release" + $downloadedArtifacts + + $expandedRoot = New-Item -Path "$(Pipeline.Workspace)/expanded" -ItemType Directory -Verbose + $symbolsRoot = New-Item -Path "$(Pipeline.Workspace)/symbols" -ItemType Directory -Verbose + + $downloadedArtifacts | Where-Object { $_.Name -like 'x*-symbols.zip'} | ForEach-Object { + $destFolder = New-Item -Path "$expandedRoot/$($_.BaseName)/" -ItemType Directory -Verbose + Expand-Archive -Path $_.FullName -DestinationPath $destFolder -Force + + Get-ChildItem -Path $destFolder -Recurse -Filter '*.pdb' | ForEach-Object { + $runtimeFolder = New-Item -Path "$symbolsRoot/$($destFolder.BaseName)/" -ItemType Directory -Verbose + Copy-Item -Path $_.FullName -Destination $runtimeFolder -Verbose + } + } + + Write-Verbose -Verbose "Enumerating $symbolsRoot" + Get-ChildItem -Path $symbolsRoot -Recurse + + $vstsCommandString = "vso[task.setvariable variable=SymbolsPath]$symbolsRoot" + Write-Verbose -Message "$vstsCommandString" -Verbose + Write-Host -Object "##$vstsCommandString" + displayName: Expand and capture symbols folders + + - task: PublishSymbols@2 + inputs: + symbolsFolder: '$(SymbolsPath)' + searchPattern: '**/*.pdb' + indexSources: false + publishSymbols: true + symbolServerType: teamServices + detailedLog: true + - stage: compliance displayName: Compliance dependsOn: Build jobs: - job: Compliance_Job pool: - name: Package ES Standard Build + name: PowerShell1ES + demands: + - ImageOverride -equals PSMMS2019-PS-Native-Secure steps: - checkout: self clean: true @@ -166,6 +232,7 @@ stages: softwareName: 'PowerShellNative' softwareNameFolder: '$(Pipeline.Workspace)/uncompressed' softwareVersion: '$(PackageVersion)' + connectionString: RunAs=App;AppId=$(APIScanClient);TenantId=$(APIScanTenant);AppKey=$(APIScanSecret) APIScan: true # set to false when not using Windows APIs. - template: publish.yml diff --git a/tools/releaseBuild/yaml/upload.yml b/tools/releaseBuild/yaml/upload.yml index 256bde8..af7100a 100644 --- a/tools/releaseBuild/yaml/upload.yml +++ b/tools/releaseBuild/yaml/upload.yml @@ -3,12 +3,44 @@ parameters: sourcePath: '' steps: -- task: AzureFileCopy@2 + +- powershell: | + [System.Net.ServicePointManager]::SecurityProtocol = + [System.Net.ServicePointManager]::SecurityProtocol -bor + [System.Security.Authentication.SslProtocols]::Tls12 -bor + [System.Security.Authentication.SslProtocols]::Tls11 + + Get-PackageProvider -Name NuGet -ForceBootstrap + Install-Module PowerShellGet -RequiredVersion 2.2.5 -SkipPublisherCheck -Force + displayName: Update PowerShell Get + +- powershell: | + $modules = 'Az.Accounts', 'Az.Storage', 'Az.Resources' + foreach($module in $modules) { + if(!(get-module $module -listavailable)) { + Write-Verbose "installing $module..." -verbose + Install-Module $module -force -AllowClobber + } else { + #Write-Verbose "updating $module..." -verbose + #Update-Module $module -verbose + } + } + displayName: Install PowerShell modules + +- powershell: | + $azureRMModule = Get-Module -Name AzureRM -ListAvailable -ErrorAction SilentlyContinue + if ($azureRMModule) { + Uninstall-AzureRm + } + displayName: Uninstall Uninstall-AzureRm + continueOnError: true + +- task: AzureFileCopy@4 displayName: 'Upload ${{ parameters.fileName }} to azure blob' inputs: SourcePath: '${{ parameters.sourcePath }}' - azureSubscription: 'MGMT-Powershell-CICDInfra (94cf12ad-4fe9-490b-b281-0a260198a4e0)' + azureSubscription: '$(AzureFileCopySubscription)' Destination: AzureBlob - storage: pscoretestdata + storage: '$(StorageAccount)' ContainerName: 'PowerShell-Native-Symbols' BlobPrefix: '$(Build.SourceBranchName)' diff --git a/tools/releaseBuild/yaml/windows-build.yml b/tools/releaseBuild/yaml/windows-build.yml index a5c9cd2..f1ed011 100644 --- a/tools/releaseBuild/yaml/windows-build.yml +++ b/tools/releaseBuild/yaml/windows-build.yml @@ -1,8 +1,4 @@ steps: - - powershell: | - choco install cmake.install --installargs 'ADD_CMAKE_TO_PATH=System' - displayName: Install cmake - condition: succeeded() - powershell: | $vsPath = ${Env:ProgramFiles(x86)} + '\Microsoft Visual Studio\2017\BuildTools\MSBuild\15.0\Bin' Write-Host "##vso[task.prependpath]$vsPath"