Merge pull request #39004 from markuskramerIgitt/compile_offline3

Compile offline3
This commit is contained in:
Mike Place 2017-02-02 13:20:38 -07:00 committed by GitHub
commit 8a24cad919
7 changed files with 182 additions and 76 deletions

View File

@ -1,7 +1,11 @@
@echo off
@echo Salt Windows Build Script
@echo Salt Windows Build Script, which calls the other *.ps1 scripts.
@echo ---------------------------------------------------------------------
@echo.
:: To activate caching, set environment variables
:: SALTREPO_LOCAL_CACHE for resources from saltstack.com/...
:: SALT_REQ_LOCAL_CACHE for pip resources specified in req.txt
:: SALT_PIP_LOCAL_CACHE for pip resources specified in req_pip.txt
:: Make sure the script is run as Admin
@echo Administrative permissions required. Detecting permissions...
@ -40,7 +44,7 @@ for /f "delims=" %%a in ('git rev-parse --show-toplevel') do @set "SrcDir=%%a"
:: Get the version from git if not passed
if [%1]==[] (
for /f "delims=" %%a in ('git describe') do @set "Version=%%a"
echo ... Version from git describe == %Version%
echo ... Version from git describe == %Version%
) else (
set "Version=%~1"
)
@ -68,6 +72,9 @@ if not %errorLevel%==0 (
@echo %0 :: Install Current Version of salt...
@echo ---------------------------------------------------------------------
"%PyDir%\python.exe" "%SrcDir%\setup.py" --quiet install --force
if not %errorLevel%==0 (
goto eof
)
@echo.
:: Build the Salt Package

View File

@ -25,7 +25,6 @@ param(
[switch]$Silent
)
# Clear-Host
Write-Output "================================================================="
Write-Output ""
Write-Output " Development Environment Installation"
@ -45,6 +44,11 @@ Write-Output ""
$script_path = dir "$($myInvocation.MyCommand.Definition)"
$script_path = $script_path.DirectoryName
#==============================================================================
# Get the name of actual script
#==============================================================================
$script_name = $MyInvocation.MyCommand.Name
#==============================================================================
# Import Modules
#==============================================================================
@ -76,7 +80,6 @@ If (!(Get-IsAdministrator)) {
# Exit from the current, unelevated, process
Exit
} Else {
Throw "You must be administrator to run this script"
}
@ -91,29 +94,26 @@ $ini = Get-Settings
# Create Directories
#------------------------------------------------------------------------------
$p = New-Item $ini['Settings']['DownloadDir'] -ItemType Directory -Force
$p = New-Item "$($ini['Settings']['DownloadDir'])\64" -ItemType Directory -Force
$p = New-Item "$($ini['Settings']['DownloadDir'])\32" -ItemType Directory -Force
$p = New-Item $ini['Settings']['SaltDir'] -ItemType Directory -Force
#------------------------------------------------------------------------------
# Determine Architecture (32 or 64 bit) and assign variables
#------------------------------------------------------------------------------
If ([System.IntPtr]::Size -ne 4) {
Write-Output "Detected 64bit Architecture..."
$bitDLLs = "64bitDLLs"
$bitPaths = "64bitPaths"
$bitPrograms = "64bitPrograms"
$bitFolder = "64"
} Else {
} Else {
Write-Output "Detected 32bit Architecture"
$bitDLLs = "32bitDLLs"
$bitPaths = "32bitPaths"
$bitPrograms = "32bitPrograms"
$bitFolder = "32"
}
#------------------------------------------------------------------------------
@ -121,12 +121,9 @@ If ([System.IntPtr]::Size -ne 4) {
#------------------------------------------------------------------------------
Write-Output " - Checking for NSIS installation . . ."
If (Test-Path "$($ini[$bitPaths]['NSISDir'])\NSIS.exe") {
# Found NSIS, do nothing
Write-Output " - NSIS Found . . ."
} Else {
# NSIS not found, install
Write-Output " - NSIS Not Found . . ."
Write-Output " - Downloading $($ini['Prerequisites']['NSIS']) . . ."
@ -139,7 +136,6 @@ If (Test-Path "$($ini[$bitPaths]['NSISDir'])\NSIS.exe") {
Write-Output " - Installing $($ini['Prerequisites']['NSIS']) . . ."
$file = "$($ini['Settings']['DownloadDir'])\$($ini['Prerequisites']['NSIS'])"
$p = Start-Process $file -ArgumentList '/S' -Wait -NoNewWindow -PassThru
}
#------------------------------------------------------------------------------
@ -147,12 +143,9 @@ If (Test-Path "$($ini[$bitPaths]['NSISDir'])\NSIS.exe") {
#------------------------------------------------------------------------------
Write-Output " - Checking for VC Compiler for Python 2.7 installation . . ."
If (Test-Path "$($ini[$bitPaths]['VCforPythonDir'])\vcvarsall.bat") {
# Found Microsoft Visual C++ for Python2.7, do nothing
Write-Output " - Microsoft Visual C++ for Python 2.7 Found . . ."
} Else {
# Microsoft Visual C++ for Python2.7 not found, install
Write-Output " - Microsoft Visual C++ for Python2.7 Not Found . . ."
Write-Output " - Downloading $($ini['Prerequisites']['VCforPython']) . . ."
@ -165,7 +158,6 @@ If (Test-Path "$($ini[$bitPaths]['VCforPythonDir'])\vcvarsall.bat") {
Write-Output " - Installing $($ini['Prerequisites']['VCforPython']) . . ."
$file = "$($ini['Settings']['DownloadDir'])\$($ini['Prerequisites']['VCforPython'])"
$p = Start-Process msiexec.exe -ArgumentList "/i $file /qb ALLUSERS=1" -Wait -NoNewWindow -PassThru
}
#------------------------------------------------------------------------------
@ -173,22 +165,17 @@ If (Test-Path "$($ini[$bitPaths]['VCforPythonDir'])\vcvarsall.bat") {
#------------------------------------------------------------------------------
Write-Output " - Checking for Python 2.7 installation . . ."
If (Test-Path "$($ini['Settings']['PythonDir'])\python.exe") {
# Found Python2.7, do nothing
Write-Output " - Python 2.7 Found . . ."
} Else {
Write-Output " - Downloading $($ini[$bitPrograms]['Python']) . . ."
$file = "$($ini[$bitPrograms]['Python'])"
$url = "$($ini['Settings']['SaltRepo'])/$bitFolder/$file"
$file = "$($ini['Settings']['DownloadDir'])\$file"
$file = "$($ini['Settings']['DownloadDir'])\$bitFolder\$file"
DownloadFileWithProgress $url $file
Write-Output " - Installing $($ini[$bitPrograms]['Python']) . . ."
$file = "$($ini['Settings']['DownloadDir'])\$($ini[$bitPrograms]['Python'])"
$p = Start-Process msiexec -ArgumentList "/i $file /qb ADDLOCAL=DefaultFeature,Extensions,pip_feature,PrependPath TARGETDIR=$($ini['Settings']['PythonDir'])" -Wait -NoNewWindow -PassThru
Write-Output " - $script_name :: Installing $($ini[$bitPrograms]['Python']) . . ."
$p = Start-Process msiexec -ArgumentList "/i $file /qb ADDLOCAL=DefaultFeature,Extensions,pip_feature,PrependPath TARGETDIR=$($ini['Settings']['PythonDir'])" -Wait -NoNewWindow -PassThru
}
#------------------------------------------------------------------------------
@ -204,58 +191,81 @@ If (!($Path.ToLower().Contains("$($ini['Settings']['ScriptsDir'])".ToLower())))
#==============================================================================
# Update PIP and SetupTools
# caching depends on environmant variable SALT_PIP_LOCAL_CACHE
#==============================================================================
Write-Output " ----------------------------------------------------------------"
Write-Output " - Updating PIP and SetupTools . . ."
Write-Output " - $script_name :: Updating PIP and SetupTools . . ."
Write-Output " ----------------------------------------------------------------"
Start_Process_and_test_exitcode "$($ini['Settings']['PythonDir'])\python.exe" "-m pip --no-cache-dir install -r $($script_path)\req_pip.txt" "python pip"
if ( ! [bool]$Env:SALT_PIP_LOCAL_CACHE) {
Start_Process_and_test_exitcode "$($ini['Settings']['PythonDir'])\python.exe" "-m pip --no-cache-dir install -r $($script_path)\req_pip.txt" "python pip"
} else {
$p = New-Item $Env:SALT_PIP_LOCAL_CACHE -ItemType Directory -Force # Ensure directory exists
if ( (Get-ChildItem $Env:SALT_PIP_LOCAL_CACHE | Measure-Object).Count -eq 0 ) {
# folder empty
Write-Output " pip download from req_pip.txt into empty local cache SALT_REQ_PIP $Env:SALT_PIP_LOCAL_CACHE"
Start_Process_and_test_exitcode "$($ini['Settings']['PythonDir'])\python.exe" "-m pip download --dest $Env:SALT_PIP_LOCAL_CACHE -r $($script_path)\req_pip.txt" "pip download"
}
Write-Output " reading from local pip cache $Env:SALT_PIP_LOCAL_CACHE"
Write-Output " If a (new) ressource is missing, please delete all files in this cache, go online and repeat"
Start_Process_and_test_exitcode "$($ini['Settings']['PythonDir'])\python.exe" "-m pip install --no-index --find-links=$Env:SALT_PIP_LOCAL_CACHE -r $($script_path)\req_pip.txt" "pip install"
}
#==============================================================================
# Install pypi resources using pip
# caching depends on environmant variable SALT_REQ_LOCAL_CACHE
#==============================================================================
Write-Output " ----------------------------------------------------------------"
Write-Output " - Installing pypi resources using pip . . ."
Write-Output " - $script_name :: Installing pypi resources using pip . . ."
Write-Output " ----------------------------------------------------------------"
Start_Process_and_test_exitcode "$($ini['Settings']['ScriptsDir'])\pip.exe" "--no-cache-dir install -r $($script_path)\req.txt" "pip install"
if ( ! [bool]$Env:SALT_REQ_LOCAL_CACHE) {
Start_Process_and_test_exitcode "$($ini['Settings']['ScriptsDir'])\pip.exe" "--no-cache-dir install -r $($script_path)\req.txt" "pip install"
} else {
if ( (Get-ChildItem $Env:SALT_REQ_LOCAL_CACHE | Measure-Object).Count -eq 0 ) {
# folder empty
Write-Output " pip download from req.txt into empty local cache SALT_REQ $Env:SALT_REQ_LOCAL_CACHE"
Start_Process_and_test_exitcode "$($ini['Settings']['PythonDir'])\python.exe" "-m pip download --dest $Env:SALT_REQ_LOCAL_CACHE -r $($script_path)\req.txt" "pip download"
}
Write-Output " reading from local pip cache $Env:SALT_REQ_LOCAL_CACHE"
Write-Output " If a (new) ressource is missing, please delete all files in this cache, go online and repeat"
Start_Process_and_test_exitcode "$($ini['Settings']['PythonDir'])\python.exe" "-m pip install --no-index --find-links=$Env:SALT_REQ_LOCAL_CACHE -r $($script_path)\req.txt" "pip install"
}
#==============================================================================
# Install PyYAML with CLoader
# This has to be a compiled binary to get the CLoader
#==============================================================================
Write-Output " ----------------------------------------------------------------"
Write-Output " - Installing PyYAML . . ."
Write-Output " - $script_name :: Installing PyYAML . . ."
Write-Output " ----------------------------------------------------------------"
# Download
$file = "$($ini[$bitPrograms]['PyYAML'])"
$url = "$($ini['Settings']['SaltRepo'])/$bitFolder/$file"
$file = "$($ini['Settings']['DownloadDir'])\$file"
$file = "$($ini['Settings']['DownloadDir'])\$bitFolder\$file"
DownloadFileWithProgress $url $file
# Install
$file = "$($ini['Settings']['DownloadDir'])\$($ini[$bitPrograms]['PyYAML'])"
Start_Process_and_test_exitcode "$($ini['Settings']['ScriptsDir'])\easy_install.exe" "-Z $file " "easy_install PyYAML"
#==============================================================================
# Install PyCrypto from wheel file
#==============================================================================
Write-Output " ----------------------------------------------------------------"
Write-Output " - Installing PyCrypto . . ."
Write-Output " - $script_name :: Installing PyCrypto . . ."
Write-Output " ----------------------------------------------------------------"
# Download
$file = "$($ini[$bitPrograms]['PyCrypto'])"
$url = "$($ini['Settings']['SaltRepo'])/$bitFolder/$file"
$file = "$($ini['Settings']['DownloadDir'])\$file"
$file = "$($ini['Settings']['DownloadDir'])\$bitFolder\$file"
DownloadFileWithProgress $url $file
# Install
$file = "$($ini['Settings']['DownloadDir'])\$($ini[$bitPrograms]['PyCrypto'])"
Start_Process_and_test_exitcode "$($ini['Settings']['ScriptsDir'])\pip.exe" "install --no-index --find-links=$($ini['Settings']['DownloadDir']) $file " "pip install PyCrypto"
#==============================================================================
# Copy DLLs to Python Directory
#==============================================================================
Write-Output " ----------------------------------------------------------------"
Write-Output " - Copying DLLs . . ."
Write-Output " - $script_name :: Copying DLLs . . ."
Write-Output " ----------------------------------------------------------------"
# Architecture Specific DLL's
ForEach($key in $ini[$bitDLLs].Keys) {
@ -263,8 +273,9 @@ ForEach($key in $ini[$bitDLLs].Keys) {
Write-Output " - $key . . ."
$file = "$($ini[$bitDLLs][$key])"
$url = "$($ini['Settings']['SaltRepo'])/$bitFolder/$file"
$file = "$($ini['Settings']['PythonDir'])\$file"
$file = "$($ini['Settings']['DownloadDir'])\$bitFolder\$file"
DownloadFileWithProgress $url $file
Copy-Item $file -destination $($ini['Settings']['PythonDir'])
}
}
@ -272,7 +283,7 @@ ForEach($key in $ini[$bitDLLs].Keys) {
# Script complete
#------------------------------------------------------------------------------
Write-Output "================================================================="
Write-Output "Salt Stack Dev Environment Script Complete"
Write-Output " $script_name :: Salt Stack Dev Environment Script Complete"
Write-Output "================================================================="
Write-Output ""
@ -286,7 +297,9 @@ If (-Not $Silent) {
# Remove the temporary download directory
#------------------------------------------------------------------------------
Write-Output " ----------------------------------------------------------------"
Write-Output " - Cleaning up downloaded files"
Write-Output " - $script_name :: Cleaning up downloaded files unless you use SALTREPO_LOCAL_CACHE"
Write-Output " ----------------------------------------------------------------"
Write-Output ""
Remove-Item $($ini['Settings']['DownloadDir']) -Force -Recurse
if ( ! [bool]$Env:SALTREPO_LOCAL_CACHE ) {
Remove-Item $($ini['Settings']['DownloadDir']) -Force -Recurse
}

View File

@ -70,9 +70,9 @@ If NOT Exist "%PreDir%" mkdir "%PreDir%"
Set Url64="http://repo.saltstack.com/windows/dependencies/64/vcredist_x64_2008_mfc.exe"
Set Url32="http://repo.saltstack.com/windows/dependencies/32/vcredist_x86_2008_mfc.exe"
If Defined ProgramFiles(x86) (
bitsadmin /transfer "VCRedist 2008 MFC AMD64" "%Url64%" "%PreDir%\vcredist.exe"
powershell -ExecutionPolicy RemoteSigned -File download_url_file.ps1 -url "%Url64%" -file "%PreDir%\vcredist.exe"
) Else (
bitsadmin /transfer "VCRedist 2008 MFC x86" "%Url32%" "%PreDir%\vcredist.exe"
powershell -ExecutionPolicy RemoteSigned -File download_url_file.ps1 -url "%Url32%" -file "%PreDir%\vcredist.exe"
)
@echo.

View File

@ -0,0 +1,39 @@
#
# Download url to file. Optionally, store in cache
#
#
Param(
[Parameter(Mandatory=$true)][string]$url,
[Parameter(Mandatory=$true)][string]$file
)
$VerbosePreference = 'Continue'
Import-Module ./Modules/download-module.psm1
if ( [bool]$Env:SALTREPO_LOCAL_CACHE) {
Write-Verbose "found SALTREPO_LOCAL_CACHE environment variable $Env:SALTREPO_LOCAL_CACHE"
} else {
Write-Verbose "no SALTREPO_LOCAL_CACHE environment variable "
}
$saltrepo_url = "http://repo.saltstack.com/windows/dependencies/"
if ( [bool]$Env:SALTREPO_LOCAL_CACHE -And $url.StartsWith($saltrepo_url) ) {
Write-Verbose "found SALTREPO_LOCAL_CACHE environment variable and url is saltrepo"
$url_relative__slash = $url -replace [regex]::Escape($saltrepo_url), ""
$url_relative__backslash = $url_relative__slash -replace [regex]::Escape("/"), "\\"
$localCacheFile = Join-Path $Env:SALTREPO_LOCAL_CACHE $url_relative__backslash
if (-Not (Test-Path $localCacheFile)) {
Write-Verbose "downloading to cache $localCacheFile"
DownloadFileWithProgress $url $localCacheFile
}
Write-Verbose "copying from cache $file"
Copy-Item $localCacheFile -destination $file
} else {
Write-Verbose "no SALTREPO_LOCAL_CACHE environment variable, or URL not saltrepo, downloading directly"
DownloadFileWithProgress $url $file
}

View File

@ -8,51 +8,69 @@ Function DownloadFileWithProgress {
# $url - the file source
# $localfile - the file destination on the local machine
# Originally, DownLoadDir is deleted for each install, therefore
# this function did not expect that the file exists.
# You may want to set an environment variable SALTREPO_LOCAL_CACHE, a cache which lives as long as you decide.
# Therefore this function must test if the file exists.
param(
[Parameter(Mandatory=$true)]
[String] $url,
[Parameter(Mandatory=$false)]
[String] $localFile = (Join-Path $pwd.Path $url.SubString($url.LastIndexOf('/')))
[String] $localFile = (Join-Path $pwd.Path $url.SubString($url.LastIndexOf('/')))
)
begin {
$client = New-Object System.Net.WebClient
$Global:downloadComplete = $false
$eventDataComplete = Register-ObjectEvent $client DownloadFileCompleted `
$Global:NEED_PROCESS_AND_END = $true
Write-Verbose " **** DownloadFileWithProgress looking for **** $localFile ********"
if ( [bool]$Env:SALTREPO_LOCAL_CACHE -and (Test-Path $localFile) ) {
Write-Verbose " **** found **** $localFile ********"
$Global:NEED_PROCESS_AND_END = $false
} else {
Write-Verbose " ++++++ BEGIN DOWNLOADING ++++++ $localFile +++++++"
$client = New-Object System.Net.WebClient
$Global:downloadComplete = $false
$eventDataComplete = Register-ObjectEvent $client DownloadFileCompleted `
-SourceIdentifier WebClient.DownloadFileComplete `
-Action {$Global:downloadComplete = $true}
$eventDataProgress = Register-ObjectEvent $client DownloadProgressChanged `
$eventDataProgress = Register-ObjectEvent $client DownloadProgressChanged `
-SourceIdentifier WebClient.DownloadProgressChanged `
-Action { $Global:DPCEventArgs = $EventArgs }
}
process {
Write-Progress -Activity 'Downloading file' -Status $url
$client.DownloadFileAsync($url, $localFile)
while (!($Global:downloadComplete)) {
$pc = $Global:DPCEventArgs.ProgressPercentage
if ($pc -ne $null) {
Write-Progress -Activity 'Downloading file' -Status $url -PercentComplete $pc
}
}
Write-Progress -Activity 'Downloading file' -Status $url -Complete
}
process {
if ( $Global:NEED_PROCESS_AND_END ) {
Write-Verbose " ++++++ actually DOWNLOADING ++++++ $localFile +++++++"
Write-Progress -Activity 'Downloading file' -Status $url
$client.DownloadFileAsync($url, $localFile)
while (!($Global:downloadComplete)) {
$pc = $Global:DPCEventArgs.ProgressPercentage
if ($pc -ne $null) {
Write-Progress -Activity 'Downloading file' -Status $url -PercentComplete $pc
}
}
Write-Progress -Activity 'Downloading file' -Status $url -Complete
}
}
end {
Unregister-Event -SourceIdentifier WebClient.DownloadProgressChanged
Unregister-Event -SourceIdentifier WebClient.DownloadFileComplete
$client.Dispose()
$Global:downloadComplete = $null
$Global:DPCEventArgs = $null
Remove-Variable client
Remove-Variable eventDataComplete
Remove-Variable eventDataProgress
[GC]::Collect()
# 2016-07-06 mkr Errorchecking added. nice-to-have: integration into the above code.
If (!((Test-Path "$localfile") -and ((Get-Item "$localfile").length -gt 0kb))) {
Write-Error "Exiting because download missing or zero-length: $localfile"
exit 2
if ( $Global:NEED_PROCESS_AND_END ) {
Unregister-Event -SourceIdentifier WebClient.DownloadProgressChanged
Unregister-Event -SourceIdentifier WebClient.DownloadFileComplete
$client.Dispose()
$Global:downloadComplete = $null
$Global:DPCEventArgs = $null
Remove-Variable client
Remove-Variable eventDataComplete
Remove-Variable eventDataProgress
[GC]::Collect()
# Errorchecking
If (!((Test-Path "$localfile") -and ((Get-Item "$localfile").length -gt 0kb))) {
Write-Error "download-module.psm1 exits in error, download is missing or has zero-length: $localfile"
exit 2
}
}
}
}

View File

@ -4,7 +4,7 @@ Function Get-Settings {
Param()
Begin
{Write-Verbose "$($MyInvocation.MyCommand.Name):: Function started"}
{Write-Verbose "$($MyInvocation.MyCommand.Name):: Function started"}
Process
{
@ -20,7 +20,14 @@ Function Get-Settings {
"ScriptsDir" = "C:\Python27\Scripts"
"DownloadDir" = "$env:Temp\DevSalt"
}
# The script deletes the DownLoadDir (above) for each install.
# You may want to set an environment variable SALTREPO_LOCAL_CACHE, a cache which lives as long as you decide.
if ( [bool]$Env:SALTREPO_LOCAL_CACHE ) {
$Settings.Set_Item("DownloadDir", "$Env:SALTREPO_LOCAL_CACHE")
}
$ini.Add("Settings", $Settings)
Write-Verbose "DownloadDir === $($ini['Settings']['DownloadDir']) ==="
# Prerequisite software
$Prerequisites = @{

View File

@ -395,7 +395,28 @@ class InstallPyCryptoWindowsWheel(Command):
with indent_log():
call_subprocess(call_arguments)
def uri_to_resource(resource_file):
### Returns the URI for a resource
# The basic case is that the resource is on saltstack.com
# It could be the case that the resource is cached.
salt_uri = 'https://repo.saltstack.com/windows/dependencies/' + resource_file
if os.getenv('SALTREPO_LOCAL_CACHE') == None:
# if environment variable not set, return the basic case
return salt_uri
if not os.path.isdir(os.getenv('SALTREPO_LOCAL_CACHE')):
# if environment variable is not a directory, return the basic case
return salt_uri
cached_resource = os.path.join(os.getenv('SALTREPO_LOCAL_CACHE'), resource_file)
cached_resource = cached_resource.replace('/', '\\')
if not os.path.isfile(cached_resource):
# if file does not exist, return the basic case
return salt_uri
if os.path.getsize(cached_resource) == 0:
# if file has zero size, return the basic case
return salt_uri
return cached_resource
class InstallCompiledPyYaml(Command):
description = 'Install PyYAML on Windows'
@ -405,6 +426,7 @@ class InstallCompiledPyYaml(Command):
def finalize_options(self):
pass
def run(self):
if getattr(self.distribution, 'salt_installing_pyyaml_windows', None) is None:
@ -417,11 +439,11 @@ class InstallCompiledPyYaml(Command):
call_arguments = ['easy_install', '-Z']
if platform_bits == '64bit':
call_arguments.append(
'https://repo.saltstack.com/windows/dependencies/64/PyYAML-3.11.win-amd64-py2.7.exe'
uri_to_resource('64/PyYAML-3.11.win-amd64-py2.7.exe')
)
else:
call_arguments.append(
'https://repo.saltstack.com/windows/dependencies/32/PyYAML-3.11.win32-py2.7.exe'
uri_to_resource('32/PyYAML-3.11.win32-py2.7.exe')
)
with indent_log():
call_subprocess(call_arguments)