Rewrite script (#87)

This commit is contained in:
Dengke Tang
2023-02-27 15:33:00 -08:00
committed by GitHub
parent c0ab7116b0
commit ff20f95232
12 changed files with 289 additions and 261 deletions
+5 -5
View File
@@ -80,7 +80,7 @@ jobs:
phpize
./configure
make
./dev-scripts/run_tests
./dev-scripts/run_tests.sh
# linux-arm:
# name: ARM (${{ matrix.arch }})
@@ -123,7 +123,7 @@ jobs:
- run: .\configure --with-prefix=${{steps.setup-php-sdk.outputs.prefix}} --enable-awscrt=shared --enable-cli --enable-openssl
- run: nmake
- run: nmake generate-php-ini
- run: .\dev-scripts\run_tests ${{steps.setup-php-sdk.outputs.prefix}}\php
- run: .\dev-scripts\run_tests.bat ${{steps.setup-php-sdk.outputs.prefix}}\php
# windows-vc14:
# runs-on: windows-latest
@@ -167,7 +167,7 @@ jobs:
./builder build -p ${{ env.PACKAGE_NAME }} --spec=downstream
pecl-package-test:
runs-on: macos-latest
runs-on: ubuntu-latest
steps:
- name: Setup PHP
uses: shivammathur/setup-php@v2
@@ -179,9 +179,9 @@ jobs:
with:
submodules: recursive
- name: Run tests
- name: Test PECL package build
run: |
./dev-scripts/prepare_release.sh --name aws-crt --user aws-crt --email [email protected] --version 1.0.0 --notes NOTES
python3 dev-scripts/prepare_pecl_release.py --name aws-crt --user aws-crt --version 1.0.0
tar -zxf *.tgz
cd awscrt-1.0.0
phpize
+4 -1
View File
@@ -1,4 +1,3 @@
# Created by https://www.toptal.com/developers/gitignore/api/autotools,cmake,phpstorm
# Edit at https://www.toptal.com/developers/gitignore?templates=autotools,cmake,phpstorm
@@ -182,6 +181,8 @@ fabric.properties
build/
configure.in
configure.ac
configure.bat
configure.js
mkinstalldirs
run-tests.php
Makefile.global
@@ -191,6 +192,7 @@ modules/
*.lo
config.h
config.nice
config.nice.bat
*.la
Makefile*
!Makefile.am
@@ -204,6 +206,7 @@ PHP-Parser*/
src/*.so
src/*.dylib
src/*.dll
/x64/
# ignoring output of package.xml as it needs to be generated from ./prepare_release.sh in each publishing
package.xml
+1 -1
View File
@@ -32,6 +32,6 @@
"NO_INTERACTION": "1"
},
"test_steps": [
["./dev-scripts/run_tests"]
["./dev-scripts/run_tests.sh"]
]
}
-48
View File
@@ -1,48 +0,0 @@
#!/usr/bin/env bash
BASENAME=${0}
TOOLS_DIR=$(dirname ${BASENAME})
WORK_DIR="${TOOLS_DIR}/.."
cd "${WORK_DIR}"
find . -name \*.tgz -a |xargs rm -rf
## get rid of anything that phpize, configure and make as well
rm -rf .deps \
.libs \
Makefile \
Makefile.fragments \
Makefile.global \
Makefile.objects \
acinclude.m4 \
aclocal.m4 \
autom4te.cache \
build \
config.guess \
config.h \
config.h.in* \
config.log \
config.nice \
config.status \
config.sub \
configure \
configure.in \
configure.ac \
include \
install-sh \
libtool \
ltmain.sh \
missing \
mkinstalldirs \
modules \
run-tests.php \
awscrt.la \
vendor \
composer.lock \
ext/awscrt.stub.php \
find . -name \*.lo -o -name \*.o | xargs rm -f
find . -name \*.la -o -name \*.a | xargs rm -f
find . -name \*.so | xargs rm -f
find . -name .libs -a -type d|xargs rm -rf
+59
View File
@@ -0,0 +1,59 @@
import os
import glob
import shutil
TOOLS_DIR = os.path.dirname(os.path.abspath(__file__))
WORK_DIR = os.path.join(TOOLS_DIR, '..')
# Remove specified directories
DIRS_TO_REMOVE = [
'.deps',
'.libs',
'build',
'include',
'modules',
'vendor',
'autom4te.cache']
# Remove specified files
FILES_TO_REMOVE = [
'Makefile',
'Makefile.fragments',
'Makefile.global',
'Makefile.objects',
'config.guess',
'config.h',
'config.h.in',
'config.log',
'config.nice',
'config.status',
'config.sub',
'configure',
'configure.in',
'configure.ac',
'install-sh',
'libtool',
'ltmain.sh',
'missing',
'mkinstalldirs',
'run-tests.php',
'awscrt.la',
'composer.lock',
'ext/awscrt.stub.php',
'acinclude.m4',
'aclocal.m4',
'**/*.lo',
'**/*.o',
'**/*.la',
'**/*.a',
'*.tgz']
os.chdir(WORK_DIR)
for directory in DIRS_TO_REMOVE:
shutil.rmtree(directory, ignore_errors=True)
for pattern in FILES_TO_REMOVE:
for filepath in glob.glob(pattern):
os.remove(filepath)
-152
View File
@@ -1,152 +0,0 @@
#!/bin/zsh
zparseopts -A opts -name: -user: -email: -version: -notes:
if [[ $# -lt 10 ]]
then
echo "Usage ${0} --name NAME, --user USER --email EMAIL --version VERSION --notes NOTES"
exit 1
fi
NAME="${opts[--name]}"
USER="${opts[--user]}"
EMAIL="${opts[--email]}"
VERSION="${opts[--version]}"
NOTES="${opts[--notes]}"
BASENAME=${0}
TOOLS_DIR=$(dirname ${BASENAME})
TODAY=$(date -u +%Y-%m-%d)
cat $TOOLS_DIR/package.xml-template_pre \
| sed "s/{{{NAME}}}/$NAME/g" \
| sed "s/{{{USER}}}/$USER/g" \
| sed "s/{{{EMAIL}}}/$EMAIL/g" \
| sed "s/{{{TODAY}}}/$TODAY/g" \
| sed "s/{{{VERSION}}}/$VERSION/g" \
| sed "s/{{{NOTES}}}/$NOTES/g"
source_ext='(c|cc|h|cpp|hpp|m4|w32|ini|frag|cmake|inl|in|py|gnu|yaml|def|pl|S|s|errordata|go|lds|num|asm|mod|peg|mk|rs|toml|sh)'
doc_ext='(md|json|html|dot|graphml|png|gn|sha1|css|rst|)'
special_docs='(LICENSE*|NOTICE|changelog.txt|CHANGELOG|THIRD-PARTY|README*|readme|METADATA|CONTRIBUTORS|UPDATING|doc.config|THIRD-PARTY-LICENSES.txt)'
special_tests='(ci-test.sh|format-check.sh|run_tests*|sanitizer-blacklist.txt|run-clang-tidy.sh|benchmark-build-run.sh|break-tests.sh|generate-coverage.sh|test.xml)'
special_src='(gen_api.php|gen_stub.php|CMakeLists.txt|post.sh|postun.sh|Makefile*|build-buildspec.sh|build-deps.sh|objects.txt|go.*|BUILD*|DEPS|install_and_run.sh|codemod.sh|requirements.txt)'
skip_files='(package.xml*|prepare_release.sh|codereview.settings|*.o|*.a|*.obj|*.lib|break-tests-android.sh|whitespace.txt|prepare_package_xml.sh|crypto_test_data.cc|*.pdf|*.svg|*.docx|cbmc-proof.txt|codecov*|litani*|*.toml|module.modulemap|cleanup.sh)'
special_scripts='(awscrt.stub.php)'
skip_directories='(tests|test|AWSCRTAndroidTestRunner|docker-images|codebuild|fuzz|third_party|docs|generated-src|aws-lc|aws-crt-sys|ecdsa-fuzz-corpus|bin|examples|compliance|verification)'
process_file() {
if (( $# == 0 ))
then
echo "ERROR: filename not passed"
exit 1
fi
if [[ $1 = $~skip_files ]]
then
# This file is not part of the release bundle
return 0
fi
echo -n '<file name="'"$1"'" role="'
# Special cases
case ${a} in
$~special_scripts)
echo -n 'script'
;;
$~special_docs)
echo -n 'doc'
;;
$~special_tests)
echo -n 'test'
;;
$~special_src)
echo -n 'src'
;;
*)
# Extension based cases
case ${a:t:e} in
$~source_ext)
echo -n 'src'
;;
$~doc_ext)
echo -n 'doc'
;;
php)
echo -n 'script'
;;
*)
echo "${a:t:e} - ${a} - FAIL TO RECOGNIZE"
exit 1
esac
esac
echo '"/>'
return 0
}
process_dir() {
if (( $# == 0 ))
then
echo "WARNING: dirname not passed"
exit 1
fi
if [[ "${1}" = $~skip_directories ]]
then
return 0
fi
echo '<dir name="'"$1"'">'
cd "$1"
for a in *
do
if [[ -f ${a} ]]
then process_file "${a}"
else process_dir "${a}"
fi
done
# Special cases for compiler features placed in tests directories in and s2n
if [[ "${1}" = "s2n" && -d tests ]]
then
echo '<dir name="tests">'
echo '<dir name="features">'
cd tests/features
for a in *
do
process_file "${a}"
done
cd ../..
echo '</dir>'
echo '</dir>'
fi
echo '</dir>'
cd ..
return 0
}
echo '<dir name="/">'
for a in *
do
if [[ ${a} == 'tests' ]]
then
echo '<dir name="tests">'
for b in tests/*
do
echo '<file name="'$( basename "${b}" )'" role="test" />'
done
echo '</dir>'
continue
fi
# skip the src directory as it is not part of the native extension
if [[ ${a} == 'src' ]]
then
continue
fi
if [[ -f ${a} ]]
then process_file "${a}"
else process_dir "${a}"
fi
done
echo '</dir>'
cat $TOOLS_DIR/package.xml-template_post
+146
View File
@@ -0,0 +1,146 @@
#!/usr/bin/env python3
import argparse
import os
import re
from datetime import datetime
parser = argparse.ArgumentParser(
description="Generate package.xml file for a release bundle.")
parser.add_argument("--name", help="Name of the package", required=True)
parser.add_argument(
"--user", help="Username of the package maintainer", required=True)
parser.add_argument(
"--email", help="Email address of the package maintainer", default='[email protected]')
parser.add_argument(
"--version", help="Version number of the package", required=True)
parser.add_argument(
"--notes", help="Release notes for the package", default='New release')
args = parser.parse_args()
name = args.name
user = args.user
email = args.email
version = args.version
notes = args.notes
tools_dir = os.path.dirname(os.path.abspath(__file__))
work_dir = os.path.join(tools_dir, '..')
os.chdir(work_dir)
today = datetime.utcnow().strftime("%Y-%m-%d")
source_ext = "(c|cc|h|cpp|hpp|m4|w32|ini|frag|cmake|inl|in|py|gnu|yaml|def|pl|S|s|errordata|go|lds|num|asm|mod|peg|mk|rs|toml|sh)"
doc_ext = "(md|json|html|dot|graphml|png|gn|sha1|css|rst)"
special_docs = "(LICENSE.*|NOTICE|changelog.txt|CHANGELOG|THIRD-PARTY|README.*|readme|METADATA|CONTRIBUTORS|UPDATING|doc.config|THIRD-PARTY-LICENSES.txt)"
special_tests = "(ci-test.sh|format-check.sh|run_tests.*|sanitizer-blacklist.txt|run-clang-tidy.sh|benchmark-build-run.sh|break-tests.sh|generate-coverage.sh|test.xml)"
special_src = "(gen_api.php|gen_stub.php|CMakeLists.txt|post.sh|postun.sh|Makefile.*|build-buildspec.sh|build-deps.sh|objects.txt|go.*|BUILD.*|DEPS|install_and_run.sh|codemod.sh|requirements.txt)"
skip_files = "(package.xml.*|prepare_release.sh|codereview.settings|.*\\.o|.*\\.a|.*\\.obj|.*\\.lib|break-tests-android.sh|whitespace.txt|prepare_package_xml.sh|crypto_test_data.cc|.*\\.pdf|.*\\.svg|.*\\.docx|cbmc-proof.txt|codecov.*|litani.*|.*\\.toml|module\\.modulemap|cleanup.sh|^\..*)"
special_scripts = "(awscrt.stub.php)"
skip_directories = "(tests|test|AWSCRTAndroidTestRunner|docker-images|codebuild|fuzz|third_party|docs|generated-src|aws-lc|aws-crt-sys|ecdsa-fuzz-corpus|bin|examples|compliance|verification|^\..*)"
output_file_name = os.path.join(work_dir, 'package.xml')
def process_file(file_name, f):
if re.match(skip_files, file_name):
# This file is not part of the release bundle
return
f.write(f'<file name="{file_name}" role="')
# Special cases
if re.match(special_scripts, file_name):
f.write('script')
elif re.match(special_docs, file_name):
f.write('doc')
elif re.match(special_tests, file_name):
f.write('test')
elif re.match(special_src, file_name):
f.write('src')
else:
# Extension based cases
ext = file_name.split('.')[-1]
if re.match(source_ext, ext):
f.write('src')
elif re.match(doc_ext, ext):
f.write('doc')
elif ext == 'php':
f.write('script')
else:
f.write(f'{ext} - {file_name} - FAIL TO RECOGNIZE')
print(f'{ext} - {file_name} - FAIL TO RECOGNIZE')
exit(1)
f.write('"/>\n')
def process_dir(dir_name, f):
if re.match(skip_directories, dir_name):
return
f.write(f'<dir name="{dir_name}">\n')
os.chdir(dir_name)
for file_name in os.listdir():
if os.path.isfile(file_name):
process_file(file_name, f)
else:
process_dir(file_name, f)
# Special cases for compiler features placed in tests directories in and s2n
if dir_name == 's2n' and os.path.isdir('tests'):
f.write('<dir name="tests">')
f.write('<dir name="features">')
os.chdir('tests/features')
for a in os.listdir():
process_file(a, f)
os.chdir('../..')
f.write('</dir>')
f.write('</dir>')
f.write('</dir>')
os.chdir('..')
# Main code
with open(os.path.join(tools_dir, "package.xml-template_pre"), 'r') as f:
template_pre = f.read()
template_pre = re.sub(r"{{{NAME}}}", name, template_pre)
template_pre = re.sub(r"{{{USER}}}", user, template_pre)
template_pre = re.sub(r"{{{EMAIL}}}", email, template_pre)
template_pre = re.sub(r"{{{TODAY}}}", today, template_pre)
template_pre = re.sub(r"{{{VERSION}}}", version, template_pre)
template_pre = re.sub(r"{{{NOTES}}}", notes, template_pre)
with open(output_file_name, "w") as f:
f.write(template_pre)
f.write('<dir name="/">\n')
with open(output_file_name, "a") as f:
for file_name in os.listdir():
if file_name == 'tests':
f.write('<dir name="tests">\n')
for test_file in os.listdir('tests'):
f.write(
f'<file name="{os.path.basename(test_file)}" role="test" />\n')
f.write('</dir>\n')
continue
if file_name == 'src':
# skip the src directory as it is not part of the native extension
continue
if os.path.isfile(file_name):
process_file(file_name, f)
else:
process_dir(file_name, f)
with open(output_file_name, "a") as f:
f.write('</dir>\n')
with open(os.path.join(tools_dir, "package.xml-template_post"), 'r') as template:
f.write(template.read())
+58
View File
@@ -0,0 +1,58 @@
import argparse
import os
import subprocess
import xml.dom.minidom
import sys
parser = argparse.ArgumentParser(description='PECL Package generator')
parser.add_argument("--name", help="Name of the package", required=True)
parser.add_argument(
"--user", help="Username of the package maintainer", required=True)
parser.add_argument(
"--email", help="Email address of the package maintainer", default='[email protected]')
parser.add_argument(
"--version", help="Version number of the package", required=True)
parser.add_argument(
"--notes", help="Release notes for the package", default='New release')
args = parser.parse_args()
PACKAGE = 'awscrt'
NAME = args.name
USER = args.user
EMAIL = args.email
VERSION = args.version
NOTES = args.notes
TOOLS_DIR = os.path.dirname(os.path.abspath(__file__))
WORK_DIR = os.path.join(TOOLS_DIR, '..')
def run(args):
print(f"$ {subprocess.list2cmdline(args)}")
subprocess.check_call(args)
run(['python3', f'{TOOLS_DIR}/cleanup_build.py'])
os.chdir(WORK_DIR)
run(['git', 'submodule', 'update', '--init', '--recursive'])
try:
run(['python3', f'{TOOLS_DIR}/prepare_pecl_package_xml.py', '--name', NAME, '--user', USER,
'--email', EMAIL, '--version', VERSION, '--notes', NOTES])
except subprocess.CalledProcessError as e:
sys.exit(f'ERROR PROCESSING review package.xml: {e}')
with open('package.xml', 'r') as f:
package_xml = f.read()
doc = xml.dom.minidom.parse('package.xml')
doc.encoding = 'UTF-8'
xml_str = doc.toprettyxml(indent=' ', newl='')
with open('package.xml', 'w') as f:
f.write(xml_str)
run(['pear', 'package-validate'])
run(['pear', 'package'])
print(f'Size of {PACKAGE}-{VERSION}.tgz: {os.path.getsize(f"{PACKAGE}-{VERSION}.tgz") / 1024 / 1024:.2f} MB')
-41
View File
@@ -1,41 +0,0 @@
#!/bin/zsh
zparseopts -A opts -name: -user: -email: -version: -notes:
if [[ $# -lt 10 ]]; then
echo "Usage ${0} --name NAME --user USER --email EMAIL --version VERSION --notes NOTES"
exit 1
fi
PACKAGE='awscrt'
NAME="${opts[--name]}"
USER="${opts[--user]}"
EMAIL="${opts[--email]}"
VERSION="${opts[--version]}"
NOTES="${opts[--notes]}"
BASENAME=${0}
TOOLS_DIR=$(dirname ${BASENAME})
WORK_DIR="${TOOLS_DIR}/.."
cd "${WORK_DIR}"
git submodule update --init --recursive
$TOOLS_DIR/cleanup.sh
$TOOLS_DIR/prepare_package_xml.sh --name "${NAME}" --user "${USER}" --email "${EMAIL}" --version "${VERSION}" --notes "${NOTES}" >package.xml
if [[ $? -ne 0 ]]; then
echo "ERROR PROCESSING review package.xml"
exit 1
fi
tidy -xml -m -i package.xml
pear package-validate
if [[ $? -ne 0 ]]; then
echo "ERROR VALIDATING review package.xml"
exit 1
fi
pear package
if [[ $? -ne 0 ]]; then
echo "ERROR PROCESSING review package.xml"
exit 1
fi
echo "Size of ${PACKAGE}-${VERSION}.tgz: " $(du -h "${PACKAGE}-${VERSION}.tgz")
+14 -11
View File
@@ -1,9 +1,6 @@
@echo on
setlocal
REM Run command to get composer_dir
set "composer_dir="
for /f "usebackq delims=" %%i in (`where composer.phar`) do set "composer_dir=%%i"
set "PHP_BINARY=php"
@@ -11,18 +8,24 @@ if not "%~1"=="" (
set "PHP_BINARY=%~1"
)
REM Check if composer_dir was found
if "%composer_dir%"=="" (
echo No composer found.
exit /b 1
)
set "script_dir=%~dp0"
set "work_dir=%script_dir%/.."
cd %work_dir%
call %PHP_BINARY% -c php-win.ini %composer_dir% update
set vendor_path=vendor
REM Run command to get composer_dir
set "composer_dir="
for /f "usebackq delims=" %%i in (`where composer.phar`) do set "composer_dir=%%i"
if not exist "%vendor_path%" (
REM Check if composer_dir was found
if "%composer_dir%"=="" (
echo No composer found.
exit /b 1
)
call %PHP_BINARY% -c php-win.ini %composer_dir% update
)
call %PHP_BINARY% -c php-win.ini vendor/bin/phpunit tests --debug
endlocal