From ff20f95232a2f71dcd6fc877039c09f72d46b41b Mon Sep 17 00:00:00 2001 From: Dengke Tang <815825145@qq.com> Date: Mon, 27 Feb 2023 15:33:00 -0800 Subject: [PATCH] Rewrite script (#87) --- .github/workflows/ci.yml | 10 +- .gitignore | 5 +- builder.json | 2 +- dev-scripts/cleanup.sh | 48 -------- dev-scripts/cleanup_build.py | 59 +++++++++ dev-scripts/prepare_package_xml.sh | 152 ------------------------ dev-scripts/prepare_pecl_package_xml.py | 146 +++++++++++++++++++++++ dev-scripts/prepare_pecl_release.py | 58 +++++++++ dev-scripts/prepare_release.sh | 41 ------- dev-scripts/run_tests.bat | 27 +++-- dev-scripts/{run_tests => run_tests.sh} | 0 php-win.ini | 2 +- 12 files changed, 289 insertions(+), 261 deletions(-) delete mode 100755 dev-scripts/cleanup.sh create mode 100644 dev-scripts/cleanup_build.py delete mode 100755 dev-scripts/prepare_package_xml.sh create mode 100644 dev-scripts/prepare_pecl_package_xml.py create mode 100644 dev-scripts/prepare_pecl_release.py delete mode 100755 dev-scripts/prepare_release.sh rename dev-scripts/{run_tests => run_tests.sh} (100%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e67912e..4b6f9dd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -80,7 +80,7 @@ jobs: phpize ./configure make - ./dev-scripts/run_tests + ./dev-scripts/run_tests.sh # linux-arm: # name: ARM (${{ matrix.arch }}) @@ -123,7 +123,7 @@ jobs: - run: .\configure --with-prefix=${{steps.setup-php-sdk.outputs.prefix}} --enable-awscrt=shared --enable-cli --enable-openssl - run: nmake - run: nmake generate-php-ini - - run: .\dev-scripts\run_tests ${{steps.setup-php-sdk.outputs.prefix}}\php + - run: .\dev-scripts\run_tests.bat ${{steps.setup-php-sdk.outputs.prefix}}\php # windows-vc14: # runs-on: windows-latest @@ -167,7 +167,7 @@ jobs: ./builder build -p ${{ env.PACKAGE_NAME }} --spec=downstream pecl-package-test: - runs-on: macos-latest + runs-on: ubuntu-latest steps: - name: Setup PHP uses: shivammathur/setup-php@v2 @@ -179,9 +179,9 @@ jobs: with: submodules: recursive - - name: Run tests + - name: Test PECL package build run: | - ./dev-scripts/prepare_release.sh --name aws-crt --user aws-crt --email aws-sdk-common-runtime@amazon.com --version 1.0.0 --notes NOTES + python3 dev-scripts/prepare_pecl_release.py --name aws-crt --user aws-crt --version 1.0.0 tar -zxf *.tgz cd awscrt-1.0.0 phpize diff --git a/.gitignore b/.gitignore index 7b7b8bf..f7413f5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ - # Created by https://www.toptal.com/developers/gitignore/api/autotools,cmake,phpstorm # Edit at https://www.toptal.com/developers/gitignore?templates=autotools,cmake,phpstorm @@ -182,6 +181,8 @@ fabric.properties build/ configure.in configure.ac +configure.bat +configure.js mkinstalldirs run-tests.php Makefile.global @@ -191,6 +192,7 @@ modules/ *.lo config.h config.nice +config.nice.bat *.la Makefile* !Makefile.am @@ -204,6 +206,7 @@ PHP-Parser*/ src/*.so src/*.dylib src/*.dll +/x64/ # ignoring output of package.xml as it needs to be generated from ./prepare_release.sh in each publishing package.xml diff --git a/builder.json b/builder.json index 14763d0..ec45884 100644 --- a/builder.json +++ b/builder.json @@ -32,6 +32,6 @@ "NO_INTERACTION": "1" }, "test_steps": [ - ["./dev-scripts/run_tests"] + ["./dev-scripts/run_tests.sh"] ] } diff --git a/dev-scripts/cleanup.sh b/dev-scripts/cleanup.sh deleted file mode 100755 index ad43122..0000000 --- a/dev-scripts/cleanup.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash - -BASENAME=${0} -TOOLS_DIR=$(dirname ${BASENAME}) -WORK_DIR="${TOOLS_DIR}/.." - -cd "${WORK_DIR}" - -find . -name \*.tgz -a |xargs rm -rf - -## get rid of anything that phpize, configure and make as well -rm -rf .deps \ -.libs \ -Makefile \ -Makefile.fragments \ -Makefile.global \ -Makefile.objects \ -acinclude.m4 \ -aclocal.m4 \ -autom4te.cache \ -build \ -config.guess \ -config.h \ -config.h.in* \ -config.log \ -config.nice \ -config.status \ -config.sub \ -configure \ -configure.in \ -configure.ac \ -include \ -install-sh \ -libtool \ -ltmain.sh \ -missing \ -mkinstalldirs \ -modules \ -run-tests.php \ -awscrt.la \ -vendor \ -composer.lock \ -ext/awscrt.stub.php \ - -find . -name \*.lo -o -name \*.o | xargs rm -f -find . -name \*.la -o -name \*.a | xargs rm -f -find . -name \*.so | xargs rm -f -find . -name .libs -a -type d|xargs rm -rf diff --git a/dev-scripts/cleanup_build.py b/dev-scripts/cleanup_build.py new file mode 100644 index 0000000..6df535c --- /dev/null +++ b/dev-scripts/cleanup_build.py @@ -0,0 +1,59 @@ +import os +import glob +import shutil + +TOOLS_DIR = os.path.dirname(os.path.abspath(__file__)) +WORK_DIR = os.path.join(TOOLS_DIR, '..') + +# Remove specified directories +DIRS_TO_REMOVE = [ + '.deps', + '.libs', + 'build', + 'include', + 'modules', + 'vendor', + 'autom4te.cache'] + +# Remove specified files +FILES_TO_REMOVE = [ + 'Makefile', + 'Makefile.fragments', + 'Makefile.global', + 'Makefile.objects', + 'config.guess', + 'config.h', + 'config.h.in', + 'config.log', + 'config.nice', + 'config.status', + 'config.sub', + 'configure', + 'configure.in', + 'configure.ac', + 'install-sh', + 'libtool', + 'ltmain.sh', + 'missing', + 'mkinstalldirs', + 'run-tests.php', + 'awscrt.la', + 'composer.lock', + 'ext/awscrt.stub.php', + 'acinclude.m4', + 'aclocal.m4', + '**/*.lo', + '**/*.o', + '**/*.la', + '**/*.a', + '*.tgz'] + +os.chdir(WORK_DIR) + + +for directory in DIRS_TO_REMOVE: + shutil.rmtree(directory, ignore_errors=True) + +for pattern in FILES_TO_REMOVE: + for filepath in glob.glob(pattern): + os.remove(filepath) diff --git a/dev-scripts/prepare_package_xml.sh b/dev-scripts/prepare_package_xml.sh deleted file mode 100755 index 29fcff8..0000000 --- a/dev-scripts/prepare_package_xml.sh +++ /dev/null @@ -1,152 +0,0 @@ -#!/bin/zsh -zparseopts -A opts -name: -user: -email: -version: -notes: -if [[ $# -lt 10 ]] -then - echo "Usage ${0} --name NAME, --user USER --email EMAIL --version VERSION --notes NOTES" - exit 1 -fi -NAME="${opts[--name]}" -USER="${opts[--user]}" -EMAIL="${opts[--email]}" -VERSION="${opts[--version]}" -NOTES="${opts[--notes]}" - -BASENAME=${0} -TOOLS_DIR=$(dirname ${BASENAME}) - -TODAY=$(date -u +%Y-%m-%d) - -cat $TOOLS_DIR/package.xml-template_pre \ - | sed "s/{{{NAME}}}/$NAME/g" \ - | sed "s/{{{USER}}}/$USER/g" \ - | sed "s/{{{EMAIL}}}/$EMAIL/g" \ - | sed "s/{{{TODAY}}}/$TODAY/g" \ - | sed "s/{{{VERSION}}}/$VERSION/g" \ - | sed "s/{{{NOTES}}}/$NOTES/g" - -source_ext='(c|cc|h|cpp|hpp|m4|w32|ini|frag|cmake|inl|in|py|gnu|yaml|def|pl|S|s|errordata|go|lds|num|asm|mod|peg|mk|rs|toml|sh)' -doc_ext='(md|json|html|dot|graphml|png|gn|sha1|css|rst|)' - -special_docs='(LICENSE*|NOTICE|changelog.txt|CHANGELOG|THIRD-PARTY|README*|readme|METADATA|CONTRIBUTORS|UPDATING|doc.config|THIRD-PARTY-LICENSES.txt)' -special_tests='(ci-test.sh|format-check.sh|run_tests*|sanitizer-blacklist.txt|run-clang-tidy.sh|benchmark-build-run.sh|break-tests.sh|generate-coverage.sh|test.xml)' -special_src='(gen_api.php|gen_stub.php|CMakeLists.txt|post.sh|postun.sh|Makefile*|build-buildspec.sh|build-deps.sh|objects.txt|go.*|BUILD*|DEPS|install_and_run.sh|codemod.sh|requirements.txt)' -skip_files='(package.xml*|prepare_release.sh|codereview.settings|*.o|*.a|*.obj|*.lib|break-tests-android.sh|whitespace.txt|prepare_package_xml.sh|crypto_test_data.cc|*.pdf|*.svg|*.docx|cbmc-proof.txt|codecov*|litani*|*.toml|module.modulemap|cleanup.sh)' - -special_scripts='(awscrt.stub.php)' - -skip_directories='(tests|test|AWSCRTAndroidTestRunner|docker-images|codebuild|fuzz|third_party|docs|generated-src|aws-lc|aws-crt-sys|ecdsa-fuzz-corpus|bin|examples|compliance|verification)' - -process_file() { - if (( $# == 0 )) - then - echo "ERROR: filename not passed" - exit 1 - fi - if [[ $1 = $~skip_files ]] - then - # This file is not part of the release bundle - return 0 - fi - - echo -n '' - return 0 -} - - -process_dir() { - if (( $# == 0 )) - then - echo "WARNING: dirname not passed" - exit 1 - fi - if [[ "${1}" = $~skip_directories ]] - then - return 0 - fi - echo '' - cd "$1" - for a in * - do - if [[ -f ${a} ]] - then process_file "${a}" - else process_dir "${a}" - fi - done - # Special cases for compiler features placed in tests directories in and s2n - if [[ "${1}" = "s2n" && -d tests ]] - then - echo '' - echo '' - cd tests/features - for a in * - do - process_file "${a}" - done - cd ../.. - echo '' - echo '' - fi - echo '' - cd .. - return 0 -} - -echo '' -for a in * -do - if [[ ${a} == 'tests' ]] - then - echo '' - for b in tests/* - do - echo '' - done - echo '' - continue - fi - # skip the src directory as it is not part of the native extension - if [[ ${a} == 'src' ]] - then - continue - fi - - if [[ -f ${a} ]] - then process_file "${a}" - else process_dir "${a}" - fi -done -echo '' - -cat $TOOLS_DIR/package.xml-template_post diff --git a/dev-scripts/prepare_pecl_package_xml.py b/dev-scripts/prepare_pecl_package_xml.py new file mode 100644 index 0000000..3acbf58 --- /dev/null +++ b/dev-scripts/prepare_pecl_package_xml.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +import argparse +import os +import re +from datetime import datetime + +parser = argparse.ArgumentParser( + description="Generate package.xml file for a release bundle.") +parser.add_argument("--name", help="Name of the package", required=True) +parser.add_argument( + "--user", help="Username of the package maintainer", required=True) +parser.add_argument( + "--email", help="Email address of the package maintainer", default='aws-sdk-common-runtime@amazon.com') +parser.add_argument( + "--version", help="Version number of the package", required=True) +parser.add_argument( + "--notes", help="Release notes for the package", default='New release') +args = parser.parse_args() + + +name = args.name +user = args.user +email = args.email +version = args.version +notes = args.notes + + +tools_dir = os.path.dirname(os.path.abspath(__file__)) +work_dir = os.path.join(tools_dir, '..') + +os.chdir(work_dir) + +today = datetime.utcnow().strftime("%Y-%m-%d") + +source_ext = "(c|cc|h|cpp|hpp|m4|w32|ini|frag|cmake|inl|in|py|gnu|yaml|def|pl|S|s|errordata|go|lds|num|asm|mod|peg|mk|rs|toml|sh)" +doc_ext = "(md|json|html|dot|graphml|png|gn|sha1|css|rst)" + +special_docs = "(LICENSE.*|NOTICE|changelog.txt|CHANGELOG|THIRD-PARTY|README.*|readme|METADATA|CONTRIBUTORS|UPDATING|doc.config|THIRD-PARTY-LICENSES.txt)" +special_tests = "(ci-test.sh|format-check.sh|run_tests.*|sanitizer-blacklist.txt|run-clang-tidy.sh|benchmark-build-run.sh|break-tests.sh|generate-coverage.sh|test.xml)" +special_src = "(gen_api.php|gen_stub.php|CMakeLists.txt|post.sh|postun.sh|Makefile.*|build-buildspec.sh|build-deps.sh|objects.txt|go.*|BUILD.*|DEPS|install_and_run.sh|codemod.sh|requirements.txt)" +skip_files = "(package.xml.*|prepare_release.sh|codereview.settings|.*\\.o|.*\\.a|.*\\.obj|.*\\.lib|break-tests-android.sh|whitespace.txt|prepare_package_xml.sh|crypto_test_data.cc|.*\\.pdf|.*\\.svg|.*\\.docx|cbmc-proof.txt|codecov.*|litani.*|.*\\.toml|module\\.modulemap|cleanup.sh|^\..*)" + + +special_scripts = "(awscrt.stub.php)" + +skip_directories = "(tests|test|AWSCRTAndroidTestRunner|docker-images|codebuild|fuzz|third_party|docs|generated-src|aws-lc|aws-crt-sys|ecdsa-fuzz-corpus|bin|examples|compliance|verification|^\..*)" + +output_file_name = os.path.join(work_dir, 'package.xml') + + +def process_file(file_name, f): + if re.match(skip_files, file_name): + # This file is not part of the release bundle + return + f.write(f'\n') + + +def process_dir(dir_name, f): + if re.match(skip_directories, dir_name): + return + + f.write(f'\n') + os.chdir(dir_name) + for file_name in os.listdir(): + if os.path.isfile(file_name): + process_file(file_name, f) + else: + process_dir(file_name, f) + + # Special cases for compiler features placed in tests directories in and s2n + if dir_name == 's2n' and os.path.isdir('tests'): + f.write('') + f.write('') + os.chdir('tests/features') + for a in os.listdir(): + process_file(a, f) + os.chdir('../..') + f.write('') + f.write('') + + f.write('') + os.chdir('..') + + +# Main code +with open(os.path.join(tools_dir, "package.xml-template_pre"), 'r') as f: + template_pre = f.read() + +template_pre = re.sub(r"{{{NAME}}}", name, template_pre) +template_pre = re.sub(r"{{{USER}}}", user, template_pre) +template_pre = re.sub(r"{{{EMAIL}}}", email, template_pre) +template_pre = re.sub(r"{{{TODAY}}}", today, template_pre) +template_pre = re.sub(r"{{{VERSION}}}", version, template_pre) +template_pre = re.sub(r"{{{NOTES}}}", notes, template_pre) + +with open(output_file_name, "w") as f: + f.write(template_pre) + f.write('\n') + +with open(output_file_name, "a") as f: + for file_name in os.listdir(): + if file_name == 'tests': + f.write('\n') + for test_file in os.listdir('tests'): + f.write( + f'\n') + f.write('\n') + continue + + if file_name == 'src': + # skip the src directory as it is not part of the native extension + continue + + if os.path.isfile(file_name): + process_file(file_name, f) + else: + process_dir(file_name, f) + +with open(output_file_name, "a") as f: + f.write('\n') + with open(os.path.join(tools_dir, "package.xml-template_post"), 'r') as template: + f.write(template.read()) diff --git a/dev-scripts/prepare_pecl_release.py b/dev-scripts/prepare_pecl_release.py new file mode 100644 index 0000000..fd4874b --- /dev/null +++ b/dev-scripts/prepare_pecl_release.py @@ -0,0 +1,58 @@ +import argparse +import os +import subprocess +import xml.dom.minidom +import sys + + +parser = argparse.ArgumentParser(description='PECL Package generator') +parser.add_argument("--name", help="Name of the package", required=True) +parser.add_argument( + "--user", help="Username of the package maintainer", required=True) +parser.add_argument( + "--email", help="Email address of the package maintainer", default='aws-sdk-common-runtime@amazon.com') +parser.add_argument( + "--version", help="Version number of the package", required=True) +parser.add_argument( + "--notes", help="Release notes for the package", default='New release') +args = parser.parse_args() + +PACKAGE = 'awscrt' +NAME = args.name +USER = args.user +EMAIL = args.email +VERSION = args.version +NOTES = args.notes + +TOOLS_DIR = os.path.dirname(os.path.abspath(__file__)) +WORK_DIR = os.path.join(TOOLS_DIR, '..') + + +def run(args): + print(f"$ {subprocess.list2cmdline(args)}") + subprocess.check_call(args) + + +run(['python3', f'{TOOLS_DIR}/cleanup_build.py']) + +os.chdir(WORK_DIR) + +run(['git', 'submodule', 'update', '--init', '--recursive']) + +try: + run(['python3', f'{TOOLS_DIR}/prepare_pecl_package_xml.py', '--name', NAME, '--user', USER, + '--email', EMAIL, '--version', VERSION, '--notes', NOTES]) +except subprocess.CalledProcessError as e: + sys.exit(f'ERROR PROCESSING review package.xml: {e}') + +with open('package.xml', 'r') as f: + package_xml = f.read() +doc = xml.dom.minidom.parse('package.xml') +doc.encoding = 'UTF-8' +xml_str = doc.toprettyxml(indent=' ', newl='') +with open('package.xml', 'w') as f: + f.write(xml_str) +run(['pear', 'package-validate']) +run(['pear', 'package']) + +print(f'Size of {PACKAGE}-{VERSION}.tgz: {os.path.getsize(f"{PACKAGE}-{VERSION}.tgz") / 1024 / 1024:.2f} MB') diff --git a/dev-scripts/prepare_release.sh b/dev-scripts/prepare_release.sh deleted file mode 100755 index 0cf237d..0000000 --- a/dev-scripts/prepare_release.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/zsh -zparseopts -A opts -name: -user: -email: -version: -notes: -if [[ $# -lt 10 ]]; then - echo "Usage ${0} --name NAME --user USER --email EMAIL --version VERSION --notes NOTES" - exit 1 -fi -PACKAGE='awscrt' -NAME="${opts[--name]}" -USER="${opts[--user]}" -EMAIL="${opts[--email]}" -VERSION="${opts[--version]}" -NOTES="${opts[--notes]}" - - -BASENAME=${0} -TOOLS_DIR=$(dirname ${BASENAME}) -WORK_DIR="${TOOLS_DIR}/.." - -cd "${WORK_DIR}" - -git submodule update --init --recursive - -$TOOLS_DIR/cleanup.sh -$TOOLS_DIR/prepare_package_xml.sh --name "${NAME}" --user "${USER}" --email "${EMAIL}" --version "${VERSION}" --notes "${NOTES}" >package.xml -if [[ $? -ne 0 ]]; then - echo "ERROR PROCESSING review package.xml" - exit 1 -fi -tidy -xml -m -i package.xml -pear package-validate -if [[ $? -ne 0 ]]; then - echo "ERROR VALIDATING review package.xml" - exit 1 -fi -pear package -if [[ $? -ne 0 ]]; then - echo "ERROR PROCESSING review package.xml" - exit 1 -fi - -echo "Size of ${PACKAGE}-${VERSION}.tgz: " $(du -h "${PACKAGE}-${VERSION}.tgz") diff --git a/dev-scripts/run_tests.bat b/dev-scripts/run_tests.bat index e92f1cb..a071c24 100644 --- a/dev-scripts/run_tests.bat +++ b/dev-scripts/run_tests.bat @@ -1,9 +1,6 @@ @echo on setlocal -REM Run command to get composer_dir -set "composer_dir=" -for /f "usebackq delims=" %%i in (`where composer.phar`) do set "composer_dir=%%i" set "PHP_BINARY=php" @@ -11,18 +8,24 @@ if not "%~1"=="" ( set "PHP_BINARY=%~1" ) -REM Check if composer_dir was found -if "%composer_dir%"=="" ( - echo No composer found. - exit /b 1 -) - set "script_dir=%~dp0" set "work_dir=%script_dir%/.." cd %work_dir% -call %PHP_BINARY% -c php-win.ini %composer_dir% update +set vendor_path=vendor +REM Run command to get composer_dir +set "composer_dir=" +for /f "usebackq delims=" %%i in (`where composer.phar`) do set "composer_dir=%%i" + +if not exist "%vendor_path%" ( + REM Check if composer_dir was found + if "%composer_dir%"=="" ( + echo No composer found. + exit /b 1 + ) + call %PHP_BINARY% -c php-win.ini %composer_dir% update +) + call %PHP_BINARY% -c php-win.ini vendor/bin/phpunit tests --debug - -endlocal \ No newline at end of file +endlocal diff --git a/dev-scripts/run_tests b/dev-scripts/run_tests.sh similarity index 100% rename from dev-scripts/run_tests rename to dev-scripts/run_tests.sh diff --git a/php-win.ini b/php-win.ini index fb0f704..a1bdb78 100644 --- a/php-win.ini +++ b/php-win.ini @@ -1,3 +1,3 @@ extension=php_awscrt.dll extension=php_openssl.dll -extension=php_mbstring.dll \ No newline at end of file +extension=php_mbstring.dll