Merge branch 'native-extension-rewrite' of github.com:awslabs/aws-crt-php into native-extension-rewrite

.github/workflows/ci.yml

@@ -80,7 +80,7 @@ jobs:
         phpize
         ./configure
         make
-        ./dev-scripts/run_tests
+        ./dev-scripts/run_tests.sh
 
     # linux-arm:
     #   name: ARM (${{ matrix.arch }})
@@ -123,7 +123,7 @@ jobs:
     - run: .\configure --with-prefix=${{steps.setup-php-sdk.outputs.prefix}} --enable-awscrt=shared --enable-cli --enable-openssl
     - run: nmake
     - run: nmake generate-php-ini
-    - run: .\dev-scripts\run_tests ${{steps.setup-php-sdk.outputs.prefix}}\php
+    - run: .\dev-scripts\run_tests.bat ${{steps.setup-php-sdk.outputs.prefix}}\php
 
   # windows-vc14:
   #   runs-on: windows-latest
@@ -167,7 +167,7 @@ jobs:
           ./builder build -p ${{ env.PACKAGE_NAME }} --spec=downstream
 
   pecl-package-test:
-    runs-on: macos-latest
+    runs-on: ubuntu-latest
     steps:
     - name: Setup PHP
       uses: shivammathur/setup-php@v2
@@ -179,9 +179,9 @@ jobs:
       with:
         submodules: recursive
 
-    - name: Run tests
+    - name: Test PECL package build
       run: |
-        ./dev-scripts/prepare_release.sh --name aws-crt --user aws-crt --email [email protected] --version 1.0.0 --notes NOTES
+        python3 dev-scripts/prepare_pecl_release.py --name aws-crt --user aws-crt --version 1.0.0
         tar -zxf *.tgz
         cd awscrt-1.0.0
         phpize

.gitignore

@@ -1,4 +1,3 @@
-
 # Created by https://www.toptal.com/developers/gitignore/api/autotools,cmake,phpstorm
 # Edit at https://www.toptal.com/developers/gitignore?templates=autotools,cmake,phpstorm
 
@@ -182,6 +181,8 @@ fabric.properties
 build/
 configure.in
 configure.ac
+configure.bat
+configure.js
 mkinstalldirs
 run-tests.php
 Makefile.global
@@ -191,6 +192,7 @@ modules/
 *.lo
 config.h
 config.nice
+config.nice.bat
 *.la
 Makefile*
 !Makefile.am
@@ -204,6 +206,7 @@ PHP-Parser*/
 src/*.so
 src/*.dylib
 src/*.dll
+/x64/
 
 # ignoring output of package.xml as it needs to be generated from ./prepare_release.sh in each publishing
 package.xml

builder.json

@@ -32,6 +32,6 @@
         "NO_INTERACTION": "1"
     },
     "test_steps": [
-        ["./dev-scripts/run_tests"]
+        ["./dev-scripts/run_tests.sh"]
     ]
 }

dev-scripts/cleanup.sh

@@ -1,48 +0,0 @@
-#!/usr/bin/env bash
-
-BASENAME=${0}
-TOOLS_DIR=$(dirname ${BASENAME})
-WORK_DIR="${TOOLS_DIR}/.."
-
-cd "${WORK_DIR}"
-
-find . -name \*.tgz -a |xargs rm -rf
-
-## get rid of anything that phpize, configure and make as well
-rm -rf .deps \
-.libs \
-Makefile \
-Makefile.fragments \
-Makefile.global \
-Makefile.objects \
-acinclude.m4 \
-aclocal.m4 \
-autom4te.cache \
-build \
-config.guess \
-config.h \
-config.h.in* \
-config.log \
-config.nice \
-config.status \
-config.sub \
-configure \
-configure.in \
-configure.ac \
-include \
-install-sh \
-libtool \
-ltmain.sh \
-missing \
-mkinstalldirs \
-modules \
-run-tests.php \
-awscrt.la \
-vendor \
-composer.lock \
-ext/awscrt.stub.php \
-
-find . -name \*.lo -o -name \*.o | xargs rm -f
-find . -name \*.la -o -name \*.a | xargs rm -f
-find . -name \*.so | xargs rm -f
-find . -name .libs -a -type d|xargs rm -rf

dev-scripts/cleanup_build.py

@@ -0,0 +1,59 @@
+import os
+import glob
+import shutil
+
+TOOLS_DIR = os.path.dirname(os.path.abspath(__file__))
+WORK_DIR = os.path.join(TOOLS_DIR, '..')
+
+# Remove specified directories
+DIRS_TO_REMOVE = [
+    '.deps',
+    '.libs',
+    'build',
+    'include',
+    'modules',
+    'vendor',
+    'autom4te.cache']
+
+# Remove specified files
+FILES_TO_REMOVE = [
+    'Makefile',
+    'Makefile.fragments',
+    'Makefile.global',
+    'Makefile.objects',
+    'config.guess',
+    'config.h',
+    'config.h.in',
+    'config.log',
+    'config.nice',
+    'config.status',
+    'config.sub',
+    'configure',
+    'configure.in',
+    'configure.ac',
+    'install-sh',
+    'libtool',
+    'ltmain.sh',
+    'missing',
+    'mkinstalldirs',
+    'run-tests.php',
+    'awscrt.la',
+    'composer.lock',
+    'ext/awscrt.stub.php',
+    'acinclude.m4',
+    'aclocal.m4',
+    '**/*.lo',
+    '**/*.o',
+    '**/*.la',
+    '**/*.a',
+    '*.tgz']
+
+os.chdir(WORK_DIR)
+
+
+for directory in DIRS_TO_REMOVE:
+    shutil.rmtree(directory, ignore_errors=True)
+
+for pattern in FILES_TO_REMOVE:
+    for filepath in glob.glob(pattern):
+        os.remove(filepath)

dev-scripts/prepare_package_xml.sh

@@ -1,152 +0,0 @@
-#!/bin/zsh
-zparseopts -A opts -name: -user: -email: -version: -notes:
-if [[ $# -lt 10 ]]
-then
-  echo "Usage ${0} --name NAME, --user USER --email EMAIL --version VERSION --notes NOTES"
-  exit 1
-fi
-NAME="${opts[--name]}"
-USER="${opts[--user]}"
-EMAIL="${opts[--email]}"
-VERSION="${opts[--version]}"
-NOTES="${opts[--notes]}"
-
-BASENAME=${0}
-TOOLS_DIR=$(dirname ${BASENAME})
-
-TODAY=$(date -u +%Y-%m-%d)
-
-cat $TOOLS_DIR/package.xml-template_pre \
-    | sed "s/{{{NAME}}}/$NAME/g" \
-    | sed "s/{{{USER}}}/$USER/g" \
-    | sed "s/{{{EMAIL}}}/$EMAIL/g" \
-    | sed "s/{{{TODAY}}}/$TODAY/g" \
-    | sed "s/{{{VERSION}}}/$VERSION/g" \
-    | sed "s/{{{NOTES}}}/$NOTES/g"
-
-source_ext='(c|cc|h|cpp|hpp|m4|w32|ini|frag|cmake|inl|in|py|gnu|yaml|def|pl|S|s|errordata|go|lds|num|asm|mod|peg|mk|rs|toml|sh)'
-doc_ext='(md|json|html|dot|graphml|png|gn|sha1|css|rst|)'
-
-special_docs='(LICENSE*|NOTICE|changelog.txt|CHANGELOG|THIRD-PARTY|README*|readme|METADATA|CONTRIBUTORS|UPDATING|doc.config|THIRD-PARTY-LICENSES.txt)'
-special_tests='(ci-test.sh|format-check.sh|run_tests*|sanitizer-blacklist.txt|run-clang-tidy.sh|benchmark-build-run.sh|break-tests.sh|generate-coverage.sh|test.xml)'
-special_src='(gen_api.php|gen_stub.php|CMakeLists.txt|post.sh|postun.sh|Makefile*|build-buildspec.sh|build-deps.sh|objects.txt|go.*|BUILD*|DEPS|install_and_run.sh|codemod.sh|requirements.txt)'
-skip_files='(package.xml*|prepare_release.sh|codereview.settings|*.o|*.a|*.obj|*.lib|break-tests-android.sh|whitespace.txt|prepare_package_xml.sh|crypto_test_data.cc|*.pdf|*.svg|*.docx|cbmc-proof.txt|codecov*|litani*|*.toml|module.modulemap|cleanup.sh)'
-
-special_scripts='(awscrt.stub.php)'
-
-skip_directories='(tests|test|AWSCRTAndroidTestRunner|docker-images|codebuild|fuzz|third_party|docs|generated-src|aws-lc|aws-crt-sys|ecdsa-fuzz-corpus|bin|examples|compliance|verification)'
-
-process_file() {
-    if (( $# == 0 ))
-    then
-      echo "ERROR: filename not passed"
-      exit 1
-    fi
-    if [[ $1 = $~skip_files ]]
-    then
-      # This file is not part of the release bundle
-      return 0
-    fi
-
-    echo -n '<file name="'"$1"'" role="'
-    # Special cases
-    case ${a} in
-    $~special_scripts)
-      echo -n 'script'
-    ;;
-    $~special_docs)
-      echo -n 'doc'
-    ;;
-    $~special_tests)
-      echo -n 'test'
-    ;;
-    $~special_src)
-      echo -n 'src'
-    ;;
-    *)
-      # Extension based cases
-      case ${a:t:e} in
-      $~source_ext)
-        echo -n 'src'
-      ;;
-      $~doc_ext)
-         echo -n 'doc'
-      ;;
-      php)
-         echo -n 'script'
-      ;;
-      *)
-         echo "${a:t:e} - ${a} - FAIL TO RECOGNIZE"
-         exit 1
-      esac
-    esac
-    echo '"/>'
-    return 0
-}
-
-
-process_dir() {
-  if (( $# == 0 ))
-  then
-    echo "WARNING: dirname not passed"
-    exit 1
-  fi
-  if [[ "${1}" = $~skip_directories ]]
-  then
-    return 0
-  fi
-  echo '<dir name="'"$1"'">'
-  cd "$1"
-  for a in *
-  do
-    if [[ -f ${a} ]]
-    then process_file "${a}"
-    else process_dir "${a}"
-    fi
-  done
-  # Special cases for compiler features placed in tests directories in and s2n
-  if [[ "${1}" = "s2n" && -d tests ]]
-  then
-      echo '<dir name="tests">'
-      echo '<dir name="features">'
-      cd tests/features
-      for a in *
-      do
-          process_file "${a}"
-      done
-      cd ../..
-      echo '</dir>'
-      echo '</dir>'
-  fi
-  echo '</dir>'
-  cd ..
-  return 0
-}
-
-echo '<dir name="/">'
-for a in *
-do
-  if [[ ${a} == 'tests' ]]
-  then
-    echo '<dir name="tests">'
-    for b in tests/*
-    do
-      echo '<file name="'$( basename "${b}" )'" role="test" />'
-    done
-    echo '</dir>'
-    continue
-  fi
-  # skip the src directory as it is not part of the native extension
-  if [[ ${a} == 'src' ]]
-  then
-    continue
-  fi
-
-  if [[ -f ${a} ]]
-  then process_file "${a}"
-  else process_dir "${a}"
-  fi
-done
-echo '</dir>'
-
-cat $TOOLS_DIR/package.xml-template_post

dev-scripts/prepare_pecl_package_xml.py

@@ -0,0 +1,146 @@
+#!/usr/bin/env python3
+import argparse
+import os
+import re
+from datetime import datetime
+
+parser = argparse.ArgumentParser(
+    description="Generate package.xml file for a release bundle.")
+parser.add_argument("--name", help="Name of the package", required=True)
+parser.add_argument(
+    "--user", help="Username of the package maintainer", required=True)
+parser.add_argument(
+    "--email", help="Email address of the package maintainer", default='[email protected]')
+parser.add_argument(
+    "--version", help="Version number of the package", required=True)
+parser.add_argument(
+    "--notes", help="Release notes for the package", default='New release')
+args = parser.parse_args()
+
+
+name = args.name
+user = args.user
+email = args.email
+version = args.version
+notes = args.notes
+
+
+tools_dir = os.path.dirname(os.path.abspath(__file__))
+work_dir = os.path.join(tools_dir, '..')
+
+os.chdir(work_dir)
+
+today = datetime.utcnow().strftime("%Y-%m-%d")
+
+source_ext = "(c|cc|h|cpp|hpp|m4|w32|ini|frag|cmake|inl|in|py|gnu|yaml|def|pl|S|s|errordata|go|lds|num|asm|mod|peg|mk|rs|toml|sh)"
+doc_ext = "(md|json|html|dot|graphml|png|gn|sha1|css|rst)"
+
+special_docs = "(LICENSE.*|NOTICE|changelog.txt|CHANGELOG|THIRD-PARTY|README.*|readme|METADATA|CONTRIBUTORS|UPDATING|doc.config|THIRD-PARTY-LICENSES.txt)"
+special_tests = "(ci-test.sh|format-check.sh|run_tests.*|sanitizer-blacklist.txt|run-clang-tidy.sh|benchmark-build-run.sh|break-tests.sh|generate-coverage.sh|test.xml)"
+special_src = "(gen_api.php|gen_stub.php|CMakeLists.txt|post.sh|postun.sh|Makefile.*|build-buildspec.sh|build-deps.sh|objects.txt|go.*|BUILD.*|DEPS|install_and_run.sh|codemod.sh|requirements.txt)"
+skip_files = "(package.xml.*|prepare_release.sh|codereview.settings|.*\\.o|.*\\.a|.*\\.obj|.*\\.lib|break-tests-android.sh|whitespace.txt|prepare_package_xml.sh|crypto_test_data.cc|.*\\.pdf|.*\\.svg|.*\\.docx|cbmc-proof.txt|codecov.*|litani.*|.*\\.toml|module\\.modulemap|cleanup.sh|^\..*)"
+
+
+special_scripts = "(awscrt.stub.php)"
+
+skip_directories = "(tests|test|AWSCRTAndroidTestRunner|docker-images|codebuild|fuzz|third_party|docs|generated-src|aws-lc|aws-crt-sys|ecdsa-fuzz-corpus|bin|examples|compliance|verification|^\..*)"
+
+output_file_name = os.path.join(work_dir, 'package.xml')
+
+
+def process_file(file_name, f):
+    if re.match(skip_files, file_name):
+        # This file is not part of the release bundle
+        return
+    f.write(f'<file name="{file_name}" role="')
+    # Special cases
+    if re.match(special_scripts, file_name):
+        f.write('script')
+    elif re.match(special_docs, file_name):
+        f.write('doc')
+    elif re.match(special_tests, file_name):
+        f.write('test')
+    elif re.match(special_src, file_name):
+        f.write('src')
+    else:
+        # Extension based cases
+        ext = file_name.split('.')[-1]
+        if re.match(source_ext, ext):
+            f.write('src')
+        elif re.match(doc_ext, ext):
+            f.write('doc')
+        elif ext == 'php':
+            f.write('script')
+        else:
+            f.write(f'{ext} - {file_name} - FAIL TO RECOGNIZE')
+            print(f'{ext} - {file_name} - FAIL TO RECOGNIZE')
+            exit(1)
+
+    f.write('"/>\n')
+
+
+def process_dir(dir_name, f):
+    if re.match(skip_directories, dir_name):
+        return
+
+    f.write(f'<dir name="{dir_name}">\n')
+    os.chdir(dir_name)
+    for file_name in os.listdir():
+        if os.path.isfile(file_name):
+            process_file(file_name, f)
+        else:
+            process_dir(file_name, f)
+
+    # Special cases for compiler features placed in tests directories in and s2n
+    if dir_name == 's2n' and os.path.isdir('tests'):
+        f.write('<dir name="tests">')
+        f.write('<dir name="features">')
+        os.chdir('tests/features')
+        for a in os.listdir():
+            process_file(a, f)
+        os.chdir('../..')
+        f.write('</dir>')
+        f.write('</dir>')
+
+    f.write('</dir>')
+    os.chdir('..')
+
+
+# Main code
+with open(os.path.join(tools_dir, "package.xml-template_pre"), 'r') as f:
+    template_pre = f.read()
+
+template_pre = re.sub(r"{{{NAME}}}", name, template_pre)
+template_pre = re.sub(r"{{{USER}}}", user, template_pre)
+template_pre = re.sub(r"{{{EMAIL}}}", email, template_pre)
+template_pre = re.sub(r"{{{TODAY}}}", today, template_pre)
+template_pre = re.sub(r"{{{VERSION}}}", version, template_pre)
+template_pre = re.sub(r"{{{NOTES}}}", notes, template_pre)
+
+with open(output_file_name, "w") as f:
+    f.write(template_pre)
+    f.write('<dir name="/">\n')
+
+with open(output_file_name, "a") as f:
+    for file_name in os.listdir():
+        if file_name == 'tests':
+            f.write('<dir name="tests">\n')
+            for test_file in os.listdir('tests'):
+                f.write(
+                    f'<file name="{os.path.basename(test_file)}" role="test" />\n')
+            f.write('</dir>\n')
+            continue
+
+        if file_name == 'src':
+            # skip the src directory as it is not part of the native extension
+            continue
+
+        if os.path.isfile(file_name):
+            process_file(file_name, f)
+        else:
+            process_dir(file_name, f)
+
+with open(output_file_name, "a") as f:
+    f.write('</dir>\n')
+    with open(os.path.join(tools_dir, "package.xml-template_post"), 'r') as template:
+        f.write(template.read())

dev-scripts/prepare_pecl_release.py

@@ -0,0 +1,58 @@
+import argparse
+import os
+import subprocess
+import xml.dom.minidom
+import sys
+
+
+parser = argparse.ArgumentParser(description='PECL Package generator')
+parser.add_argument("--name", help="Name of the package", required=True)
+parser.add_argument(
+    "--user", help="Username of the package maintainer", required=True)
+parser.add_argument(
+    "--email", help="Email address of the package maintainer", default='[email protected]')
+parser.add_argument(
+    "--version", help="Version number of the package", required=True)
+parser.add_argument(
+    "--notes", help="Release notes for the package", default='New release')
+args = parser.parse_args()
+
+PACKAGE = 'awscrt'
+NAME = args.name
+USER = args.user
+EMAIL = args.email
+VERSION = args.version
+NOTES = args.notes
+
+TOOLS_DIR = os.path.dirname(os.path.abspath(__file__))
+WORK_DIR = os.path.join(TOOLS_DIR, '..')
+
+
+def run(args):
+    print(f"$ {subprocess.list2cmdline(args)}")
+    subprocess.check_call(args)
+
+
+run(['python3', f'{TOOLS_DIR}/cleanup_build.py'])
+
+os.chdir(WORK_DIR)
+
+run(['git', 'submodule', 'update', '--init', '--recursive'])
+
+try:
+    run(['python3', f'{TOOLS_DIR}/prepare_pecl_package_xml.py', '--name', NAME, '--user', USER,
+                    '--email', EMAIL, '--version', VERSION, '--notes', NOTES])
+except subprocess.CalledProcessError as e:
+    sys.exit(f'ERROR PROCESSING review package.xml: {e}')
+
+with open('package.xml', 'r') as f:
+    package_xml = f.read()
+doc = xml.dom.minidom.parse('package.xml')
+doc.encoding = 'UTF-8'
+xml_str = doc.toprettyxml(indent=' ', newl='')
+with open('package.xml', 'w') as f:
+    f.write(xml_str)
+run(['pear', 'package-validate'])
+run(['pear', 'package'])
+
+print(f'Size of {PACKAGE}-{VERSION}.tgz: {os.path.getsize(f"{PACKAGE}-{VERSION}.tgz") / 1024 / 1024:.2f} MB')

dev-scripts/prepare_release.sh

@@ -1,41 +0,0 @@
-#!/bin/zsh
-zparseopts -A opts -name: -user: -email: -version: -notes:
-if [[ $# -lt 10 ]]; then
-  echo "Usage ${0} --name NAME --user USER --email EMAIL --version VERSION --notes NOTES"
-  exit 1
-fi
-PACKAGE='awscrt'
-NAME="${opts[--name]}"
-USER="${opts[--user]}"
-EMAIL="${opts[--email]}"
-VERSION="${opts[--version]}"
-NOTES="${opts[--notes]}"
-
-
-BASENAME=${0}
-TOOLS_DIR=$(dirname ${BASENAME})
-WORK_DIR="${TOOLS_DIR}/.."
-
-cd "${WORK_DIR}"
-
-git submodule update --init --recursive
-
-$TOOLS_DIR/cleanup.sh
-$TOOLS_DIR/prepare_package_xml.sh --name "${NAME}" --user "${USER}" --email "${EMAIL}" --version "${VERSION}" --notes "${NOTES}" >package.xml
-if [[ $? -ne 0 ]]; then
-  echo "ERROR PROCESSING review package.xml"
-  exit 1
-fi
-tidy -xml -m -i package.xml
-pear package-validate
-if [[ $? -ne 0 ]]; then
-  echo "ERROR VALIDATING review package.xml"
-  exit 1
-fi
-pear package
-if [[ $? -ne 0 ]]; then
-  echo "ERROR PROCESSING review package.xml"
-  exit 1
-fi
-
-echo "Size of ${PACKAGE}-${VERSION}.tgz: " $(du -h "${PACKAGE}-${VERSION}.tgz")

dev-scripts/run_tests.bat

@@ -1,9 +1,6 @@
 @echo on
 setlocal
 
-REM Run command to get composer_dir
-set "composer_dir="
-for /f "usebackq delims=" %%i in (`where composer.phar`) do set "composer_dir=%%i"
 
 set "PHP_BINARY=php"
 
@@ -11,18 +8,24 @@ if not "%~1"=="" (
   set "PHP_BINARY=%~1"
 )
 
-REM Check if composer_dir was found
-if "%composer_dir%"=="" (
-  echo No composer found.
-  exit /b 1
-)
-
 set "script_dir=%~dp0"
 set "work_dir=%script_dir%/.."
 cd %work_dir%
 
-call %PHP_BINARY% -c php-win.ini %composer_dir% update
+set vendor_path=vendor
+REM Run command to get composer_dir
+set "composer_dir="
+for /f "usebackq delims=" %%i in (`where composer.phar`) do set "composer_dir=%%i"
+
+if not exist "%vendor_path%" (
+  REM Check if composer_dir was found
+  if "%composer_dir%"=="" (
+    echo No composer found.
+    exit /b 1
+  )
+  call %PHP_BINARY% -c php-win.ini %composer_dir% update
+)
+
 call %PHP_BINARY% -c php-win.ini vendor/bin/phpunit tests --debug
 
-
 endlocal