update helper scripts and more

.github/workflows/ci.yml

@@ -167,7 +167,7 @@ jobs:
           ./builder build -p ${{ env.PACKAGE_NAME }} --spec=downstream
 
   pecl-package-test:
-    runs-on: macos-latest
+    runs-on: ubuntu-latest
     steps:
     - name: Setup PHP
       uses: shivammathur/setup-php@v2
@@ -179,9 +179,9 @@ jobs:
       with:
         submodules: recursive
 
-    - name: Run tests
+    - name: Test PECL package build
       run: |
-        ./dev-scripts/prepare_release.sh --name aws-crt --user aws-crt --email [email protected] --version 1.0.0 --notes NOTES
+        python3 dev-scripts/prepare_pecl_release.py --name aws-crt --user aws-crt --version 1.0.0
         tar -zxf *.tgz
         cd awscrt-1.0.0
         phpize

dev-scripts/cleanup.sh

@@ -1,48 +0,0 @@
-#!/usr/bin/env bash
-
-BASENAME=${0}
-TOOLS_DIR=$(dirname ${BASENAME})
-WORK_DIR="${TOOLS_DIR}/.."
-
-cd "${WORK_DIR}"
-
-find . -name \*.tgz -a |xargs rm -rf
-
-## get rid of anything that phpize, configure and make as well
-rm -rf .deps \
-.libs \
-Makefile \
-Makefile.fragments \
-Makefile.global \
-Makefile.objects \
-acinclude.m4 \
-aclocal.m4 \
-autom4te.cache \
-build \
-config.guess \
-config.h \
-config.h.in* \
-config.log \
-config.nice \
-config.status \
-config.sub \
-configure \
-configure.in \
-configure.ac \
-include \
-install-sh \
-libtool \
-ltmain.sh \
-missing \
-mkinstalldirs \
-modules \
-run-tests.php \
-awscrt.la \
-vendor \
-composer.lock \
-ext/awscrt.stub.php \
-
-find . -name \*.lo -o -name \*.o | xargs rm -f
-find . -name \*.la -o -name \*.a | xargs rm -f
-find . -name \*.so | xargs rm -f
-find . -name .libs -a -type d|xargs rm -rf

dev-scripts/cleanup_build.py

@@ -0,0 +1,40 @@
+import os
+import glob
+import shutil
+
+TOOLS_DIR = os.path.dirname(os.path.abspath(__file__))
+WORK_DIR = os.path.join(TOOLS_DIR, '..')
+
+
+def remove_files(files):
+    for file in files:
+        if os.path.exists(file):
+            os.remove(file)
+
+
+os.chdir(WORK_DIR)
+
+# Remove specified directories
+dirs_to_remove = ['.deps', '.libs', 'build', 'include', 'modules', 'vendor']
+for directory in dirs_to_remove:
+    shutil.rmtree(directory, ignore_errors=True)
+
+# Remove specified files
+files_to_remove = ['Makefile', 'Makefile.fragments', 'Makefile.global', 'Makefile.objects',
+                   'config.guess', 'config.h', 'config.log', 'config.nice', 'config.status',
+                   'config.sub', 'configure', 'configure.in', 'configure.ac', 'install-sh',
+                   'libtool', 'ltmain.sh', 'missing', 'mkinstalldirs', 'run-tests.php',
+                   'awscrt.la', 'composer.lock', 'ext/awscrt.stub.php', 'acinclude.m4', 'aclocal.m4',
+                   'autom4te.cache']
+
+# Remove all .lo and .o files
+files_to_remove += glob.glob(os.path.join(WORK_DIR, '**/*.lo'))
+files_to_remove += glob.glob(os.path.join(WORK_DIR, '**/*.o'))
+
+# Remove all .la and .a files
+files_to_remove += glob.glob(os.path.join(WORK_DIR, '**/*.la'))
+files_to_remove += glob.glob(os.path.join(WORK_DIR, '**/*.a'))
+# Remove all .tgz files
+files_to_remove += glob.glob(os.path.join(WORK_DIR, '*.tgz'))
+
+remove_files(files_to_remove)

dev-scripts/prepare_package_xml.sh

@@ -1,152 +0,0 @@
-#!/bin/zsh
-zparseopts -A opts -name: -user: -email: -version: -notes:
-if [[ $# -lt 10 ]]
-then
-  echo "Usage ${0} --name NAME, --user USER --email EMAIL --version VERSION --notes NOTES"
-  exit 1
-fi
-NAME="${opts[--name]}"
-USER="${opts[--user]}"
-EMAIL="${opts[--email]}"
-VERSION="${opts[--version]}"
-NOTES="${opts[--notes]}"
-
-BASENAME=${0}
-TOOLS_DIR=$(dirname ${BASENAME})
-
-TODAY=$(date -u +%Y-%m-%d)
-
-cat $TOOLS_DIR/package.xml-template_pre \
-    | sed "s/{{{NAME}}}/$NAME/g" \
-    | sed "s/{{{USER}}}/$USER/g" \
-    | sed "s/{{{EMAIL}}}/$EMAIL/g" \
-    | sed "s/{{{TODAY}}}/$TODAY/g" \
-    | sed "s/{{{VERSION}}}/$VERSION/g" \
-    | sed "s/{{{NOTES}}}/$NOTES/g"
-
-source_ext='(c|cc|h|cpp|hpp|m4|w32|ini|frag|cmake|inl|in|py|gnu|yaml|def|pl|S|s|errordata|go|lds|num|asm|mod|peg|mk|rs|toml|sh)'
-doc_ext='(md|json|html|dot|graphml|png|gn|sha1|css|rst|)'
-
-special_docs='(LICENSE*|NOTICE|changelog.txt|CHANGELOG|THIRD-PARTY|README*|readme|METADATA|CONTRIBUTORS|UPDATING|doc.config|THIRD-PARTY-LICENSES.txt)'
-special_tests='(ci-test.sh|format-check.sh|run_tests*|sanitizer-blacklist.txt|run-clang-tidy.sh|benchmark-build-run.sh|break-tests.sh|generate-coverage.sh|test.xml)'
-special_src='(gen_api.php|gen_stub.php|CMakeLists.txt|post.sh|postun.sh|Makefile*|build-buildspec.sh|build-deps.sh|objects.txt|go.*|BUILD*|DEPS|install_and_run.sh|codemod.sh|requirements.txt)'
-skip_files='(package.xml*|prepare_release.sh|codereview.settings|*.o|*.a|*.obj|*.lib|break-tests-android.sh|whitespace.txt|prepare_package_xml.sh|crypto_test_data.cc|*.pdf|*.svg|*.docx|cbmc-proof.txt|codecov*|litani*|*.toml|module.modulemap|cleanup.sh)'
-
-special_scripts='(awscrt.stub.php)'
-
-skip_directories='(tests|test|AWSCRTAndroidTestRunner|docker-images|codebuild|fuzz|third_party|docs|generated-src|aws-lc|aws-crt-sys|ecdsa-fuzz-corpus|bin|examples|compliance|verification)'
-
-process_file() {
-    if (( $# == 0 ))
-    then
-      echo "ERROR: filename not passed"
-      exit 1
-    fi
-    if [[ $1 = $~skip_files ]]
-    then
-      # This file is not part of the release bundle
-      return 0
-    fi
-
-    echo -n '<file name="'"$1"'" role="'
-    # Special cases
-    case ${a} in
-    $~special_scripts)
-      echo -n 'script'
-    ;;
-    $~special_docs)
-      echo -n 'doc'
-    ;;
-    $~special_tests)
-      echo -n 'test'
-    ;;
-    $~special_src)
-      echo -n 'src'
-    ;;
-    *)
-      # Extension based cases
-      case ${a:t:e} in
-      $~source_ext)
-        echo -n 'src'
-      ;;
-      $~doc_ext)
-         echo -n 'doc'
-      ;;
-      php)
-         echo -n 'script'
-      ;;
-      *)
-         echo "${a:t:e} - ${a} - FAIL TO RECOGNIZE"
-         exit 1
-      esac
-    esac
-    echo '"/>'
-    return 0
-}
-
-
-process_dir() {
-  if (( $# == 0 ))
-  then
-    echo "WARNING: dirname not passed"
-    exit 1
-  fi
-  if [[ "${1}" = $~skip_directories ]]
-  then
-    return 0
-  fi
-  echo '<dir name="'"$1"'">'
-  cd "$1"
-  for a in *
-  do
-    if [[ -f ${a} ]]
-    then process_file "${a}"
-    else process_dir "${a}"
-    fi
-  done
-  # Special cases for compiler features placed in tests directories in and s2n
-  if [[ "${1}" = "s2n" && -d tests ]]
-  then
-      echo '<dir name="tests">'
-      echo '<dir name="features">'
-      cd tests/features
-      for a in *
-      do
-          process_file "${a}"
-      done
-      cd ../..
-      echo '</dir>'
-      echo '</dir>'
-  fi
-  echo '</dir>'
-  cd ..
-  return 0
-}
-
-echo '<dir name="/">'
-for a in *
-do
-  if [[ ${a} == 'tests' ]]
-  then
-    echo '<dir name="tests">'
-    for b in tests/*
-    do
-      echo '<file name="'$( basename "${b}" )'" role="test" />'
-    done
-    echo '</dir>'
-    continue
-  fi
-  # skip the src directory as it is not part of the native extension
-  if [[ ${a} == 'src' ]]
-  then
-    continue
-  fi
-
-  if [[ -f ${a} ]]
-  then process_file "${a}"
-  else process_dir "${a}"
-  fi
-done
-echo '</dir>'
-
-cat $TOOLS_DIR/package.xml-template_post

dev-scripts/prepare_pecl_package_xml.py

@@ -0,0 +1,147 @@
+#!/usr/bin/env python3
+import argparse
+import os
+import re
+from datetime import datetime
+
+parser = argparse.ArgumentParser(
+    description="Generate package.xml file for a release bundle.")
+parser.add_argument("--name", help="Name of the package", required=True)
+parser.add_argument(
+    "--user", help="Username of the package maintainer", required=True)
+parser.add_argument(
+    "--email", help="Email address of the package maintainer", default='[email protected]')
+parser.add_argument(
+    "--version", help="Version number of the package", required=True)
+parser.add_argument(
+    "--notes", help="Release notes for the package", default='New release')
+args = parser.parse_args()
+
+
+name = args.name
+user = args.user
+email = args.email
+version = args.version
+notes = args.notes
+
+
+tools_dir = os.path.dirname(os.path.abspath(__file__))
+work_dir = os.path.join(tools_dir, '..')
+
+os.chdir(work_dir)
+
+today = datetime.utcnow().strftime("%Y-%m-%d")
+
+source_ext = "(c|cc|h|cpp|hpp|m4|w32|ini|frag|cmake|inl|in|py|gnu|yaml|def|pl|S|s|errordata|go|lds|num|asm|mod|peg|mk|rs|toml|sh)"
+doc_ext = "(md|json|html|dot|graphml|png|gn|sha1|css|rst)"
+
+special_docs = "(LICENSE.*|NOTICE|changelog.txt|CHANGELOG|THIRD-PARTY|README.*|readme|METADATA|CONTRIBUTORS|UPDATING|doc.config|THIRD-PARTY-LICENSES.txt)"
+special_tests = "(ci-test.sh|format-check.sh|run_tests.*|sanitizer-blacklist.txt|run-clang-tidy.sh|benchmark-build-run.sh|break-tests.sh|generate-coverage.sh|test.xml)"
+special_src = "(gen_api.php|gen_stub.php|CMakeLists.txt|post.sh|postun.sh|Makefile.*|build-buildspec.sh|build-deps.sh|objects.txt|go.*|BUILD.*|DEPS|install_and_run.sh|codemod.sh|requirements.txt)"
+skip_files = "(package.xml.*|prepare_release.sh|codereview.settings|.*\\.o|.*\\.a|.*\\.obj|.*\\.lib|break-tests-android.sh|whitespace.txt|prepare_package_xml.sh|crypto_test_data.cc|.*\\.pdf|.*\\.svg|.*\\.docx|cbmc-proof.txt|codecov.*|litani.*|.*\\.toml|module\\.modulemap|cleanup.sh|^\..*)"
+
+
+special_scripts = "(awscrt.stub.php)"
+
+skip_directories = "(tests|test|AWSCRTAndroidTestRunner|docker-images|codebuild|fuzz|third_party|docs|generated-src|aws-lc|aws-crt-sys|ecdsa-fuzz-corpus|bin|examples|compliance|verification|^\..*)"
+
+output_file_name = os.path.join(work_dir, 'package.xml')
+
+
+def process_file(file_name, f):
+    if re.match(skip_files, file_name):
+        # This file is not part of the release bundle
+        return
+    f.write(f'<file name="{file_name}" role="')
+    # Special cases
+    if re.match(special_scripts, file_name):
+        f.write('script')
+    elif re.match(special_docs, file_name):
+        f.write('doc')
+    elif re.match(special_tests, file_name):
+        f.write('test')
+    elif re.match(special_src, file_name):
+        f.write('src')
+    else:
+        # Extension based cases
+        ext = file_name.split('.')[-1]
+        if re.match(source_ext, ext):
+            f.write('src')
+        elif re.match(doc_ext, ext):
+            f.write('doc')
+        elif ext == 'php':
+            f.write('script')
+        else:
+            f.write(f'{ext} - {file_name} - FAIL TO RECOGNIZE')
+            print(f'{ext} - {file_name} - FAIL TO RECOGNIZE')
+            exit(1)
+
+    f.write('"/>\n')
+
+
+def process_dir(dir_name, f):
+    if re.match(skip_directories, dir_name):
+        return
+
+    f.write(f'<dir name="{dir_name}">\n')
+
+    if dir_name == 'tests' and os.path.isdir('tests'):
+        f.write('<dir name="tests">\n')
+        f.write('<dir name="features">\n')
+        os.chdir('tests/features')
+        for file_name in os.listdir():
+            process_file(file_name)
+        os.chdir('../..')
+        f.write('</dir>\n')
+        f.write('</dir>\n')
+
+    else:
+        os.chdir(dir_name)
+        for file_name in os.listdir():
+            if os.path.isfile(file_name):
+                process_file(file_name, f)
+            else:
+                process_dir(file_name, f)
+
+    f.write('</dir>\n')
+    os.chdir('..')
+
+
+# Main code
+with open(os.path.join(tools_dir, "package.xml-template_pre"), 'r') as f:
+    template_pre = f.read()
+
+template_pre = re.sub(r"{{{NAME}}}", name, template_pre)
+template_pre = re.sub(r"{{{USER}}}", user, template_pre)
+template_pre = re.sub(r"{{{EMAIL}}}", email, template_pre)
+template_pre = re.sub(r"{{{TODAY}}}", today, template_pre)
+template_pre = re.sub(r"{{{VERSION}}}", version, template_pre)
+template_pre = re.sub(r"{{{NOTES}}}", notes, template_pre)
+
+with open(output_file_name, "w") as f:
+    f.write(template_pre)
+    f.write('<dir name="/">\n')
+
+with open(output_file_name, "a") as f:
+    for file_name in os.listdir():
+        if file_name == 'tests':
+            f.write('<dir name="tests">\n')
+            for test_file in os.listdir('tests'):
+                f.write(
+                    f'<file name="{os.path.basename(test_file)}" role="test" />\n')
+            f.write('</dir>\n')
+            continue
+
+        if file_name == 'src':
+            # skip the src directory as it is not part of the native extension
+            continue
+
+        if os.path.isfile(file_name):
+            process_file(file_name, f)
+        else:
+            process_dir(file_name, f)
+
+with open(output_file_name, "a") as f:
+    f.write('</dir>\n')
+    with open(os.path.join(tools_dir, "package.xml-template_post"), 'r') as template:
+        f.write(template.read())

dev-scripts/prepare_pecl_release.py

@@ -0,0 +1,47 @@
+import argparse
+import os
+import subprocess
+
+
+parser = argparse.ArgumentParser(description='PECL Package generator')
+parser.add_argument("--name", help="Name of the package", required=True)
+parser.add_argument(
+    "--user", help="Username of the package maintainer", required=True)
+parser.add_argument(
+    "--email", help="Email address of the package maintainer", default='[email protected]')
+parser.add_argument(
+    "--version", help="Version number of the package", required=True)
+parser.add_argument(
+    "--notes", help="Release notes for the package", default='New release')
+args = parser.parse_args()
+
+PACKAGE = 'awscrt'
+NAME = args.name
+USER = args.user
+EMAIL = args.email
+VERSION = args.version
+NOTES = args.notes
+
+TOOLS_DIR = os.path.dirname(os.path.abspath(__file__))
+WORK_DIR = os.path.join(TOOLS_DIR, '..')
+
+subprocess.run(['python3', f'{TOOLS_DIR}/cleanup_build.py'], check=True)
+
+os.chdir(WORK_DIR)
+
+subprocess.run(['git', 'submodule', 'update',
+               '--init', '--recursive'], check=True)
+
+subprocess.run(['python3', f'{TOOLS_DIR}/prepare_pecl_package_xml.py', '--name', NAME, '--user', USER,
+                '--email', EMAIL, '--version', VERSION, '--notes', NOTES], check=True)
+try:
+    with open('package.xml', 'r') as f:
+        package_xml = f.read()
+    subprocess.run(['tidy', '-xml', '-m', '-i', 'package.xml'], check=True)
+    subprocess.run(['pear', 'package-validate'], check=True)
+    subprocess.run(['pear', 'package'], check=True)
+except subprocess.CalledProcessError as e:
+    print(f'ERROR PROCESSING review package.xml: {e}')
+    exit(1)
+
+print(f'Size of {PACKAGE}-{VERSION}.tgz: {os.path.getsize(f"{PACKAGE}-{VERSION}.tgz") / 1024 / 1024:.2f} MB')

dev-scripts/prepare_release.sh

@@ -1,41 +0,0 @@
-#!/bin/zsh
-zparseopts -A opts -name: -user: -email: -version: -notes:
-if [[ $# -lt 10 ]]; then
-  echo "Usage ${0} --name NAME --user USER --email EMAIL --version VERSION --notes NOTES"
-  exit 1
-fi
-PACKAGE='awscrt'
-NAME="${opts[--name]}"
-USER="${opts[--user]}"
-EMAIL="${opts[--email]}"
-VERSION="${opts[--version]}"
-NOTES="${opts[--notes]}"
-
-
-BASENAME=${0}
-TOOLS_DIR=$(dirname ${BASENAME})
-WORK_DIR="${TOOLS_DIR}/.."
-
-cd "${WORK_DIR}"
-
-git submodule update --init --recursive
-
-$TOOLS_DIR/cleanup.sh
-$TOOLS_DIR/prepare_package_xml.sh --name "${NAME}" --user "${USER}" --email "${EMAIL}" --version "${VERSION}" --notes "${NOTES}" >package.xml
-if [[ $? -ne 0 ]]; then
-  echo "ERROR PROCESSING review package.xml"
-  exit 1
-fi
-tidy -xml -m -i package.xml
-pear package-validate
-if [[ $? -ne 0 ]]; then
-  echo "ERROR VALIDATING review package.xml"
-  exit 1
-fi
-pear package
-if [[ $? -ne 0 ]]; then
-  echo "ERROR PROCESSING review package.xml"
-  exit 1
-fi
-
-echo "Size of ${PACKAGE}-${VERSION}.tgz: " $(du -h "${PACKAGE}-${VERSION}.tgz")

dev-scripts/run_tests.bat

@@ -1,9 +1,6 @@
 @echo on
 setlocal
 
-REM Run command to get composer_dir
-set "composer_dir="
-for /f "usebackq delims=" %%i in (`where composer.phar`) do set "composer_dir=%%i"
 
 set "PHP_BINARY=php"
 
@@ -11,18 +8,25 @@ if not "%~1"=="" (
   set "PHP_BINARY=%~1"
 )
 
+set "script_dir=%~dp0"
+set "work_dir=%script_dir%/.."
+cd %work_dir%
+
+set vendor_path=vendor
+
+if not exist "%vendor_path%" (
+  REM Run command to get composer_dir
+  set "composer_dir="
+  for /f "usebackq delims=" %%i in (`where composer.phar`) do set "composer_dir=%%i"
+
   REM Check if composer_dir was found
   if "%composer_dir%"=="" (
     echo No composer found.
     exit /b 1
   )
-
+)
-set "script_dir=%~dp0"
-set "work_dir=%script_dir%/.."
-cd %work_dir%
 
 call %PHP_BINARY% -c php-win.ini %composer_dir% update
 call %PHP_BINARY% -c php-win.ini vendor/bin/phpunit tests --debug
 
-
 endlocal

dev-scripts/test.py

@@ -0,0 +1,9 @@
+import os
+
+# get the absolute path of the script file
+script_path = os.path.abspath(__file__)
+
+# get the directory of the script file
+script_dir = os.path.dirname(script_path)
+
+print("Directory of the script file:", script_dir)