diff --git a/tools/CI/ConstructReleaseDoc.py b/tools/CI/ConstructReleaseDoc.py deleted file mode 100644 index 3d1d0db9859..00000000000 --- a/tools/CI/ConstructReleaseDoc.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/python - -import argparse -import shutil -import subprocess -import re -import subprocess -import os -import zipfile -import io -import json - -def Main(): - - parser = argparse.ArgumentParser(description="Creates a release doc based on a list of changes.") - parser.add_argument("--changesList", action="store") - - args = vars( parser.parse_args() ) - - changes = args["changesList"] - - changeDoc = {} - - changeList = changes.split() - releases = [] - release = {} - features = [] - - for change in changeList: - feature = {} - feature["service-name"] = change.replace("aws-cpp-sdk-", "") - features.append(feature) - - release["features"] = features - releases.append(release) - changeDoc["releases"] = releases - - print(json.dumps(changeDoc)) - -Main() diff --git a/tools/CI/DetectDirectoryChanges b/tools/CI/DetectDirectoryChanges deleted file mode 100644 index 9459ae8046f..00000000000 --- a/tools/CI/DetectDirectoryChanges +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -FILES_CHANGED_STR=`git diff --name-only $1@{1} $1` -FILES_CHANGED=${FILES_CHANGED_STR} - -declare -A DIRS_SET - -for FILE in $FILES_CHANGED ; do -DIR=`echo $FILE | cut -d "/" -f1` - -if test "${DIRS_SET[${DIR}]+isset}" - then - continue - else - echo $DIR -fi - DIRS_SET[${DIR}]="" -done - - diff --git a/tools/CI/DetermineBuilds b/tools/CI/DetermineBuilds deleted file mode 100644 index 5de153ef74b..00000000000 --- a/tools/CI/DetermineBuilds +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -CURDIR="$(dirname "$(readlink -f "$0")")" -CHANGED_DIRS=`$CURDIR/DetectDirectoryChanges $1` - -case $CHANGED_DIRS in - *"aws-cpp-sdk-core"*) - ;& - *"CMakeLists.txt"*) - ;& - *"cmake"*) - ;& - *"code-generation"*) - echo "-DBUILD_ONLY=\"\"" - exit 0 - ;; -*) -esac - -BUILD_ONLY_OUT="-DBUILD_ONLY=\"${CHANGED_DIRS//$'\n'/';'}\"" -echo ${BUILD_ONLY_OUT//$'aws-cpp-sdk-'/''} diff --git a/tools/CI/ExtractBuildArgs.py b/tools/CI/ExtractBuildArgs.py deleted file mode 100755 index bc4eb5a169a..00000000000 --- a/tools/CI/ExtractBuildArgs.py +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/python -import sys -import json - -if len(sys.argv) != 2: - print >> sys.stderr, " Usage: python ExtractBuildArgs.py " - exit (-1) - -try: - data = json.load(open('BuildSpec.json')) - if sys.argv[1] == "cmakeFlags" and data["cmakeFlags"] != "": - print(data["cmakeFlags"]) - elif sys.argv[1] == "branch" and data["branch"] != "": - print(data["branch"]) -except: - print >> sys.stderr, "No related args found in BuildSpec.json" - exit(-1) diff --git a/tools/CI/PackSourceBySpec.sh b/tools/CI/PackSourceBySpec.sh deleted file mode 100755 index 9123b97aeef..00000000000 --- a/tools/CI/PackSourceBySpec.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -branch =$(python aws-sdk-cpp/CI/ExtractBuildArgs.py branch) -git clone git@github.com:awslabs/aws-sdk-cpp-staging.git aws-sdk-cpp -cd aws-sdk-cpp -git reset --hard HEAD -git checkout master -git pull -git checkout $branch diff --git a/tools/CI/check_not_a_release b/tools/CI/check_not_a_release deleted file mode 100644 index d693903052b..00000000000 --- a/tools/CI/check_not_a_release +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -rm -f ./not_a_release -aws s3 cp --quiet s3://aws-sdk-cpp-pipeline-sdks-team/not_a_release ./not_a_release - -if [ -f ./not_a_release ]; then - aws s3 rm s3://aws-sdk-cpp-pipeline-sdks-team/not_a_release - exit 1 -fi - -exit 0 - diff --git a/tools/CI/dev-pipeline/build_custom_client_linux.yml b/tools/CI/dev-pipeline/build_custom_client_linux.yml deleted file mode 100644 index d1d5c61cb24..00000000000 --- a/tools/CI/dev-pipeline/build_custom_client_linux.yml +++ /dev/null @@ -1,49 +0,0 @@ -version: 0.2 -phases: - build: - commands: - - export SDK_ROOT=$CODEBUILD_SRC_DIR/aws-sdk-cpp - - cd $SDK_ROOT - # Testing the first approach to build custom client as a separate package, which means you have to build and install aws-sdk-cpp first. - # Generate custom client source code under custom-service/ with API description file located at code-generation/api-description/custom-service. - - python scripts/generate_sdks.py --pathToApiDefinitions=code-generation/api-descriptions/custom-service --outputLocation custom-service --serviceName custom-service --apiVersion 2017-11-03 --namespace Custom --prepareTool --standalone - - # Build and install aws-cpp-sdk-core - - mkdir -p $SDK_ROOT/build/AWSSDK - - mkdir -p $SDK_ROOT/install - - cd $SDK_ROOT/build/AWSSDK - - cmake $SDK_ROOT -DBUILD_ONLY="core" -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX="$SDK_ROOT/install" -DBUILD_SHARED_LIBS=ON - - make -j 8 - - make install - - # Build custom-service - - mkdir -p $SDK_ROOT/build/custom-service - - cd $SDK_ROOT/build/custom-service - - cmake $SDK_ROOT/custom-service/aws-cpp-sdk-custom-service -DCMAKE_BUILD_TYPE=Debug -DCMAKE_PREFIX_PATH="$SDK_ROOT/install" -DAWSSDK_ROOT_DIR="$SDK_ROOT/install" -DBUILD_SHARED_LIBS=ON - - make -j 8 - - # Build and run custom-service integration tests - - mkdir -p $SDK_ROOT/build/custom-service-integration-tests - - cd $SDK_ROOT/build/custom-service-integration-tests - - cmake $SDK_ROOT/aws-cpp-sdk-custom-service-integration-tests -DCMAKE_BUILD_TYPE=Debug -DCMAKE_PREFIX_PATH="$SDK_ROOT/install;$SDK_ROOT/build/custom-service" -DAWSSDK_ROOT_DIR="$SDK_ROOT/install" -DBUILD_SHARED_LIBS=ON -DSTANDALONE=ON - - make -j 8 - - $SDK_ROOT/build/custom-service-integration-tests/aws-cpp-sdk-custom-service-integration-tests - - # Testing the second approach to build custom client along with AWS C++ SDK, which means we will build everything altogether at the same time. - # Copy the c2j model to code-generation/api-descriptions - - cp $SDK_ROOT/code-generation/api-descriptions/custom-service/custom-service-2017-11-03.normal.json $SDK_ROOT/code-generation/api-descriptions/petstore-2017-11-03.normal.json - - # Build and install aws-cpp-sdk-core and aws-cpp-sdk-petstore - - mkdir -p $SDK_ROOT/build_all - - mkdir -p $SDK_ROOT/install_all - - cd $SDK_ROOT/build_all - - cmake $SDK_ROOT -DBUILD_ONLY=core -DADD_CUSTOM_CLIENTS="serviceName=petstore, version=2017-11-03" -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX=$SDK_ROOT/install_all -DBUILD_SHARED_LIBS=ON - - make -j 8 - - make install - - # Build and run petstore integration tests - - mkdir -p $SDK_ROOT/build_tests - - cd $SDK_ROOT/build_tests - - cmake $SDK_ROOT/aws-cpp-sdk-custom-service-integration-tests -DCMAKE_BUILD_TYPE=Debug -DCMAKE_PREFIX_PATH="$SDK_ROOT/install_all" -DAWSSDK_ROOT_DIR="$SDK_ROOT/install_all" -DBUILD_SHARED_LIBS=ON -DSTANDALONE=OFF - - make -j 8 - - $SDK_ROOT/build_tests/aws-cpp-sdk-custom-service-integration-tests \ No newline at end of file diff --git a/tools/CI/dev-pipeline/build_custom_client_windows.yml b/tools/CI/dev-pipeline/build_custom_client_windows.yml deleted file mode 100755 index 2931a13107c..00000000000 --- a/tools/CI/dev-pipeline/build_custom_client_windows.yml +++ /dev/null @@ -1,50 +0,0 @@ -version: 0.2 -phases: - build: - commands: - - $SDK_ROOT="$Env:CODEBUILD_SRC_DIR/aws-sdk-cpp" - - cd $SDK_ROOT - # Testing the first approach to build custom client as a separate package, which means you have to build and install aws-sdk-cpp first. - # Generate custom client source code under custom-service/ with API description file located at code-generation/api-description/custom-service. - - python scripts/generate_sdks.py --pathToApiDefinitions=code-generation/api-descriptions/custom-service --outputLocation custom-service --serviceName custom-service --apiVersion 2017-11-03 --namespace Custom --prepareTool --standalone - - # Build and install aws-cpp-sdk-core - - mkdir -p $SDK_ROOT/build/AWSSDK - - mkdir -p $SDK_ROOT/install - - cd $SDK_ROOT/build/AWSSDK - - cmake $SDK_ROOT -DBUILD_ONLY="core" -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX="$SDK_ROOT/install" -DBUILD_SHARED_LIBS=ON - - MSBuild.exe ALL_BUILD.vcxproj -p:Configuration=Debug -m - - MSBuild.exe INSTALL.vcxproj -p:Configuration=Debug - - # Build custom-service - - mkdir -p $SDK_ROOT/build/custom-service - - cd $SDK_ROOT/build/custom-service - - cmake $SDK_ROOT/custom-service/aws-cpp-sdk-custom-service -DCMAKE_PREFIX_PATH="$SDK_ROOT/install" -DAWSSDK_ROOT_DIR="$SDK_ROOT/install" -DCMAKE_INSTALL_PREFIX="$SDK_ROOT/install" -DBUILD_SHARED_LIBS=ON -DUSE_WINDOWS_DLL_SEMANTICS=ON - - MSBuild.exe ALL_BUILD.vcxproj -p:Configuration=Debug -m - - MSBuild.exe INSTALL.vcxproj -p:Configuration=Debug - - # Build and run custom-service integration tests - - mkdir -p $SDK_ROOT/build/custom-service-integration-tests - - cd $SDK_ROOT/build/custom-service-integration-tests - - cmake $SDK_ROOT/aws-cpp-sdk-custom-service-integration-tests -DCMAKE_PREFIX_PATH="$SDK_ROOT/install" -DAWSSDK_ROOT_DIR="$SDK_ROOT/install" -DBUILD_SHARED_LIBS=ON -DSTANDALONE=ON - - MSBuild.exe ALL_BUILD.vcxproj -p:Configuration=Debug -m - - ./Debug/aws-cpp-sdk-custom-service-integration-tests - - # Testing the second approach to build custom client along with AWS C++ SDK, which means we will build everything altogether at the same time. - # Copy the c2j model to code-generation/api-descriptions - - cp $SDK_ROOT/code-generation/api-descriptions/custom-service/custom-service-2017-11-03.normal.json $SDK_ROOT/code-generation/api-descriptions/petstore-2017-11-03.normal.json - - # Build and install aws-cpp-sdk-core and aws-cpp-sdk-petstore - - mkdir -p $SDK_ROOT/build_all - - mkdir -p $SDK_ROOT/install_all - - cd $SDK_ROOT/build_all - - cmake $SDK_ROOT -DBUILD_ONLY=core -DADD_CUSTOM_CLIENTS="serviceName=petstore, version=2017-11-03" -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX="$SDK_ROOT/install_all" -DBUILD_SHARED_LIBS=ON - - MSBuild.exe ALL_BUILD.vcxproj -p:Configuration=Debug -m - - MSBuild.exe INSTALL.vcxproj -p:Configuration=Debug - - # Build and run petstore integration tests - - mkdir -p $SDK_ROOT/build_tests - - cd $SDK_ROOT/build_tests - - cmake $SDK_ROOT/aws-cpp-sdk-custom-service-integration-tests -DCMAKE_PREFIX_PATH="$SDK_ROOT/install_all" -DAWSSDK_ROOT_DIR="$SDK_ROOT/install_all" -DBUILD_SHARED_LIBS=ON -DSTANDALONE=OFF - - MSBuild.exe ALL_BUILD.vcxproj -p:Configuration=Debug -m - - ./Debug/aws-cpp-sdk-custom-service-integration-tests.exe \ No newline at end of file diff --git a/tools/CI/handle_release_notification.py b/tools/CI/handle_release_notification.py deleted file mode 100644 index a8dbc5705ee..00000000000 --- a/tools/CI/handle_release_notification.py +++ /dev/null @@ -1,57 +0,0 @@ - -from __future__ import print_function - -import json -import zipfile -import boto3 -from botocore.exceptions import ClientError - -print('Loading function') -bucket_name = 'aws-sdk-cpp-pipeline-sdks-team' -key = 'pending-releases.zip' -temp_archive_file = '/tmp/pending_releases.zip' -artifact = 'pending_releases' -temp_artifact_file = '/tmp/pending_releases' -s3 = boto3.client('s3') - -def lambda_handler(event, context): - message = event['Records'][0]['Sns']['Message'] - print("From SNS: " + message) - - releasesDoc = {} - releasesDoc['releases'] = [] - - pendingReleases = None - - try: - pendingReleases = s3.get_object(Bucket=bucket_name, Key=key) - body_stream_to_file(pendingReleases["Body"].read()) - releasesDoc = read_zipped_release_doc() - except ClientError as e: - print("Couldn't pull doc, assuming it is empty. exception " + e.message) - - releasesDoc['releases'].append(json.loads(message)["release"]) - write_zipped_release_doc(releasesDoc) - with open(temp_archive_file) as archive: - s3.put_object(Bucket=bucket_name, Key=key, Body=archive.read()) - - return message - -def read_zipped_release_doc(): - archive = zipfile.ZipFile(temp_archive_file, 'r') - with archive.open(artifact) as artifactFile: - return json.loads(artifactFile.read()) - -def write_zipped_release_doc(doc): - releasesDocStr = json.dumps(doc) - print("New Release Doc: " + releasesDocStr) - with open(temp_artifact_file, "w") as artifactFile: - artifactFile.write(releasesDocStr) - - with zipfile.ZipFile(temp_archive_file, 'w') as archiveStream: - archiveStream.write(temp_artifact_file, artifact) - -def body_stream_to_file(body): - with open(temp_archive_file, 'w') as archiveFile: - archiveFile.write(body) - diff --git a/tools/CI/move_release_doc_to_models.py b/tools/CI/move_release_doc_to_models.py deleted file mode 100644 index fbbf27a8930..00000000000 --- a/tools/CI/move_release_doc_to_models.py +++ /dev/null @@ -1,89 +0,0 @@ - -from __future__ import print_function - -import json -import zipfile -import boto3 -import os -import re -import sys -import argparse -from botocore.exceptions import ClientError -import requests - -import requests.packages.urllib3 -requests.packages.urllib3.disable_warnings() - -temp_archive_file = 'models.zip' - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument('-r', '--releaseDoc') - parser.add_argument('-m', '--modelsDir') - args = parser.parse_args() - - releaseDocPath = args.releaseDoc - modelsDir = args.modelsDir - - print('Release Doc path {0}'.format(releaseDocPath)) - print('Models Directory {0}'.format(modelsDir)) - - releaseDoc = {} - - pendingReleases = None - - with open(releaseDocPath, "r") as releaseDocFileStream: - releaseDoc = json.loads(releaseDocFileStream.read()) - - if(len(releaseDoc) == 0 or len(releaseDoc["releases"]) == 0): - return - - for release in releaseDoc["releases"]: - for feature in release["features"]: - if feature["c2jModels"] != None: - response = requests.get(feature["c2jModels"]) - if response.status_code != 200: - print("Error downloading {0} artifacts skipping.", json.dumps(feature)) - continue - - body_stream_to_file(response.content) - copy_model_files(modelsDir) - cat_release_notes(feature["releaseNotes"], modelsDir) - - cat_pending_releases(release["id"], modelsDir) - - emptyReleaseDoc = "{ \"releases\": []}" - - with open(releaseDocPath, "w") as emptyReleaseFile: - emptyReleaseFile.write(emptyReleaseDoc) - -def copy_model_files(models_dir): - archive = zipfile.ZipFile(temp_archive_file, 'r') - archive.debug = 3 - for info in archive.infolist(): - print(info.filename) - if re.match(r'output/.*\.normal\.json', info.filename): - outputPath = os.path.join(models_dir, os.path.basename(info.filename)) - print("copying {0} to {1}".format(info.filename, outputPath)) - fileHandle = archive.open(info.filename, 'r') - fileOutput = fileHandle.read() - - with open(outputPath, 'wb') as destination: - destination.write(fileOutput) - - fileHandle.close() - -def body_stream_to_file(body): - with open(temp_archive_file, 'w') as archiveFile: - archiveFile.write(body) - -def cat_release_notes(releaseNotes, models_path): - with open(os.path.join(models_path, "release_notes"), "a") as releaseNotesFile: - releaseNotesFile.write(releaseNotes + "\n\n") - -def cat_pending_releases(release_guid, models_path): - with open(os.path.join(models_path, "pending_releases"), "a") as pendingReleasesFile: - pendingReleasesFile.write(release_guid + "\n") - -if __name__ == "__main__": - main() diff --git a/tools/CI/publish_release_notification b/tools/CI/publish_release_notification deleted file mode 100644 index 8420b2d335c..00000000000 --- a/tools/CI/publish_release_notification +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -IFS=$'\n' read -d '' -r -a releases < $1/pending_releases -for i in "${releases[@]}" -do - aws sqs send-message --debug --message-group-id "needlessField" --queue-url "$4" --message-body "{ \"releaseId\": \"$i\", \"language\": \"CPP\", \"releaseState\":\"$2\", \"statusMessage\":\"$3\" }" --region us-west-2 -done - diff --git a/tools/CI/reset_models b/tools/CI/reset_models deleted file mode 100644 index f3a850829a4..00000000000 --- a/tools/CI/reset_models +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -e - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -if [ -f ./models/pending_releases ] && [ -s ./models/pending_releases ]; -then - aws s3 cp s3://aws-sdk-cpp-pipeline-sdks-team/modelsSnapshot.zip modelsLatest.zip --region us-east-1 - unzip modelsLatest.zip -d modelsLatest - rm modelsLatest.zip - - grep -vf ./models/pending_releases ./modelsLatest/models/pending_releases | xargs | tee ./modelsLatest/models/pending_releases - grep -vf ./models/release_notes ./modelsLatest/models/release_notes | xargs | tee ./modelsLatest/models/release_notes - touch ./not_a_release - aws s3 cp not_a_release s3://aws-sdk-cpp-pipeline-sdks-team/not_a_release --region us-east-1 - rm -rf ./models - mkdir ./models - cp -r ./modelsLatest/models/* ./models - zip -r modelsSnapshot.zip ./models - aws s3 cp modelsSnapshot.zip s3://aws-sdk-cpp-pipeline-sdks-team/modelsSnapshot.zip --region us-east-1 -fi diff --git a/tools/CI/run_release_doc_models_move b/tools/CI/run_release_doc_models_move deleted file mode 100755 index 46c0f4b5e02..00000000000 --- a/tools/CI/run_release_doc_models_move +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -set -e -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -aws s3 cp s3://aws-sdk-cpp-pipeline-sdks-team/modelsSnapshot.zip models.zip --region us-east-1 -unzip models.zip -rm models.zip -cp $1 $1_cpy -python $DIR/move_release_doc_to_models.py --modelsDir="./models" --releaseDoc="$1" -rm models.zip -zip -r modelsSnapshot.zip ./models -aws s3 cp modelsSnapshot.zip s3://aws-sdk-cpp-pipeline-sdks-team/modelsSnapshot.zip --region us-east-1 -zip -r pending-releases.zip -j $1 -aws s3 cp pending-releases.zip s3://aws-sdk-cpp-pipeline-sdks-team/pending-releases.zip --region us-east-1 diff --git a/tools/CI/validate_install.sh b/tools/CI/validate_install.sh deleted file mode 100755 index 4a6df8e3cd9..00000000000 --- a/tools/CI/validate_install.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -# This script validates that every files in the install manifest generated by cmake is in an expected location -# We expect to use this script as a validation step in CI for Linux x86_64 platforms. -# This acts also as a driver to make thes list of locations more compact, expect reductions here in the future. - -set -e -MANIFEST=install_manifest.txt -TOTAL_FILES=$(wc -l $MANIFEST) -UNEXPECTED_FILES=$( cat $MANIFEST \ - | grep -v '^/usr/local/include/aws' \ - | grep -v '^/usr/local/include/s2n.h' \ - | grep -v '^/usr/local/lib\(64\)\?/cmake' \ - | grep -v '^/usr/local/lib\(64\)\?/libaws-' \ - | grep -v '^/usr/local/lib\(64\)\?/pkgconfig/aws-cpp-sdk-' \ - | grep -v '^/usr/local/lib\(64\)\?/aws-crt-cpp' \ - | grep -v '^/usr/local/lib\(64\)\?/aws-c' \ - | grep -v '^/usr/local/lib\(64\)\?/s2n' \ - | grep -v '^/usr/local/lib\(64\)\?/libs2n' \ - | grep -v '^/usr/local/lib\(64\)\?/libtesting-resources' \ - | grep -v '^/usr/local/lib\(64\)\?/pkgconfig/testing-resources.pc' ) \ - || [[ $? == 1 ]] # 1 is grep error code for empty result -if [[ ! -z $UNEXPECTED_FILES ]] -then - TOTAL_UNEXPECTED_FILES=$( echo $UNEXPECTED_FILES | wc -l) - echo $TOTAL_UNEXPECTED_FILES ' / ' $TOTAL_FILES ' are unexpected to be installed from manifest' - echo $UNEXPECTED_FILES - exit 1 -fi