-
Notifications
You must be signed in to change notification settings - Fork 2.2k
Expand file tree
/
Copy pathrelease-s3.sh
More file actions
executable file
·162 lines (137 loc) · 6.07 KB
/
Copy pathrelease-s3.sh
File metadata and controls
executable file
·162 lines (137 loc) · 6.07 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
#!/usr/bin/env bash
set -euo pipefail
# release-s3.sh
#
# SUMMARY
#
# Uploads archives and packages to S3
vdev_cmd="${VDEV:-cargo vdev}"
CHANNEL="${CHANNEL:-"$($vdev_cmd release channel)"}"
VERSION="${VECTOR_VERSION:-"$($vdev_cmd version)"}"
DATE="${DATE:-"$(date -u +%Y-%m-%d)"}"
VERIFY_TIMEOUT="${VERIFY_TIMEOUT:-"30"}" # seconds
VERIFY_RETRIES="${VERIFY_RETRIES:-"2"}"
export AWS_REGION=us-east-1
#
# Setup
#
td="$(mktemp -d)"
cp -av "target/artifacts/." "$td"
td_nightly="$(mktemp -d)"
cp -av "target/artifacts/." "$td_nightly"
for f in "$td_nightly"/*; do
a="$(echo "$f" | sed -r -e "s/$VERSION/nightly/")"
mv "$f" "$a"
done
ls "$td_nightly"
#
# A helper function for verifying a published artifact.
#
# Retries a content mismatch as well as a 404, since packages.timber.io is
# fronted by a CDN and an object we just overwrote via `aws s3 rm` + `cp` can
# serve stale bytes at the edge for a while.
verify_artifact() {
local URL="$1"
local FILENAME="$2"
local attempts=7
local delay=1
echo "Verifying $URL"
for ((attempt = 1; attempt <= attempts; attempt++)); do
if cmp <(wget -qO- --retry-on-http-error=404 --wait 10 --tries "$VERIFY_RETRIES" "$URL") "$FILENAME"; then
return 0
fi
if (( attempt < attempts )); then
echo "Attempt $attempt/$attempts did not match (likely stale CDN cache); retrying in ${delay}s"
sleep "$delay"
delay=$((delay * 2))
fi
done
echo "Verification of $URL failed after $attempts attempts"
return 1
}
#
# Upload
#
if [[ "$CHANNEL" == "nightly" ]]; then
# Add nightly files with the $DATE for posterity
echo "Uploading all artifacts to s3://packages.timber.io/vector/nightly/$DATE"
aws s3 cp "$td_nightly" "s3://packages.timber.io/vector/nightly/$DATE" --recursive --sse --acl public-read
echo "Uploaded archives"
# Add "latest" nightly files
echo "Uploading all artifacts to s3://packages.timber.io/vector/nightly/latest"
aws s3 rm --recursive "s3://packages.timber.io/vector/nightly/latest"
aws s3 cp "$td_nightly" "s3://packages.timber.io/vector/nightly/latest" --recursive --sse --acl public-read
echo "Uploaded archives"
# Verify that the files exist and can be downloaded
echo "Waiting for $VERIFY_TIMEOUT seconds before running the verifications"
sleep "$VERIFY_TIMEOUT"
verify_artifact \
"https://packages.timber.io/vector/nightly/$DATE/vector-nightly-x86_64-unknown-linux-musl.tar.gz" \
"$td_nightly/vector-nightly-x86_64-unknown-linux-musl.tar.gz"
verify_artifact \
"https://packages.timber.io/vector/nightly/latest/vector-nightly-x86_64-unknown-linux-musl.tar.gz" \
"$td_nightly/vector-nightly-x86_64-unknown-linux-musl.tar.gz"
verify_artifact \
"https://packages.timber.io/vector/nightly/latest/vector-nightly-x86_64-unknown-linux-gnu.tar.gz" \
"$td_nightly/vector-nightly-x86_64-unknown-linux-gnu.tar.gz"
elif [[ "$CHANNEL" == "release" ]]; then
VERSION_EXACT="$VERSION"
# shellcheck disable=SC2001
VERSION_MINOR_X="$(echo "$VERSION" | sed 's/\.[0-9]*$/.X/g')"
# shellcheck disable=SC2001
VERSION_MAJOR_X="$(echo "$VERSION" | sed 's/\.[0-9]*\.[0-9]*$/.X/g')"
for i in "$VERSION_EXACT" "$VERSION_MINOR_X" "$VERSION_MAJOR_X" "latest"; do
# Upload the specific version
echo "Uploading artifacts to s3://packages.timber.io/vector/$i/"
aws s3 cp "$td" "s3://packages.timber.io/vector/$i/" --recursive --sse --acl public-read
if [[ "$i" == "${VERSION_MAJOR_X}" || "$i" == "${VERSION_MINOR_X}" || "$i" == "latest" ]] ; then
# Delete anything that isn't the current version
echo "Deleting old artifacts from s3://packages.timber.io/vector/$i/"
aws s3 rm "s3://packages.timber.io/vector/$i/" --recursive --exclude "*$VERSION_EXACT*"
echo "Deleted old versioned artifacts"
fi
echo "Redirecting old artifact names in s3://packages.timber.io/vector/$i/"
for file in $(aws s3api list-objects-v2 --bucket packages.timber.io --prefix "vector/$i/" --query 'Contents[*].Key' --output text | tr "\t" "\n" | grep "\-$VERSION_EXACT"); do
file=$(basename "$file")
# vector-$version-amd64.deb -> vector-amd64.deb
echo -n "" | aws s3 cp - "s3://packages.timber.io/vector/$i/${file/-$VERSION_EXACT/}" --website-redirect "/vector/$i/$file" --acl public-read
done
echo "Redirected old artifact names"
done
echo "Add latest symlinks"
find "$td" -maxdepth 1 -type f -print0 | while read -r -d $'\0' file ; do
file=$(basename "$file")
# vector-$version-amd64.deb -> vector-latest-amd64.deb
echo -n "" | aws s3 cp - "s3://packages.timber.io/vector/latest/${file/$VERSION_EXACT/latest}" --website-redirect "/vector/latest/$file" --acl public-read
# vector-$version-amd64.deb -> vector-amd64.deb
echo -n "" | aws s3 cp - "s3://packages.timber.io/vector/latest/${file/$VERSION_EXACT-/}" --website-redirect "/vector/latest/$file" --acl public-read
done
echo "Added latest symlinks"
# Verify that the files exist and can be downloaded
echo "Waiting for $VERIFY_TIMEOUT seconds before running the verifications"
sleep "$VERIFY_TIMEOUT"
for i in "$VERSION_EXACT" "$VERSION_MINOR_X" "$VERSION_MAJOR_X" "latest"; do
verify_artifact \
"https://packages.timber.io/vector/$i/vector-$VERSION-x86_64-unknown-linux-musl.tar.gz" \
"$td/vector-$VERSION-x86_64-unknown-linux-musl.tar.gz"
done
verify_artifact \
"https://packages.timber.io/vector/latest/vector-latest-x86_64-unknown-linux-gnu.tar.gz" \
"$td/vector-$VERSION-x86_64-unknown-linux-gnu.tar.gz"
elif [[ "$CHANNEL" == "custom" ]]; then
# Add custom files
echo "Uploading all artifacts to s3://packages.timber.io/vector/custom"
aws s3 cp "$td" "s3://packages.timber.io/vector/custom/$VERSION" --recursive --sse --acl public-read
echo "Uploaded archives"
# Verify that the files exist and can be downloaded
echo "Waiting for $VERIFY_TIMEOUT seconds before running the verifications"
sleep "$VERIFY_TIMEOUT"
verify_artifact \
"https://packages.timber.io/vector/custom/$VERSION/vector-$VERSION-x86_64-unknown-linux-gnu.tar.gz" \
"$td/vector-$VERSION-x86_64-unknown-linux-gnu.tar.gz"
fi
#
# Cleanup
#
rm -rf "$td"
rm -rf "$td_nightly"