Line data Source code
1 : use aws_sdk_s3::primitives::ByteStream;
2 : use chrono::prelude::*;
3 : use futures::StreamExt;
4 : use std::{collections::HashMap, io::Read};
5 : use std::path::PathBuf;
6 :
7 : use crate::{
8 : badge::build_version_badge,
9 : git_info::GitInfo,
10 : release_metadata::{
11 : Architecture, Bundle, BundleTarget, BundleType, Platform, ReleaseMetadata, UpdateArtifact,
12 : UpdateManifest,
13 : },
14 : release_version::ReleaseVersion,
15 : remote_paths::{derive_remote_paths, electron_channel_manifest},
16 : };
17 :
18 : const LATEST_BADGE_PATH: &str = "channels/latest/badge.svg";
19 :
20 : /// A local file to upload into the release bucket
21 : #[derive(Debug)]
22 : pub struct FileUpload {
23 : /// The local path of the file
24 : pub source_path: PathBuf,
25 : /// The object key in the release bucket
26 : pub remote_path: String,
27 : }
28 :
29 : /// A release prepared for upload, with its artifacts, metadata and channel pointers
30 : #[derive(Default, Debug)]
31 : pub struct Release {
32 : /// The artifact uploads, keyed by remote path
33 : pub file_uploads: HashMap<String, FileUpload>,
34 : /// The metadata of the release
35 : pub release_metadata: ReleaseMetadata,
36 : /// The object key of the release metadata
37 : pub release_metadata_path: String,
38 : /// The update manifest of the release
39 : pub release_update_manifest: UpdateManifest,
40 : /// The object key of the release update manifest
41 : pub release_update_manifest_path: String,
42 : /// The channel metadata key that is pointed to this release
43 : pub channel_metadata_path: &'static str,
44 : /// The channel update manifest key that is pointed to this release
45 : pub channel_update_manifest_path: &'static str,
46 : /// The electron-builder channel manifests as `(object key, content)` pairs
47 : pub electron_channel_manifests: Vec<(String, Vec<u8>)>,
48 : /// The SVG version badge, only a release of the latest channel has one
49 : pub latest_badge: Option<Vec<u8>>,
50 : }
51 :
52 : /// The inputs for building a [`Release`]
53 : pub struct ReleaseArgs {
54 : /// The public base URL of the release bucket, e.g. `https://get.dashql.app`
55 : pub remote_base_url: String,
56 : /// The git information of the released commit
57 : pub git_repo: GitInfo,
58 : /// The version, id and channel of the release
59 : pub release_version: ReleaseVersion,
60 : /// The path of the arm64 macOS DMG
61 : pub macos_arm64_dmg_path: PathBuf,
62 : /// The path of the arm64 macOS ZIP used for auto updates
63 : pub macos_arm64_zip_path: PathBuf,
64 : /// The path of the blockmap of the arm64 macOS ZIP
65 : pub macos_arm64_zip_blockmap_path: PathBuf,
66 : /// The path of the electron-builder update manifest for arm64
67 : pub macos_arm64_update_manifest_path: PathBuf,
68 : /// The path of the x64 macOS DMG
69 : pub macos_x64_dmg_path: PathBuf,
70 : /// The path of the x64 macOS ZIP used for auto updates
71 : pub macos_x64_zip_path: PathBuf,
72 : /// The path of the blockmap of the x64 macOS ZIP
73 : pub macos_x64_zip_blockmap_path: PathBuf,
74 : /// The path of the electron-builder update manifest for x64
75 : pub macos_x64_update_manifest_path: PathBuf,
76 : }
77 :
78 172 : fn file_name(path: &PathBuf) -> anyhow::Result<String> {
79 172 : path.file_name().and_then(|value| value.to_str()).map(str::to_owned)
80 172 : .ok_or_else(|| anyhow::anyhow!("artifact path has no valid file name: {:?}", path))
81 172 : }
82 :
83 : impl Release {
84 18 : pub async fn build(args: ReleaseArgs) -> anyhow::Result<Self> {
85 18 : let remote_paths = derive_remote_paths(&args.release_version);
86 :
87 18 : let mut release = Release::default();
88 18 : let pub_date = Utc::now();
89 :
90 : // Prepare release metadata
91 18 : release.release_metadata_path = remote_paths.release_metadata.clone();
92 18 : release.release_metadata.release_id = args.release_version.id.to_string();
93 18 : release.release_metadata.version = args.release_version.version.clone();
94 18 : release.release_metadata.pub_date = pub_date.clone();
95 18 : release.release_metadata.update_manifest_url = format!(
96 : "{}/{}",
97 18 : &args.remote_base_url,
98 18 : remote_paths.release_update.clone()
99 : );
100 18 : release.release_metadata.git_commit_hash = args.git_repo.version.short_hash.clone();
101 18 : release.release_metadata.git_commit_url = format!(
102 : "https://github.com/ankoh/dashql/tree/{}",
103 18 : &args.git_repo.version.short_hash
104 : );
105 :
106 : // Prepare update manifest
107 18 : release.release_update_manifest_path = remote_paths.release_update.clone();
108 18 : release.release_update_manifest.version = args.release_version.version.clone();
109 18 : release.release_update_manifest.pub_date = pub_date;
110 18 : release.release_update_manifest.notes = "".to_string(); // XXX Get from commit info
111 18 : if remote_paths.channel == "latest" {
112 4 : release.latest_badge = Some(build_version_badge(&args.release_version.version.to_string()).into_bytes());
113 14 : }
114 :
115 : // Prepare channel paths
116 18 : release.channel_metadata_path = remote_paths.channel_metadata;
117 18 : release.channel_update_manifest_path = remote_paths.channel_update;
118 :
119 35 : for (arch, architecture, dmg, zip, blockmap, manifest) in [
120 18 : ("arm64", Architecture::Aarch64, &args.macos_arm64_dmg_path, &args.macos_arm64_zip_path, &args.macos_arm64_zip_blockmap_path, &args.macos_arm64_update_manifest_path),
121 18 : ("x64", Architecture::X86_64, &args.macos_x64_dmg_path, &args.macos_x64_zip_path, &args.macos_x64_zip_blockmap_path, &args.macos_x64_update_manifest_path),
122 18 : ] {
123 105 : for artifact in [dmg, zip, blockmap] {
124 105 : if !artifact.is_file() {
125 1 : return Err(anyhow::anyhow!("missing Electron artifact: {:?}", artifact));
126 104 : }
127 104 : let remote_path = format!("{}/macos/{}/{}", remote_paths.release_directory, arch, file_name(artifact)?);
128 104 : release.file_uploads.insert(remote_path.clone(), FileUpload {source_path: artifact.clone(), remote_path});
129 : }
130 :
131 34 : let dmg_name = file_name(dmg)?;
132 34 : let remote_path = format!("{}/macos/{}/{}", remote_paths.release_directory, arch, dmg_name);
133 34 : let remote_url = format!("{}/{}", &args.remote_base_url, remote_path);
134 34 : let bundle = Bundle {
135 34 : url: remote_url.clone(),
136 34 : signature: None,
137 34 : name: dmg_name,
138 34 : bundle_type: BundleType::Dmg,
139 34 : targets: vec![BundleTarget {platform: Platform::Darwin, arch: architecture}],
140 34 : };
141 34 : release.release_metadata.bundles.push(bundle);
142 :
143 34 : let zip_name = file_name(zip)?;
144 34 : let zip_remote_path = format!("{}/macos/{}/{}", remote_paths.release_directory, arch, zip_name);
145 34 : let update_artifact = UpdateArtifact {
146 34 : url: format!("{}/{}", &args.remote_base_url, zip_remote_path),
147 34 : signature: String::new(),
148 34 : };
149 34 : release.release_update_manifest.platforms.insert(BundleTarget {platform: Platform::Darwin, arch: architecture}, update_artifact);
150 :
151 34 : let manifest_text = std::fs::read_to_string(manifest)?;
152 33 : let artifact_base = format!("{}/{}/macos/{}/", args.remote_base_url, remote_paths.release_directory, arch);
153 33 : let manifest_text = manifest_text
154 33 : .replace(&format!("url: {}", zip_name), &format!("url: {}{}", artifact_base, zip_name))
155 33 : .replace(&format!("path: {}", zip_name), &format!("path: {}{}", artifact_base, zip_name));
156 33 : release.electron_channel_manifests.push((electron_channel_manifest(remote_paths.channel, arch), manifest_text.into_bytes()));
157 : }
158 :
159 16 : log::info!("{:?}", &release);
160 16 : Ok(release)
161 18 : }
162 :
163 10 : pub async fn publish(&self, client: &aws_sdk_s3::Client) -> anyhow::Result<()> {
164 : // Upload files one by one first to work around R2 upload issue
165 57 : for (_, file_upload) in self.file_uploads.iter() {
166 57 : let path = file_upload.remote_path.clone();
167 57 : let client = client.clone();
168 57 : log::info!("upload started, path={}", &path);
169 :
170 57 : let result = multipart_upload(&client, &file_upload.source_path, &path).await;
171 57 : match result {
172 : Ok(_) => {
173 56 : log::info!("multipart upload finished, path={}", &path);
174 : }
175 1 : Err(e) => {
176 1 : log::error!("multipart upload failed, path={}, error={}", &path, &e);
177 1 : return Err(e);
178 : }
179 : }
180 : }
181 :
182 : // Serialize release metadata and update manifest and abort after serialization errors
183 9 : let release_metadata = serde_json::to_string_pretty(&self.release_metadata)?
184 9 : .as_bytes()
185 9 : .to_vec();
186 9 : let update_manifest = serde_json::to_string_pretty(&self.release_update_manifest)?
187 9 : .as_bytes()
188 9 : .to_vec();
189 :
190 : // Collect json file uploads
191 9 : let mut pending_uploads = vec![
192 9 : (self.release_metadata_path.clone(), &release_metadata),
193 9 : (self.release_update_manifest_path.clone(), &update_manifest),
194 : ];
195 :
196 : // Spawn json uploads for release files
197 9 : let mut upload_futures = futures::stream::FuturesUnordered::new();
198 18 : for (path, metadata) in pending_uploads.drain(..) {
199 18 : let path = path.clone();
200 18 : let bytes = ByteStream::from(metadata.to_vec());
201 18 : let client = client.clone();
202 18 : let content_type = if path.ends_with(".yml") {"application/yaml"} else {"application/json"};
203 18 : log::info!("upload started, path={}", &path);
204 18 : upload_futures.push(tokio::spawn(async move {
205 18 : client
206 18 : .put_object()
207 18 : .bucket("dashql-get")
208 18 : .key(&path)
209 18 : .body(bytes)
210 18 : .content_type(content_type)
211 18 : // Versioned release files are immutable, cache them aggressively.
212 18 : .cache_control("public, max-age=31536000, immutable")
213 18 : .send()
214 18 : .await
215 18 : .map_err(|e| (path.clone(), e))
216 18 : .map(|_| path.clone())
217 18 : }));
218 : }
219 :
220 : // Join all uploads
221 9 : let mut upload_error: Option<anyhow::Error> = None;
222 9 : let mut channel_upload_error: Option<anyhow::Error> = None;
223 27 : while let Some(next) = upload_futures.next().await {
224 18 : match next {
225 17 : Ok(Ok(path)) => {
226 17 : log::info!("upload finished, path={}", &path);
227 : }
228 1 : Ok(Err((path, e))) => {
229 1 : log::error!("upload failed, path={}, error={}", &path, &e);
230 1 : upload_error = Some(e.into());
231 : }
232 0 : Err(e) => {
233 0 : log::error!("failed to join upload task, error={}", &e);
234 0 : upload_error = Some(anyhow::format_err!(
235 0 : "failed to join upload task, error={}",
236 0 : &e
237 0 : ));
238 : }
239 : }
240 : }
241 : // Don't update the top-level release metadata if any of the release uploads failed
242 9 : if let Some(e) = upload_error {
243 1 : return Err(e);
244 8 : }
245 :
246 : // Update mutable channel pointers only after every immutable artifact is available.
247 8 : pending_uploads.push((self.channel_metadata_path.to_string(), &release_metadata));
248 8 : pending_uploads.push((self.channel_update_manifest_path.to_string(), &update_manifest));
249 16 : for (path, manifest) in &self.electron_channel_manifests {
250 16 : pending_uploads.push((path.clone(), manifest));
251 16 : }
252 8 : if let Some(badge) = &self.latest_badge {
253 3 : pending_uploads.push((LATEST_BADGE_PATH.to_string(), badge));
254 5 : }
255 35 : for (path, metadata) in pending_uploads.drain(..) {
256 35 : let path = path.clone();
257 35 : let bytes = ByteStream::from(metadata.to_vec());
258 35 : let client = client.clone();
259 35 : log::info!("upload started, path={}", &path);
260 35 : upload_futures.push(tokio::spawn(async move {
261 35 : client
262 35 : .put_object()
263 35 : .bucket("dashql-get")
264 35 : .key(&path)
265 35 : .body(bytes)
266 35 : .content_type(content_type(&path))
267 35 : // Channel pointers (stable.json/latest.json + update manifests) are mutable.
268 35 : // Force revalidation on every load so clients never serve a stale channel
269 35 : // manifest from heuristic browser caching (revalidates cheaply via ETag).
270 35 : .cache_control("no-cache")
271 35 : .send()
272 35 : .await
273 35 : .map_err(|e| (path.clone(), e))
274 35 : .map(|_| path.clone())
275 35 : }));
276 : }
277 :
278 : // Join all uploads
279 43 : while let Some(next) = upload_futures.next().await {
280 35 : match next {
281 34 : Ok(Ok(path)) => {
282 34 : log::info!("upload finished, path={}", &path);
283 : }
284 1 : Ok(Err((path, e))) => {
285 1 : log::error!("upload failed, path={}, error={}", &path, &e);
286 1 : channel_upload_error = Some(e.into());
287 : }
288 0 : Err(e) => {
289 0 : log::error!("failed to join upload task, error={}", &e);
290 0 : channel_upload_error = Some(e.into());
291 : }
292 : }
293 : }
294 8 : if let Some(error) = channel_upload_error {
295 1 : return Err(error);
296 7 : }
297 7 : Ok(())
298 10 : }
299 : }
300 :
301 38 : fn content_type(path: &str) -> &'static str {
302 38 : if path.ends_with(".svg") {
303 4 : "image/svg+xml"
304 34 : } else if path.ends_with(".yml") {
305 17 : "application/yaml"
306 : } else {
307 17 : "application/json"
308 : }
309 38 : }
310 :
311 : #[cfg(test)]
312 : mod tests {
313 : use super::*;
314 : use crate::git_info::GitVersion;
315 : use crate::release_version::{build_release_version, ReleaseChannel};
316 : use crate::test_s3::{FakeS3, TempDir};
317 : use semver::Version;
318 :
319 : #[test]
320 1 : fn detects_upload_content_type() {
321 1 : assert_eq!(content_type("channels/latest/badge.svg"), "image/svg+xml");
322 1 : assert_eq!(content_type("latest-mac.yml"), "application/yaml");
323 1 : assert_eq!(content_type("latest.json"), "application/json");
324 1 : }
325 :
326 : const IMMUTABLE: &str = "public, max-age=31536000, immutable";
327 : const ARM64_DMG: &str = "DashQL-arm64.dmg";
328 : const ARM64_ZIP: &str = "DashQL-arm64.zip";
329 :
330 : /// The artifacts of a release in a directory
331 13 : fn release_args(dir: &TempDir, iteration: u32) -> ReleaseArgs {
332 26 : let manifest = |zip: &str| format!("version: 1.2.3\nfiles:\n - url: {zip}\n sha512: abc\npath: {zip}\nsha512: abc\n");
333 13 : let git_version = GitVersion {
334 13 : description: format!("v1.2.3-{iteration}-gabc1234"),
335 13 : major: 1,
336 13 : minor: 2,
337 13 : patch: 3,
338 13 : iteration,
339 13 : short_hash: "abc1234".to_string(),
340 13 : };
341 13 : let channel = if iteration == 0 { ReleaseChannel::Stable } else { ReleaseChannel::Latest };
342 13 : ReleaseArgs {
343 13 : remote_base_url: "https://get.example".to_string(),
344 13 : release_version: build_release_version(git_version.as_semver(), channel),
345 13 : git_repo: GitInfo { version: git_version },
346 13 : macos_arm64_dmg_path: dir.write(ARM64_DMG, b"arm64 dmg"),
347 13 : macos_arm64_zip_path: dir.write(ARM64_ZIP, b"arm64 zip"),
348 13 : macos_arm64_zip_blockmap_path: dir.write("DashQL-arm64.zip.blockmap", b"arm64 blockmap"),
349 13 : macos_arm64_update_manifest_path: dir.write("arm64.yml", manifest(ARM64_ZIP).as_bytes()),
350 13 : macos_x64_dmg_path: dir.write("DashQL-x64.dmg", b"x64 dmg"),
351 13 : macos_x64_zip_path: dir.write("DashQL-x64.zip", b"x64 zip"),
352 13 : macos_x64_zip_blockmap_path: dir.write("DashQL-x64.zip.blockmap", b"x64 blockmap"),
353 13 : macos_x64_update_manifest_path: dir.write("x64.yml", manifest("DashQL-x64.zip").as_bytes()),
354 13 : }
355 13 : }
356 :
357 : /// The mutable pointers that clients read to find a release
358 47 : fn is_channel_pointer(key: &str) -> bool {
359 47 : !key.starts_with("releases/")
360 47 : }
361 :
362 : #[tokio::test]
363 1 : async fn builds_the_paths_of_a_stable_release() {
364 1 : let dir = TempDir::new();
365 1 : let release = Release::build(release_args(&dir, 0)).await.unwrap();
366 :
367 1 : assert_eq!(release.release_metadata.version, Version::new(1, 2, 3));
368 1 : assert_eq!(release.release_metadata_path, "releases/1.2.3/metadata.json");
369 1 : assert_eq!(release.release_metadata.update_manifest_url, "https://get.example/releases/1.2.3/update.json");
370 1 : assert_eq!(release.release_metadata.git_commit_url, "https://github.com/ankoh/dashql/tree/abc1234");
371 1 : assert_eq!(release.channel_metadata_path, "stable.json");
372 1 : assert_eq!(release.channel_update_manifest_path, "stable-update.json");
373 :
374 1 : let mut uploads: Vec<&String> = release.file_uploads.keys().collect();
375 1 : uploads.sort();
376 1 : assert_eq!(uploads, vec![
377 : "releases/1.2.3/macos/arm64/DashQL-arm64.dmg",
378 1 : "releases/1.2.3/macos/arm64/DashQL-arm64.zip",
379 1 : "releases/1.2.3/macos/arm64/DashQL-arm64.zip.blockmap",
380 1 : "releases/1.2.3/macos/x64/DashQL-x64.dmg",
381 1 : "releases/1.2.3/macos/x64/DashQL-x64.zip",
382 1 : "releases/1.2.3/macos/x64/DashQL-x64.zip.blockmap",
383 : ]);
384 1 : assert_eq!(release.release_metadata.bundles[0].url, "https://get.example/releases/1.2.3/macos/arm64/DashQL-arm64.dmg");
385 1 : assert_eq!(release.release_metadata.bundles.len(), 2);
386 1 : assert_eq!(release.release_update_manifest.platforms[&BundleTarget {platform: Platform::Darwin, arch: Architecture::Aarch64}].url,
387 : "https://get.example/releases/1.2.3/macos/arm64/DashQL-arm64.zip");
388 1 : assert_eq!(release.release_update_manifest.platforms[&BundleTarget {platform: Platform::Darwin, arch: Architecture::X86_64}].url,
389 1 : "https://get.example/releases/1.2.3/macos/x64/DashQL-x64.zip");
390 1 : }
391 :
392 : #[tokio::test]
393 1 : async fn a_stable_release_leaves_the_latest_channel_alone() {
394 1 : let dir = TempDir::new();
395 1 : let release = Release::build(release_args(&dir, 0)).await.unwrap();
396 2 : let feeds: Vec<&str> = release.electron_channel_manifests.iter().map(|(path, _)| path.as_str()).collect();
397 1 : assert_eq!(feeds, vec![
398 : "channels/stable/macos/arm64/latest-mac.yml",
399 1 : "channels/stable/macos/x64/latest-mac.yml",
400 : ]);
401 1 : assert_eq!(release.latest_badge, None);
402 1 : }
403 :
404 : #[tokio::test]
405 1 : async fn a_prerelease_leaves_the_stable_channel_alone() {
406 1 : let dir = TempDir::new();
407 1 : let release = Release::build(release_args(&dir, 4)).await.unwrap();
408 1 : assert_eq!(release.release_metadata.version.to_string(), "1.2.4-dev.4");
409 1 : assert_eq!(release.channel_metadata_path, "latest.json");
410 1 : assert_eq!(release.channel_update_manifest_path, "latest-update.json");
411 2 : assert!(release.electron_channel_manifests.iter().all(|(path, _)| path.starts_with("channels/latest/")));
412 1 : }
413 :
414 : #[tokio::test]
415 1 : async fn the_electron_feed_points_to_the_versioned_artifacts() {
416 1 : let dir = TempDir::new();
417 1 : let release = Release::build(release_args(&dir, 0)).await.unwrap();
418 1 : let (_, manifest) = &release.electron_channel_manifests[0];
419 1 : let manifest = String::from_utf8(manifest.clone()).unwrap();
420 1 : let artifact = "https://get.example/releases/1.2.3/macos/arm64/DashQL-arm64.zip";
421 1 : assert!(manifest.contains(&format!(" - url: {artifact}\n")), "{manifest}");
422 1 : assert!(manifest.contains(&format!("\npath: {artifact}\n")), "{manifest}");
423 : // The checksums of the file stay valid
424 1 : assert_eq!(manifest.matches("sha512: abc").count(), 2);
425 1 : }
426 :
427 : #[tokio::test]
428 1 : async fn refuses_a_release_with_a_missing_artifact() {
429 1 : let dir = TempDir::new();
430 1 : let mut args = release_args(&dir, 0);
431 1 : args.macos_x64_zip_blockmap_path = dir.path().join("missing.blockmap");
432 1 : let error = Release::build(args).await.unwrap_err();
433 1 : assert!(error.to_string().contains("missing Electron artifact"), "{error}");
434 1 : }
435 :
436 : #[tokio::test]
437 1 : async fn refuses_a_release_with_a_missing_update_manifest() {
438 1 : let dir = TempDir::new();
439 1 : let mut args = release_args(&dir, 0);
440 1 : args.macos_arm64_update_manifest_path = dir.path().join("missing.yml");
441 1 : assert!(Release::build(args).await.is_err());
442 1 : }
443 :
444 : #[tokio::test]
445 1 : async fn publishes_every_file_of_a_release() {
446 1 : let dir = TempDir::new();
447 1 : let release = Release::build(release_args(&dir, 0)).await.unwrap();
448 1 : let s3 = FakeS3::start().await;
449 1 : release.publish(&s3.client()).await.unwrap();
450 :
451 1 : assert_eq!(s3.object_keys(), vec![
452 : "channels/stable/macos/arm64/latest-mac.yml",
453 1 : "channels/stable/macos/x64/latest-mac.yml",
454 1 : "releases/1.2.3/macos/arm64/DashQL-arm64.dmg",
455 1 : "releases/1.2.3/macos/arm64/DashQL-arm64.zip",
456 1 : "releases/1.2.3/macos/arm64/DashQL-arm64.zip.blockmap",
457 1 : "releases/1.2.3/macos/x64/DashQL-x64.dmg",
458 1 : "releases/1.2.3/macos/x64/DashQL-x64.zip",
459 1 : "releases/1.2.3/macos/x64/DashQL-x64.zip.blockmap",
460 1 : "releases/1.2.3/metadata.json",
461 1 : "releases/1.2.3/update.json",
462 1 : "stable-update.json",
463 1 : "stable.json",
464 : ]);
465 1 : assert_eq!(s3.object("releases/1.2.3/macos/arm64/DashQL-arm64.dmg").unwrap(), b"arm64 dmg");
466 1 : let pointer: serde_json::Value = serde_json::from_slice(&s3.object("stable.json").unwrap()).unwrap();
467 1 : assert_eq!(pointer["version"], "1.2.3");
468 1 : assert_eq!(s3.object("stable.json"), s3.object("releases/1.2.3/metadata.json"));
469 1 : assert_eq!(s3.object("stable-update.json"), s3.object("releases/1.2.3/update.json"));
470 1 : }
471 :
472 : #[tokio::test]
473 1 : async fn a_prerelease_moves_the_latest_channel_and_its_badge() {
474 1 : let dir = TempDir::new();
475 1 : let release = Release::build(release_args(&dir, 4)).await.unwrap();
476 1 : let s3 = FakeS3::start().await;
477 1 : release.publish(&s3.client()).await.unwrap();
478 :
479 13 : let pointers: Vec<String> = s3.object_keys().into_iter().filter(|key| is_channel_pointer(key)).collect();
480 1 : assert_eq!(pointers, vec![
481 : "channels/latest/badge.svg",
482 1 : "channels/latest/macos/arm64/latest-mac.yml",
483 1 : "channels/latest/macos/x64/latest-mac.yml",
484 1 : "latest-update.json",
485 1 : "latest.json",
486 : ]);
487 25 : let badge = s3.requests().into_iter().find(|request| request.key == "channels/latest/badge.svg").unwrap();
488 1 : assert_eq!(badge.headers.get("cache-control").map(String::as_str), Some("no-cache"));
489 1 : assert_eq!(badge.headers.get("content-type").map(String::as_str), Some("image/svg+xml"));
490 1 : }
491 :
492 : #[tokio::test]
493 1 : async fn moves_the_channel_pointers_only_after_every_immutable_file() {
494 1 : let dir = TempDir::new();
495 1 : let release = Release::build(release_args(&dir, 0)).await.unwrap();
496 1 : let s3 = FakeS3::start().await;
497 1 : release.publish(&s3.client()).await.unwrap();
498 :
499 1 : let requests = s3.requests();
500 21 : let first_pointer = requests.iter().position(|request| is_channel_pointer(&request.key)).unwrap();
501 4 : assert!(requests[first_pointer..].iter().all(|request| is_channel_pointer(&request.key)));
502 1 : assert_eq!(requests[first_pointer..].len(), 4);
503 1 : }
504 :
505 : #[tokio::test]
506 1 : async fn caches_versioned_files_forever_and_revalidates_pointers() {
507 1 : let dir = TempDir::new();
508 1 : let release = Release::build(release_args(&dir, 0)).await.unwrap();
509 1 : let s3 = FakeS3::start().await;
510 1 : release.publish(&s3.client()).await.unwrap();
511 :
512 7 : let header = |key: &str, name: &str| s3.requests().iter()
513 147 : .find(|request| request.key == key && (request.method == "PUT" || request.query.contains_key("uploads")))
514 7 : .and_then(|request| request.headers.get(name).cloned());
515 2 : for key in ["releases/1.2.3/metadata.json", "releases/1.2.3/macos/arm64/DashQL-arm64.dmg"] {
516 2 : assert_eq!(header(key, "cache-control").as_deref(), Some(IMMUTABLE), "{key}");
517 : }
518 3 : for key in ["stable.json", "stable-update.json", "channels/stable/macos/x64/latest-mac.yml"] {
519 3 : assert_eq!(header(key, "cache-control").as_deref(), Some("no-cache"), "{key}");
520 : }
521 1 : assert_eq!(header("channels/stable/macos/x64/latest-mac.yml", "content-type").as_deref(), Some("application/yaml"));
522 1 : assert_eq!(header("stable.json", "content-type").as_deref(), Some("application/json"));
523 1 : }
524 :
525 : #[tokio::test]
526 1 : async fn a_failed_artifact_upload_leaves_the_channels_alone() {
527 1 : let dir = TempDir::new();
528 1 : let release = Release::build(release_args(&dir, 0)).await.unwrap();
529 1 : let s3 = FakeS3::start().await;
530 1 : s3.deny_upload("releases/1.2.3/macos/x64/DashQL-x64.zip");
531 1 : assert!(release.publish(&s3.client()).await.is_err());
532 2 : assert!(s3.object_keys().iter().all(|key| !is_channel_pointer(key)), "{:?}", s3.object_keys());
533 1 : assert_eq!(s3.object("releases/1.2.3/metadata.json"), None);
534 1 : }
535 :
536 : #[tokio::test]
537 1 : async fn a_failed_metadata_upload_leaves_the_channels_alone() {
538 1 : let dir = TempDir::new();
539 1 : let release = Release::build(release_args(&dir, 0)).await.unwrap();
540 1 : let s3 = FakeS3::start().await;
541 1 : s3.deny_upload("releases/1.2.3/update.json");
542 1 : assert!(release.publish(&s3.client()).await.is_err());
543 7 : assert!(s3.object_keys().iter().all(|key| !is_channel_pointer(key)), "{:?}", s3.object_keys());
544 1 : }
545 :
546 : #[tokio::test]
547 1 : async fn reports_a_failed_channel_upload_after_trying_the_others() {
548 1 : let dir = TempDir::new();
549 1 : let release = Release::build(release_args(&dir, 0)).await.unwrap();
550 1 : let s3 = FakeS3::start().await;
551 1 : s3.deny_upload("stable-update.json");
552 1 : assert!(release.publish(&s3.client()).await.is_err());
553 1 : assert!(s3.object("stable.json").is_some());
554 1 : assert_eq!(s3.object("stable-update.json"), None);
555 1 : }
556 :
557 : #[tokio::test]
558 1 : async fn uploads_large_artifacts_in_parts_of_five_megabytes() {
559 1 : let s3 = FakeS3::start().await;
560 1 : let dir = TempDir::new();
561 1 : let content = vec![7u8; 5 * 1024 * 1024 + 1];
562 1 : let path = dir.write("large.dmg", &content);
563 1 : multipart_upload(&s3.client(), &path, "releases/1.2.3/large.dmg").await.unwrap();
564 :
565 1 : let parts: Vec<(String, usize)> = s3.requests().iter()
566 4 : .filter(|request| request.query.contains_key("partNumber"))
567 2 : .map(|request| (request.query["partNumber"].clone(), request.body.len()))
568 1 : .collect();
569 1 : assert_eq!(parts, vec![("1".to_string(), 5 * 1024 * 1024), ("2".to_string(), 1)]);
570 1 : assert_eq!(s3.object("releases/1.2.3/large.dmg").unwrap(), content);
571 1 : }
572 :
573 : #[tokio::test]
574 1 : async fn fails_for_an_artifact_that_vanished() {
575 1 : let s3 = FakeS3::start().await;
576 1 : let dir = TempDir::new();
577 1 : assert!(multipart_upload(&s3.client(), &dir.path().join("gone.dmg"), "releases/gone.dmg").await.is_err());
578 1 : assert_eq!(s3.object("releases/gone.dmg"), None);
579 1 : }
580 : }
581 :
582 59 : async fn multipart_upload(
583 59 : client: &aws_sdk_s3::Client,
584 59 : source_path: &PathBuf,
585 59 : remote_path: &str,
586 59 : ) -> anyhow::Result<()> {
587 : // Create multipart upload
588 59 : let create_upload = client
589 59 : .create_multipart_upload()
590 59 : .bucket("dashql-get")
591 59 : .key(remote_path)
592 59 : .content_type("application/octet-stream")
593 59 : .cache_control("public, max-age=31536000, immutable")
594 59 : .send()
595 59 : .await?;
596 58 : let upload_id = create_upload.upload_id().unwrap();
597 :
598 : // Read file in chunks
599 58 : let file = std::fs::File::open(source_path)?;
600 57 : let mut reader = std::io::BufReader::new(file);
601 57 : let mut part_number = 1;
602 57 : let mut parts = Vec::new();
603 57 : let mut buffer = vec![0; 5 * 1024 * 1024]; // 5MB chunks
604 :
605 : loop {
606 115 : let bytes_read = reader.read(&mut buffer)?;
607 115 : if bytes_read == 0 {
608 57 : break;
609 58 : }
610 :
611 58 : let part = client
612 58 : .upload_part()
613 58 : .bucket("dashql-get")
614 58 : .key(remote_path)
615 58 : .upload_id(upload_id)
616 58 : .part_number(part_number)
617 58 : .body(ByteStream::from(buffer[..bytes_read].to_vec()))
618 58 : .send()
619 58 : .await?;
620 :
621 58 : parts.push(aws_sdk_s3::types::CompletedPart::builder()
622 58 : .part_number(part_number)
623 58 : .e_tag(part.e_tag().unwrap())
624 58 : .build());
625 :
626 58 : part_number += 1;
627 : }
628 :
629 : // Complete multipart upload
630 57 : client
631 57 : .complete_multipart_upload()
632 57 : .bucket("dashql-get")
633 57 : .key(remote_path)
634 57 : .upload_id(upload_id)
635 57 : .multipart_upload(aws_sdk_s3::types::CompletedMultipartUpload::builder()
636 57 : .set_parts(Some(parts))
637 57 : .build())
638 57 : .send()
639 57 : .await?;
640 :
641 57 : Ok(())
642 59 : }
|
